{ "benchmark": "multilingual-legal-attribution-2026-08", "runDate": "2026-08-30", "scope": "English extracted contract text; questions and grounded answers translated into the target language; PDF and OCR excluded", "methodology": { "generationModel": "openai/gpt-5.6-luna", "translationJudgeModel": "anthropic/claude-sonnet-4.5", "generationTemperature": 0, "seed": 20260830, "attributionBackend": "TokenPath API", "externalCallsCached": true, "secretValuesStored": false, "internationalConfigSha256": "b8ce54d682003bbbb35187d4d0e29ce7ff7dbe3295b4c6fd79e3ac3de0246e8c", "internationalCorpusSha256": "2ef85553e0491f051ed7764d387a563781371f743209732f6e904de6ec66c555", "indiaConfigSha256": "7193bb7d9a00bffea03894ed2b6edd8cd14dcaf0d0866258aa4b36dcd14f3930", "indiaCorpusSha256": "26bd292a9b807ff7f196f96b49fa0aa078b11339c2868ee7d908f2f562508526" }, "international": { "documents": 4, "claimsPerLanguage": 32, "targetLanguages": 23, "translatedClaims": 736, "validTranslatedClaims": 724, "tokenpathGoldClauseHits": 720, "baselineTop1GoldClauseHits": 636, "clearAdvantageVsTop1": 15, "baseline": { "embeddingModel": "openai/text-embedding-3-large", "topK": 5, "rerankerModel": "BAAI/bge-reranker-base", "rerankerRevision": "2cfc18c9415c912f9d8155881c133215df768a70", "reportedEmbeddingCostUsd": 0.00401505, "selection": "Return the highest-ranked reranker candidate for every claim" }, "statistics": { "method": "Paired bootstrap over claim-level hit-rate differences", "resamples": 10000, "confidenceLevel": 0.95, "clearAdvantageRule": "The lower bound of the paired 95% bootstrap confidence interval is greater than zero" }, "languages": [ {"id":"en","language":"English","valid":32,"tokenpathHit":100.0,"top1Hit":87.5,"meanIou":43.1}, {"id":"es","language":"Spanish","valid":32,"tokenpathHit":100.0,"top1Hit":96.9,"meanIou":42.7}, {"id":"fr","language":"French","valid":32,"tokenpathHit":100.0,"top1Hit":84.4,"meanIou":40.4}, {"id":"de","language":"German","valid":31,"tokenpathHit":100.0,"top1Hit":83.9,"meanIou":37.9}, {"id":"pt","language":"Portuguese","valid":32,"tokenpathHit":96.9,"top1Hit":84.4,"meanIou":39.2}, {"id":"it","language":"Italian","valid":32,"tokenpathHit":100.0,"top1Hit":87.5,"meanIou":39.1}, {"id":"nl","language":"Dutch","valid":32,"tokenpathHit":100.0,"top1Hit":87.5,"meanIou":40.0}, {"id":"pl","language":"Polish","valid":31,"tokenpathHit":100.0,"top1Hit":74.2,"meanIou":38.4}, {"id":"ru","language":"Russian","valid":32,"tokenpathHit":100.0,"top1Hit":84.4,"meanIou":39.2}, {"id":"uk","language":"Ukrainian","valid":32,"tokenpathHit":100.0,"top1Hit":81.2,"meanIou":39.6}, {"id":"ar","language":"Arabic","valid":30,"tokenpathHit":100.0,"top1Hit":86.7,"meanIou":35.5}, {"id":"fa","language":"Persian","valid":32,"tokenpathHit":100.0,"top1Hit":81.2,"meanIou":32.7}, {"id":"tr","language":"Turkish","valid":32,"tokenpathHit":96.9,"top1Hit":81.2,"meanIou":33.5}, {"id":"he","language":"Hebrew","valid":30,"tokenpathHit":100.0,"top1Hit":83.3,"meanIou":37.2}, {"id":"hi","language":"Hindi","valid":30,"tokenpathHit":96.7,"top1Hit":100.0,"meanIou":32.5}, {"id":"bn","language":"Bengali","valid":32,"tokenpathHit":100.0,"top1Hit":84.4,"meanIou":27.6}, {"id":"zh-Hans","language":"Chinese (Simplified)","valid":32,"tokenpathHit":100.0,"top1Hit":100.0,"meanIou":39.2}, {"id":"zh-Hant","language":"Chinese (Traditional)","valid":32,"tokenpathHit":100.0,"top1Hit":100.0,"meanIou":36.7}, {"id":"ja","language":"Japanese","valid":30,"tokenpathHit":100.0,"top1Hit":100.0,"meanIou":35.3}, {"id":"ko","language":"Korean","valid":31,"tokenpathHit":100.0,"top1Hit":96.8,"meanIou":32.5}, {"id":"vi","language":"Vietnamese","valid":32,"tokenpathHit":96.9,"top1Hit":81.2,"meanIou":35.6}, {"id":"id","language":"Indonesian","valid":32,"tokenpathHit":100.0,"top1Hit":87.5,"meanIou":35.1}, {"id":"th","language":"Thai","valid":32,"tokenpathHit":100.0,"top1Hit":93.8,"meanIou":28.3}, {"id":"sw","language":"Swahili","valid":31,"tokenpathHit":100.0,"top1Hit":80.6,"meanIou":31.9} ] }, "india": { "documents": 3, "claimsPerLanguage": 18, "scheduledLanguages": 22, "translatedClaims": 396, "validTranslatedClaims": 371, "goldClauseHits": 371, "languages": [ {"id":"en-Latn","language":"English","valid":18,"clauseHit":100.0,"meanIou":71.1}, {"id":"hi-Deva","language":"Hindi","valid":18,"clauseHit":100.0,"meanIou":75.4}, {"id":"bn-Beng","language":"Bengali","valid":18,"clauseHit":100.0,"meanIou":74.9}, {"id":"mr-Deva","language":"Marathi","valid":18,"clauseHit":100.0,"meanIou":71.0}, {"id":"ta-Taml","language":"Tamil","valid":18,"clauseHit":100.0,"meanIou":63.4}, {"id":"te-Telu","language":"Telugu","valid":18,"clauseHit":100.0,"meanIou":62.1}, {"id":"gu-Gujr","language":"Gujarati","valid":18,"clauseHit":100.0,"meanIou":61.5}, {"id":"kn-Knda","language":"Kannada","valid":18,"clauseHit":100.0,"meanIou":73.2}, {"id":"ml-Mlym","language":"Malayalam","valid":18,"clauseHit":100.0,"meanIou":73.7}, {"id":"pa-Guru","language":"Punjabi","valid":15,"clauseHit":100.0,"meanIou":81.0}, {"id":"ur-Arab","language":"Urdu","valid":17,"clauseHit":100.0,"meanIou":72.4}, {"id":"as-Beng","language":"Assamese","valid":18,"clauseHit":100.0,"meanIou":66.4}, {"id":"brx-Deva","language":"Bodo","valid":18,"clauseHit":100.0,"meanIou":74.1}, {"id":"doi-Deva","language":"Dogri","valid":18,"clauseHit":100.0,"meanIou":78.0}, {"id":"ks-Arab","language":"Kashmiri","valid":18,"clauseHit":100.0,"meanIou":75.1}, {"id":"kok-Deva","language":"Konkani","valid":18,"clauseHit":100.0,"meanIou":80.5}, {"id":"mai-Deva","language":"Maithili","valid":17,"clauseHit":100.0,"meanIou":71.1}, {"id":"mni-Mtei","language":"Manipuri (Meitei)","valid":6,"clauseHit":100.0,"meanIou":79.5}, {"id":"ne-Deva","language":"Nepali","valid":17,"clauseHit":100.0,"meanIou":60.6}, {"id":"or-Orya","language":"Odia","valid":18,"clauseHit":100.0,"meanIou":72.1}, {"id":"sa-Deva","language":"Sanskrit","valid":18,"clauseHit":100.0,"meanIou":75.1}, {"id":"sat-Olck","language":"Santali","valid":11,"clauseHit":100.0,"meanIou":69.4}, {"id":"sd-Arab","language":"Sindhi","valid":18,"clauseHit":100.0,"meanIou":73.6} ] }, "metrics": { "goldClauseHit": "The predicted source span has any character overlap with the exact gold source clause.", "meanIou": "Mean character intersection-over-union between the predicted source span and exact gold source clause.", "translationGate": "Only answers that passed independent semantic, language, and script validation enter attribution-only metrics." }, "limitations": [ "Synthetic short English contracts, not customer documents", "Extracted text only; PDF and OCR are not tested", "Model-generated translations with an independent LLM judge", "The international baseline always returns its highest-ranked candidate", "The Indian pilot has only 6 to 18 valid claims per language and no matched retrieval baseline" ] }