{ "schema_version": 3, "evaluation_date_kst": "2026-10-07", "evaluation_date_utc": "2026-10-07", "dataset": { "file": "evaluation-v2.json", "sha256": "ff2a2f979ad3aa849af10488dd506e6036cc5de43272dc1d7792c9b87a2bc03b", "cases": 48 }, "selection": "Explicit immutable run paths; no latest-directory selection", "models": { "Julia-1": { "directory": "results/20261007T014249", "dataset_sha256": "ff2a2f979ad3aa849af10488dd506e6036cc5de43272dc1d7792c9b87a2bc03b", "first_request": { "id": "eval_relation_hours_supports", "wall_ms": 261.06991700362414 }, "load": { "seconds": 11.145837042015046, "peak_rss_bytes": 1134444544, "device": "cpu" }, "tasks": { "evidence_relation": { "status": "complete", "planned": 24, "completed": 24, "correct": 7, "accuracy": 0.2916666666666667, "latency_all": { "n": 24, "median_ms": 24.808728994685225, "p95_ms": 29.34570799698122 }, "latency_after_first_overall": { "n": 23, "median_ms": 24.610540975118056, "p95_ms": 28.240832994924858 }, "rows": [ { "id": "eval_relation_hours_supports", "status": "completed", "expected": "supports", "choice": "insufficient", "match": false, "seconds": 0.26106991700362414 }, { "id": "eval_relation_hours_refutes", "status": "completed", "expected": "refutes", "choice": "insufficient", "match": false, "seconds": 0.022124665993032977 }, { "id": "eval_relation_hours_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.022809334011981264 }, { "id": "eval_relation_count_supports", "status": "completed", "expected": "supports", "choice": "insufficient", "match": false, "seconds": 0.02710158401168883 }, { "id": "eval_relation_count_refutes", "status": "completed", "expected": "refutes", "choice": "insufficient", "match": false, "seconds": 0.024386208009673283 }, { "id": "eval_relation_count_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.024610540975118056 }, { "id": "eval_relation_negation_supports", "status": "completed", "expected": "supports", "choice": "insufficient", "match": false, "seconds": 0.02676274999976158 }, { "id": "eval_relation_negation_refutes", "status": "completed", "expected": "refutes", "choice": "supports", "match": false, "seconds": 0.02326779198483564 }, { "id": "eval_relation_negation_insufficient", "status": "completed", "expected": "insufficient", "choice": "supports", "match": false, "seconds": 0.024116332991980016 }, { "id": "eval_relation_time_supports", "status": "completed", "expected": "supports", "choice": "insufficient", "match": false, "seconds": 0.02563566601020284 }, { "id": "eval_relation_time_refutes", "status": "completed", "expected": "refutes", "choice": "insufficient", "match": false, "seconds": 0.023820542002795264 }, { "id": "eval_relation_time_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.023011040990240872 }, { "id": "eval_relation_rule_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.02934570799698122 }, { "id": "eval_relation_rule_refutes", "status": "completed", "expected": "refutes", "choice": "supports", "match": false, "seconds": 0.025377207988640293 }, { "id": "eval_relation_rule_insufficient", "status": "completed", "expected": "insufficient", "choice": "supports", "match": false, "seconds": 0.025377415993716568 }, { "id": "eval_relation_exception_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.026403625000966713 }, { "id": "eval_relation_exception_refutes", "status": "completed", "expected": "refutes", "choice": "supports", "match": false, "seconds": 0.023746207996737212 }, { "id": "eval_relation_exception_insufficient", "status": "completed", "expected": "insufficient", "choice": "supports", "match": false, "seconds": 0.027240334020461887 }, { "id": "eval_relation_comparison_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.02442066601361148 }, { "id": "eval_relation_comparison_refutes", "status": "completed", "expected": "refutes", "choice": "insufficient", "match": false, "seconds": 0.022555625007953495 }, { "id": "eval_relation_comparison_insufficient", "status": "completed", "expected": "insufficient", "choice": "supports", "match": false, "seconds": 0.023664500011363998 }, { "id": "eval_relation_scope_supports", "status": "completed", "expected": "supports", "choice": "insufficient", "match": false, "seconds": 0.028240832994924858 }, { "id": "eval_relation_scope_refutes", "status": "completed", "expected": "refutes", "choice": "insufficient", "match": false, "seconds": 0.025006917014252394 }, { "id": "eval_relation_scope_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.028024250001180917 } ] }, "intent_routing": { "status": "complete", "planned": 24, "completed": 24, "correct": 19, "accuracy": 0.7916666666666666, "latency_all": { "n": 24, "median_ms": 21.43714549310971, "p95_ms": 24.052291992120445 }, "latency_after_first_overall": { "n": 24, "median_ms": 21.43714549310971, "p95_ms": 24.052291992120445 }, "rows": [ { "id": "eval_route_billing_01", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.02761099999770522 }, { "id": "eval_route_billing_02", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.023410458001308143 }, { "id": "eval_route_billing_03", "status": "completed", "expected": "billing", "choice": "access", "match": false, "seconds": 0.024052291992120445 }, { "id": "eval_route_billing_04", "status": "completed", "expected": "billing", "choice": "access", "match": false, "seconds": 0.021492165978997946 }, { "id": "eval_route_billing_05", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.02099058299791068 }, { "id": "eval_route_billing_06", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.023808041005395353 }, { "id": "eval_route_billing_07", "status": "completed", "expected": "billing", "choice": "access", "match": false, "seconds": 0.022156416001962498 }, { "id": "eval_route_billing_08", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.022438916988903657 }, { "id": "eval_route_shipping_01", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.021794416999910027 }, { "id": "eval_route_shipping_02", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.020132500008912757 }, { "id": "eval_route_shipping_03", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.01959820798947476 }, { "id": "eval_route_shipping_04", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.019952083006501198 }, { "id": "eval_route_shipping_05", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.018888499995227903 }, { "id": "eval_route_shipping_06", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.021563332993537188 }, { "id": "eval_route_shipping_07", "status": "completed", "expected": "shipping", "choice": "access", "match": false, "seconds": 0.021382125007221475 }, { "id": "eval_route_shipping_08", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.019558708008844405 }, { "id": "eval_route_access_01", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.020409207994816825 }, { "id": "eval_route_access_02", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.019103041995549574 }, { "id": "eval_route_access_03", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.018937000015284866 }, { "id": "eval_route_access_04", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.02173937499173917 }, { "id": "eval_route_access_05", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.022753583994926885 }, { "id": "eval_route_access_06", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.02109724999172613 }, { "id": "eval_route_access_07", "status": "completed", "expected": "access", "choice": "shipping", "match": false, "seconds": 0.02353004098404199 }, { "id": "eval_route_access_08", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.02064970799256116 } ] } }, "supervisor": "results/julia-rerun-retry-supervisor.json" }, "Kev-0.8B": { "directory": "results/kev-20261007T014408", "dataset_sha256": "ff2a2f979ad3aa849af10488dd506e6036cc5de43272dc1d7792c9b87a2bc03b", "first_request": { "id": "eval_relation_hours_supports", "wall_ms": 131.55287501285784 }, "load": { "seconds": 7.747803667007247, "peak_rss_bytes": 2235580416, "backend": "mlx", "dtype": "bf16", "mlx_peak_bytes": 2572572364 }, "tasks": { "evidence_relation": { "status": "complete", "planned": 24, "completed": 24, "correct": 14, "accuracy": 0.5833333333333334, "latency_all": { "n": 24, "median_ms": 49.09497899643611, "p95_ms": 50.24254199815914 }, "latency_after_first_overall": { "n": 23, "median_ms": 49.078249983722344, "p95_ms": 49.84095800318755 }, "rows": [ { "id": "eval_relation_hours_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.13155287501285784 }, { "id": "eval_relation_hours_refutes", "status": "completed", "expected": "refutes", "choice": "insufficient", "match": false, "seconds": 0.0488400409813039 }, { "id": "eval_relation_hours_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.04926450000493787 }, { "id": "eval_relation_count_supports", "status": "completed", "expected": "supports", "choice": "refutes", "match": false, "seconds": 0.04984095800318755 }, { "id": "eval_relation_count_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.05024254199815914 }, { "id": "eval_relation_count_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.04953941598068923 }, { "id": "eval_relation_negation_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.04876504198182374 }, { "id": "eval_relation_negation_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.04630308298510499 }, { "id": "eval_relation_negation_insufficient", "status": "completed", "expected": "insufficient", "choice": "refutes", "match": false, "seconds": 0.04805079198558815 }, { "id": "eval_relation_time_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.048708666989114136 }, { "id": "eval_relation_time_refutes", "status": "completed", "expected": "refutes", "choice": "supports", "match": false, "seconds": 0.048621958994772285 }, { "id": "eval_relation_time_insufficient", "status": "completed", "expected": "insufficient", "choice": "refutes", "match": false, "seconds": 0.04852300000493415 }, { "id": "eval_relation_rule_supports", "status": "completed", "expected": "supports", "choice": "refutes", "match": false, "seconds": 0.04934062500251457 }, { "id": "eval_relation_rule_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.049612625007284805 }, { "id": "eval_relation_rule_insufficient", "status": "completed", "expected": "insufficient", "choice": "refutes", "match": false, "seconds": 0.04911170800914988 }, { "id": "eval_relation_exception_supports", "status": "completed", "expected": "supports", "choice": "refutes", "match": false, "seconds": 0.04871275002369657 }, { "id": "eval_relation_exception_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.048397625010693446 }, { "id": "eval_relation_exception_insufficient", "status": "completed", "expected": "insufficient", "choice": "refutes", "match": false, "seconds": 0.04841687498264946 }, { "id": "eval_relation_comparison_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.04925091698532924 }, { "id": "eval_relation_comparison_refutes", "status": "completed", "expected": "refutes", "choice": "supports", "match": false, "seconds": 0.04922200000146404 }, { "id": "eval_relation_comparison_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.04888637500698678 }, { "id": "eval_relation_scope_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.04952595799113624 }, { "id": "eval_relation_scope_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.049286708992440253 }, { "id": "eval_relation_scope_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.049078249983722344 } ] }, "intent_routing": { "status": "complete", "planned": 24, "completed": 24, "correct": 24, "accuracy": 1.0, "latency_all": { "n": 24, "median_ms": 49.66637499455828, "p95_ms": 50.870833976659924 }, "latency_after_first_overall": { "n": 24, "median_ms": 49.66637499455828, "p95_ms": 50.870833976659924 }, "rows": [ { "id": "eval_route_billing_01", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.05016554199391976 }, { "id": "eval_route_billing_02", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.05084691700176336 }, { "id": "eval_route_billing_03", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.050776374991983175 }, { "id": "eval_route_billing_04", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.04903712499071844 }, { "id": "eval_route_billing_05", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.049023125000530854 }, { "id": "eval_route_billing_06", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.050870833976659924 }, { "id": "eval_route_billing_07", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.04940925000119023 }, { "id": "eval_route_billing_08", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.050697750004474074 }, { "id": "eval_route_shipping_01", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.049685749982018024 }, { "id": "eval_route_shipping_02", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.04909045799286105 }, { "id": "eval_route_shipping_03", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.048951458011288196 }, { "id": "eval_route_shipping_04", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.04919162500300445 }, { "id": "eval_route_shipping_05", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.04944066601456143 }, { "id": "eval_route_shipping_06", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.050448708003386855 }, { "id": "eval_route_shipping_07", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.05060079199029133 }, { "id": "eval_route_shipping_08", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.049149957980262116 }, { "id": "eval_route_access_01", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.05032370801200159 }, { "id": "eval_route_access_02", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.04906320801819675 }, { "id": "eval_route_access_03", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.04964700000709854 }, { "id": "eval_route_access_04", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.048798375006299466 }, { "id": "eval_route_access_05", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.0504062500258442 }, { "id": "eval_route_access_06", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.05077949998667464 }, { "id": "eval_route_access_07", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.05662920800386928 }, { "id": "eval_route_access_08", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.04926508400239982 } ] } }, "supervisor": "results/kev-rerun-retry-supervisor.json" }, "mDeBERTa": { "directory": "results/nli-20261007T014309", "dataset_sha256": "ff2a2f979ad3aa849af10488dd506e6036cc5de43272dc1d7792c9b87a2bc03b", "first_request": { "id": "eval_relation_hours_supports", "wall_ms": 77.66683399677277 }, "load": { "seconds": 6.344097083027009, "peak_rss_bytes": 2292875264, "revision": "b5113eb38ab63efdd7f280f8c144ea8b13f978ce", "device": "cpu", "torch": "2.14.1" }, "tasks": { "evidence_relation": { "status": "complete", "planned": 24, "completed": 24, "correct": 16, "accuracy": 0.6666666666666666, "latency_all": { "n": 24, "median_ms": 39.719875014270656, "p95_ms": 43.9085419930052 }, "latency_after_first_overall": { "n": 23, "median_ms": 39.64316702331416, "p95_ms": 43.21041697403416 }, "rows": [ { "id": "eval_relation_hours_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.07766683399677277 }, { "id": "eval_relation_hours_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.03964316702331416 }, { "id": "eval_relation_hours_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.03904433298157528 }, { "id": "eval_relation_count_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.03764158400008455 }, { "id": "eval_relation_count_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.0373495839885436 }, { "id": "eval_relation_count_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.03680341600556858 }, { "id": "eval_relation_negation_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.037587458005873486 }, { "id": "eval_relation_negation_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.037312792002921924 }, { "id": "eval_relation_negation_insufficient", "status": "completed", "expected": "insufficient", "choice": "refutes", "match": false, "seconds": 0.04042066598776728 }, { "id": "eval_relation_time_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.041463083995040506 }, { "id": "eval_relation_time_refutes", "status": "completed", "expected": "refutes", "choice": "supports", "match": false, "seconds": 0.040412249974906445 }, { "id": "eval_relation_time_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.04014299999107607 }, { "id": "eval_relation_rule_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.042178791016340256 }, { "id": "eval_relation_rule_refutes", "status": "completed", "expected": "refutes", "choice": "supports", "match": false, "seconds": 0.03995524998754263 }, { "id": "eval_relation_rule_insufficient", "status": "completed", "expected": "insufficient", "choice": "supports", "match": false, "seconds": 0.040586041985079646 }, { "id": "eval_relation_exception_supports", "status": "completed", "expected": "supports", "choice": "refutes", "match": false, "seconds": 0.038979750010184944 }, { "id": "eval_relation_exception_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.038358541001798585 }, { "id": "eval_relation_exception_insufficient", "status": "completed", "expected": "insufficient", "choice": "refutes", "match": false, "seconds": 0.03813358399202116 }, { "id": "eval_relation_comparison_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.037161875021411106 }, { "id": "eval_relation_comparison_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.036899291997542605 }, { "id": "eval_relation_comparison_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.03979658300522715 }, { "id": "eval_relation_scope_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.0439085419930052 }, { "id": "eval_relation_scope_refutes", "status": "completed", "expected": "refutes", "choice": "supports", "match": false, "seconds": 0.04283762499107979 }, { "id": "eval_relation_scope_insufficient", "status": "completed", "expected": "insufficient", "choice": "refutes", "match": false, "seconds": 0.04321041697403416 } ] }, "intent_routing": { "status": "not_applicable", "planned": 24, "completed": 0, "correct": null, "accuracy": null } }, "supervisor": "results/nli-rerun-supervisor.json" }, "Qwen3.5-4B": { "directory": "results/qwen-20261007T014431", "dataset_sha256": "ff2a2f979ad3aa849af10488dd506e6036cc5de43272dc1d7792c9b87a2bc03b", "first_request": { "id": "eval_relation_hours_supports", "wall_ms": 2880.852458998561 }, "load": null, "tasks": { "evidence_relation": { "status": "complete", "planned": 24, "completed": 24, "correct": 20, "accuracy": 0.8333333333333334, "latency_all": { "n": 24, "median_ms": 652.6293334900402, "p95_ms": 877.1946669730823 }, "latency_after_first_overall": { "n": 23, "median_ms": 652.5422499980778, "p95_ms": 784.6413340012077 }, "rows": [ { "id": "eval_relation_hours_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 2.880852458998561 }, { "id": "eval_relation_hours_refutes", "status": "completed", "expected": "refutes", "choice": "insufficient", "match": false, "seconds": 0.6129258330038283 }, { "id": "eval_relation_hours_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.6145015000074636 }, { "id": "eval_relation_count_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.5974443329905625 }, { "id": "eval_relation_count_refutes", "status": "completed", "expected": "refutes", "choice": "supports", "match": false, "seconds": 0.6527164169820026 }, { "id": "eval_relation_count_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.6138416670146398 }, { "id": "eval_relation_negation_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.8771946669730823 }, { "id": "eval_relation_negation_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.7846413340012077 }, { "id": "eval_relation_negation_insufficient", "status": "completed", "expected": "insufficient", "choice": "refutes", "match": false, "seconds": 0.63212416699389 }, { "id": "eval_relation_time_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.636215541017009 }, { "id": "eval_relation_time_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.6478652080113534 }, { "id": "eval_relation_time_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.6638764579838607 }, { "id": "eval_relation_rule_supports", "status": "completed", "expected": "supports", "choice": "insufficient", "match": false, "seconds": 0.7040598330204375 }, { "id": "eval_relation_rule_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.6896367500012275 }, { "id": "eval_relation_rule_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.6671232090156991 }, { "id": "eval_relation_exception_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.6525422499980778 }, { "id": "eval_relation_exception_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.6596703339891974 }, { "id": "eval_relation_exception_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.6653472500038333 }, { "id": "eval_relation_comparison_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.6285494999901857 }, { "id": "eval_relation_comparison_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.6576389159890823 }, { "id": "eval_relation_comparison_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.6667014590057079 }, { "id": "eval_relation_scope_supports", "status": "completed", "expected": "supports", "choice": "supports", "match": true, "seconds": 0.6126253329857718 }, { "id": "eval_relation_scope_refutes", "status": "completed", "expected": "refutes", "choice": "refutes", "match": true, "seconds": 0.6323799579986371 }, { "id": "eval_relation_scope_insufficient", "status": "completed", "expected": "insufficient", "choice": "insufficient", "match": true, "seconds": 0.6301925420120824 } ] }, "intent_routing": { "status": "complete", "planned": 24, "completed": 24, "correct": 24, "accuracy": 1.0, "latency_all": { "n": 24, "median_ms": 556.6631664987653, "p95_ms": 599.3522500211839 }, "latency_after_first_overall": { "n": 24, "median_ms": 556.6631664987653, "p95_ms": 599.3522500211839 }, "rows": [ { "id": "eval_route_billing_01", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.5770360410097055 }, { "id": "eval_route_billing_02", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.5993522500211839 }, { "id": "eval_route_billing_03", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.5998682499921415 }, { "id": "eval_route_billing_04", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.5868212089990266 }, { "id": "eval_route_billing_05", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.554119417007314 }, { "id": "eval_route_billing_06", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.5512136250035837 }, { "id": "eval_route_billing_07", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.5542368330061436 }, { "id": "eval_route_billing_08", "status": "completed", "expected": "billing", "choice": "billing", "match": true, "seconds": 0.5862738750001881 }, { "id": "eval_route_shipping_01", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.5648920419916976 }, { "id": "eval_route_shipping_02", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.5682052499905694 }, { "id": "eval_route_shipping_03", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.5590894999913871 }, { "id": "eval_route_shipping_04", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.5504433340101968 }, { "id": "eval_route_shipping_05", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.5440740409831051 }, { "id": "eval_route_shipping_06", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.536726209014887 }, { "id": "eval_route_shipping_07", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.570380874996772 }, { "id": "eval_route_shipping_08", "status": "completed", "expected": "shipping", "choice": "shipping", "match": true, "seconds": 0.5471575840201695 }, { "id": "eval_route_access_01", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.5432694999908563 }, { "id": "eval_route_access_02", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.5654047499992885 }, { "id": "eval_route_access_03", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.5403822080115788 }, { "id": "eval_route_access_04", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.5606280830106698 }, { "id": "eval_route_access_05", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.5392709159932565 }, { "id": "eval_route_access_06", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.5843786250043195 }, { "id": "eval_route_access_07", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.5469394590181764 }, { "id": "eval_route_access_08", "status": "completed", "expected": "access", "choice": "access", "match": true, "seconds": 0.5371789160126355 } ] } }, "supervisor": "results/qwen-rerun-v2-supervisor.json", "memory": { "sampled_daemon_group_peak_rss_bytes": 5279170560, "sampling_interval_seconds": 0.5 }, "first_request_api": { "model": "qwen3.5:4b", "created_at": "2026-10-07T01:44:34.058802Z", "message": { "role": "assistant", "content": "{\"choice\": \"supports\"}" }, "done": true, "done_reason": "stop", "total_duration": 2879602167, "load_duration": 2304803542, "prompt_eval_count": 137, "prompt_eval_cached_count": 0, "prompt_eval_duration": 411645000, "eval_count": 7, "eval_duration": 158414000 } } }, "failed_attempts": { "Julia-package-path": { "directory": "results/20261007T014232", "dataset_sha256": "ff2a2f979ad3aa849af10488dd506e6036cc5de43272dc1d7792c9b87a2bc03b", "first_request": null, "load": null, "tasks": { "evidence_relation": { "status": "load_failed", "planned": 24, "completed": 0, "correct": null, "accuracy": null, "latency_all": { "n": 0, "median_ms": null, "p95_ms": null }, "latency_after_first_overall": { "n": 0, "median_ms": null, "p95_ms": null }, "rows": [ { "id": "eval_relation_hours_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_hours_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_hours_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_count_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_count_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_count_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_negation_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_negation_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_negation_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_time_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_time_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_time_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_rule_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_rule_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_rule_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_exception_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_exception_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_exception_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_comparison_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_comparison_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_comparison_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_scope_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_scope_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_scope_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null } ] }, "intent_routing": { "status": "load_failed", "planned": 24, "completed": 0, "correct": null, "accuracy": null, "latency_all": { "n": 0, "median_ms": null, "p95_ms": null }, "latency_after_first_overall": { "n": 0, "median_ms": null, "p95_ms": null }, "rows": [ { "id": "eval_route_billing_01", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_02", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_03", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_04", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_05", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_06", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_07", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_08", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_01", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_02", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_03", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_04", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_05", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_06", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_07", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_08", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_01", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_02", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_03", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_04", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_05", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_06", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_07", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_08", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null } ] } } }, "Kev-tokenizer-cache-path": { "directory": "results/kev-20261007T014327", "dataset_sha256": "ff2a2f979ad3aa849af10488dd506e6036cc5de43272dc1d7792c9b87a2bc03b", "first_request": null, "load": null, "tasks": { "evidence_relation": { "status": "load_failed", "planned": 24, "completed": 0, "correct": null, "accuracy": null, "latency_all": { "n": 0, "median_ms": null, "p95_ms": null }, "latency_after_first_overall": { "n": 0, "median_ms": null, "p95_ms": null }, "rows": [ { "id": "eval_relation_hours_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_hours_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_hours_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_count_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_count_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_count_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_negation_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_negation_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_negation_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_time_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_time_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_time_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_rule_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_rule_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_rule_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_exception_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_exception_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_exception_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_comparison_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_comparison_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_comparison_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_scope_supports", "status": "not_run", "expected": "supports", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_scope_refutes", "status": "not_run", "expected": "refutes", "choice": null, "match": null, "seconds": null }, { "id": "eval_relation_scope_insufficient", "status": "not_run", "expected": "insufficient", "choice": null, "match": null, "seconds": null } ] }, "intent_routing": { "status": "load_failed", "planned": 24, "completed": 0, "correct": null, "accuracy": null, "latency_all": { "n": 0, "median_ms": null, "p95_ms": null }, "latency_after_first_overall": { "n": 0, "median_ms": null, "p95_ms": null }, "rows": [ { "id": "eval_route_billing_01", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_02", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_03", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_04", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_05", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_06", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_07", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_billing_08", "status": "not_run", "expected": "billing", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_01", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_02", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_03", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_04", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_05", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_06", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_07", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_shipping_08", "status": "not_run", "expected": "shipping", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_01", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_02", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_03", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_04", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_05", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_06", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_07", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null }, { "id": "eval_route_access_08", "status": "not_run", "expected": "access", "choice": null, "match": null, "seconds": null } ] } } } }, "measurement_note": "Single fixed-order pass; first request excluded only from after_first_overall distribution; quantile nearest rank; accuracy field is synthetic label agreement." }