{
  "payload": {
    "schema_version": "1",
    "run_id": "20261101T003952Z-a4ec8e",
    "mode": "hosted",
    "started_at": "2026-11-01T00:39:52.942625Z",
    "finished_at": "2026-11-01T00:57:55.762378Z",
    "tool": {
      "version": "0.1.0",
      "commit": "4fabdb2909e645622f9bb514c1180e2d0c7e7130",
      "image_digest": "sha256:2e89150c73f10947df66c017dd905ae0208b9a839d87f26a940f441bccdbeb9f"
    },
    "dataset": {
      "name": "locomo",
      "version": "v1",
      "hash": "sha256:9c7812f2f45ce12b06cd9d3fc55736c3a8a83d2b263c11f28c292c475f6ddf38",
      "curated": true,
      "license": "CC-BY-NC-4.0",
      "upstream_sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
      "errata_applied": 0,
      "selection": null,
      "conversation_ids": null,
      "questions": 1986
    },
    "connector": {
      "id": "baselines/bm25",
      "code_hash": "sha256:1ef7f15e03d56b16b7c8535b4862e2ac398cf13925bee541a221e44179597b0d",
      "target": "BM25 keyword retrieval over raw turns (baseline)",
      "label": "maintainer-written",
      "sdk": null,
      "settings": {
        "k1": 1.5,
        "b": 0.75
      },
      "models": {},
      "endpoint": null,
      "custom_endpoint": false
    },
    "judge": {
      "route": "anthropic",
      "model": "claude-sonnet-5-5",
      "upstream_provider": null,
      "effort": "medium",
      "max_output_tokens": 4000,
      "prompt_version": "v3",
      "prompt_hash": "sha256:ce6d9750331c79d1bb3a0737109c458fef4ba9fd8cf929c75c1c1cf9407a4b78",
      "served_by": [
        "claude-sonnet-5-5"
      ],
      "official": true,
      "season": "1"
    },
    "config": {
      "budget_tokens": 8192,
      "tokenizer": "o200k_base",
      "concurrency": 4,
      "judge_concurrency": 8
    },
    "metrics": {
      "top10": {
        "overall": {
          "questions": 1986,
          "correct": 1275,
          "accuracy": 0.6419939577039275,
          "accuracy_ci95": [
            0.6287722920354274,
            0.6551394315433309
          ],
          "context_tokens_mean": 451.0558912386707,
          "context_tokens_p95": 586.0,
          "efficiency": 142.3313540902682,
          "retrieval_latency_p50_s": 0.003209389000005558,
          "retrieval_latency_p95_s": 0.005959355000001665,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9551569506726457
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 1977,
            "value": 0.5909404528717025
          },
          "truncation_rate": 0.0,
          "flagged": 20,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 426,
            "accuracy": 0.9551569506726457,
            "accuracy_ci95": [
              0.9396374473254623,
              0.971764705882353
            ],
            "context_tokens_mean": 453.46188340807174,
            "context_tokens_p95": 577.0,
            "efficiency": 210.63665671169477,
            "retrieval_latency_p50_s": 0.003122416499991232,
            "retrieval_latency_p95_s": 0.005953151749992003,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9551569506726457
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 446,
              "value": 0.6681614349775785
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 55,
            "accuracy": 0.1950354609929078,
            "accuracy_ci95": [
              0.14705030660913015,
              0.23935380642263646
            ],
            "context_tokens_mean": 452.0070921985816,
            "context_tokens_p95": 601.9,
            "efficiency": 43.148761238290994,
            "retrieval_latency_p50_s": 0.0032746100000053957,
            "retrieval_latency_p95_s": 0.005906985050037859,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 281,
              "value": 0.2735061529592341
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 30,
            "accuracy": 0.3125,
            "accuracy_ci95": [
              0.189873417721519,
              0.4315849282296651
            ],
            "context_tokens_mean": 424.1041666666667,
            "context_tokens_p95": 555.75,
            "efficiency": 73.6847276121236,
            "retrieval_latency_p50_s": 0.0035785935000092195,
            "retrieval_latency_p95_s": 0.0063421599999600176,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 89,
              "value": 0.25674958440986195
            },
            "truncation_rate": 0.0,
            "flagged": 1,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 556,
            "accuracy": 0.6611177170035671,
            "accuracy_ci95": [
              0.6297408694749949,
              0.6979870552018203
            ],
            "context_tokens_mean": 453.62068965517244,
            "context_tokens_p95": 586.0,
            "efficiency": 145.74240815738082,
            "retrieval_latency_p50_s": 0.00314272200000687,
            "retrieval_latency_p95_s": 0.005840485000021545,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 841,
              "value": 0.6603250099088387
            },
            "truncation_rate": 0.0,
            "flagged": 8,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 208,
            "accuracy": 0.6479750778816199,
            "accuracy_ci95": [
              0.5800540263932217,
              0.7173943748464751
            ],
            "context_tokens_mean": 448.2180685358255,
            "context_tokens_p95": 595.0,
            "efficiency": 144.5669247556958,
            "retrieval_latency_p50_s": 0.003232402000037382,
            "retrieval_latency_p95_s": 0.0060840640000492385,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 320,
              "value": 0.67265625
            },
            "truncation_rate": 0.0,
            "flagged": 7,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 0.26187452099998154,
          "per_turn_s": 4.4521339850387886e-05,
          "per_session_p50_s": 0.00041579899999533154,
          "per_session_p95_s": 0.0008529090499990842,
          "api_calls": 0,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 10.167292,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q161",
          "conv-26:q5",
          "conv-26:q94",
          "conv-30:q57",
          "conv-30:q95",
          "conv-41:q48",
          "conv-41:q68",
          "conv-42:q119",
          "conv-42:q73",
          "conv-43:q135",
          "conv-43:q164",
          "conv-43:q234",
          "conv-43:q6",
          "conv-43:q60",
          "conv-44:q5",
          "conv-47:q72",
          "conv-48:q116",
          "conv-49:q161",
          "conv-49:q74",
          "conv-49:q80"
        ]
      },
      "top30": {
        "overall": {
          "questions": 1986,
          "correct": 1405,
          "accuracy": 0.7074521651560927,
          "accuracy_ci95": [
            0.6937749929819134,
            0.7218182562958428
          ],
          "context_tokens_mean": 1330.714501510574,
          "context_tokens_p95": 1696.75,
          "efficiency": 53.16333175546078,
          "retrieval_latency_p50_s": 0.006425280499996688,
          "retrieval_latency_p95_s": 0.012062503750001952,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9327354260089686
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 1977,
            "value": 0.6862622154879401
          },
          "truncation_rate": 0.0,
          "flagged": 22,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 416,
            "accuracy": 0.9327354260089686,
            "accuracy_ci95": [
              0.9051700350156048,
              0.9562225123092288
            ],
            "context_tokens_mean": 1351.865470852018,
            "context_tokens_p95": 1682.0,
            "efficiency": 68.99617203930129,
            "retrieval_latency_p50_s": 0.005567306500012137,
            "retrieval_latency_p95_s": 0.011848754000013173,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9327354260089686
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 446,
              "value": 0.7645739910313901
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 86,
            "accuracy": 0.3049645390070922,
            "accuracy_ci95": [
              0.23942713520178308,
              0.36239370581579045
            ],
            "context_tokens_mean": 1295.5212765957447,
            "context_tokens_p95": 1682.6499999999999,
            "efficiency": 23.539909727183396,
            "retrieval_latency_p50_s": 0.007112251499989952,
            "retrieval_latency_p95_s": 0.012061978550019599,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 281,
              "value": 0.38353039127528754
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 36,
            "accuracy": 0.375,
            "accuracy_ci95": [
              0.22075424575424574,
              0.51
            ],
            "context_tokens_mean": 1231.4479166666667,
            "context_tokens_p95": 1690.25,
            "efficiency": 30.45195780712068,
            "retrieval_latency_p50_s": 0.006407176000010395,
            "retrieval_latency_p95_s": 0.013218978499992318,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 89,
              "value": 0.33953213563259826
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 636,
            "accuracy": 0.7562425683709869,
            "accuracy_ci95": [
              0.7222813478840078,
              0.7922263884278777
            ],
            "context_tokens_mean": 1357.3412604042805,
            "context_tokens_p95": 1705.0,
            "efficiency": 55.71499153762864,
            "retrieval_latency_p50_s": 0.006340613000020312,
            "retrieval_latency_p95_s": 0.011919027999965692,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 841,
              "value": 0.7615933412604042
            },
            "truncation_rate": 0.0,
            "flagged": 9,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 231,
            "accuracy": 0.719626168224299,
            "accuracy_ci95": [
              0.6478385481231715,
              0.7949526813880127
            ],
            "context_tokens_mean": 1292.1713395638628,
            "context_tokens_p95": 1753.0,
            "efficiency": 55.69123429615683,
            "retrieval_latency_p50_s": 0.006953363000036461,
            "retrieval_latency_p95_s": 0.012444528999992599,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 320,
              "value": 0.7414062499999998
            },
            "truncation_rate": 0.0,
            "flagged": 9,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 0.26187452099998154,
          "per_turn_s": 4.4521339850387886e-05,
          "per_session_p50_s": 0.00041579899999533154,
          "per_session_p95_s": 0.0008529090499990842,
          "api_calls": 0,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 15.292102,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q161",
          "conv-26:q5",
          "conv-30:q57",
          "conv-30:q95",
          "conv-41:q129",
          "conv-41:q48",
          "conv-42:q119",
          "conv-42:q24",
          "conv-43:q135",
          "conv-43:q164",
          "conv-43:q234",
          "conv-43:q6",
          "conv-43:q60",
          "conv-44:q5",
          "conv-47:q72",
          "conv-48:q116",
          "conv-49:q144",
          "conv-49:q24",
          "conv-49:q74",
          "conv-49:q80",
          "conv-50:q151",
          "conv-50:q165"
        ]
      }
    },
    "price_table_version": "2026-10-04",
    "artifacts": [
      {
        "name": "conversations.jsonl",
        "sha256": "sha256:355db4834e174b0b5630afea7a818e966d4403453ba9e5f74dea9568410c8973",
        "url": "https://artifacts.memjudge.org/runs/20261101T003952Z-a4ec8e/conversations.jsonl"
      },
      {
        "name": "questions-top10.jsonl",
        "sha256": "sha256:d34d5d2cf1fd0e19cca5f9087bd46d9782a047c52f73dd05791f2311e748e889",
        "url": "https://artifacts.memjudge.org/runs/20261101T003952Z-a4ec8e/questions-top10.jsonl"
      },
      {
        "name": "questions-top30.jsonl",
        "sha256": "sha256:6dd8f5027eda5262ea62e1d19ed756e2cdead2428ef9ab24228b1806fe0f9818",
        "url": "https://artifacts.memjudge.org/runs/20261101T003952Z-a4ec8e/questions-top30.jsonl"
      }
    ],
    "affiliation": {
      "affiliated": false,
      "system": null
    },
    "reused_ingestion": null,
    "rejudged_from": null,
    "resumes": 0,
    "history": [],
    "key_id": "478df5eb628bfa74"
  },
  "signature": {
    "alg": "Ed25519",
    "key_id": "478df5eb628bfa74",
    "value": "uksxGoP/V+nxgK3yKymOq5UraAXvn2OHoM183CF+i2eApnHXe/Ebu7/QI1MgqdkcIZNK0fdTuNvVntUzEV/2DQ=="
  }
}