{
  "payload": {
    "schema_version": "1",
    "run_id": "20261101T190122Z-157588",
    "mode": "hosted",
    "started_at": "2026-11-01T19:01:22.354209Z",
    "finished_at": "2026-11-01T22:14:34.661524Z",
    "tool": {
      "version": "0.1.0",
      "commit": "a73cf373d9587b7ade768783d2ac75a520c9c1ed",
      "image_digest": "sha256:8119890e8697d791a388f2397208dbc8cb236a01319e1d74e36fd616660fce2c"
    },
    "dataset": {
      "name": "locomo",
      "version": "v1",
      "hash": "sha256:9c7812f2f45ce12b06cd9d3fc55736c3a8a83d2b263c11f28c292c475f6ddf38",
      "curated": true,
      "license": "CC-BY-NC-4.0",
      "upstream_sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
      "errata_applied": 0,
      "selection": null,
      "conversation_ids": null,
      "questions": 1986
    },
    "connector": {
      "id": "recalld/cloud",
      "code_hash": "sha256:2c22ea086202c1a0314c13ce277063b23d34661d5224589f4409344c6a2094c4",
      "target": "Recalld (hosted API, v1)",
      "label": "vendor-maintained",
      "sdk": null,
      "settings": {
        "retrieval": "search",
        "decompose_model": "openai-gpt-6-luna",
        "evaluate_model": "openai-gpt-6-luna",
        "select_model": "openai-gpt-6-luna",
        "embedding_model": "gemini-embedding-002-vertex",
        "base_url": null,
        "max_retries": 4,
        "timeout_s": 300,
        "batch": false,
        "allow_non_eu": false,
        "batch_poll_s": 60
      },
      "models": {
        "decompose_model": "openai-gpt-6-luna",
        "evaluate_model": "openai-gpt-6-luna",
        "select_model": "openai-gpt-6-luna",
        "embedding_model": "gemini-embedding-002-vertex"
      },
      "endpoint": "https://us.recalld.ai",
      "custom_endpoint": false
    },
    "judge": {
      "route": "anthropic",
      "model": "claude-sonnet-5-5",
      "upstream_provider": null,
      "effort": "medium",
      "max_output_tokens": 4000,
      "prompt_version": "v3",
      "prompt_hash": "sha256:ce6d9750331c79d1bb3a0737109c458fef4ba9fd8cf929c75c1c1cf9407a4b78",
      "served_by": [
        "claude-sonnet-5-5"
      ],
      "official": true,
      "season": "1"
    },
    "config": {
      "budget_tokens": 8192,
      "tokenizer": "o200k_base",
      "concurrency": 4,
      "judge_concurrency": 8
    },
    "metrics": {
      "top10": {
        "overall": {
          "questions": 1986,
          "correct": 1599,
          "accuracy": 0.8051359516616314,
          "accuracy_ci95": [
            0.787100968091346,
            0.8248306891073979
          ],
          "context_tokens_mean": 403.6737160120846,
          "context_tokens_p95": 503.75,
          "efficiency": 199.4521614177943,
          "retrieval_latency_p50_s": 0.2888262735002627,
          "retrieval_latency_p95_s": 2.3471462847503517,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9394618834080718
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 25,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 419,
            "accuracy": 0.9394618834080718,
            "accuracy_ci95": [
              0.9207030976854765,
              0.9585572145930178
            ],
            "context_tokens_mean": 390.0291479820628,
            "context_tokens_p95": 480.25,
            "efficiency": 240.86966019557008,
            "retrieval_latency_p50_s": 0.2962490109994178,
            "retrieval_latency_p95_s": 2.365322077999963,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9394618834080718
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 3,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 135,
            "accuracy": 0.4787234042553192,
            "accuracy_ci95": [
              0.4,
              0.5592788200245576
            ],
            "context_tokens_mean": 391.8936170212766,
            "context_tokens_p95": 489.95,
            "efficiency": 122.15646886367338,
            "retrieval_latency_p50_s": 0.28110425150021,
            "retrieval_latency_p95_s": 0.410543059201154,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 1,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 45,
            "accuracy": 0.46875,
            "accuracy_ci95": [
              0.2765686061658706,
              0.6585566872152239
            ],
            "context_tokens_mean": 384.2083333333333,
            "context_tokens_p95": 487.0,
            "efficiency": 122.00412102808806,
            "retrieval_latency_p50_s": 0.2815868094999132,
            "retrieval_latency_p95_s": 0.9403790482497243,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 739,
            "accuracy": 0.8787158145065398,
            "accuracy_ci95": [
              0.837800740839357,
              0.9151996378705294
            ],
            "context_tokens_mean": 405.3674197384067,
            "context_tokens_p95": 509.0,
            "efficiency": 216.77021184228278,
            "retrieval_latency_p50_s": 0.28813349100028063,
            "retrieval_latency_p95_s": 2.356495503999213,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 7,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 261,
            "accuracy": 0.8130841121495327,
            "accuracy_ci95": [
              0.7546583850931677,
              0.8695753129627137
            ],
            "context_tokens_mean": 434.3644859813084,
            "context_tokens_p95": 530.0,
            "efficiency": 187.18936248036664,
            "retrieval_latency_p50_s": 0.28414042300028086,
            "retrieval_latency_p95_s": 0.39696171099967614,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 14,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 34844.638361579004,
          "per_turn_s": 5.923943958105917,
          "per_session_p50_s": 116.36512003750022,
          "per_session_p95_s": 214.87939649764962,
          "api_calls": 5902,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 9.990134,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q161",
          "conv-26:q5",
          "conv-26:q94",
          "conv-30:q34",
          "conv-30:q95",
          "conv-41:q48",
          "conv-41:q85",
          "conv-42:q127",
          "conv-42:q24",
          "conv-42:q244",
          "conv-42:q35",
          "conv-42:q67",
          "conv-43:q136",
          "conv-43:q55",
          "conv-43:q6",
          "conv-43:q60",
          "conv-44:q16",
          "conv-44:q5",
          "conv-48:q174",
          "conv-48:q179",
          "conv-48:q188",
          "conv-48:q71",
          "conv-48:q81",
          "conv-49:q39",
          "conv-49:q80"
        ]
      },
      "top30": {
        "overall": {
          "questions": 1986,
          "correct": 1703,
          "accuracy": 0.8575025176233635,
          "accuracy_ci95": [
            0.8456625963522438,
            0.8717447530806309
          ],
          "context_tokens_mean": 1198.8690835850957,
          "context_tokens_p95": 1403.0,
          "efficiency": 71.52595136235307,
          "retrieval_latency_p50_s": 0.2874704224996094,
          "retrieval_latency_p95_s": 2.3500681894997797,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9237668161434978
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 19,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 412,
            "accuracy": 0.9237668161434978,
            "accuracy_ci95": [
              0.9069210823642918,
              0.9402333122429611
            ],
            "context_tokens_mean": 1175.0515695067265,
            "context_tokens_p95": 1385.75,
            "efficiency": 78.61500210848489,
            "retrieval_latency_p50_s": 0.2924492040001496,
            "retrieval_latency_p95_s": 2.3762780882500465,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9237668161434978
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 182,
            "accuracy": 0.6453900709219859,
            "accuracy_ci95": [
              0.589504290876242,
              0.6977611940298507
            ],
            "context_tokens_mean": 1166.627659574468,
            "context_tokens_p95": 1358.9,
            "efficiency": 55.32099857442043,
            "retrieval_latency_p50_s": 0.28064783300033014,
            "retrieval_latency_p95_s": 0.4768093634004338,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 1,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 54,
            "accuracy": 0.5625,
            "accuracy_ci95": [
              0.3957589285714286,
              0.7473710792131845
            ],
            "context_tokens_mean": 1142.9583333333333,
            "context_tokens_p95": 1323.25,
            "efficiency": 49.214392475666216,
            "retrieval_latency_p50_s": 0.288158735500474,
            "retrieval_latency_p95_s": 0.9744836330000908,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 775,
            "accuracy": 0.9215219976218787,
            "accuracy_ci95": [
              0.8970588235294118,
              0.9450180412790982
            ],
            "context_tokens_mean": 1203.2271105826396,
            "context_tokens_p95": 1422.0,
            "efficiency": 76.5875360949646,
            "retrieval_latency_p50_s": 0.2941071130007913,
            "retrieval_latency_p95_s": 2.3581778400002804,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 3,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 280,
            "accuracy": 0.8722741433021807,
            "accuracy_ci95": [
              0.8388035628310062,
              0.9041982913154583
            ],
            "context_tokens_mean": 1265.588785046729,
            "context_tokens_p95": 1424.0,
            "efficiency": 68.92239830253978,
            "retrieval_latency_p50_s": 0.27771087099972647,
            "retrieval_latency_p95_s": 2.18311945700043,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 11,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 34844.638361579004,
          "per_turn_s": 5.923943958105917,
          "per_session_p50_s": 116.36512003750022,
          "per_session_p95_s": 214.87939649764962,
          "api_calls": 5902,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 14.450086,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q161",
          "conv-26:q184",
          "conv-26:q5",
          "conv-30:q57",
          "conv-30:q95",
          "conv-41:q48",
          "conv-41:q85",
          "conv-42:q35",
          "conv-42:q52",
          "conv-43:q234",
          "conv-43:q56",
          "conv-43:q6",
          "conv-43:q60",
          "conv-44:q5",
          "conv-47:q31",
          "conv-48:q188",
          "conv-48:q81",
          "conv-49:q74",
          "conv-49:q80"
        ]
      }
    },
    "price_table_version": "2026-10-04",
    "artifacts": [
      {
        "name": "conversations.jsonl",
        "sha256": "sha256:1f72a97a1b27c550770222751705b0acf9d1ad17a86945ad3809ff7cb59322eb",
        "url": "https://artifacts.memjudge.org/runs/20261101T190122Z-157588/conversations.jsonl"
      },
      {
        "name": "questions-top10.jsonl",
        "sha256": "sha256:5b9703f445b4bec92a443bd6357a46377821104218c41969b657bdcabe40d91a",
        "url": "https://artifacts.memjudge.org/runs/20261101T190122Z-157588/questions-top10.jsonl"
      },
      {
        "name": "questions-top30.jsonl",
        "sha256": "sha256:d64dd97190865648c1de54af3d5846de5405974e2160d9652017cccf9190cb90",
        "url": "https://artifacts.memjudge.org/runs/20261101T190122Z-157588/questions-top30.jsonl"
      }
    ],
    "affiliation": {
      "affiliated": true,
      "system": "Recalld"
    },
    "reused_ingestion": null,
    "rejudged_from": null,
    "resumes": 0,
    "history": [],
    "key_id": "478df5eb628bfa74"
  },
  "signature": {
    "alg": "Ed25519",
    "key_id": "478df5eb628bfa74",
    "value": "jMCT5sOjafus0n551G8WYT2UcRWs8e5YPTL5r0E723yeIUo8BT//Ua/Vie2zbbVVfDfqviSfu4JPf3Q11rdlDg=="
  }
}