{
  "payload": {
    "schema_version": "1",
    "run_id": "20261101T011204Z-108368",
    "mode": "hosted",
    "started_at": "2026-11-01T01:12:04.064642Z",
    "finished_at": "2026-11-01T11:36:46.647141Z",
    "tool": {
      "version": "0.1.0",
      "commit": "896c4f7b13f7baa49cf6a779134b154c957a3cf7",
      "image_digest": "sha256:8671c6e11fb4468e182e776556e616e221df73a4f77051716772e8cd4238222e"
    },
    "dataset": {
      "name": "locomo",
      "version": "v1",
      "hash": "sha256:9c7812f2f45ce12b06cd9d3fc55736c3a8a83d2b263c11f28c292c475f6ddf38",
      "curated": true,
      "license": "CC-BY-NC-4.0",
      "upstream_sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
      "errata_applied": 0,
      "selection": null,
      "conversation_ids": null,
      "questions": 1986
    },
    "connector": {
      "id": "recalld/cloud",
      "code_hash": "sha256:2c22ea086202c1a0314c13ce277063b23d34661d5224589f4409344c6a2094c4",
      "target": "Recalld (hosted API, v1)",
      "label": "vendor-maintained",
      "sdk": null,
      "settings": {
        "retrieval": "recall",
        "decompose_model": "openai-gpt-6-luna",
        "evaluate_model": "openai-gpt-6-luna",
        "select_model": "openai-gpt-6-luna",
        "embedding_model": "gemini-embedding-002-vertex",
        "base_url": null,
        "max_retries": 4,
        "timeout_s": 300,
        "batch": false,
        "allow_non_eu": false,
        "batch_poll_s": 60
      },
      "models": {
        "decompose_model": "openai-gpt-6-luna",
        "evaluate_model": "openai-gpt-6-luna",
        "select_model": "openai-gpt-6-luna",
        "embedding_model": "gemini-embedding-002-vertex"
      },
      "endpoint": "https://us.recalld.ai",
      "custom_endpoint": false
    },
    "judge": {
      "route": "anthropic",
      "model": "claude-sonnet-5-5",
      "upstream_provider": null,
      "effort": "medium",
      "max_output_tokens": 4000,
      "prompt_version": "v3",
      "prompt_hash": "sha256:ce6d9750331c79d1bb3a0737109c458fef4ba9fd8cf929c75c1c1cf9407a4b78",
      "served_by": [
        "claude-sonnet-5-5"
      ],
      "official": true,
      "season": "1"
    },
    "config": {
      "budget_tokens": 8192,
      "tokenizer": "o200k_base",
      "concurrency": 4,
      "judge_concurrency": 8
    },
    "metrics": {
      "top10": {
        "overall": {
          "questions": 1986,
          "correct": 1722,
          "accuracy": 0.8670694864048338,
          "accuracy_ci95": [
            0.8503231283180026,
            0.884317609370797
          ],
          "context_tokens_mean": 420.4989929506546,
          "context_tokens_p95": 522.0,
          "efficiency": 206.2001338744191,
          "retrieval_latency_p50_s": 1.5747747640000398,
          "retrieval_latency_p95_s": 2.264117726498853,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9237668161434978
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 18,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 412,
            "accuracy": 0.9237668161434978,
            "accuracy_ci95": [
              0.8964629402129403,
              0.9480566534914361
            ],
            "context_tokens_mean": 404.109865470852,
            "context_tokens_p95": 495.5,
            "efficiency": 228.5929879655779,
            "retrieval_latency_p50_s": 1.6067921155004115,
            "retrieval_latency_p95_s": 2.2099664387501434,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9237668161434978
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 195,
            "accuracy": 0.6914893617021277,
            "accuracy_ci95": [
              0.6503456619426201,
              0.7399492438675875
            ],
            "context_tokens_mean": 439.0851063829787,
            "context_tokens_p95": 561.75,
            "efficiency": 157.4841304453167,
            "retrieval_latency_p50_s": 1.587817399500068,
            "retrieval_latency_p95_s": 2.3272244251502117,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 1,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 54,
            "accuracy": 0.5625,
            "accuracy_ci95": [
              0.36840592861464005,
              0.7272947925860548
            ],
            "context_tokens_mean": 415.4895833333333,
            "context_tokens_p95": 495.25,
            "efficiency": 135.38245543660844,
            "retrieval_latency_p50_s": 1.7000221184991915,
            "retrieval_latency_p95_s": 2.530320023999252,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 779,
            "accuracy": 0.9262782401902497,
            "accuracy_ci95": [
              0.9126310204349827,
              0.9418453583739451
            ],
            "context_tokens_mean": 414.25089179548155,
            "context_tokens_p95": 520.0,
            "efficiency": 223.60319761183752,
            "retrieval_latency_p50_s": 1.5678709329999947,
            "retrieval_latency_p95_s": 2.283688692999931,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 2,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 282,
            "accuracy": 0.8785046728971962,
            "accuracy_ci95": [
              0.8344140905107162,
              0.9179860911958704
            ],
            "context_tokens_mean": 444.809968847352,
            "context_tokens_p95": 545.0,
            "efficiency": 197.50112057373377,
            "retrieval_latency_p50_s": 1.5028399989996615,
            "retrieval_latency_p95_s": 2.013169437000215,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 11,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 28196.522013407,
          "per_turn_s": 4.7936963640610335,
          "per_session_p50_s": 96.12856999700034,
          "per_session_p95_s": 198.5485151874498,
          "api_calls": 5902,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 10.077956,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q184",
          "conv-41:q48",
          "conv-41:q80",
          "conv-42:q22",
          "conv-42:q24",
          "conv-42:q35",
          "conv-43:q165",
          "conv-43:q234",
          "conv-43:q236",
          "conv-43:q6",
          "conv-43:q60",
          "conv-44:q5",
          "conv-47:q31",
          "conv-48:q71",
          "conv-49:q54",
          "conv-49:q74",
          "conv-49:q80",
          "conv-50:q200"
        ]
      },
      "top30": {
        "overall": {
          "questions": 1986,
          "correct": 1744,
          "accuracy": 0.878147029204431,
          "accuracy_ci95": [
            0.8605877503012552,
            0.8948735886107021
          ],
          "context_tokens_mean": 459.92598187311177,
          "context_tokens_p95": 671.75,
          "efficiency": 190.93225080002145,
          "retrieval_latency_p50_s": 1.558453209999925,
          "retrieval_latency_p95_s": 2.24298101050033,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9304932735426009
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 21,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 415,
            "accuracy": 0.9304932735426009,
            "accuracy_ci95": [
              0.9060625752054366,
              0.951862084332704
            ],
            "context_tokens_mean": 439.2937219730942,
            "context_tokens_p95": 616.0,
            "efficiency": 211.815745821105,
            "retrieval_latency_p50_s": 1.578018866499633,
            "retrieval_latency_p95_s": 2.254368642750137,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9304932735426009
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 211,
            "accuracy": 0.74822695035461,
            "accuracy_ci95": [
              0.7014388489208633,
              0.792771665154265
            ],
            "context_tokens_mean": 525.1312056737588,
            "context_tokens_p95": 871.9999999999998,
            "efficiency": 142.48381019265705,
            "retrieval_latency_p50_s": 1.6167056495005454,
            "retrieval_latency_p95_s": 2.3347212547995695,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 1,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 55,
            "accuracy": 0.5729166666666666,
            "accuracy_ci95": [
              0.3917525773195876,
              0.75
            ],
            "context_tokens_mean": 500.0625,
            "context_tokens_p95": 961.0,
            "efficiency": 114.56901220680749,
            "retrieval_latency_p50_s": 1.633824268000808,
            "retrieval_latency_p95_s": 2.4394798359999186,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 783,
            "accuracy": 0.9310344827586207,
            "accuracy_ci95": [
              0.9134728067562162,
              0.9487496786091089
            ],
            "context_tokens_mean": 446.84304399524376,
            "context_tokens_p95": 614.0,
            "efficiency": 208.35828044545562,
            "retrieval_latency_p50_s": 1.5580175640006928,
            "retrieval_latency_p95_s": 2.2436794389996066,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 280,
            "accuracy": 0.8722741433021807,
            "accuracy_ci95": [
              0.8276801470588235,
              0.9159212438244697
            ],
            "context_tokens_mean": 453.58255451713393,
            "context_tokens_p95": 570.0,
            "efficiency": 192.30769230769232,
            "retrieval_latency_p50_s": 1.5010635170001478,
            "retrieval_latency_p95_s": 1.9577473700001065,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 12,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 28196.522013407,
          "per_turn_s": 4.7936963640610335,
          "per_session_p50_s": 96.12856999700034,
          "per_session_p95_s": 198.5485151874498,
          "api_calls": 5902,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 10.328274,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q138",
          "conv-26:q161",
          "conv-26:q184",
          "conv-41:q35",
          "conv-41:q48",
          "conv-42:q119",
          "conv-42:q24",
          "conv-42:q35",
          "conv-43:q136",
          "conv-43:q164",
          "conv-43:q234",
          "conv-43:q6",
          "conv-43:q60",
          "conv-44:q16",
          "conv-47:q31",
          "conv-47:q62",
          "conv-48:q71",
          "conv-49:q74",
          "conv-49:q80",
          "conv-50:q196",
          "conv-50:q2"
        ]
      }
    },
    "price_table_version": "2026-10-04",
    "artifacts": [
      {
        "name": "conversations.jsonl",
        "sha256": "sha256:e8f47a73ea371de297bd3547fbf5233b1b6b06c80bc9f6cb8d9c9946208cadbc",
        "url": "https://artifacts.memjudge.org/runs/20261101T011204Z-108368/conversations.jsonl"
      },
      {
        "name": "questions-top10.jsonl",
        "sha256": "sha256:1ea9bd871f422efc91608d499ada2139a330fca923ad44dd958ac5adce3242fe",
        "url": "https://artifacts.memjudge.org/runs/20261101T011204Z-108368/questions-top10.jsonl"
      },
      {
        "name": "questions-top30.jsonl",
        "sha256": "sha256:845feb91d240e9da396d0ac09af7fb1ec3b2f223bbe2ea670d305f93aaf4d370",
        "url": "https://artifacts.memjudge.org/runs/20261101T011204Z-108368/questions-top30.jsonl"
      }
    ],
    "affiliation": {
      "affiliated": true,
      "system": "Recalld"
    },
    "reused_ingestion": null,
    "rejudged_from": null,
    "resumes": 1,
    "history": [
      {
        "at": "2026-11-01T04:14:13.895005Z",
        "event": "stopped",
        "cause": "judge"
      },
      {
        "at": "2026-11-01T11:22:37.483976Z",
        "event": "resumed",
        "cause": null
      }
    ],
    "key_id": "478df5eb628bfa74"
  },
  "signature": {
    "alg": "Ed25519",
    "key_id": "478df5eb628bfa74",
    "value": "ad//47CtmXGBii2tGD33IPexTdDkZF9Y10Ud+UsVePPjd1+IR2/vl4ki0BYBXWIwGwoX5gMjqrQbCDP+ZzSeCQ=="
  }
}