{
  "payload": {
    "schema_version": "1",
    "run_id": "20261101T114118Z-534f50",
    "mode": "hosted",
    "started_at": "2026-11-01T11:41:18.847891Z",
    "finished_at": "2026-11-01T15:13:25.272300Z",
    "tool": {
      "version": "0.1.0",
      "commit": "785596c2bacb762d6e2429fdac7a24d7abfaf283",
      "image_digest": "sha256:e65efdddd807b5126c7abe5ccb9f1b4cad374a1f6c54c388b216e387da442467"
    },
    "dataset": {
      "name": "locomo",
      "version": "v1",
      "hash": "sha256:9c7812f2f45ce12b06cd9d3fc55736c3a8a83d2b263c11f28c292c475f6ddf38",
      "curated": true,
      "license": "CC-BY-NC-4.0",
      "upstream_sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
      "errata_applied": 0,
      "selection": null,
      "conversation_ids": null,
      "questions": 1986
    },
    "connector": {
      "id": "letta/cloud",
      "code_hash": "sha256:073c87f21cf055b89bf56a9bb39d7a206f53725f8e247318bf4d46b95e6b8f7b",
      "target": "Letta (hosted API, v1; agent-managed memory)",
      "label": "maintainer-written",
      "sdk": "letta-client==1.12.1",
      "settings": {
        "model": "openai/gpt-6-luna",
        "embedding": "openai/text-embedding-3-small",
        "base_url": null
      },
      "models": {
        "model": "openai/gpt-6-luna",
        "embedding": "openai/text-embedding-3-small"
      },
      "endpoint": "https://api.letta.com",
      "custom_endpoint": false
    },
    "judge": {
      "route": "anthropic",
      "model": "claude-sonnet-5-5",
      "upstream_provider": null,
      "effort": "medium",
      "max_output_tokens": 4000,
      "prompt_version": "v3",
      "prompt_hash": "sha256:ce6d9750331c79d1bb3a0737109c458fef4ba9fd8cf929c75c1c1cf9407a4b78",
      "served_by": [
        "claude-sonnet-5-5"
      ],
      "official": true,
      "season": "1"
    },
    "config": {
      "budget_tokens": 8192,
      "tokenizer": "o200k_base",
      "concurrency": 3,
      "judge_concurrency": 8
    },
    "metrics": {
      "top10": {
        "overall": {
          "questions": 1986,
          "correct": 1096,
          "accuracy": 0.5518630412890232,
          "accuracy_ci95": [
            0.531111848743137,
            0.5737028539226342
          ],
          "context_tokens_mean": 509.6535750251762,
          "context_tokens_p95": 655.0,
          "efficiency": 108.2819915982659,
          "retrieval_latency_p50_s": 1.6950138464999327,
          "retrieval_latency_p95_s": 2.814056284000344,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9125560538116592
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 8,
          "errors": 41
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 407,
            "accuracy": 0.9125560538116592,
            "accuracy_ci95": [
              0.8841441322021916,
              0.9383267455192806
            ],
            "context_tokens_mean": 505.1502242152466,
            "context_tokens_p95": 644.75,
            "efficiency": 180.65043032086538,
            "retrieval_latency_p50_s": 1.710855531499874,
            "retrieval_latency_p95_s": 2.7766928000000917,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9125560538116592
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 5,
            "errors": 7
          },
          "multi-hop": {
            "questions": 282,
            "correct": 71,
            "accuracy": 0.25177304964539005,
            "accuracy_ci95": [
              0.18017567567567566,
              0.3379973586602226
            ],
            "context_tokens_mean": 502.1808510638298,
            "context_tokens_p95": 658.95,
            "efficiency": 50.13593192811495,
            "retrieval_latency_p50_s": 1.6920051625000951,
            "retrieval_latency_p95_s": 2.77453039194952,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 4
          },
          "open-domain": {
            "questions": 96,
            "correct": 32,
            "accuracy": 0.3333333333333333,
            "accuracy_ci95": [
              0.18604651162790697,
              0.46534653465346537
            ],
            "context_tokens_mean": 489.0729166666667,
            "context_tokens_p95": 635.25,
            "efficiency": 68.1561628080339,
            "retrieval_latency_p50_s": 1.7136199110000234,
            "retrieval_latency_p95_s": 2.817699026749551,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 3
          },
          "single-hop": {
            "questions": 841,
            "correct": 571,
            "accuracy": 0.6789536266349584,
            "accuracy_ci95": [
              0.6429336655888955,
              0.7151782569453617
            ],
            "context_tokens_mean": 517.9964328180737,
            "context_tokens_p95": 658.0,
            "efficiency": 131.07303132209304,
            "retrieval_latency_p50_s": 1.688800478000303,
            "retrieval_latency_p95_s": 2.817751825999949,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 3,
            "errors": 20
          },
          "temporal": {
            "questions": 321,
            "correct": 15,
            "accuracy": 0.04672897196261682,
            "accuracy_ci95": [
              0.025732937405731522,
              0.06764943074003796
            ],
            "context_tokens_mean": 506.7725856697819,
            "context_tokens_p95": 657.0,
            "efficiency": 9.22089577928864,
            "retrieval_latency_p50_s": 1.6940144560001045,
            "retrieval_latency_p95_s": 2.828045812999335,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 7
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 24689.194870637002,
          "per_turn_s": 4.197414972906665,
          "per_session_p50_s": 80.88461753599995,
          "per_session_p95_s": 167.4171360764,
          "api_calls": 5913,
          "input_tokens": 301717465,
          "output_tokens": 759914,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 10.633684,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4203,
          "value": 0.009754936949797764
        },
        "flagged_question_ids": [
          "conv-26:q161",
          "conv-26:q184",
          "conv-43:q165",
          "conv-43:q234",
          "conv-43:q86",
          "conv-47:q72",
          "conv-50:q165",
          "conv-50:q200"
        ]
      },
      "top30": {
        "overall": {
          "questions": 1986,
          "correct": 1222,
          "accuracy": 0.6153071500503524,
          "accuracy_ci95": [
            0.5914804683016414,
            0.6392514482943271
          ],
          "context_tokens_mean": 1438.514098690836,
          "context_tokens_p95": 1792.75,
          "efficiency": 42.773800452170164,
          "retrieval_latency_p50_s": 1.714071337000405,
          "retrieval_latency_p95_s": 2.792571880250307,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.8654708520179372
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 3,
          "errors": 43
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 386,
            "accuracy": 0.8654708520179372,
            "accuracy_ci95": [
              0.8381733538195435,
              0.8921587055874982
            ],
            "context_tokens_mean": 1423.5717488789237,
            "context_tokens_p95": 1757.75,
            "efficiency": 60.7957310686661,
            "retrieval_latency_p50_s": 1.7023502739998548,
            "retrieval_latency_p95_s": 2.371947138250107,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.8654708520179372
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 3,
            "errors": 11
          },
          "multi-hop": {
            "questions": 282,
            "correct": 122,
            "accuracy": 0.4326241134751773,
            "accuracy_ci95": [
              0.3782747701736466,
              0.4891709000130816
            ],
            "context_tokens_mean": 1409.8687943262412,
            "context_tokens_p95": 1800.6499999999999,
            "efficiency": 30.685416629986698,
            "retrieval_latency_p50_s": 1.7356435354995483,
            "retrieval_latency_p95_s": 2.813213026549965,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 6
          },
          "open-domain": {
            "questions": 96,
            "correct": 39,
            "accuracy": 0.40625,
            "accuracy_ci95": [
              0.24443798449612403,
              0.556840034965035
            ],
            "context_tokens_mean": 1380.9479166666667,
            "context_tokens_p95": 1780.5,
            "efficiency": 29.41819855021083,
            "retrieval_latency_p50_s": 1.7398107520002668,
            "retrieval_latency_p95_s": 2.851588715499247,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 3
          },
          "single-hop": {
            "questions": 841,
            "correct": 658,
            "accuracy": 0.7824019024970273,
            "accuracy_ci95": [
              0.7417935549731768,
              0.8287461773700305
            ],
            "context_tokens_mean": 1469.0047562425684,
            "context_tokens_p95": 1812.0,
            "efficiency": 53.26067864465333,
            "retrieval_latency_p50_s": 1.7063240950001273,
            "retrieval_latency_p95_s": 2.7849735750005493,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 16
          },
          "temporal": {
            "questions": 321,
            "correct": 17,
            "accuracy": 0.0529595015576324,
            "accuracy_ci95": [
              0.02517929052984028,
              0.08181951089845826
            ],
            "context_tokens_mean": 1421.7725856697818,
            "context_tokens_p95": 1734.0,
            "efficiency": 3.724892580671314,
            "retrieval_latency_p50_s": 1.7192664449999029,
            "retrieval_latency_p95_s": 2.8149058030012384,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 7
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 24689.194870637002,
          "per_turn_s": 4.197414972906665,
          "per_session_p50_s": 80.88461753599995,
          "per_session_p95_s": 167.4171360764,
          "api_calls": 5913,
          "input_tokens": 301717465,
          "output_tokens": 759914,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 16.090826,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4201,
          "value": 0.010235658176624612
        },
        "flagged_question_ids": [
          "conv-26:q184",
          "conv-47:q176",
          "conv-50:q200"
        ]
      }
    },
    "price_table_version": "2026-10-04",
    "artifacts": [
      {
        "name": "conversations.jsonl",
        "sha256": "sha256:79fd3c3becc90de37ae19256eae336fdff90ff7f85015b30e1ccbbc098db461f",
        "url": "https://artifacts.memjudge.org/runs/20261101T114118Z-534f50/conversations.jsonl"
      },
      {
        "name": "questions-top10.jsonl",
        "sha256": "sha256:56bca9aecd015032332e07e3050a210192f34b6ec37a113d1dbb0f0f630788aa",
        "url": "https://artifacts.memjudge.org/runs/20261101T114118Z-534f50/questions-top10.jsonl"
      },
      {
        "name": "questions-top30.jsonl",
        "sha256": "sha256:d54ec99348369abd312d176b37de733a4187176d44f093c7cee47462559de8e0",
        "url": "https://artifacts.memjudge.org/runs/20261101T114118Z-534f50/questions-top30.jsonl"
      }
    ],
    "affiliation": {
      "affiliated": false,
      "system": null
    },
    "reused_ingestion": null,
    "rejudged_from": null,
    "resumes": 0,
    "history": [],
    "key_id": "478df5eb628bfa74"
  },
  "signature": {
    "alg": "Ed25519",
    "key_id": "478df5eb628bfa74",
    "value": "d1pHAOncEgwE+DTmJ/WFTGt3kK9SFs2kUgKlWmpF+TJuUaqwwf/kzRKqgCMxgxvuar1acEyAQybdihIbT+8QBA=="
  }
}