{
  "payload": {
    "schema_version": "1",
    "run_id": "20261101T151858Z-33a8ed",
    "mode": "hosted",
    "started_at": "2026-11-01T15:18:58.258371Z",
    "finished_at": "2026-11-01T19:38:56.255287Z",
    "tool": {
      "version": "0.1.0",
      "commit": "a73cf373d9587b7ade768783d2ac75a520c9c1ed",
      "image_digest": "sha256:8119890e8697d791a388f2397208dbc8cb236a01319e1d74e36fd616660fce2c"
    },
    "dataset": {
      "name": "locomo",
      "version": "v1",
      "hash": "sha256:9c7812f2f45ce12b06cd9d3fc55736c3a8a83d2b263c11f28c292c475f6ddf38",
      "curated": true,
      "license": "CC-BY-NC-4.0",
      "upstream_sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
      "errata_applied": 0,
      "selection": null,
      "conversation_ids": null,
      "questions": 1986
    },
    "connector": {
      "id": "zep/graphiti",
      "code_hash": "sha256:d1a55436d110c08a023de10be6bd4d4af17998d8249e65770d42fb63f9f1db2b",
      "target": "Zep Graphiti open source (in-process, local FalkorDB)",
      "label": "maintainer-written",
      "sdk": "graphiti-core[falkordb]==0.31.1",
      "settings": {
        "llm_model": "gpt-6-luna",
        "small_model": "gpt-6-luna",
        "embedder_model": "text-embedding-3-small",
        "falkordb_host": "localhost",
        "falkordb_port": 6379
      },
      "models": {
        "llm_model": "gpt-6-luna",
        "small_model": "gpt-6-luna",
        "embedder_model": "text-embedding-3-small"
      },
      "endpoint": null,
      "custom_endpoint": false
    },
    "judge": {
      "route": "anthropic",
      "model": "claude-sonnet-5-5",
      "upstream_provider": null,
      "effort": "medium",
      "max_output_tokens": 4000,
      "prompt_version": "v3",
      "prompt_hash": "sha256:ce6d9750331c79d1bb3a0737109c458fef4ba9fd8cf929c75c1c1cf9407a4b78",
      "served_by": [
        "claude-sonnet-5-5"
      ],
      "official": true,
      "season": "1"
    },
    "config": {
      "budget_tokens": 8192,
      "tokenizer": "o200k_base",
      "concurrency": 4,
      "judge_concurrency": 8
    },
    "metrics": {
      "top10": {
        "overall": {
          "questions": 1986,
          "correct": 978,
          "accuracy": 0.49244712990936557,
          "accuracy_ci95": [
            0.4644842495367511,
            0.5196039870888168
          ],
          "context_tokens_mean": 426.3897280966767,
          "context_tokens_p95": 731.75,
          "efficiency": 115.49225918446876,
          "retrieval_latency_p50_s": 1.6032329774998288,
          "retrieval_latency_p95_s": 2.343595080250452,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9439461883408071
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 8,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 421,
            "accuracy": 0.9439461883408071,
            "accuracy_ci95": [
              0.9275346392737697,
              0.9624730960264901
            ],
            "context_tokens_mean": 410.65695067264573,
            "context_tokens_p95": 694.75,
            "efficiency": 229.86246471529267,
            "retrieval_latency_p50_s": 1.5474235560000125,
            "retrieval_latency_p95_s": 2.469778584999972,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9439461883408071
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 2,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 75,
            "accuracy": 0.26595744680851063,
            "accuracy_ci95": [
              0.20628903539208884,
              0.3295505457803845
            ],
            "context_tokens_mean": 433.77659574468083,
            "context_tokens_p95": 759.8,
            "efficiency": 61.312078479460446,
            "retrieval_latency_p50_s": 1.6050888690001557,
            "retrieval_latency_p95_s": 2.3054970323507176,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 34,
            "accuracy": 0.3541666666666667,
            "accuracy_ci95": [
              0.21567046173308035,
              0.5185363247863248
            ],
            "context_tokens_mean": 427.8958333333333,
            "context_tokens_p95": 746.25,
            "efficiency": 82.76936559715665,
            "retrieval_latency_p50_s": 1.6313276959999712,
            "retrieval_latency_p95_s": 2.2005241679999017,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 320,
            "accuracy": 0.380499405469679,
            "accuracy_ci95": [
              0.3461453744493392,
              0.41562888704707796
            ],
            "context_tokens_mean": 428.5077288941736,
            "context_tokens_p95": 734.0,
            "efficiency": 88.79639264654874,
            "retrieval_latency_p50_s": 1.6324189670003761,
            "retrieval_latency_p95_s": 2.4197183259998383,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 2,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 128,
            "accuracy": 0.3987538940809969,
            "accuracy_ci95": [
              0.31510934442510596,
              0.47417800792322673
            ],
            "context_tokens_mean": 435.7601246105919,
            "context_tokens_p95": 732.0,
            "efficiency": 91.50766019202311,
            "retrieval_latency_p50_s": 1.59140631199989,
            "retrieval_latency_p95_s": 2.163652842000374,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 23163.799504199,
          "per_turn_s": 3.938082200645869,
          "per_session_p50_s": 80.18204221849999,
          "per_session_p95_s": 150.3936902897999,
          "api_calls": 5882,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 10.09575,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q161",
          "conv-26:q5",
          "conv-43:q134",
          "conv-43:q234",
          "conv-44:q5",
          "conv-47:q42",
          "conv-48:q101",
          "conv-49:q13"
        ]
      },
      "top30": {
        "overall": {
          "questions": 1986,
          "correct": 1059,
          "accuracy": 0.5332326283987915,
          "accuracy_ci95": [
            0.5133887231086551,
            0.5547201716018825
          ],
          "context_tokens_mean": 1199.6822759315207,
          "context_tokens_p95": 1567.5,
          "efficiency": 44.44782081862057,
          "retrieval_latency_p50_s": 2.8023241019998295,
          "retrieval_latency_p95_s": 3.774810440999886,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9260089686098655
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 9,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 413,
            "accuracy": 0.9260089686098655,
            "accuracy_ci95": [
              0.9081311436712809,
              0.942786211960791
            ],
            "context_tokens_mean": 1176.7959641255604,
            "context_tokens_p95": 1556.75,
            "efficiency": 78.6889993541024,
            "retrieval_latency_p50_s": 2.7109466800002338,
            "retrieval_latency_p95_s": 3.9386713712500523,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9260089686098655
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 3,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 103,
            "accuracy": 0.36524822695035464,
            "accuracy_ci95": [
              0.30881899055918666,
              0.41812148006924815
            ],
            "context_tokens_mean": 1195.1205673758866,
            "context_tokens_p95": 1582.8,
            "efficiency": 30.56162172426889,
            "retrieval_latency_p50_s": 2.867504325500022,
            "retrieval_latency_p95_s": 3.751850380550013,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 43,
            "accuracy": 0.4479166666666667,
            "accuracy_ci95": [
              0.30434782608695654,
              0.6021535923753666
            ],
            "context_tokens_mean": 1211.3854166666667,
            "context_tokens_p95": 1572.5,
            "efficiency": 36.97557032667487,
            "retrieval_latency_p50_s": 2.908889436499976,
            "retrieval_latency_p95_s": 3.634131663500284,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 354,
            "accuracy": 0.42092746730083236,
            "accuracy_ci95": [
              0.38674792846402406,
              0.45122247937807847
            ],
            "context_tokens_mean": 1204.8608799048752,
            "context_tokens_p95": 1562.0,
            "efficiency": 34.93577344249611,
            "retrieval_latency_p50_s": 2.8250342259998433,
            "retrieval_latency_p95_s": 3.7873619220004002,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 146,
            "accuracy": 0.45482866043613707,
            "accuracy_ci95": [
              0.37462212266048095,
              0.5283667763899536
            ],
            "context_tokens_mean": 1218.4205607476636,
            "context_tokens_p95": 1553.0,
            "efficiency": 37.32936517067957,
            "retrieval_latency_p50_s": 2.7862994030001573,
            "retrieval_latency_p95_s": 3.7020114500001,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 2,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 23163.799504199,
          "per_turn_s": 3.938082200645869,
          "per_session_p50_s": 80.18204221849999,
          "per_session_p95_s": 150.3936902897999,
          "api_calls": 5882,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 15.007238,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q161",
          "conv-26:q5",
          "conv-30:q95",
          "conv-43:q134",
          "conv-43:q148",
          "conv-43:q234",
          "conv-43:q74",
          "conv-47:q58",
          "conv-48:q101"
        ]
      }
    },
    "price_table_version": "2026-10-04",
    "artifacts": [
      {
        "name": "conversations.jsonl",
        "sha256": "sha256:5cead1b3569f89146e22de6a290c44c83e3b6826c2489195c810495c7f528bcc",
        "url": "https://artifacts.memjudge.org/runs/20261101T151858Z-33a8ed/conversations.jsonl"
      },
      {
        "name": "questions-top10.jsonl",
        "sha256": "sha256:007375471476b4d61feeb039fb4e86a029416d29c5f2ead7538e97680b807b57",
        "url": "https://artifacts.memjudge.org/runs/20261101T151858Z-33a8ed/questions-top10.jsonl"
      },
      {
        "name": "questions-top30.jsonl",
        "sha256": "sha256:83a60cb03d53d458dd6986aa6a9e7707941139b513000edeb02c5acba7d27b3a",
        "url": "https://artifacts.memjudge.org/runs/20261101T151858Z-33a8ed/questions-top30.jsonl"
      }
    ],
    "affiliation": {
      "affiliated": false,
      "system": null
    },
    "reused_ingestion": null,
    "rejudged_from": null,
    "resumes": 1,
    "history": [
      {
        "at": "2026-11-01T16:19:59.450618Z",
        "event": "stopped",
        "cause": "retrieval"
      },
      {
        "at": "2026-11-01T17:18:53.383841Z",
        "event": "resumed",
        "cause": null
      }
    ],
    "key_id": "478df5eb628bfa74"
  },
  "signature": {
    "alg": "Ed25519",
    "key_id": "478df5eb628bfa74",
    "value": "jN5AN6QwZYylirN/WpRN9hl/+15wg3T/TH+ffiNFpkbr1ACKKzjUWvqNoks/ggqHA9gA9n1vxrSiPOgbXZmpCQ=="
  }
}