{
  "payload": {
    "schema_version": "1",
    "run_id": "20261101T010624Z-79759b",
    "mode": "hosted",
    "started_at": "2026-11-01T01:06:24.922865Z",
    "finished_at": "2026-11-01T03:42:14.875279Z",
    "tool": {
      "version": "0.1.0",
      "commit": "896c4f7b13f7baa49cf6a779134b154c957a3cf7",
      "image_digest": "sha256:8671c6e11fb4468e182e776556e616e221df73a4f77051716772e8cd4238222e"
    },
    "dataset": {
      "name": "locomo",
      "version": "v1",
      "hash": "sha256:9c7812f2f45ce12b06cd9d3fc55736c3a8a83d2b263c11f28c292c475f6ddf38",
      "curated": true,
      "license": "CC-BY-NC-4.0",
      "upstream_sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
      "errata_applied": 0,
      "selection": null,
      "conversation_ids": null,
      "questions": 1986
    },
    "connector": {
      "id": "mem0/oss",
      "code_hash": "sha256:2bea0837a48df5082a31b032cf620319ce603c7ea13f26727ca3e455980112fd",
      "target": "Mem0 open source (in-process, full multi-signal setup)",
      "label": "maintainer-written",
      "sdk": "mem0ai[nlp]==2.2.1",
      "settings": {
        "llm_model": "gpt-6-luna",
        "llm_is_reasoning_model": true,
        "embedder_model": "text-embedding-3-small",
        "embedding_dims": 1536
      },
      "models": {
        "llm_model": "gpt-6-luna",
        "embedder_model": "text-embedding-3-small"
      },
      "endpoint": null,
      "custom_endpoint": false
    },
    "judge": {
      "route": "anthropic",
      "model": "claude-sonnet-5-5",
      "upstream_provider": null,
      "effort": "medium",
      "max_output_tokens": 4000,
      "prompt_version": "v3",
      "prompt_hash": "sha256:ce6d9750331c79d1bb3a0737109c458fef4ba9fd8cf929c75c1c1cf9407a4b78",
      "served_by": [
        "claude-sonnet-5-5"
      ],
      "official": true,
      "season": "1"
    },
    "config": {
      "budget_tokens": 8192,
      "tokenizer": "o200k_base",
      "concurrency": 4,
      "judge_concurrency": 8
    },
    "metrics": {
      "top10": {
        "overall": {
          "questions": 1986,
          "correct": 1579,
          "accuracy": 0.7950654582074521,
          "accuracy_ci95": [
            0.7713275354323248,
            0.8168538291605302
          ],
          "context_tokens_mean": 459.1289023162135,
          "context_tokens_p95": 542.75,
          "efficiency": 173.1682440805852,
          "retrieval_latency_p50_s": 0.5915046365000762,
          "retrieval_latency_p95_s": 0.8973641075000387,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9417040358744395
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 25,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 420,
            "accuracy": 0.9417040358744395,
            "accuracy_ci95": [
              0.9142843219159008,
              0.9704672375279226
            ],
            "context_tokens_mean": 446.2533632286995,
            "context_tokens_p95": 514.75,
            "efficiency": 211.02452406433238,
            "retrieval_latency_p50_s": 0.5779887235000842,
            "retrieval_latency_p95_s": 0.8990722397502395,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9417040358744395
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 131,
            "accuracy": 0.4645390070921986,
            "accuracy_ci95": [
              0.4136335377095571,
              0.520149613899614
            ],
            "context_tokens_mean": 456.42198581560285,
            "context_tokens_p95": 560.8499999999999,
            "efficiency": 101.77840277831731,
            "retrieval_latency_p50_s": 0.5569172735001757,
            "retrieval_latency_p95_s": 0.8060429287998886,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 45,
            "accuracy": 0.46875,
            "accuracy_ci95": [
              0.29472136222910217,
              0.6408099131323456
            ],
            "context_tokens_mean": 452.8229166666667,
            "context_tokens_p95": 525.25,
            "efficiency": 103.51728738699363,
            "retrieval_latency_p50_s": 0.6134098804998303,
            "retrieval_latency_p95_s": 0.8195098792501767,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 722,
            "accuracy": 0.8585017835909632,
            "accuracy_ci95": [
              0.8326169259045626,
              0.8929615102711912
            ],
            "context_tokens_mean": 460.5398335315101,
            "context_tokens_p95": 542.0,
            "efficiency": 186.41205843320924,
            "retrieval_latency_p50_s": 0.600320358999852,
            "retrieval_latency_p95_s": 0.905108957000266,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 11,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 261,
            "accuracy": 0.8130841121495327,
            "accuracy_ci95": [
              0.7530378919860627,
              0.8655050165349476
            ],
            "context_tokens_mean": 477.5856697819315,
            "context_tokens_p95": 559.0,
            "efficiency": 170.24885033103942,
            "retrieval_latency_p50_s": 0.6014491229998384,
            "retrieval_latency_p95_s": 0.915068250999866,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 10,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 26991.128810634007,
          "per_turn_s": 4.588767223841212,
          "per_session_p50_s": 89.56950723300014,
          "per_session_p95_s": 177.18170835970002,
          "api_calls": 5882,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 10.24146,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q184",
          "conv-26:q5",
          "conv-26:q94",
          "conv-30:q34",
          "conv-41:q48",
          "conv-41:q69",
          "conv-42:q119",
          "conv-42:q24",
          "conv-42:q35",
          "conv-43:q135",
          "conv-43:q136",
          "conv-43:q164",
          "conv-43:q208",
          "conv-43:q234",
          "conv-43:q56",
          "conv-44:q5",
          "conv-47:q31",
          "conv-47:q37",
          "conv-48:q174",
          "conv-48:q179",
          "conv-48:q188",
          "conv-49:q80",
          "conv-50:q108",
          "conv-50:q151",
          "conv-50:q200"
        ]
      },
      "top30": {
        "overall": {
          "questions": 1986,
          "correct": 1703,
          "accuracy": 0.8575025176233635,
          "accuracy_ci95": [
            0.8385544824000167,
            0.877011963427147
          ],
          "context_tokens_mean": 1357.1213494461228,
          "context_tokens_p95": 1546.0,
          "efficiency": 63.185397383464114,
          "retrieval_latency_p50_s": 0.5917619185001968,
          "retrieval_latency_p95_s": 0.8999329037499137,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9304932735426009
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 20,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 415,
            "accuracy": 0.9304932735426009,
            "accuracy_ci95": [
              0.9050620386760629,
              0.9575660990844529
            ],
            "context_tokens_mean": 1329.567264573991,
            "context_tokens_p95": 1490.0,
            "efficiency": 69.98467082752235,
            "retrieval_latency_p50_s": 0.5730812109999306,
            "retrieval_latency_p95_s": 0.8974388009996801,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9304932735426009
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 179,
            "accuracy": 0.6347517730496454,
            "accuracy_ci95": [
              0.5719837178916536,
              0.7010957842010732
            ],
            "context_tokens_mean": 1350.0425531914893,
            "context_tokens_p95": 1563.3999999999999,
            "efficiency": 47.017167832902565,
            "retrieval_latency_p50_s": 0.5817086679999193,
            "retrieval_latency_p95_s": 0.8159202321005751,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 47,
            "accuracy": 0.4895833333333333,
            "accuracy_ci95": [
              0.291128201354136,
              0.6881899641577062
            ],
            "context_tokens_mean": 1327.1770833333333,
            "context_tokens_p95": 1524.0,
            "efficiency": 36.8890737702988,
            "retrieval_latency_p50_s": 0.6086230850000902,
            "retrieval_latency_p95_s": 0.8092694007499404,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 785,
            "accuracy": 0.9334126040428062,
            "accuracy_ci95": [
              0.9190050305285544,
              0.9538916674874
            ],
            "context_tokens_mean": 1360.909631391201,
            "context_tokens_p95": 1554.0,
            "efficiency": 68.58740525545531,
            "retrieval_latency_p50_s": 0.5999549890002527,
            "retrieval_latency_p95_s": 0.9192872120001994,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 8,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 277,
            "accuracy": 0.8629283489096573,
            "accuracy_ci95": [
              0.8198757763975155,
              0.904320987654321
            ],
            "context_tokens_mean": 1400.6542056074766,
            "context_tokens_p95": 1567.0,
            "efficiency": 61.60894997887058,
            "retrieval_latency_p50_s": 0.5934140929994101,
            "retrieval_latency_p95_s": 0.8883453920007014,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 8,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 26991.128810634007,
          "per_turn_s": 4.588767223841212,
          "per_session_p50_s": 89.56950723300014,
          "per_session_p95_s": 177.18170835970002,
          "api_calls": 5882,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 15.363512,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q161",
          "conv-26:q184",
          "conv-26:q94",
          "conv-30:q13",
          "conv-30:q34",
          "conv-30:q57",
          "conv-41:q48",
          "conv-41:q85",
          "conv-43:q135",
          "conv-43:q208",
          "conv-43:q234",
          "conv-43:q6",
          "conv-43:q60",
          "conv-47:q31",
          "conv-48:q116",
          "conv-48:q174",
          "conv-48:q188",
          "conv-49:q74",
          "conv-49:q80",
          "conv-50:q151"
        ]
      }
    },
    "price_table_version": "2026-10-04",
    "artifacts": [
      {
        "name": "conversations.jsonl",
        "sha256": "sha256:b56d353cb45209bb866c411f79fba4db8c108d93281c92c323ab45c2a680e8f8",
        "url": "https://artifacts.memjudge.org/runs/20261101T010624Z-79759b/conversations.jsonl"
      },
      {
        "name": "questions-top10.jsonl",
        "sha256": "sha256:be7695c9f9557bf898e7a707c81387234763daaf61e2fea5a6f0b7c2d92cbe61",
        "url": "https://artifacts.memjudge.org/runs/20261101T010624Z-79759b/questions-top10.jsonl"
      },
      {
        "name": "questions-top30.jsonl",
        "sha256": "sha256:cd1e2c795f0278155f22e9cd8f073dea2623a7cdf997d2a0a55654ed59a3839a",
        "url": "https://artifacts.memjudge.org/runs/20261101T010624Z-79759b/questions-top30.jsonl"
      }
    ],
    "affiliation": {
      "affiliated": false,
      "system": null
    },
    "reused_ingestion": null,
    "rejudged_from": null,
    "resumes": 0,
    "history": [],
    "key_id": "478df5eb628bfa74"
  },
  "signature": {
    "alg": "Ed25519",
    "key_id": "478df5eb628bfa74",
    "value": "zh+3DdPgPz8Yv1CL+Obl0g84ogFTT3Ke+Xhj32M6ybGJ6h1DzIvO4YB+WNZBlqBBWRn9vjTznRkpgR16hygTCw=="
  }
}