{
  "payload": {
    "schema_version": "1",
    "run_id": "20261101T114259Z-05d6c0",
    "mode": "hosted",
    "started_at": "2026-11-01T11:42:59.256644Z",
    "finished_at": "2026-11-01T13:06:50.683430Z",
    "tool": {
      "version": "0.1.0",
      "commit": "896c4f7b13f7baa49cf6a779134b154c957a3cf7",
      "image_digest": "sha256:8671c6e11fb4468e182e776556e616e221df73a4f77051716772e8cd4238222e"
    },
    "dataset": {
      "name": "locomo",
      "version": "v1",
      "hash": "sha256:9c7812f2f45ce12b06cd9d3fc55736c3a8a83d2b263c11f28c292c475f6ddf38",
      "curated": true,
      "license": "CC-BY-NC-4.0",
      "upstream_sha256": "79fa87e90f04081343b8c8debecb80a9a6842b76a7aa537dc9fdf651ea698ff4",
      "errata_applied": 0,
      "selection": null,
      "conversation_ids": null,
      "questions": 1986
    },
    "connector": {
      "id": "supermemory/cloud",
      "code_hash": "sha256:c9e53310a709c5efaf804ced8772eb54d8e4c994f20ebc3901c16ec48a901d43",
      "target": "Supermemory (hosted API, v5)",
      "label": "maintainer-written",
      "sdk": "supermemory==5.0.0",
      "settings": {
        "dreaming": "instant",
        "poll_interval_s": 1.0,
        "ingest_timeout_s": 3600,
        "base_url": null
      },
      "models": "vendor",
      "endpoint": "https://api.supermemory.ai",
      "custom_endpoint": false
    },
    "judge": {
      "route": "anthropic",
      "model": "claude-sonnet-5-5",
      "upstream_provider": null,
      "effort": "medium",
      "max_output_tokens": 4000,
      "prompt_version": "v3",
      "prompt_hash": "sha256:ce6d9750331c79d1bb3a0737109c458fef4ba9fd8cf929c75c1c1cf9407a4b78",
      "served_by": [
        "claude-sonnet-5-5"
      ],
      "official": true,
      "season": "1"
    },
    "config": {
      "budget_tokens": 8192,
      "tokenizer": "o200k_base",
      "concurrency": 4,
      "judge_concurrency": 8
    },
    "metrics": {
      "top10": {
        "overall": {
          "questions": 1986,
          "correct": 1477,
          "accuracy": 0.743705941591138,
          "accuracy_ci95": [
            0.7102188405107279,
            0.7734313661770441
          ],
          "context_tokens_mean": 761.337361530715,
          "context_tokens_p95": 946.0,
          "efficiency": 97.68415149046042,
          "retrieval_latency_p50_s": 0.3006829835001099,
          "retrieval_latency_p95_s": 0.7285009707501331,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9372197309417041
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 17,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 418,
            "accuracy": 0.9372197309417041,
            "accuracy_ci95": [
              0.9256896625925288,
              0.9498609079750994
            ],
            "context_tokens_mean": 768.1278026905829,
            "context_tokens_p95": 1050.75,
            "efficiency": 122.01351489411388,
            "retrieval_latency_p50_s": 0.29859461299997747,
            "retrieval_latency_p95_s": 0.7184176792501376,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9372197309417041
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 4,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 140,
            "accuracy": 0.49645390070921985,
            "accuracy_ci95": [
              0.4217637054821929,
              0.5680440878641533
            ],
            "context_tokens_mean": 749.2588652482269,
            "context_tokens_p95": 850.6499999999999,
            "efficiency": 66.25932955024113,
            "retrieval_latency_p50_s": 0.30140682249998463,
            "retrieval_latency_p95_s": 0.7547436544496686,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 1,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 45,
            "accuracy": 0.46875,
            "accuracy_ci95": [
              0.3067947860962567,
              0.6055538883649149
            ],
            "context_tokens_mean": 745.1354166666666,
            "context_tokens_p95": 1012.0,
            "efficiency": 62.908028462387996,
            "retrieval_latency_p50_s": 0.3006016384999839,
            "retrieval_latency_p95_s": 0.616701265500069,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 739,
            "accuracy": 0.8787158145065398,
            "accuracy_ci95": [
              0.8487045407219589,
              0.9113514404067031
            ],
            "context_tokens_mean": 773.8632580261593,
            "context_tokens_p95": 1050.0,
            "efficiency": 113.5492356553819,
            "retrieval_latency_p50_s": 0.3017438859999402,
            "retrieval_latency_p95_s": 0.7285566740001741,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 7,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 135,
            "accuracy": 0.4205607476635514,
            "accuracy_ci95": [
              0.3312484927652733,
              0.5129701791755419
            ],
            "context_tokens_mean": 734.5420560747664,
            "context_tokens_p95": 834.0,
            "efficiency": 57.25482212835258,
            "retrieval_latency_p50_s": 0.29729450800005,
            "retrieval_latency_p95_s": 0.8452304729999014,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 5,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 11438.308644542,
          "per_turn_s": 1.9446291473209791,
          "per_session_p50_s": 39.344587058,
          "per_session_p95_s": 68.87810135514972,
          "api_calls": 9929,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 12.460026,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q144",
          "conv-26:q161",
          "conv-26:q184",
          "conv-26:q5",
          "conv-30:q57",
          "conv-42:q119",
          "conv-42:q52",
          "conv-43:q113",
          "conv-43:q135",
          "conv-43:q164",
          "conv-43:q234",
          "conv-43:q60",
          "conv-47:q31",
          "conv-48:q174",
          "conv-49:q74",
          "conv-49:q80",
          "conv-50:q200"
        ]
      },
      "top30": {
        "overall": {
          "questions": 1986,
          "correct": 1607,
          "accuracy": 0.8091641490433031,
          "accuracy_ci95": [
            0.7867602770427206,
            0.8326581154755474
          ],
          "context_tokens_mean": 2333.8524672708963,
          "context_tokens_p95": 3255.25,
          "efficiency": 34.670749774920594,
          "retrieval_latency_p50_s": 0.31946438650015807,
          "retrieval_latency_p95_s": 0.7754823449999435,
          "abstention_accuracy": {
            "n": 446,
            "value": 0.9394618834080718
          },
          "stale_fact_rate": {
            "n": 0,
            "value": null
          },
          "evidence_recall": {
            "n": 0,
            "value": null
          },
          "truncation_rate": 0.0,
          "flagged": 26,
          "errors": 0
        },
        "by_category": {
          "adversarial": {
            "questions": 446,
            "correct": 419,
            "accuracy": 0.9394618834080718,
            "accuracy_ci95": [
              0.9220477728285078,
              0.9610099189801025
            ],
            "context_tokens_mean": 2297.836322869955,
            "context_tokens_p95": 3194.75,
            "efficiency": 40.88463020876532,
            "retrieval_latency_p50_s": 0.3161276820001149,
            "retrieval_latency_p95_s": 0.8404537342501044,
            "abstention_accuracy": {
              "n": 446,
              "value": 0.9394618834080718
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 6,
            "errors": 0
          },
          "multi-hop": {
            "questions": 282,
            "correct": 202,
            "accuracy": 0.7163120567375887,
            "accuracy_ci95": [
              0.6579693007897907,
              0.7841796527442357
            ],
            "context_tokens_mean": 2315.3971631205673,
            "context_tokens_p95": 3110.7,
            "efficiency": 30.936897917426055,
            "retrieval_latency_p50_s": 0.3183432449998236,
            "retrieval_latency_p95_s": 0.7846002538999751,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 2,
            "errors": 0
          },
          "open-domain": {
            "questions": 96,
            "correct": 53,
            "accuracy": 0.5520833333333334,
            "accuracy_ci95": [
              0.3614302571109179,
              0.72
            ],
            "context_tokens_mean": 2405.5104166666665,
            "context_tokens_p95": 3954.75,
            "efficiency": 22.950777078669205,
            "retrieval_latency_p50_s": 0.3161963369999512,
            "retrieval_latency_p95_s": 0.6848921072499934,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 0,
            "errors": 0
          },
          "single-hop": {
            "questions": 841,
            "correct": 782,
            "accuracy": 0.929845422116528,
            "accuracy_ci95": [
              0.9151826241974841,
              0.9467818100691604
            ],
            "context_tokens_mean": 2389.860879904875,
            "context_tokens_p95": 3547.0,
            "efficiency": 38.90793099862528,
            "retrieval_latency_p50_s": 0.3209891209999114,
            "retrieval_latency_p95_s": 0.7699835099999746,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 11,
            "errors": 0
          },
          "temporal": {
            "questions": 321,
            "correct": 151,
            "accuracy": 0.470404984423676,
            "accuracy_ci95": [
              0.36081472441805396,
              0.581976856316297
            ],
            "context_tokens_mean": 2231.93769470405,
            "context_tokens_p95": 2447.0,
            "efficiency": 21.07608046317129,
            "retrieval_latency_p50_s": 0.3211912229999143,
            "retrieval_latency_p95_s": 0.7397302880000325,
            "abstention_accuracy": {
              "n": 0,
              "value": null
            },
            "stale_fact_rate": {
              "n": 0,
              "value": null
            },
            "evidence_recall": {
              "n": 0,
              "value": null
            },
            "truncation_rate": 0.0,
            "flagged": 7,
            "errors": 0
          }
        },
        "ingestion": {
          "conversations": 10,
          "sessions": 272,
          "turns": 5882,
          "total_s": 11438.308644542,
          "per_turn_s": 1.9446291473209791,
          "per_session_p50_s": 39.344587058,
          "per_session_p95_s": 68.87810135514972,
          "api_calls": 9929,
          "input_tokens": null,
          "output_tokens": null,
          "mode": "per-session"
        },
        "cost": {
          "ingestion_usd": null,
          "retrieval_usd": null,
          "judge_usd": 22.406258,
          "total_usd": null,
          "per_1k_questions_usd": null,
          "ingestion_credits": null,
          "retrieval_credits": null,
          "judge_cost_source": "price-table"
        },
        "error_rate": {
          "n": 4244,
          "value": 0.0
        },
        "flagged_question_ids": [
          "conv-26:q184",
          "conv-26:q5",
          "conv-26:q94",
          "conv-30:q57",
          "conv-30:q95",
          "conv-42:q119",
          "conv-42:q52",
          "conv-43:q113",
          "conv-43:q135",
          "conv-43:q164",
          "conv-43:q234",
          "conv-43:q55",
          "conv-43:q60",
          "conv-44:q30",
          "conv-47:q31",
          "conv-48:q116",
          "conv-48:q155",
          "conv-48:q174",
          "conv-49:q144",
          "conv-49:q161",
          "conv-49:q54",
          "conv-49:q74",
          "conv-49:q80",
          "conv-50:q147",
          "conv-50:q196",
          "conv-50:q200"
        ]
      }
    },
    "price_table_version": "2026-10-04",
    "artifacts": [
      {
        "name": "conversations.jsonl",
        "sha256": "sha256:cd9bfd31da9a64a0d6c8291639d37b5e3bddb5097eddede6b0e11df29868d3b2",
        "url": "https://artifacts.memjudge.org/runs/20261101T114259Z-05d6c0/conversations.jsonl"
      },
      {
        "name": "questions-top10.jsonl",
        "sha256": "sha256:e2b8044e66f3c894adc8444e7adc0bd1723949690997f03aca0c4c5de0dcde9b",
        "url": "https://artifacts.memjudge.org/runs/20261101T114259Z-05d6c0/questions-top10.jsonl"
      },
      {
        "name": "questions-top30.jsonl",
        "sha256": "sha256:3fd804e76facb4e4db3b398ef2ae90082376d6a839b91e88e7c1fcfeef52b484",
        "url": "https://artifacts.memjudge.org/runs/20261101T114259Z-05d6c0/questions-top30.jsonl"
      }
    ],
    "affiliation": {
      "affiliated": false,
      "system": null
    },
    "reused_ingestion": null,
    "rejudged_from": null,
    "resumes": 0,
    "history": [],
    "key_id": "478df5eb628bfa74"
  },
  "signature": {
    "alg": "Ed25519",
    "key_id": "478df5eb628bfa74",
    "value": "m8Ei4hcuAg4C62BFdIAo7nzl4y+ONoeXOe7ihgkGz6XRIjnK/ISGpK5tp5avlocP7gG4/8kt/syRRtRbP7DcAQ=="
  }
}