{
  "benchmark_id": "OTM-Mem-Utility",
  "version": "v1.1",
  "run_id": "2026-05-26T02:01:43.838478+00:00",
  "completed_at": "2026-05-26T02:01:43.838478+00:00",
  "agent_version": "v0.2.0-oos-clean",
  "n_samples": 25,
  "seeds": [
    42
  ],
  "metrics": {
    "primary": {
      "name": "memory_utility",
      "value": -0.064,
      "ci_low": -0.276,
      "ci_high": 0.128,
      "n": 25
    },
    "secondary": [
      {
        "name": "mean_score_with_memory",
        "value": 3.768
      },
      {
        "name": "mean_score_without_memory",
        "value": 3.832
      },
      {
        "name": "total_cost_usd",
        "value": 1.3862
      },
      {
        "name": "utility_factual_domain",
        "value": 0.14,
        "ci_low": -0.1,
        "ci_high": 0.32,
        "n": 5
      },
      {
        "name": "utility_historical_comparison",
        "value": 0.06,
        "ci_low": -0.56,
        "ci_high": 0.64,
        "n": 5
      },
      {
        "name": "utility_pattern_recognition",
        "value": -0.38,
        "ci_low": -0.98,
        "ci_high": 0.12,
        "n": 5
      },
      {
        "name": "utility_real_time",
        "value": -0.08,
        "ci_low": -0.3,
        "ci_high": 0.14,
        "n": 5
      },
      {
        "name": "utility_reasoning",
        "value": -0.06,
        "ci_low": -0.4,
        "ci_high": 0.28,
        "n": 5
      }
    ]
  },
  "baselines": [
    {
      "name": "no_memory",
      "primary_value": 0.0,
      "description": "Zero utility \u2014 memory has no effect"
    }
  ],
  "notes": "EXPLORATORY — not pre-registered. Executed post-hoc at reduced scale (designed n=500, actual n=25) due to budget constraints. Results statistically underpowered for subcategory analysis. Real agent calls with LLM-as-judge (Claude Haiku) blind scoring. Total cost: $1.39.",
  "protocol": "exploratory_post_hoc_not_pre_registered",
  "designed_n": 500,
  "data_filter": "Queries span 5 categories; judge order randomized per query."
}