{
  "benchmark_id": "OTM-Skill-Retention",
  "version": "v1.0",
  "run_id": "2026-05-25T09:09:42.521470+00:00",
  "completed_at": "2026-05-25T09:09:42.521470+00:00",
  "agent_version": "v0.2.0-oos-clean",
  "n_samples": 60,
  "seeds": [
    42
  ],
  "metrics": {
    "primary": {
      "name": "plan_inclusion",
      "value": 0.8167,
      "ci_low": 0.7167,
      "ci_high": 0.9167,
      "n": 60
    },
    "secondary": [
      {
        "name": "top1_accuracy",
        "value": 0.2167,
        "ci_low": 0.1167,
        "ci_high": 0.3167,
        "description": "Expected skill is first step in plan (strict positional)"
      },
      {
        "name": "top3_accuracy",
        "value": 0.5833,
        "ci_low": 0.45,
        "ci_high": 0.7
      },
      {
        "name": "total_probes",
        "value": 60
      },
      {
        "name": "total_agent_runs",
        "value": 60
      },
      {
        "name": "total_cost_usd",
        "value": 1.65
      },
      {
        "name": "plan_alert",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "top1_alert",
        "value": 0.0,
        "n": 3
      },
      {
        "name": "plan_cloud_run",
        "value": 0.5,
        "n": 6
      },
      {
        "name": "top1_cloud_run",
        "value": 0.5,
        "n": 6
      },
      {
        "name": "plan_compute",
        "value": 0.9333,
        "n": 15
      },
      {
        "name": "top1_compute",
        "value": 0.0,
        "n": 15
      },
      {
        "name": "plan_llm",
        "value": 1.0,
        "n": 6
      },
      {
        "name": "top1_llm",
        "value": 0.0,
        "n": 6
      },
      {
        "name": "plan_ml",
        "value": 1.0,
        "n": 9
      },
      {
        "name": "top1_ml",
        "value": 0.0,
        "n": 9
      },
      {
        "name": "plan_query",
        "value": 0.6111,
        "n": 18
      },
      {
        "name": "top1_query",
        "value": 0.5556,
        "n": 18
      },
      {
        "name": "plan_trade",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "top1_trade",
        "value": 0.0,
        "n": 3
      },
      {
        "name": "plan_skill_alert_on_threshold",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_backtest_strategy",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_compare_to_history",
        "value": 0.3333,
        "n": 3
      },
      {
        "name": "plan_skill_compose_signal",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_compute_fill_signal",
        "value": 0.6667,
        "n": 3
      },
      {
        "name": "plan_skill_detect_anomaly",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_estimate_volatility_regime",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_evaluate_pnl",
        "value": 0.6667,
        "n": 3
      },
      {
        "name": "plan_skill_explain_decision",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_fetch_inventory_baseline",
        "value": 0.0,
        "n": 3
      },
      {
        "name": "plan_skill_fetch_market_context",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_fetch_tank_features",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_forecast_var_spread",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_place_paper_trade",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_predict_eia_surprise",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_predict_regime",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_predict_spread_direction",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "plan_skill_query_goii",
        "value": 0.6667,
        "n": 3
      },
      {
        "name": "plan_skill_sense_tank_state",
        "value": 0.0,
        "n": 3
      },
      {
        "name": "plan_skill_summarize_to_persona",
        "value": 1.0,
        "n": 3
      },
      {
        "name": "mean_plan_length",
        "value": 3.3
      },
      {
        "name": "median_plan_length",
        "value": 4.0
      }
    ]
  },
  "baselines": [
    {
      "name": "random_plan_inclusion",
      "primary_value": 0.1572,
      "description": "Random chance of 1 target skill appearing in a 3.3-step plan from 20 skills (15.7%)"
    },
    {
      "name": "random_top1",
      "primary_value": 0.05,
      "description": "Random skill as first step from 20 skills (5%)"
    }
  ],
  "notes": "Production skill-selection benchmark using real OTMAgent planner. 60 probes (3 per skill, 20 skills, 7 categories). Primary metric: plan inclusion (expected skill anywhere in plan). Single-turn protocol (v1.0); multi-turn retention decay deferred to v0.2. Total LLM cost: $1.65.",
  "data_filter": "Probes are canonical queries; no OOS data used."
}