{
  "schema": "skillreducer/research-table/1",
  "id": "delegation",
  "study": "delegation",
  "title": "Delegation pilot, F5, three repetitions per arm",
  "evidence_class": "measured-local",
  "date": "2026-09-22",
  "commit": "da8b53ad95e89365b23f2d5316e2e056d3a5e7b2",
  "source_documents": [
    {
      "path": "docs/experiments/DELEGATION-2026-09-22.md",
      "commit": "da8b53ad95e89365b23f2d5316e2e056d3a5e7b2",
      "committed": "2026-09-22"
    }
  ],
  "n_meaning": "Repetitions per arm",
  "columns": [
    {
      "name": "task_id",
      "description": "Pilot task"
    },
    {
      "name": "arm",
      "description": "A original, B delegated to a cheaper model, C delegated to code"
    },
    {
      "name": "label",
      "description": "What the arm does"
    },
    {
      "name": "score",
      "description": "Blind grade"
    },
    {
      "name": "quality_ok",
      "description": "Within 0.05 of arm A on this task"
    },
    {
      "name": "main_tokens",
      "description": "Main-session tokens"
    },
    {
      "name": "main_token_reduction",
      "description": "Main-session token reduction against A; negative is dearer"
    },
    {
      "name": "total_tokens",
      "description": "Main-session plus delegate tokens"
    },
    {
      "name": "total_token_reduction",
      "description": "Total token reduction against A"
    },
    {
      "name": "cost_reduction",
      "description": "Cost reduction against A"
    },
    {
      "name": "tool_calls",
      "description": "Median tool calls"
    },
    {
      "name": "tool_call_change",
      "description": "Change in tool calls against A"
    },
    {
      "name": "delegate_calls",
      "description": "Calls to the delegate"
    },
    {
      "name": "delegate_tokens",
      "description": "Tokens the delegate model spent"
    },
    {
      "name": "n",
      "description": "Repetitions per arm"
    },
    {
      "name": "evidence_class",
      "description": "Evidence class of the figure (SPEC.md section 20)"
    },
    {
      "name": "date",
      "description": "UTC day the study's last measured run landed"
    },
    {
      "name": "commit",
      "description": "Commit the figures come from: the committed data when the study has some, otherwise the latest commit of the document that reports it"
    }
  ],
  "rows": [
    {
      "task_id": "product-table",
      "arm": "A",
      "label": "original, all on the main model",
      "score": 1,
      "quality_ok": null,
      "main_tokens": 42858,
      "main_token_reduction": null,
      "total_tokens": 42858,
      "total_token_reduction": null,
      "cost_reduction": null,
      "tool_calls": 3,
      "tool_call_change": null,
      "delegate_calls": 0,
      "delegate_tokens": null,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-22",
      "commit": "da8b53ad95e89365b23f2d5316e2e056d3a5e7b2"
    },
    {
      "task_id": "product-table",
      "arm": "B",
      "label": "delegated to model (Sonnet + haiku)",
      "score": 0.8,
      "quality_ok": false,
      "main_tokens": 63649,
      "main_token_reduction": -0.4851,
      "total_tokens": 90116,
      "total_token_reduction": -1.1027,
      "cost_reduction": -0.532,
      "tool_calls": 6,
      "tool_call_change": 3,
      "delegate_calls": 4,
      "delegate_tokens": 26467,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-22",
      "commit": "da8b53ad95e89365b23f2d5316e2e056d3a5e7b2"
    },
    {
      "task_id": "product-table",
      "arm": "C",
      "label": "delegated to code (lookup script, no model)",
      "score": 0.8,
      "quality_ok": false,
      "main_tokens": 176628,
      "main_token_reduction": -3.1212,
      "total_tokens": 176628,
      "total_token_reduction": -3.1212,
      "cost_reduction": -0.9839,
      "tool_calls": 13,
      "tool_call_change": 10,
      "delegate_calls": 0,
      "delegate_tokens": null,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-22",
      "commit": "da8b53ad95e89365b23f2d5316e2e056d3a5e7b2"
    },
    {
      "task_id": "invoice-guard",
      "arm": "A",
      "label": "original, all on the main model",
      "score": 0.8,
      "quality_ok": null,
      "main_tokens": 35619,
      "main_token_reduction": null,
      "total_tokens": 35619,
      "total_token_reduction": null,
      "cost_reduction": null,
      "tool_calls": 3,
      "tool_call_change": null,
      "delegate_calls": 0,
      "delegate_tokens": null,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-22",
      "commit": "da8b53ad95e89365b23f2d5316e2e056d3a5e7b2"
    },
    {
      "task_id": "invoice-guard",
      "arm": "B",
      "label": "delegated to model (Sonnet + haiku)",
      "score": 0.8,
      "quality_ok": true,
      "main_tokens": 81071,
      "main_token_reduction": -1.2761,
      "total_tokens": 107336,
      "total_token_reduction": -2.0134,
      "cost_reduction": -3.8187,
      "tool_calls": 6,
      "tool_call_change": 3,
      "delegate_calls": 4,
      "delegate_tokens": 26652,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-22",
      "commit": "da8b53ad95e89365b23f2d5316e2e056d3a5e7b2"
    },
    {
      "task_id": "invoice-guard",
      "arm": "C",
      "label": "delegated to code (lookup script, no model)",
      "score": 0.8,
      "quality_ok": true,
      "main_tokens": 138539,
      "main_token_reduction": -2.8895,
      "total_tokens": 138539,
      "total_token_reduction": -2.8895,
      "cost_reduction": -2.0265,
      "tool_calls": 11,
      "tool_call_change": 8,
      "delegate_calls": 0,
      "delegate_tokens": null,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-22",
      "commit": "da8b53ad95e89365b23f2d5316e2e056d3a5e7b2"
    }
  ]
}