{
  "schema": "skillreducer/research-table/1",
  "id": "compiler-loop-comparisons",
  "study": "compiler-loop",
  "title": "Random-draw loop, the comparisons",
  "evidence_class": "measured-local",
  "date": "2026-09-19",
  "commit": "0d3b836079506c06e00a5ae5fa4d0eede75aa628",
  "source_documents": [
    {
      "path": "docs/experiments/COMPILER-LOOP-NULL-2026-09-19.md",
      "commit": "0d3b836079506c06e00a5ae5fa4d0eede75aa628",
      "committed": "2026-09-19"
    }
  ],
  "n_meaning": "Runs per arm behind the row: repetitions times tasks compared, or the graded runs of the eligibility check when the skill was not compared",
  "columns": [
    {
      "name": "skill",
      "description": "Skill slug in the AirMarket catalog"
    },
    {
      "name": "name",
      "description": "Skill name as listed"
    },
    {
      "name": "shape",
      "description": "Document shape (SPEC-003)"
    },
    {
      "name": "installs",
      "description": "Installs reported by the catalog"
    },
    {
      "name": "status",
      "description": "How far the skill got: compared, or the reason it stopped"
    },
    {
      "name": "status_superseded",
      "description": "Status under a superseded scoring rule, when a correction moved it"
    },
    {
      "name": "decision",
      "description": "Acceptance decision for a compared skill (SPEC-031 section 4)"
    },
    {
      "name": "accepted",
      "description": "Fewer total tokens, mission kept, no added tool calls, three repetitions agreeing"
    },
    {
      "name": "artifacts",
      "description": "Artifact kinds shipped, separated by semicolons"
    },
    {
      "name": "skill_md_unchanged",
      "description": "Whether SKILL.md stayed byte-identical"
    },
    {
      "name": "token_reduction",
      "description": "Median share of complete execution tokens saved; positive is cheaper"
    },
    {
      "name": "spread_min",
      "description": "Smallest per-repetition token reduction"
    },
    {
      "name": "spread_max",
      "description": "Largest per-repetition token reduction"
    },
    {
      "name": "straddles_zero",
      "description": "The repetitions disagree in sign, so the row is inconclusive"
    },
    {
      "name": "net_call_change",
      "description": "Change in median tool calls; positive means more calls"
    },
    {
      "name": "tokens_none",
      "description": "Median complete execution tokens, no skill"
    },
    {
      "name": "tokens_original",
      "description": "Median complete execution tokens, original skill"
    },
    {
      "name": "tokens_optimized",
      "description": "Median complete execution tokens, improved skill"
    },
    {
      "name": "score_none",
      "description": "Blind grade, no skill"
    },
    {
      "name": "score_original",
      "description": "Blind grade, original skill"
    },
    {
      "name": "score_optimized",
      "description": "Blind grade, improved skill"
    },
    {
      "name": "mission_preserved",
      "description": "The improved skill kept the original's grade on every task"
    },
    {
      "name": "retention_band",
      "description": "Diagnostic only: how much of the skill's contribution was kept"
    },
    {
      "name": "R",
      "description": "Repetitions per task per arm"
    },
    {
      "name": "tasks",
      "description": "Tasks compared, or tasks graded when the skill was not compared"
    },
    {
      "name": "n",
      "description": "Runs per arm behind the row: repetitions times tasks compared, or the graded runs of the eligibility check when the skill was not compared"
    },
    {
      "name": "evidence_class",
      "description": "Evidence class of the figure (SPEC.md section 20)"
    },
    {
      "name": "date",
      "description": "UTC day the study's last measured run landed"
    },
    {
      "name": "commit",
      "description": "Commit the figures come from: the committed data when the study has some, otherwise the latest commit of the document that reports it"
    }
  ],
  "rows": [
    {
      "skill": "webiny-api-architect",
      "name": "API Architecture Patterns",
      "shape": "reference-lookup",
      "installs": 1,
      "status": "compared",
      "status_superseded": null,
      "decision": "inconclusive_repetitions_straddle_zero",
      "accepted": false,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": 0.229,
      "spread_min": -0.2595,
      "spread_max": 0.4176,
      "straddles_zero": true,
      "net_call_change": -2,
      "tokens_none": 74934,
      "tokens_original": 129805,
      "tokens_optimized": 100075,
      "score_none": 0,
      "score_original": 0.6,
      "score_optimized": 0.8,
      "mission_preserved": true,
      "retention_band": "full",
      "R": 3,
      "tasks": 1,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-19",
      "commit": "0d3b836079506c06e00a5ae5fa4d0eede75aa628"
    },
    {
      "skill": "axiom-swift",
      "name": "Swift Language & Platform",
      "shape": "reference-lookup",
      "installs": null,
      "status": "compared",
      "status_superseded": null,
      "decision": "inconclusive_repetitions_straddle_zero",
      "accepted": false,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": 0.0057,
      "spread_min": -0.0057,
      "spread_max": 0.0206,
      "straddles_zero": true,
      "net_call_change": -1,
      "tokens_none": 21050,
      "tokens_original": 55930,
      "tokens_optimized": 55610,
      "score_none": 0.4,
      "score_original": 1,
      "score_optimized": 1,
      "mission_preserved": true,
      "retention_band": "full",
      "R": 3,
      "tasks": 1,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-19",
      "commit": "0d3b836079506c06e00a5ae5fa4d0eede75aa628"
    },
    {
      "skill": "aws-cloudformation",
      "name": "CloudFormation",
      "shape": "workflow",
      "installs": null,
      "status": "compared",
      "status_superseded": null,
      "decision": "rejected_not_cheaper",
      "accepted": false,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": -0.4342,
      "spread_min": -0.7728,
      "spread_max": -0.2091,
      "straddles_zero": false,
      "net_call_change": 0,
      "tokens_none": 92266,
      "tokens_original": 215759,
      "tokens_optimized": 309437,
      "score_none": 0.8,
      "score_original": 1,
      "score_optimized": 1,
      "mission_preserved": true,
      "retention_band": "full",
      "R": 3,
      "tasks": 2,
      "n": 6,
      "evidence_class": "measured-local",
      "date": "2026-09-19",
      "commit": "0d3b836079506c06e00a5ae5fa4d0eede75aa628"
    }
  ]
}