{
  "schema": "skillreducer/research-table/1",
  "id": "rule-checks",
  "study": "rule-checks",
  "title": "Rule checks, lane K, three repetitions per arm",
  "evidence_class": "measured-local",
  "date": "2026-09-18",
  "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e",
  "source_documents": [
    {
      "path": "docs/experiments/COMPILER-2026-09-19.md",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e",
      "committed": "2026-09-19"
    }
  ],
  "n_meaning": "Runs per arm behind the row: repetitions times tasks compared, or the graded runs of the eligibility check when the skill was not compared",
  "columns": [
    {
      "name": "skill",
      "description": "Skill slug in the AirMarket catalog"
    },
    {
      "name": "name",
      "description": "Skill name as listed"
    },
    {
      "name": "shape",
      "description": "Document shape (SPEC-003)"
    },
    {
      "name": "installs",
      "description": "Installs reported by the catalog"
    },
    {
      "name": "status",
      "description": "How far the skill got: compared, or the reason it stopped"
    },
    {
      "name": "status_superseded",
      "description": "Status under a superseded scoring rule, when a correction moved it"
    },
    {
      "name": "decision",
      "description": "Acceptance decision for a compared skill (SPEC-031 section 4)"
    },
    {
      "name": "accepted",
      "description": "Fewer total tokens, mission kept, no added tool calls, three repetitions agreeing"
    },
    {
      "name": "artifacts",
      "description": "Artifact kinds shipped, separated by semicolons"
    },
    {
      "name": "skill_md_unchanged",
      "description": "Whether SKILL.md stayed byte-identical"
    },
    {
      "name": "token_reduction",
      "description": "Median share of complete execution tokens saved; positive is cheaper"
    },
    {
      "name": "spread_min",
      "description": "Smallest per-repetition token reduction"
    },
    {
      "name": "spread_max",
      "description": "Largest per-repetition token reduction"
    },
    {
      "name": "straddles_zero",
      "description": "The repetitions disagree in sign, so the row is inconclusive"
    },
    {
      "name": "net_call_change",
      "description": "Change in median tool calls; positive means more calls"
    },
    {
      "name": "tokens_none",
      "description": "Median complete execution tokens, no skill"
    },
    {
      "name": "tokens_original",
      "description": "Median complete execution tokens, original skill"
    },
    {
      "name": "tokens_optimized",
      "description": "Median complete execution tokens, improved skill"
    },
    {
      "name": "score_none",
      "description": "Blind grade, no skill"
    },
    {
      "name": "score_original",
      "description": "Blind grade, original skill"
    },
    {
      "name": "score_optimized",
      "description": "Blind grade, improved skill"
    },
    {
      "name": "mission_preserved",
      "description": "The improved skill kept the original's grade on every task"
    },
    {
      "name": "retention_band",
      "description": "Diagnostic only: how much of the skill's contribution was kept"
    },
    {
      "name": "R",
      "description": "Repetitions per task per arm"
    },
    {
      "name": "tasks",
      "description": "Tasks compared, or tasks graded when the skill was not compared"
    },
    {
      "name": "n",
      "description": "Runs per arm behind the row: repetitions times tasks compared, or the graded runs of the eligibility check when the skill was not compared"
    },
    {
      "name": "evidence_class",
      "description": "Evidence class of the figure (SPEC.md section 20)"
    },
    {
      "name": "date",
      "description": "UTC day the study's last measured run landed"
    },
    {
      "name": "commit",
      "description": "Commit the figures come from: the committed data when the study has some, otherwise the latest commit of the document that reports it"
    }
  ],
  "rows": [
    {
      "skill": "ce-work",
      "name": "Work Execution Command",
      "shape": "workflow",
      "installs": null,
      "status": "compared",
      "status_superseded": null,
      "decision": "accepted",
      "accepted": true,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": 0.4937,
      "spread_min": 0.1249,
      "spread_max": 0.6397,
      "straddles_zero": false,
      "net_call_change": -1.5,
      "tokens_none": 236619,
      "tokens_original": 1065055,
      "tokens_optimized": 539226,
      "score_none": 0.8,
      "score_original": 1,
      "score_optimized": 1,
      "mission_preserved": true,
      "retention_band": "full",
      "R": 3,
      "tasks": 2,
      "n": 6,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "code-review-expert",
      "name": "Code Review Expert",
      "shape": "reference-lookup",
      "installs": 13180,
      "status": "compared",
      "status_superseded": null,
      "decision": "accepted",
      "accepted": true,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": 0.3028,
      "spread_min": 0.1089,
      "spread_max": 0.3028,
      "straddles_zero": false,
      "net_call_change": -1,
      "tokens_none": 23913,
      "tokens_original": 44482,
      "tokens_optimized": 31013,
      "score_none": 0.8,
      "score_original": 1,
      "score_optimized": 1,
      "mission_preserved": true,
      "retention_band": "full",
      "R": 3,
      "tasks": 1,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "rds-sqlserver",
      "name": "Amazon RDS for SQL Server",
      "shape": "workflow",
      "installs": 3348,
      "status": "compared",
      "status_superseded": null,
      "decision": "accepted",
      "accepted": true,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": 0.2341,
      "spread_min": 0.0043,
      "spread_max": 0.3872,
      "straddles_zero": false,
      "net_call_change": -1,
      "tokens_none": 16663,
      "tokens_original": 76880,
      "tokens_optimized": 58879,
      "score_none": 0.2,
      "score_original": 1,
      "score_optimized": 1,
      "mission_preserved": true,
      "retention_band": "full",
      "R": 3,
      "tasks": 1,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "github-project-triage",
      "name": "GitHub Project Triage",
      "shape": "reference-lookup",
      "installs": 89,
      "status": "compared",
      "status_superseded": null,
      "decision": "accepted",
      "accepted": true,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": 0.0378,
      "spread_min": 0.0199,
      "spread_max": 0.2618,
      "straddles_zero": false,
      "net_call_change": 0,
      "tokens_none": 21476,
      "tokens_original": 51049,
      "tokens_optimized": 49117,
      "score_none": 0.6,
      "score_original": 0.8,
      "score_optimized": 1,
      "mission_preserved": true,
      "retention_band": "full",
      "R": 3,
      "tasks": 1,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "write-memo",
      "name": "Write Investment Memo Skill",
      "shape": "reference-prose",
      "installs": 7,
      "status": "compared",
      "status_superseded": null,
      "decision": "rejected_mission_failed",
      "accepted": false,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": 0.2381,
      "spread_min": 0.2171,
      "spread_max": 0.5397,
      "straddles_zero": false,
      "net_call_change": 0,
      "tokens_none": 102254,
      "tokens_original": 214300,
      "tokens_optimized": 163284,
      "score_none": 0.8,
      "score_original": 1,
      "score_optimized": 0.6,
      "mission_preserved": false,
      "retention_band": "lost",
      "R": 3,
      "tasks": 2,
      "n": 6,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "cloudflare-one-migrations",
      "name": "Cloudflare One Migrations",
      "shape": "reference-prose",
      "installs": 55775,
      "status": "compared",
      "status_superseded": null,
      "decision": "inconclusive_repetitions_straddle_zero",
      "accepted": false,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": -0.0493,
      "spread_min": -0.572,
      "spread_max": 0.2857,
      "straddles_zero": true,
      "net_call_change": 0,
      "tokens_none": 26654,
      "tokens_original": 51907,
      "tokens_optimized": 54465,
      "score_none": 0.2,
      "score_original": 0.8,
      "score_optimized": 0.8,
      "mission_preserved": true,
      "retention_band": "full",
      "R": 3,
      "tasks": 1,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "dev-commit",
      "name": "Dev Commit",
      "shape": "workflow",
      "installs": 2,
      "status": "compared",
      "status_superseded": null,
      "decision": "rejected_not_cheaper",
      "accepted": false,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": -0.5188,
      "spread_min": -1.3633,
      "spread_max": -0.2421,
      "straddles_zero": false,
      "net_call_change": 5,
      "tokens_none": 149257,
      "tokens_original": 169655,
      "tokens_optimized": 257674,
      "score_none": 0,
      "score_original": 0.8,
      "score_optimized": 0,
      "mission_preserved": false,
      "retention_band": "lost",
      "R": 3,
      "tasks": 2,
      "n": 6,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "prp-commit",
      "name": "Commit Intended Work",
      "shape": "workflow",
      "installs": null,
      "status": "compared",
      "status_superseded": null,
      "decision": "rejected_not_cheaper",
      "accepted": false,
      "artifacts": "hook",
      "skill_md_unchanged": true,
      "token_reduction": -0.7969,
      "spread_min": -2.117,
      "spread_max": -0.6558,
      "straddles_zero": false,
      "net_call_change": 3,
      "tokens_none": 47001,
      "tokens_original": 65366,
      "tokens_optimized": 117453,
      "score_none": 0,
      "score_original": 0.2,
      "score_optimized": 0,
      "mission_preserved": false,
      "retention_band": "lost",
      "R": 3,
      "tasks": 1,
      "n": 3,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "architecture-decision",
      "name": "ADR-[NNNN]: [Title]",
      "shape": "reference-lookup",
      "installs": null,
      "status": "no_baseline_advantage",
      "status_superseded": null,
      "decision": null,
      "accepted": false,
      "artifacts": null,
      "skill_md_unchanged": null,
      "token_reduction": null,
      "spread_min": null,
      "spread_max": null,
      "straddles_zero": null,
      "net_call_change": null,
      "tokens_none": null,
      "tokens_original": null,
      "tokens_optimized": null,
      "score_none": null,
      "score_original": null,
      "score_optimized": null,
      "mission_preserved": null,
      "retention_band": null,
      "R": 3,
      "tasks": 2,
      "n": 6,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "azure-reliability",
      "name": "Azure Reliability Assessment & Configuration",
      "shape": "reference-lookup",
      "installs": 266100,
      "status": "unmeasurable",
      "status_superseded": null,
      "decision": null,
      "accepted": false,
      "artifacts": null,
      "skill_md_unchanged": null,
      "token_reduction": null,
      "spread_min": null,
      "spread_max": null,
      "straddles_zero": null,
      "net_call_change": null,
      "tokens_none": null,
      "tokens_original": null,
      "tokens_optimized": null,
      "score_none": null,
      "score_original": null,
      "score_optimized": null,
      "mission_preserved": null,
      "retention_band": null,
      "R": 3,
      "tasks": 0,
      "n": 0,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "ce-work-beta",
      "name": "Work Execution Command",
      "shape": "workflow",
      "installs": null,
      "status": "no_baseline_advantage",
      "status_superseded": null,
      "decision": null,
      "accepted": false,
      "artifacts": null,
      "skill_md_unchanged": null,
      "token_reduction": null,
      "spread_min": null,
      "spread_max": null,
      "straddles_zero": null,
      "net_call_change": null,
      "tokens_none": null,
      "tokens_original": null,
      "tokens_optimized": null,
      "score_none": null,
      "score_original": null,
      "score_optimized": null,
      "mission_preserved": null,
      "retention_band": null,
      "R": 3,
      "tasks": 2,
      "n": 6,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "do",
      "name": "Do Plan",
      "shape": "workflow",
      "installs": 6528,
      "status": "no_baseline_advantage",
      "status_superseded": null,
      "decision": null,
      "accepted": false,
      "artifacts": null,
      "skill_md_unchanged": null,
      "token_reduction": null,
      "spread_min": null,
      "spread_max": null,
      "straddles_zero": null,
      "net_call_change": null,
      "tokens_none": null,
      "tokens_original": null,
      "tokens_optimized": null,
      "score_none": null,
      "score_original": null,
      "score_optimized": null,
      "mission_preserved": null,
      "retention_band": null,
      "R": 3,
      "tasks": 2,
      "n": 6,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "docs-authoring",
      "name": "Docs Authoring",
      "shape": "workflow",
      "installs": null,
      "status": "no_baseline_advantage",
      "status_superseded": null,
      "decision": null,
      "accepted": false,
      "artifacts": null,
      "skill_md_unchanged": null,
      "token_reduction": null,
      "spread_min": null,
      "spread_max": null,
      "straddles_zero": null,
      "net_call_change": null,
      "tokens_none": null,
      "tokens_original": null,
      "tokens_optimized": null,
      "score_none": null,
      "score_original": null,
      "score_optimized": null,
      "mission_preserved": null,
      "retention_band": null,
      "R": 3,
      "tasks": 2,
      "n": 6,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "migrate-to-codex",
      "name": "Migrate to Codex",
      "shape": "workflow",
      "installs": 2068,
      "status": "unmeasurable",
      "status_superseded": null,
      "decision": null,
      "accepted": false,
      "artifacts": null,
      "skill_md_unchanged": null,
      "token_reduction": null,
      "spread_min": null,
      "spread_max": null,
      "straddles_zero": null,
      "net_call_change": null,
      "tokens_none": null,
      "tokens_original": null,
      "tokens_optimized": null,
      "score_none": null,
      "score_original": null,
      "score_optimized": null,
      "mission_preserved": null,
      "retention_band": null,
      "R": 3,
      "tasks": 0,
      "n": 0,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    },
    {
      "skill": "plan-payroll",
      "name": "Plan Payroll",
      "shape": "workflow",
      "installs": 1533,
      "status": "no_baseline_advantage",
      "status_superseded": null,
      "decision": null,
      "accepted": false,
      "artifacts": null,
      "skill_md_unchanged": null,
      "token_reduction": null,
      "spread_min": null,
      "spread_max": null,
      "straddles_zero": null,
      "net_call_change": null,
      "tokens_none": null,
      "tokens_original": null,
      "tokens_optimized": null,
      "score_none": null,
      "score_original": null,
      "score_optimized": null,
      "mission_preserved": null,
      "retention_band": null,
      "R": 3,
      "tasks": 2,
      "n": 6,
      "evidence_class": "measured-local",
      "date": "2026-09-18",
      "commit": "5658cf4666bb9158f16708aa0a48dfb3c59d752e"
    }
  ]
}