{
  "schema": "skillreducer/research-table/1",
  "id": "leaderboard",
  "study": "leaderboard",
  "title": "Optimization leaderboard, SPEC-027",
  "evidence_class": "measured-local",
  "date": "2026-09-11",
  "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd",
  "source_documents": [
    {
      "path": "docs/specs/SPEC-027-optimization-leaderboard.md",
      "commit": "5e9cb0eb430d62c9ccdfa333ba45608653b215d9",
      "committed": "2026-09-15"
    },
    {
      "path": "docs/experiments/MEASUREMENT-INTEGRITY.md",
      "commit": "8fce0d59659a22f7e8ebaa831b0be8bfacd26390",
      "committed": "2026-09-15"
    }
  ],
  "n_meaning": "Successful runs behind the row, both versions together; for an excluded skill, every recorded run",
  "columns": [
    {
      "name": "skill",
      "description": "Skill id in the benchmark"
    },
    {
      "name": "outcome",
      "description": "won, did_not_win, or excluded"
    },
    {
      "name": "excluded_reason",
      "description": "Why the board excludes the skill (SPEC-027 section 4)"
    },
    {
      "name": "skill_tokens_original",
      "description": "Mean skill tokens, original"
    },
    {
      "name": "skill_tokens_optimized",
      "description": "Mean skill tokens, improved"
    },
    {
      "name": "skill_tokens_change",
      "description": "Change in skill tokens; the sort key; negative is better"
    },
    {
      "name": "cost_change",
      "description": "Change in billed money"
    },
    {
      "name": "seconds_original",
      "description": "Mean wall-clock seconds, original"
    },
    {
      "name": "seconds_optimized",
      "description": "Mean wall-clock seconds, improved"
    },
    {
      "name": "seconds_change",
      "description": "Change in wall-clock time"
    },
    {
      "name": "session_tokens_original",
      "description": "Mean session tokens, original"
    },
    {
      "name": "session_tokens_optimized",
      "description": "Mean session tokens, improved"
    },
    {
      "name": "session_tokens_change",
      "description": "Change in session tokens; never the sort key"
    },
    {
      "name": "attributable_share",
      "description": "Skill tokens as a share of session tokens"
    },
    {
      "name": "correctness_original",
      "description": "Checks passed, original"
    },
    {
      "name": "correctness_optimized",
      "description": "Checks passed, improved"
    },
    {
      "name": "held",
      "description": "Improved passes at least as many checks"
    },
    {
      "name": "improved",
      "description": "Improved passes more checks"
    },
    {
      "name": "won",
      "description": "Held and leaner on skill tokens"
    },
    {
      "name": "predicted_no_win",
      "description": "Its author pre-registered no win before the test"
    },
    {
      "name": "note",
      "description": "Why a row did not win, in one clause"
    },
    {
      "name": "installs",
      "description": "Installs reported by the catalog"
    },
    {
      "name": "fixtures",
      "description": "Fixtures, the tasks, per version"
    },
    {
      "name": "cells_original",
      "description": "Successful runs, original"
    },
    {
      "name": "cells_optimized",
      "description": "Successful runs, improved"
    },
    {
      "name": "integrity_2026_09_15",
      "description": "Status under the 2026-09-15 complete-usage re-score"
    },
    {
      "name": "n",
      "description": "Successful runs behind the row, both versions together; for an excluded skill, every recorded run"
    },
    {
      "name": "evidence_class",
      "description": "Evidence class of the figure (SPEC.md section 20)"
    },
    {
      "name": "date",
      "description": "UTC day the study's last measured run landed"
    },
    {
      "name": "commit",
      "description": "Commit the figures come from: the committed data when the study has some, otherwise the latest commit of the document that reports it"
    }
  ],
  "rows": [
    {
      "skill": "stitch-loop",
      "outcome": "won",
      "excluded_reason": null,
      "skill_tokens_original": 26407.083333333332,
      "skill_tokens_optimized": 13960.666666666666,
      "skill_tokens_change": -0.4713287154646007,
      "cost_change": -0.23494176257435778,
      "seconds_original": 71.05866666666667,
      "seconds_optimized": 37.81516666666666,
      "seconds_change": -0.4678317446616881,
      "session_tokens_original": 1053650.25,
      "session_tokens_optimized": 904587.4166666667,
      "session_tokens_change": -0.14147278314918377,
      "attributable_share": 0.02061432618070023,
      "correctness_original": "12/12",
      "correctness_optimized": "12/12",
      "held": true,
      "improved": false,
      "won": true,
      "predicted_no_win": false,
      "note": null,
      "installs": 53439,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "json-canvas",
      "outcome": "won",
      "excluded_reason": null,
      "skill_tokens_original": 7693.499999999999,
      "skill_tokens_optimized": 4754.083333333333,
      "skill_tokens_change": -0.38206494659994367,
      "cost_change": -0.18642336048392374,
      "seconds_original": 18.352083333333333,
      "seconds_optimized": 36.49883333333334,
      "seconds_change": 0.9888114428425476,
      "session_tokens_original": 353763.8333333334,
      "session_tokens_optimized": 315692.4166666667,
      "session_tokens_change": -0.10761817087953691,
      "attributable_share": 0.018593572519986678,
      "correctness_original": "12/12",
      "correctness_optimized": "12/12",
      "held": true,
      "improved": false,
      "won": true,
      "predicted_no_win": false,
      "note": "leaner but slower on purpose: a script instead of reasoning",
      "installs": 67693,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "scaffold-exercises",
      "outcome": "won",
      "excluded_reason": null,
      "skill_tokens_original": 18270.416666666668,
      "skill_tokens_optimized": 13498.833333333334,
      "skill_tokens_change": -0.2611644507286369,
      "cost_change": -0.2571956597781178,
      "seconds_original": 92.12666666666665,
      "seconds_optimized": 39.56283333333334,
      "seconds_change": -0.570560460235907,
      "session_tokens_original": 1267068.8333333335,
      "session_tokens_optimized": 635188.0833333334,
      "session_tokens_change": -0.4986948880572524,
      "attributable_share": 0.016700819811274175,
      "correctness_original": "12/12",
      "correctness_optimized": "12/12",
      "held": true,
      "improved": false,
      "won": true,
      "predicted_no_win": false,
      "note": null,
      "installs": 312779,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "skill-creator",
      "outcome": "won",
      "excluded_reason": null,
      "skill_tokens_original": 54969.583333333336,
      "skill_tokens_optimized": 47205.16666666667,
      "skill_tokens_change": -0.14124932727948028,
      "cost_change": -0.04077013810568863,
      "seconds_original": 187.078,
      "seconds_optimized": 211.8094166666667,
      "seconds_change": 0.13219842347398778,
      "session_tokens_original": 1321372,
      "session_tokens_optimized": 1363780.1666666665,
      "session_tokens_change": 0.03209404063856848,
      "attributable_share": 0.03805175411225919,
      "correctness_original": "7/12",
      "correctness_optimized": "9/12",
      "held": true,
      "improved": true,
      "won": true,
      "predicted_no_win": false,
      "note": null,
      "installs": 374470,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "hyperframes",
      "outcome": "won",
      "excluded_reason": null,
      "skill_tokens_original": 21121.666666666668,
      "skill_tokens_optimized": 18219,
      "skill_tokens_change": -0.13742602383019023,
      "cost_change": 0.08371542468644565,
      "seconds_original": 87.7085,
      "seconds_optimized": 64.12191666666666,
      "seconds_change": -0.26892015407096614,
      "session_tokens_original": 656299.25,
      "session_tokens_optimized": 792686,
      "session_tokens_change": 0.2078118327881679,
      "attributable_share": 0.027150494918196492,
      "correctness_original": "11/12",
      "correctness_optimized": "11/12",
      "held": true,
      "improved": false,
      "won": true,
      "predicted_no_win": false,
      "note": "a loss on session tokens, a win on the skill's own",
      "installs": 476616,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "obsidian-cli",
      "outcome": "won",
      "excluded_reason": null,
      "skill_tokens_original": 14710.416666666668,
      "skill_tokens_optimized": 13932.833333333334,
      "skill_tokens_change": -0.05285936836142193,
      "cost_change": -0.04974492014780895,
      "seconds_original": 133.82758333333334,
      "seconds_optimized": 119.21825000000001,
      "seconds_change": -0.10916533773866992,
      "session_tokens_original": 1046660.9166666666,
      "session_tokens_optimized": 910745.9166666667,
      "session_tokens_change": -0.12985580892124318,
      "attributable_share": 0.014633263515905099,
      "correctness_original": "7/12",
      "correctness_optimized": "11/12",
      "held": true,
      "improved": true,
      "won": true,
      "predicted_no_win": false,
      "note": null,
      "installs": 73603,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "vercel-cli-with-tokens",
      "outcome": "won",
      "excluded_reason": null,
      "skill_tokens_original": 11874.166666666666,
      "skill_tokens_optimized": 11604.083333333334,
      "skill_tokens_change": -0.022745455821461053,
      "cost_change": -0.07585253900572461,
      "seconds_original": 30.2435,
      "seconds_optimized": 28.177333333333337,
      "seconds_change": -0.06831771014157303,
      "session_tokens_original": 487502.8333333333,
      "session_tokens_optimized": 479555.9166666667,
      "session_tokens_change": -0.01630127277892736,
      "attributable_share": 0.024277997588047263,
      "correctness_original": "10/12",
      "correctness_optimized": "12/12",
      "held": true,
      "improved": true,
      "won": true,
      "predicted_no_win": false,
      "note": null,
      "installs": 94038,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "deploy-to-vercel",
      "outcome": "did_not_win",
      "excluded_reason": null,
      "skill_tokens_original": 16604.916666666668,
      "skill_tokens_optimized": 14227.583333333332,
      "skill_tokens_change": -0.14317044650429855,
      "cost_change": -0.08607056283414163,
      "seconds_original": 47.1305,
      "seconds_optimized": 44.38575,
      "seconds_change": -0.05823723491157531,
      "session_tokens_original": 908259.9166666667,
      "session_tokens_optimized": 809479.25,
      "session_tokens_change": -0.10875814825033113,
      "attributable_share": 0.017949465552346663,
      "correctness_original": "8/12",
      "correctness_optimized": "6/12",
      "held": false,
      "improved": false,
      "won": false,
      "predicted_no_win": false,
      "note": "cheaper on every axis, but a claim regressed",
      "installs": 121850,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "vercel-optimize",
      "outcome": "did_not_win",
      "excluded_reason": null,
      "skill_tokens_original": 16451,
      "skill_tokens_optimized": 16854.916666666668,
      "skill_tokens_change": 0.02455271209450294,
      "cost_change": 0.09603629361529996,
      "seconds_original": 83.5005,
      "seconds_optimized": 65.82175,
      "seconds_change": -0.21172028910006527,
      "session_tokens_original": 567363.5,
      "session_tokens_optimized": 690873.25,
      "session_tokens_change": 0.21769068683480697,
      "attributable_share": 0.026470309873453203,
      "correctness_original": "9/12",
      "correctness_optimized": "11/12",
      "held": true,
      "improved": true,
      "won": false,
      "predicted_no_win": false,
      "note": "quality up, everything else up",
      "installs": 66262,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "domain-modeling",
      "outcome": "did_not_win",
      "excluded_reason": null,
      "skill_tokens_original": 15266.083333333334,
      "skill_tokens_optimized": 16676.25,
      "skill_tokens_change": 0.09237252515107014,
      "cost_change": 0.03131643921356859,
      "seconds_original": 32.375416666666666,
      "seconds_optimized": 29.506083333333333,
      "seconds_change": -0.08862691599850717,
      "session_tokens_original": 512668.9166666666,
      "session_tokens_optimized": 535701.5,
      "session_tokens_change": 0.044926818429112954,
      "attributable_share": 0.030468556557419077,
      "correctness_original": "12/12",
      "correctness_optimized": "12/12",
      "held": true,
      "improved": false,
      "won": false,
      "predicted_no_win": true,
      "note": "optimized doc is larger by design",
      "installs": 583528,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "gsap",
      "outcome": "did_not_win",
      "excluded_reason": null,
      "skill_tokens_original": 11841.833333333332,
      "skill_tokens_optimized": 13222.833333333332,
      "skill_tokens_change": 0.11662045572898341,
      "cost_change": 0.1879229361819283,
      "seconds_original": 19.710916666666666,
      "seconds_optimized": 34.55816666666667,
      "seconds_change": 0.7532501025235597,
      "session_tokens_original": 323557.75,
      "session_tokens_optimized": 435967.25,
      "session_tokens_change": 0.3474171148736199,
      "attributable_share": 0.0330004498425551,
      "correctness_original": "8/12",
      "correctness_optimized": "8/12",
      "held": true,
      "improved": false,
      "won": false,
      "predicted_no_win": true,
      "note": "a checker, not a generator",
      "installs": 95137,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "missing_usage",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "impeccable",
      "outcome": "did_not_win",
      "excluded_reason": null,
      "skill_tokens_original": 9585.916666666668,
      "skill_tokens_optimized": 18168.333333333332,
      "skill_tokens_change": 0.8953151759091024,
      "cost_change": 0.49477220598652816,
      "seconds_original": 80.282,
      "seconds_optimized": 111.3475,
      "seconds_change": 0.38695473456067364,
      "session_tokens_original": 331546.6666666667,
      "session_tokens_optimized": 551405.1666666667,
      "session_tokens_change": 0.6631298761360895,
      "attributable_share": 0.03143348136582006,
      "correctness_original": "12/12",
      "correctness_optimized": "6/12",
      "held": false,
      "improved": false,
      "won": false,
      "predicted_no_win": false,
      "note": "the optimized skill does more thinking, not less",
      "installs": 265361,
      "fixtures": 4,
      "cells_original": 12,
      "cells_optimized": 12,
      "integrity_2026_09_15": "comparable",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    },
    {
      "skill": "mcp-builder",
      "outcome": "excluded",
      "excluded_reason": "no_successful_cells",
      "skill_tokens_original": null,
      "skill_tokens_optimized": null,
      "skill_tokens_change": null,
      "cost_change": null,
      "seconds_original": null,
      "seconds_optimized": null,
      "seconds_change": null,
      "session_tokens_original": null,
      "session_tokens_optimized": null,
      "session_tokens_change": null,
      "attributable_share": null,
      "correctness_original": null,
      "correctness_optimized": null,
      "held": null,
      "improved": null,
      "won": null,
      "predicted_no_win": null,
      "note": null,
      "installs": 111815,
      "fixtures": null,
      "cells_original": null,
      "cells_optimized": null,
      "integrity_2026_09_15": "no_successful_cells",
      "n": 24,
      "evidence_class": "measured-local",
      "date": "2026-09-11",
      "commit": "2ebfd81e00220a00a6fb6f4eefc75fefc3c850bd"
    }
  ]
}