{
 "evidence_sha256": "939dff7240a4c381913dc97c5dccec034c2c198e12793637d044ba80b4a87921",
 "format": "gaents.curation.public.v1",
 "generated_at": "2026-11-19T12:29:21Z",
 "items": [
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "n=228 gives roughly +/-0.06-0.10; strict scoring understates every arm (16 of 31 disagreements defensible both ways) (Pixel)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "unknown",
     "cost_per": 1000,
     "cost_per_correct_usd": null,
     "evidence_id": "e7983fe2b7555706",
     "item_set": "placement-228cos-opus-review",
     "measured_at": "2026-10-26T06:30:13Z",
     "n": 182,
     "quality": [
      {
       "ci": "source",
       "hi": 0.88,
       "k": 151,
       "key": "precision",
       "lo": 0.77,
       "n": 182,
       "score": 0.83
      },
      {
       "ci": "wilson_here",
       "hi": 0.8165381572058469,
       "k": 151,
       "key": "recall",
       "lo": 0.6987173699005573,
       "n": 198,
       "score": 0.76
      }
     ],
     "roles": [],
     "source_file": "race_results.tsv",
     "source_owner": "Pixel",
     "state": "undecided",
     "superseded_by": [],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "240504c43f0dd83b",
       "name": "gaents/claude-sonnet",
       "offered": true
      },
      {
       "evidence_id": "a8f51c4055f6cd24",
       "name": "gaents/qwen-flash",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "Re-scored 2026-11-19 with Pixel's own score.py row() on the companies the subset arm was asked (no model calls)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "unknown",
     "cost_per": 1000,
     "cost_per_correct_usd": null,
     "evidence_id": "8004ea5a25ba9ed7",
     "item_set": "placement-subset_114-opus-review",
     "measured_at": "2026-11-19",
     "n": 72,
     "quality": [
      {
       "ci": "source",
       "hi": 0.91,
       "k": 61,
       "key": "precision",
       "lo": 0.75,
       "n": 72,
       "score": 0.85
      },
      {
       "ci": "wilson_here",
       "hi": 0.8023128049451995,
       "k": 61,
       "key": "recall",
       "lo": 0.6141587579262481,
       "n": 85,
       "score": 0.72
      }
     ],
     "roles": [
      "top score"
     ],
     "source_file": "placement-rescore-2026-11-19.json",
     "source_owner": "Hustle (re-score) / Pixel (scorer, key, stored answers)",
     "state": "undecided",
     "superseded_by": [],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "538567b2b097d20b",
       "name": "gaents/claude-sonnet",
       "offered": true
      },
      {
       "evidence_id": "9c64213a8628902d",
       "name": "[name not shown]",
       "offered": false
      },
      {
       "evidence_id": "01817e97cb685dd4",
       "name": "gaents/qwen-flash",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "Re-scored 2026-11-19 with Pixel's own score.py row() on the companies the subset arm was asked (no model calls)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "unknown",
     "cost_per": 1000,
     "cost_per_correct_usd": null,
     "evidence_id": "c3feda386210fb8e",
     "item_set": "placement-subset_48-opus-review",
     "measured_at": "2026-11-19",
     "n": 23,
     "quality": [
      {
       "ci": "source",
       "hi": 0.98,
       "k": 21,
       "key": "precision",
       "lo": 0.73,
       "n": 23,
       "score": 0.91
      },
      {
       "ci": "wilson_here",
       "hi": 0.8333543735604106,
       "k": 21,
       "key": "recall",
       "lo": 0.5212387898833981,
       "n": 30,
       "score": 0.7
      }
     ],
     "roles": [
      "top score"
     ],
     "source_file": "placement-rescore-2026-11-19.json",
     "source_owner": "Hustle (re-score) / Pixel (scorer, key, stored answers)",
     "state": "undecided",
     "superseded_by": [],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "8dbeb35e369965a1",
       "name": "gaents/claude-sonnet",
       "offered": true
      },
      {
       "evidence_id": "eaafe9c906ea1c1e",
       "name": "[name not shown]",
       "offered": false
      },
      {
       "evidence_id": "1596e1a0bfe933ab",
       "name": "gaents/qwen-flash",
       "offered": true
      },
      {
       "evidence_id": "4dc2bee7e37d0bb0",
       "name": "gaents/grok",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key and grader are both Opus, which favours Claude routes (REPORT.md).",
      "Cost covers every company called (200 API routes, 192 Claude routes); the score covers the 192 in scope."
     ],
     "correct_unit": "correct field",
     "cost_basis": "capacity_list",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.6134076797829038,
     "evidence_id": "6a12ad6648ce6793",
     "item_set": "s1-192cos-judge-key",
     "measured_at": "2026-11-18T15:16Z",
     "n": 1607,
     "quality": [
      {
       "ci": "source",
       "hi": 0.9297,
       "k": 1474,
       "key": "precision",
       "lo": 0.9028,
       "n": 1607,
       "score": 0.9172
      },
      {
       "ci": "source",
       "hi": 0.9171,
       "k": null,
       "key": "recall",
       "lo": 0.8885,
       "n": null,
       "score": 0.9037
      }
     ],
     "roles": [],
     "source_file": "tables.json",
     "source_owner": "Hustle (re-score) / Judge (scorer) / Fathom+[name not shown] (runs)",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "a92d06281136abe0",
       "name": "gaents/deepseek",
       "offered": true
      }
     ],
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)",
     "undecided_with": []
    }
   ],
   "id": "gaents/claude-haiku",
   "kept_because_used": true,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 725 items x $9.82e-05/item (gaents/glm-flash measured) x output-price ratio 1.00",
    "est_usd": 0.0712,
    "task": "roar"
   },
   "status_class": "superseded",
   "superseded_on": [
    {
     "by": "gaents/deepseek",
     "by_offered": true,
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)"
    }
   ],
   "used": true,
   "verdict": "On Company facts extraction (13 fields from a company's site), gaents/deepseek matches or beats it for less. Kept because it is already listed and used. No verdict yet on Industry placement (family level) (a cost is unknown or nothing comparable was run)."
  },
  {
   "best_for": [
    {
     "task": "coding",
     "task_name": "Agentic coding (repo tasks, hidden-test checks)"
    }
   ],
   "evidence": [
    {
     "caveats": [
      "Model and harness are measured together (Claude Code vs opencode).",
      "Mergeable = one Opus reviewer, blind, on the 9 tasks both pass (n=9).",
      "glm-flash took ~3.1x the wall time."
     ],
     "correct_unit": "passing run",
     "cost_basis": "capacity_list",
     "cost_per": 1,
     "cost_per_correct_usd": 0.9086111111111111,
     "evidence_id": "c6346ba282b51949",
     "item_set": "coding-h2h-14tasks-x3",
     "measured_at": "2026-11-09",
     "n": 42,
     "quality": [
      {
       "ci": "source",
       "hi": 0.93,
       "k": 36,
       "key": "passes",
       "lo": 0.72,
       "n": 42,
       "score": 0.8571428571428571
      },
      {
       "ci": "wilson_here",
       "hi": 1.0,
       "k": 9,
       "key": "mergeable",
       "lo": 0.7008472464490406,
       "n": 9,
       "score": 1.0
      }
     ],
     "roles": [
      "top score"
     ],
     "source_file": "RESULTS-r2-2026-11-09.md",
     "source_owner": "[name not shown] (h2h) / Judge (blind review)",
     "state": "frontier",
     "superseded_by": [],
     "task": "coding",
     "task_name": "Agentic coding (repo tasks, hidden-test checks)",
     "undecided_with": []
    }
   ],
   "id": "gaents/claude-opus",
   "kept_because_used": false,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 725 items x $9.82e-05/item (gaents/glm-flash measured) x output-price ratio 40.00",
    "est_usd": 2.8478,
    "task": "roar"
   },
   "status_class": "best-for",
   "superseded_on": [],
   "used": false,
   "verdict": "Best for Agentic coding (repo tasks, hidden-test checks) on our tests (nothing we measured beats it on both quality and cost there)."
  },
  {
   "best_for": [
    {
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)"
    }
   ],
   "evidence": [
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "n=228 gives roughly +/-0.06-0.10; strict scoring understates every arm (16 of 31 disagreements defensible both ways) (Pixel)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "capacity_list",
     "cost_per": 1000,
     "cost_per_correct_usd": 54.99305555555555,
     "evidence_id": "240504c43f0dd83b",
     "item_set": "placement-228cos-opus-review",
     "measured_at": "2026-10-26T06:30:13Z",
     "n": 182,
     "quality": [
      {
       "ci": "source",
       "hi": 0.84,
       "k": 144,
       "key": "precision",
       "lo": 0.73,
       "n": 182,
       "score": 0.79
      },
      {
       "ci": "wilson_here",
       "hi": 0.7845409439498122,
       "k": 144,
       "key": "recall",
       "lo": 0.6613532621999607,
       "n": 198,
       "score": 0.73
      }
     ],
     "roles": [],
     "source_file": "race_results.tsv",
     "source_owner": "Pixel",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "a8f51c4055f6cd24",
       "name": "gaents/qwen-flash",
       "offered": true
      }
     ],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "e7983fe2b7555706",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "Re-scored 2026-11-19 with Pixel's own score.py row() on the companies the subset arm was asked (no model calls)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "capacity_list",
     "cost_per": 1000,
     "cost_per_correct_usd": 69.46491228070175,
     "evidence_id": "538567b2b097d20b",
     "item_set": "placement-subset_114-opus-review",
     "measured_at": "2026-11-19",
     "n": 73,
     "quality": [
      {
       "ci": "source",
       "hi": 0.86,
       "k": 57,
       "key": "precision",
       "lo": 0.67,
       "n": 73,
       "score": 0.78
      },
      {
       "ci": "wilson_here",
       "hi": 0.7612237373723095,
       "k": 57,
       "key": "recall",
       "lo": 0.5651999200134196,
       "n": 85,
       "score": 0.67
      }
     ],
     "roles": [],
     "source_file": "placement-rescore-2026-11-19.json",
     "source_owner": "Hustle (re-score) / Pixel (scorer, key, stored answers)",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "01817e97cb685dd4",
       "name": "gaents/qwen-flash",
       "offered": true
      },
      {
       "evidence_id": "9c64213a8628902d",
       "name": "[name not shown]",
       "offered": false
      }
     ],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "8004ea5a25ba9ed7",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "Re-scored 2026-11-19 with Pixel's own score.py row() on the companies the subset arm was asked (no model calls)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "capacity_list",
     "cost_per": 1000,
     "cost_per_correct_usd": 79.38847117794485,
     "evidence_id": "8dbeb35e369965a1",
     "item_set": "placement-subset_48-opus-review",
     "measured_at": "2026-11-19",
     "n": 24,
     "quality": [
      {
       "ci": "source",
       "hi": 0.96,
       "k": 21,
       "key": "precision",
       "lo": 0.69,
       "n": 24,
       "score": 0.88
      },
      {
       "ci": "wilson_here",
       "hi": 0.8333543735604106,
       "k": 21,
       "key": "recall",
       "lo": 0.5212387898833981,
       "n": 30,
       "score": 0.7
      }
     ],
     "roles": [],
     "source_file": "placement-rescore-2026-11-19.json",
     "source_owner": "Hustle (re-score) / Pixel (scorer, key, stored answers)",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "1596e1a0bfe933ab",
       "name": "gaents/qwen-flash",
       "offered": true
      },
      {
       "evidence_id": "eaafe9c906ea1c1e",
       "name": "[name not shown]",
       "offered": false
      },
      {
       "evidence_id": "4dc2bee7e37d0bb0",
       "name": "gaents/grok",
       "offered": true
      }
     ],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "c3feda386210fb8e",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key and grader are both Opus, which favours Claude routes (REPORT.md).",
      "Cost covers every company called (200 API routes, 192 Claude routes); the score covers the 192 in scope."
     ],
     "correct_unit": "correct field",
     "cost_basis": "capacity_list",
     "cost_per": 1000,
     "cost_per_correct_usd": 7.636504435746902,
     "evidence_id": "85412a3ea06a625a",
     "item_set": "s1-192cos-judge-key",
     "measured_at": "2026-11-18T15:16Z",
     "n": 1698,
     "quality": [
      {
       "ci": "source",
       "hi": 0.916,
       "k": 1533,
       "key": "precision",
       "lo": 0.8878,
       "n": 1698,
       "score": 0.9028
      },
      {
       "ci": "source",
       "hi": 0.9504,
       "k": null,
       "key": "recall",
       "lo": 0.9273,
       "n": null,
       "score": 0.9399
      }
     ],
     "roles": [],
     "source_file": "tables.json",
     "source_owner": "Hustle (re-score) / Judge (scorer) / Fathom+[name not shown] (runs)",
     "state": "frontier",
     "superseded_by": [],
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)",
     "undecided_with": []
    }
   ],
   "id": "gaents/claude-sonnet",
   "kept_because_used": false,
   "kind": "alias",
   "listed": true,
   "next_test": null,
   "status_class": "best-for",
   "superseded_on": [
    {
     "by": "gaents/qwen-flash",
     "by_offered": true,
     "task": "placement",
     "task_name": "Industry placement (family level)"
    },
    {
     "by": "gaents/qwen-flash",
     "by_offered": true,
     "task": "placement",
     "task_name": "Industry placement (family level)"
    },
    {
     "by": "gaents/qwen-flash",
     "by_offered": true,
     "task": "placement",
     "task_name": "Industry placement (family level)"
    }
   ],
   "used": false,
   "verdict": "Best for Company facts extraction (13 fields from a company's site) on our tests (nothing we measured beats it on both quality and cost there). On Industry placement (family level), gaents/qwen-flash matches or beats it for less. On Industry placement (family level), gaents/qwen-flash matches or beats it for less. On Industry placement (family level), gaents/qwen-flash matches or beats it for less."
  },
  {
   "best_for": [
    {
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)"
    }
   ],
   "evidence": [
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "n=228 gives roughly +/-0.06-0.10; strict scoring understates every arm (16 of 31 disagreements defensible both ways) (Pixel)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 1.1743589743589742,
     "evidence_id": "fbd4915db46a1140",
     "item_set": "placement-228cos-opus-review",
     "measured_at": "2026-10-26T06:30:13Z",
     "n": 178,
     "quality": [
      {
       "ci": "source",
       "hi": 0.72,
       "k": 117,
       "key": "precision",
       "lo": 0.58,
       "n": 178,
       "score": 0.66
      },
      {
       "ci": "wilson_here",
       "hi": 0.6570307752937751,
       "k": 117,
       "key": "recall",
       "lo": 0.5213269071661342,
       "n": 198,
       "score": 0.59
      }
     ],
     "roles": [
      "cheapest known cost per correct output"
     ],
     "source_file": "race_results.tsv",
     "source_owner": "Pixel",
     "state": "undecided",
     "superseded_by": [],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "e7983fe2b7555706",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "Re-scored 2026-11-19 with Pixel's own score.py row() on the companies the subset arm was asked (no model calls)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 1.5266666666666666,
     "evidence_id": "40ad0ef4ab22bdff",
     "item_set": "placement-subset_114-opus-review",
     "measured_at": "2026-11-19",
     "n": 69,
     "quality": [
      {
       "ci": "source",
       "hi": 0.75,
       "k": 45,
       "key": "precision",
       "lo": 0.53,
       "n": 69,
       "score": 0.65
      },
      {
       "ci": "wilson_here",
       "hi": 0.6319400802964319,
       "k": 45,
       "key": "recall",
       "lo": 0.4243398606321421,
       "n": 85,
       "score": 0.53
      }
     ],
     "roles": [
      "cheapest known cost per correct output"
     ],
     "source_file": "placement-rescore-2026-11-19.json",
     "source_owner": "Hustle (re-score) / Pixel (scorer, key, stored answers)",
     "state": "undecided",
     "superseded_by": [],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "8004ea5a25ba9ed7",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "Re-scored 2026-11-19 with Pixel's own score.py row() on the companies the subset arm was asked (no model calls)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 2.0661654135338345,
     "evidence_id": "028f1f8722b67e86",
     "item_set": "placement-subset_48-opus-review",
     "measured_at": "2026-11-19",
     "n": 19,
     "quality": [
      {
       "ci": "source",
       "hi": 0.88,
       "k": 14,
       "key": "precision",
       "lo": 0.51,
       "n": 19,
       "score": 0.74
      },
      {
       "ci": "wilson_here",
       "hi": 0.6385798671847496,
       "k": 14,
       "key": "recall",
       "lo": 0.3023212722412823,
       "n": 30,
       "score": 0.47
      }
     ],
     "roles": [],
     "source_file": "placement-rescore-2026-11-19.json",
     "source_owner": "Hustle (re-score) / Pixel (scorer, key, stored answers)",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "1596e1a0bfe933ab",
       "name": "gaents/qwen-flash",
       "offered": true
      }
     ],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "c3feda386210fb8e",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key and grader are both Opus, which favours Claude routes (REPORT.md).",
      "Cost covers every company called (200 API routes, 192 Claude routes); the score covers the 192 in scope."
     ],
     "correct_unit": "correct field",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.5610000870147256,
     "evidence_id": "a92d06281136abe0",
     "item_set": "s1-192cos-judge-key",
     "measured_at": "2026-11-18T15:16Z",
     "n": 1651,
     "quality": [
      {
       "ci": "source",
       "hi": 0.9181,
       "k": 1494,
       "key": "precision",
       "lo": 0.8898,
       "n": 1651,
       "score": 0.9049
      },
      {
       "ci": "source",
       "hi": 0.9285,
       "k": null,
       "key": "recall",
       "lo": 0.9015,
       "n": null,
       "score": 0.916
      }
     ],
     "roles": [],
     "source_file": "tables.json",
     "source_owner": "Hustle (re-score) / Judge (scorer) / Fathom+[name not shown] (runs)",
     "state": "frontier",
     "superseded_by": [],
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)",
     "undecided_with": []
    }
   ],
   "id": "gaents/deepseek",
   "kept_because_used": true,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 725 items x $9.82e-05/item (gaents/glm-flash measured) x output-price ratio 0.76",
    "est_usd": 0.0541,
    "task": "roar"
   },
   "status_class": "best-for",
   "superseded_on": [
    {
     "by": "gaents/qwen-flash",
     "by_offered": true,
     "task": "placement",
     "task_name": "Industry placement (family level)"
    }
   ],
   "used": true,
   "verdict": "Best for Company facts extraction (13 fields from a company's site) on our tests (nothing we measured beats it on both quality and cost there). On Industry placement (family level), gaents/qwen-flash matches or beats it for less. Kept because it is already listed and used. No verdict yet on Industry placement (family level) (a cost is unknown or nothing comparable was run)."
  },
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "Labels are Sonnet's: this is agreement with Sonnet (Sonnet's own test-retest ~0.84), not truth.",
      "Interval: Wilson 95% computed here.",
      "Cost is our own gate price (includes our margin), not the provider's."
     ],
     "correct_unit": "item agreeing with Sonnet's sign",
     "cost_basis": "list_price",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.11177298311444654,
     "evidence_id": "56f61ba6b058fb80",
     "item_set": "roar-725-kimi-aligned",
     "measured_at": "2026-11-19",
     "n": 725,
     "quality": [
      {
       "ci": "wilson_here",
       "hi": 0.7659911676245971,
       "k": 533,
       "key": "sign_agreement",
       "lo": 0.7018745524440709,
       "n": 725,
       "score": 0.7352
      }
     ],
     "roles": [],
     "source_file": "report-deepseek-flash.json",
     "source_owner": "Hustle (run) / Fathom (adapter, labels)",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "f6bfcc577dd0197b",
       "name": "typesafe/jev-1.13",
       "offered": false
      }
     ],
     "task": "roar",
     "task_name": "Directed sentiment (agreement with Sonnet's labels)",
     "undecided_with": [
      {
       "evidence_id": "8bc1c8fd476bdf9b",
       "name": "fathom:roar-roberta-large-ft2",
       "offered": false
      },
      {
       "evidence_id": "ecd0d4b7ce741727",
       "name": "moonshotai/kimi-k2.6",
       "offered": false
      },
      {
       "evidence_id": "f26601ac1831ad11",
       "name": "gaents/qwen-27b",
       "offered": true
      }
     ]
    }
   ],
   "id": "gaents/deepseek-flash",
   "kept_because_used": true,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 228 items x $0.000755/item (gaents/qwen-flash measured) x output-price ratio 0.38",
    "est_usd": 0.0659,
    "task": "placement"
   },
   "status_class": "superseded",
   "superseded_on": [
    {
     "by": "typesafe/jev-1.13",
     "by_offered": false,
     "task": "roar",
     "task_name": "Directed sentiment (agreement with Sonnet's labels)"
    }
   ],
   "used": true,
   "verdict": "On Directed sentiment (agreement with Sonnet's labels), typesafe/jev-1.13 matches or beats it for less (not offered here). Kept because it is already listed and used."
  },
  {
   "best_for": [],
   "evidence": [],
   "id": "gaents/fable",
   "kept_because_used": false,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 725 items x $9.82e-05/item (gaents/glm-flash measured) x output-price ratio 80.00",
    "est_usd": 5.6956,
    "task": "roar"
   },
   "status_class": "untested",
   "superseded_on": [],
   "used": false,
   "verdict": "Not yet tested by us on any task."
  },
  {
   "best_for": [
    {
     "task": "coding",
     "task_name": "Agentic coding (repo tasks, hidden-test checks)"
    },
    {
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)"
    }
   ],
   "evidence": [
    {
     "caveats": [
      "Model and harness are measured together (Claude Code vs opencode).",
      "Mergeable = one Opus reviewer, blind, on the 9 tasks both pass (n=9).",
      "glm-flash took ~3.1x the wall time."
     ],
     "correct_unit": "passing run",
     "cost_basis": "list_price",
     "cost_per": 1,
     "cost_per_correct_usd": 0.0703030303030303,
     "evidence_id": "7b894e832947569a",
     "item_set": "coding-h2h-14tasks-x3",
     "measured_at": "2026-11-09",
     "n": 42,
     "quality": [
      {
       "ci": "source",
       "hi": 0.88,
       "k": 33,
       "key": "passes",
       "lo": 0.64,
       "n": 42,
       "score": 0.7857142857142857
      },
      {
       "ci": "wilson_here",
       "hi": 0.7333526542559703,
       "k": 4,
       "key": "mergeable",
       "lo": 0.188775429471914,
       "n": 9,
       "score": 0.4444444444444444
      }
     ],
     "roles": [
      "cheapest known cost per correct output"
     ],
     "source_file": "RESULTS-r2-2026-11-09.md",
     "source_owner": "[name not shown] (h2h) / Judge (blind review)",
     "state": "frontier",
     "superseded_by": [],
     "task": "coding",
     "task_name": "Agentic coding (repo tasks, hidden-test checks)",
     "undecided_with": []
    },
    {
     "caveats": [
      "Labels are Sonnet's: this is agreement with Sonnet (Sonnet's own test-retest ~0.84), not truth.",
      "Interval: Wilson 95% computed here from score x n."
     ],
     "correct_unit": "item agreeing with Sonnet's sign",
     "cost_basis": "list_price",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.12897294457578146,
     "evidence_id": "9b9dff029e6247a1",
     "item_set": "roar-725-kimi-aligned",
     "measured_at": "2026-10-18",
     "n": 725,
     "quality": [
      {
       "ci": "wilson_here",
       "hi": 0.790977532596503,
       "k": 552,
       "key": "sign_agreement",
       "lo": 0.7290257163947729,
       "n": 725,
       "score": 0.7614
      }
     ],
     "roles": [],
     "source_file": "model-arena.jsonl",
     "source_owner": "Fathom",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "f6bfcc577dd0197b",
       "name": "typesafe/jev-1.13",
       "offered": false
      },
      {
       "evidence_id": "56f61ba6b058fb80",
       "name": "gaents/deepseek-flash",
       "offered": true
      }
     ],
     "task": "roar",
     "task_name": "Directed sentiment (agreement with Sonnet's labels)",
     "undecided_with": [
      {
       "evidence_id": "8bc1c8fd476bdf9b",
       "name": "fathom:roar-roberta-large-ft2",
       "offered": false
      },
      {
       "evidence_id": "ecd0d4b7ce741727",
       "name": "moonshotai/kimi-k2.6",
       "offered": false
      }
     ]
    },
    {
     "caveats": [
      "Answer key and grader are both Opus, which favours Claude routes (REPORT.md).",
      "Cost covers every company called (200 API routes, 192 Claude routes); the score covers the 192 in scope."
     ],
     "correct_unit": "correct field",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.22991632440284054,
     "evidence_id": "cea4958c1119537e",
     "item_set": "s1-192cos-judge-key",
     "measured_at": "2026-11-18T15:16Z",
     "n": 1745,
     "quality": [
      {
       "ci": "source",
       "hi": 0.9017,
       "k": 1549,
       "key": "precision",
       "lo": 0.872,
       "n": 1745,
       "score": 0.8877
      },
      {
       "ci": "source",
       "hi": 0.9593,
       "k": null,
       "key": "recall",
       "lo": 0.938,
       "n": null,
       "score": 0.9497
      }
     ],
     "roles": [],
     "source_file": "tables.json",
     "source_owner": "Hustle (re-score) / Judge (scorer) / Fathom+[name not shown] (runs)",
     "state": "frontier",
     "superseded_by": [],
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)",
     "undecided_with": []
    }
   ],
   "id": "gaents/glm-flash",
   "kept_because_used": true,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 228 items x $0.000755/item (gaents/qwen-flash measured) x output-price ratio 1.06",
    "est_usd": 0.1831,
    "task": "placement"
   },
   "status_class": "best-for",
   "superseded_on": [
    {
     "by": "typesafe/jev-1.13",
     "by_offered": false,
     "task": "roar",
     "task_name": "Directed sentiment (agreement with Sonnet's labels)"
    }
   ],
   "used": true,
   "verdict": "Best for Agentic coding (repo tasks, hidden-test checks), Company facts extraction (13 fields from a company's site) on our tests (nothing we measured beats it on both quality and cost there). On Directed sentiment (agreement with Sonnet's labels), typesafe/jev-1.13 matches or beats it for less (not offered here). Kept because it is already listed and used."
  },
  {
   "best_for": [
    {
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)"
    }
   ],
   "evidence": [
    {
     "caveats": [
      "Labels are Sonnet's: this is agreement with Sonnet (Sonnet's own test-retest ~0.84), not truth.",
      "Interval: Wilson 95% computed here from score x n."
     ],
     "correct_unit": "item agreeing with Sonnet's sign",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.0696,
     "evidence_id": "672c08c3f831432a",
     "item_set": "roar-725-kimi-aligned",
     "measured_at": "2026-10-21",
     "n": 725,
     "quality": [
      {
       "ci": "wilson_here",
       "hi": 0.5561659250156447,
       "k": 377,
       "key": "sign_agreement",
       "lo": 0.48362324179371424,
       "n": 725,
       "score": 0.52
      }
     ],
     "roles": [],
     "source_file": "model-arena.jsonl",
     "source_owner": "Fathom",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "f6bfcc577dd0197b",
       "name": "typesafe/jev-1.13",
       "offered": false
      },
      {
       "evidence_id": "c024a96833c449e2",
       "name": "clef-flash",
       "offered": false
      }
     ],
     "task": "roar",
     "task_name": "Directed sentiment (agreement with Sonnet's labels)",
     "undecided_with": [
      {
       "evidence_id": "8bc1c8fd476bdf9b",
       "name": "fathom:roar-roberta-large-ft2",
       "offered": false
      },
      {
       "evidence_id": "ecd0d4b7ce741727",
       "name": "moonshotai/kimi-k2.6",
       "offered": false
      },
      {
       "evidence_id": "f26601ac1831ad11",
       "name": "gaents/qwen-27b",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key and grader are both Opus, which favours Claude routes (REPORT.md).",
      "Cost covers every company called (200 API routes, 192 Claude routes); the score covers the 192 in scope."
     ],
     "correct_unit": "correct field",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.14685165187713312,
     "evidence_id": "b978813b93055cab",
     "item_set": "s1-192cos-judge-key",
     "measured_at": "2026-11-18T15:16Z",
     "n": 1679,
     "quality": [
      {
       "ci": "source",
       "hi": 0.8876,
       "k": 1465,
       "key": "precision",
       "lo": 0.8557,
       "n": 1679,
       "score": 0.8725
      },
      {
       "ci": "source",
       "hi": 0.912,
       "k": null,
       "key": "recall",
       "lo": 0.8826,
       "n": null,
       "score": 0.8982
      }
     ],
     "roles": [
      "cheapest known cost per correct output"
     ],
     "source_file": "tables.json",
     "source_owner": "Hustle (re-score) / Judge (scorer) / Fathom+[name not shown] (runs)",
     "state": "frontier",
     "superseded_by": [],
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)",
     "undecided_with": []
    }
   ],
   "id": "gaents/gpt-oss",
   "kept_because_used": true,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 228 items x $0.000755/item (gaents/qwen-flash measured) x output-price ratio 1.28",
    "est_usd": 0.2198,
    "task": "placement"
   },
   "status_class": "best-for",
   "superseded_on": [
    {
     "by": "typesafe/jev-1.13",
     "by_offered": false,
     "task": "roar",
     "task_name": "Directed sentiment (agreement with Sonnet's labels)"
    }
   ],
   "used": true,
   "verdict": "Best for Company facts extraction (13 fields from a company's site) on our tests (nothing we measured beats it on both quality and cost there). On Directed sentiment (agreement with Sonnet's labels), typesafe/jev-1.13 matches or beats it for less (not offered here). Kept because it is already listed and used."
  },
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "n=228 gives roughly +/-0.06-0.10; strict scoring understates every arm (16 of 31 disagreements defensible both ways) (Pixel)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 13.78125,
     "evidence_id": "4dc2bee7e37d0bb0",
     "item_set": "placement-subset_48-opus-review",
     "measured_at": "2026-10-26T06:30:13Z",
     "n": 20,
     "quality": [
      {
       "ci": "source",
       "hi": 0.92,
       "k": 16,
       "key": "precision",
       "lo": 0.58,
       "n": 20,
       "score": 0.8
      },
      {
       "ci": "wilson_here",
       "hi": 0.6976787277587176,
       "k": 16,
       "key": "recall",
       "lo": 0.36142013281525043,
       "n": 30,
       "score": 0.53
      }
     ],
     "roles": [],
     "source_file": "race_results.tsv",
     "source_owner": "Pixel",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "1596e1a0bfe933ab",
       "name": "gaents/qwen-flash",
       "offered": true
      },
      {
       "evidence_id": "028f1f8722b67e86",
       "name": "gaents/deepseek",
       "offered": true
      },
      {
       "evidence_id": "eaafe9c906ea1c1e",
       "name": "[name not shown]",
       "offered": false
      }
     ],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "c3feda386210fb8e",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    }
   ],
   "id": "gaents/grok",
   "kept_because_used": false,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 725 items x $9.82e-05/item (gaents/glm-flash measured) x output-price ratio 5.00",
    "est_usd": 0.356,
    "task": "roar"
   },
   "status_class": "superseded",
   "superseded_on": [
    {
     "by": "gaents/qwen-flash",
     "by_offered": true,
     "task": "placement",
     "task_name": "Industry placement (family level)"
    }
   ],
   "used": false,
   "verdict": "On Industry placement (family level), gaents/qwen-flash matches or beats it for less."
  },
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "Labels are Sonnet's: this is agreement with Sonnet (Sonnet's own test-retest ~0.84), not truth.",
      "Interval: Wilson 95% computed here from score x n."
     ],
     "correct_unit": "item agreeing with Sonnet's sign",
     "cost_basis": "free_tier",
     "cost_per": 1000,
     "cost_per_correct_usd": null,
     "evidence_id": "f26601ac1831ad11",
     "item_set": "roar-725-kimi-aligned",
     "measured_at": "2026-10-02",
     "n": 725,
     "quality": [
      {
       "ci": "wilson_here",
       "hi": 0.7448375515247431,
       "k": 517,
       "key": "sign_agreement",
       "lo": 0.6791228810301494,
       "n": 725,
       "score": 0.7131
      }
     ],
     "roles": [],
     "source_file": "model-arena.jsonl",
     "source_owner": "[name not shown]",
     "state": "undecided",
     "superseded_by": [],
     "task": "roar",
     "task_name": "Directed sentiment (agreement with Sonnet's labels)",
     "undecided_with": [
      {
       "evidence_id": "784a9fbf1a5fce0b",
       "name": "clef",
       "offered": false
      },
      {
       "evidence_id": "c024a96833c449e2",
       "name": "clef-flash",
       "offered": false
      },
      {
       "evidence_id": "56f61ba6b058fb80",
       "name": "gaents/deepseek-flash",
       "offered": true
      },
      {
       "evidence_id": "bb2c6741e8869ab5",
       "name": "deepseek/deepseek-v4.1-flash",
       "offered": false
      },
      {
       "evidence_id": "8bc1c8fd476bdf9b",
       "name": "fathom:roar-roberta-large-ft2",
       "offered": false
      },
      {
       "evidence_id": "ecd0d4b7ce741727",
       "name": "moonshotai/kimi-k2.6",
       "offered": false
      },
      {
       "evidence_id": "02f16f511d13ea37",
       "name": "moonshotai/kimi-k3",
       "offered": false
      },
      {
       "evidence_id": "f6bfcc577dd0197b",
       "name": "typesafe/jev-1.13",
       "offered": false
      },
      {
       "evidence_id": "14e52cf76bb6cc59",
       "name": "[name not shown]",
       "offered": false
      },
      {
       "evidence_id": "9b9dff029e6247a1",
       "name": "gaents/glm-flash",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key and grader are both Opus, which favours Claude routes (REPORT.md).",
      "Cost covers every company called (200 API routes, 192 Claude routes); the score covers the 192 in scope."
     ],
     "correct_unit": "correct field",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 2.352243993084371,
     "evidence_id": "f016ccdc2c457bd1",
     "item_set": "s1-192cos-judge-key",
     "measured_at": "2026-11-18T15:16Z",
     "n": 1588,
     "quality": [
      {
       "ci": "source",
       "hi": 0.9236,
       "k": 1446,
       "key": "precision",
       "lo": 0.8955,
       "n": 1588,
       "score": 0.9106
      },
      {
       "ci": "source",
       "hi": 0.9011,
       "k": null,
       "key": "recall",
       "lo": 0.8703,
       "n": null,
       "score": 0.8866
      }
     ],
     "roles": [],
     "source_file": "tables.json",
     "source_owner": "Hustle (re-score) / Judge (scorer) / Fathom+[name not shown] (runs)",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "a92d06281136abe0",
       "name": "gaents/deepseek",
       "offered": true
      },
      {
       "evidence_id": "6a12ad6648ce6793",
       "name": "gaents/claude-haiku",
       "offered": true
      },
      {
       "evidence_id": "90294a3ec3404c51",
       "name": "gaents/qwen-flash",
       "offered": true
      }
     ],
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)",
     "undecided_with": []
    }
   ],
   "id": "gaents/qwen-27b",
   "kept_because_used": false,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 228 items x $0.000755/item (gaents/qwen-flash measured) x output-price ratio 4.68",
    "est_usd": 0.8058,
    "task": "placement"
   },
   "status_class": "superseded",
   "superseded_on": [
    {
     "by": "gaents/deepseek",
     "by_offered": true,
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)"
    }
   ],
   "used": false,
   "verdict": "On Company facts extraction (13 fields from a company's site), gaents/deepseek matches or beats it for less. No verdict yet on Directed sentiment (agreement with Sonnet's labels) (a cost is unknown or nothing comparable was run)."
  },
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "n=228 gives roughly +/-0.06-0.10; strict scoring understates every arm (16 of 31 disagreements defensible both ways) (Pixel)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 1.204195804195804,
     "evidence_id": "a8f51c4055f6cd24",
     "item_set": "placement-228cos-opus-review",
     "measured_at": "2026-10-26T06:30:13Z",
     "n": 185,
     "quality": [
      {
       "ci": "source",
       "hi": 0.83,
       "k": 143,
       "key": "precision",
       "lo": 0.71,
       "n": 185,
       "score": 0.77
      },
      {
       "ci": "wilson_here",
       "hi": 0.7799296793948676,
       "k": 143,
       "key": "recall",
       "lo": 0.6560557666182437,
       "n": 198,
       "score": 0.72
      }
     ],
     "roles": [],
     "source_file": "race_results.tsv",
     "source_owner": "Pixel",
     "state": "undecided",
     "superseded_by": [],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "e7983fe2b7555706",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "Re-scored 2026-11-19 with Pixel's own score.py row() on the companies the subset arm was asked (no model calls)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 1.5375,
     "evidence_id": "01817e97cb685dd4",
     "item_set": "placement-subset_114-opus-review",
     "measured_at": "2026-11-19",
     "n": 74,
     "quality": [
      {
       "ci": "source",
       "hi": 0.84,
       "k": 56,
       "key": "precision",
       "lo": 0.65,
       "n": 74,
       "score": 0.76
      },
      {
       "ci": "wilson_here",
       "hi": 0.7507822823823792,
       "k": 56,
       "key": "recall",
       "lo": 0.5531293986319203,
       "n": 85,
       "score": 0.66
      }
     ],
     "roles": [],
     "source_file": "placement-rescore-2026-11-19.json",
     "source_owner": "Hustle (re-score) / Pixel (scorer, key, stored answers)",
     "state": "undecided",
     "superseded_by": [],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "8004ea5a25ba9ed7",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key = six blind Opus reviewers (Opus is the key, not an arm).",
      "Re-scored 2026-11-19 with Pixel's own score.py row() on the companies the subset arm was asked (no model calls)."
     ],
     "correct_unit": "correct family placement",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 2.014035087719298,
     "evidence_id": "1596e1a0bfe933ab",
     "item_set": "placement-subset_48-opus-review",
     "measured_at": "2026-11-19",
     "n": 23,
     "quality": [
      {
       "ci": "source",
       "hi": 0.9,
       "k": 18,
       "key": "precision",
       "lo": 0.58,
       "n": 23,
       "score": 0.78
      },
      {
       "ci": "wilson_here",
       "hi": 0.754096056739166,
       "k": 18,
       "key": "recall",
       "lo": 0.42320052498273836,
       "n": 30,
       "score": 0.6
      }
     ],
     "roles": [
      "cheapest known cost per correct output"
     ],
     "source_file": "placement-rescore-2026-11-19.json",
     "source_owner": "Hustle (re-score) / Pixel (scorer, key, stored answers)",
     "state": "undecided",
     "superseded_by": [],
     "task": "placement",
     "task_name": "Industry placement (family level)",
     "undecided_with": [
      {
       "evidence_id": "c3feda386210fb8e",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ]
    },
    {
     "caveats": [
      "Answer key and grader are both Opus, which favours Claude routes (REPORT.md).",
      "Cost covers every company called (200 API routes, 192 Claude routes); the score covers the 192 in scope."
     ],
     "correct_unit": "correct field",
     "cost_basis": "cash_billed",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.6974436922015183,
     "evidence_id": "90294a3ec3404c51",
     "item_set": "s1-192cos-judge-key",
     "measured_at": "2026-11-18T15:16Z",
     "n": 1612,
     "quality": [
      {
       "ci": "source",
       "hi": 0.9127,
       "k": 1449,
       "key": "precision",
       "lo": 0.8832,
       "n": 1612,
       "score": 0.8989
      },
      {
       "ci": "source",
       "hi": 0.9028,
       "k": null,
       "key": "recall",
       "lo": 0.8722,
       "n": null,
       "score": 0.8884
      }
     ],
     "roles": [],
     "source_file": "tables.json",
     "source_owner": "Hustle (re-score) / Judge (scorer) / Fathom+[name not shown] (runs)",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "cea4958c1119537e",
       "name": "gaents/glm-flash",
       "offered": true
      },
      {
       "evidence_id": "a92d06281136abe0",
       "name": "gaents/deepseek",
       "offered": true
      },
      {
       "evidence_id": "6a12ad6648ce6793",
       "name": "gaents/claude-haiku",
       "offered": true
      }
     ],
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)",
     "undecided_with": []
    }
   ],
   "id": "gaents/qwen-flash",
   "kept_because_used": true,
   "kind": "alias",
   "listed": true,
   "next_test": {
    "basis": "E: 725 items x $9.82e-05/item (gaents/glm-flash measured) x output-price ratio 0.94",
    "est_usd": 0.0669,
    "task": "roar"
   },
   "status_class": "superseded",
   "superseded_on": [
    {
     "by": "gaents/glm-flash",
     "by_offered": true,
     "task": "s1",
     "task_name": "Company facts extraction (13 fields from a company's site)"
    }
   ],
   "used": true,
   "verdict": "On Company facts extraction (13 fields from a company's site), gaents/glm-flash matches or beats it for less. Kept because it is already listed and used. No verdict yet on Industry placement (family level) (a cost is unknown or nothing comparable was run)."
  },
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "Exact span + type vs CoNLL-2003 gold, PER/ORG/LOC only (35 MISC gold spans excluded); F1 0.6805.",
      "spaCy en_core_web_sm comparator not run (not installed here)."
     ],
     "correct_unit": "correct entity",
     "cost_basis": "list_price",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.003408450704225352,
     "evidence_id": "fa0241fc99544a49",
     "item_set": "ner-conll2003-test-first200",
     "measured_at": "2026-11-19",
     "n": 254,
     "quality": [
      {
       "ci": "wilson_here",
       "hi": 0.8787,
       "k": 213,
       "key": "precision",
       "lo": 0.7883,
       "n": 254,
       "score": 0.8386
      },
      {
       "ci": "wilson_here",
       "hi": 0.6219,
       "k": 213,
       "key": "recall",
       "lo": 0.5218,
       "n": 372,
       "score": 0.5726
      }
     ],
     "roles": [
      "top score",
      "cheapest known cost per correct output"
     ],
     "source_file": "local-jobs-2026-11-19.json",
     "source_owner": "Hustle",
     "state": "uncompared",
     "superseded_by": [],
     "task": "ner",
     "task_name": "Named entities (200 CoNLL-2003 test sentences, person/organization/location)",
     "undecided_with": []
    }
   ],
   "id": "job:ner-gliner",
   "kept_because_used": false,
   "kind": "job",
   "listed": true,
   "next_test": null,
   "status_class": "undecided",
   "superseded_on": [],
   "used": false,
   "verdict": "No verdict yet on Named entities (200 CoNLL-2003 test sentences, person/organization/location) (a cost is unknown or nothing comparable was run)."
  },
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "char_accuracy = 1 - pooled CER vs the PDF's own text layer (pdftotext); layout, math and figure labels count as errors, so this is a floor.",
      "Pages 1-12 CER 0.0666; pages 13-15 are attention-figure pages.",
      "One public document; no comparator OCR run yet."
     ],
     "correct_unit": "page at the measured character accuracy (cost per 1,000 pages / accuracy)",
     "cost_basis": "list_price",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.0031612998088516395,
     "evidence_id": "f01373f9f6f9e97e",
     "item_set": "ocr-arxiv-1706.03762-15pages-150dpi",
     "measured_at": "2026-11-19",
     "n": null,
     "quality": [
      {
       "ci": "none",
       "hi": null,
       "k": null,
       "key": "char_accuracy",
       "lo": null,
       "n": null,
       "score": 0.9068
      }
     ],
     "roles": [
      "top score",
      "cheapest known cost per correct output"
     ],
     "source_file": "local-jobs-2026-11-19.json",
     "source_owner": "Hustle",
     "state": "uncompared",
     "superseded_by": [],
     "task": "ocr",
     "task_name": "OCR (15 pages of a public paper, character accuracy vs its text layer)",
     "undecided_with": []
    }
   ],
   "id": "job:ocr-tesseract",
   "kept_because_used": false,
   "kind": "job",
   "listed": true,
   "next_test": null,
   "status_class": "undecided",
   "superseded_on": [],
   "used": true,
   "verdict": "No verdict yet on OCR (15 pages of a public paper, character accuracy vs its text layer) (a cost is unknown or nothing comparable was run)."
  },
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "word_accuracy = 1 - pooled WER over 15 short clips (clean/accented/noisy/non-English); no interval in the source.",
      "Grid rows: best-of several configs on the same 15 clips is selection on the test set."
     ],
     "correct_unit": "audio minute at the measured word accuracy",
     "cost_basis": "paid_as_buyer",
     "cost_per": 1,
     "cost_per_correct_usd": 0.06501057082452431,
     "evidence_id": "c00311c74044608a",
     "item_set": "transcriber-h2h-15clips-2026-11-18",
     "measured_at": "2026-11-18T19:47:38Z",
     "n": null,
     "quality": [
      {
       "ci": "none",
       "hi": null,
       "k": null,
       "key": "word_accuracy",
       "lo": null,
       "n": null,
       "score": 0.946
      }
     ],
     "roles": [],
     "source_file": "scores.json",
     "source_owner": "Hustle",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "ccdb4ccad4ab1379",
       "name": "[name not shown]",
       "offered": false
      },
      {
       "evidence_id": "33bde13f2a998b7a",
       "name": "apify:steadyfetch/media-transcriber",
       "offered": false
      },
      {
       "evidence_id": "cf48ce210f7fa0fd",
       "name": "job:transcribe-premium",
       "offered": true
      }
     ],
     "task": "transcription",
     "task_name": "Audio transcription (15 clips: clean, accented, noisy, non-English)",
     "undecided_with": [
      {
       "evidence_id": "2398c7de76a9eadc",
       "name": "faster-whisper:large-v3-turbo@local-cpu",
       "offered": false
      },
      {
       "evidence_id": "10b6cc3795b5c000",
       "name": "faster-whisper:medium@local-cpu",
       "offered": false
      },
      {
       "evidence_id": "785823f9fdd30012",
       "name": "faster-whisper:small@local-cpu",
       "offered": false
      },
      {
       "evidence_id": "db378b0a263566ba",
       "name": "[name not shown]",
       "offered": false
      }
     ]
    }
   ],
   "id": "job:transcribe",
   "kept_because_used": false,
   "kind": "job",
   "listed": true,
   "next_test": null,
   "status_class": "superseded",
   "superseded_on": [
    {
     "by": "[name not shown]",
     "by_offered": false,
     "task": "transcription",
     "task_name": "Audio transcription (15 clips: clean, accented, noisy, non-English)"
    }
   ],
   "used": false,
   "verdict": "On Audio transcription (15 clips: clean, accented, noisy, non-English), [name not shown]:whisper-large-v3-turbo matches or beats it for less (not offered here)."
  },
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "word_accuracy = 1 - pooled WER over the same 15 short clips; no interval in the source.",
      "Cost is our platform cost, not a buyer price; the comparison rows use list or buyer prices."
     ],
     "correct_unit": "audio minute at the measured word accuracy",
     "cost_basis": "cash_billed",
     "cost_per": 1,
     "cost_per_correct_usd": 0.023534485501038878,
     "evidence_id": "cf48ce210f7fa0fd",
     "item_set": "transcriber-h2h-15clips-2026-11-18",
     "measured_at": "2026-11-18",
     "n": null,
     "quality": [
      {
       "ci": "none",
       "hi": null,
       "k": null,
       "key": "word_accuracy",
       "lo": null,
       "n": null,
       "score": 0.9685
      }
     ],
     "roles": [],
     "source_file": "transcribe-premium-2026-11-19.json",
     "source_owner": "Hustle",
     "state": "superseded",
     "superseded_by": [
      {
       "evidence_id": "ccdb4ccad4ab1379",
       "name": "[name not shown]",
       "offered": false
      },
      {
       "evidence_id": "33bde13f2a998b7a",
       "name": "apify:steadyfetch/media-transcriber",
       "offered": false
      }
     ],
     "task": "transcription",
     "task_name": "Audio transcription (15 clips: clean, accented, noisy, non-English)",
     "undecided_with": [
      {
       "evidence_id": "2398c7de76a9eadc",
       "name": "faster-whisper:large-v3-turbo@local-cpu",
       "offered": false
      },
      {
       "evidence_id": "db378b0a263566ba",
       "name": "[name not shown]",
       "offered": false
      }
     ]
    }
   ],
   "id": "job:transcribe-premium",
   "kept_because_used": false,
   "kind": "job",
   "listed": true,
   "next_test": null,
   "status_class": "superseded",
   "superseded_on": [
    {
     "by": "[name not shown]",
     "by_offered": false,
     "task": "transcription",
     "task_name": "Audio transcription (15 clips: clean, accented, noisy, non-English)"
    }
   ],
   "used": false,
   "verdict": "On Audio transcription (15 clips: clean, accented, noisy, non-English), [name not shown]:whisper-large-v3-turbo matches or beats it for less (not offered here)."
  },
  {
   "best_for": [],
   "evidence": [
    {
     "caveats": [
      "Intelligibility only: Piper audio transcribed back by local faster-whisper large-v3-turbo, WER with the h2h normaliser; measures TTS + ASR together, not naturalness.",
      "43/50 sentences transcribed back exactly.",
      "No comparator TTS run yet."
     ],
     "correct_unit": "sentence at the measured round-trip word accuracy",
     "cost_basis": "list_price",
     "cost_per": 1000,
     "cost_per_correct_usd": 0.003389484430832057,
     "evidence_id": "3c207cd6890a7401",
     "item_set": "tts-harvard-lists1-5-50",
     "measured_at": "2026-11-19",
     "n": null,
     "quality": [
      {
       "ci": "none",
       "hi": null,
       "k": null,
       "key": "roundtrip_word_accuracy",
       "lo": null,
       "n": null,
       "score": 0.9795
      }
     ],
     "roles": [
      "top score",
      "cheapest known cost per correct output"
     ],
     "source_file": "local-jobs-2026-11-19.json",
     "source_owner": "Hustle",
     "state": "uncompared",
     "superseded_by": [],
     "task": "tts",
     "task_name": "Text to speech (50 Harvard sentences, round-trip intelligibility)",
     "undecided_with": []
    }
   ],
   "id": "job:tts-piper",
   "kept_because_used": false,
   "kind": "job",
   "listed": true,
   "next_test": null,
   "status_class": "undecided",
   "superseded_on": [],
   "used": true,
   "verdict": "No verdict yet on Text to speech (50 Harvard sentences, round-trip intelligibility) (a cost is unknown or nothing comparable was run)."
  }
 ],
 "rule": {
  "superseded.cost_tolerance": 0.0,
  "superseded.eligible_kinds": [
   "single"
  ],
  "superseded.known_cost_bases": [
   "cash_billed",
   "list_price",
   "capacity_list",
   "paid_as_buyer"
  ],
  "superseded.min_n": null,
  "superseded.missing_column": "blocks",
  "superseded.no_interval": "point",
  "superseded.quality_columns": "all_measured",
  "superseded.quality_test": "point_ge_reference_lower_bound",
  "superseded.reported_cost_as_upper_bound": true,
  "superseded.require_strict_gain": true,
  "superseded.row_pick": "best_score",
  "superseded.same_item_set_only": true,
  "superseded.unknown_cost": "undecided"
 },
 "rule_provenance": "Hustle default, unruled; Adam's words in VISION.md (Curation, 2026-11-19)",
 "settings_sha256": "7912bc65c1fc20dd1c1a6eb6ab198270fe1f96c9c5b06f17df657142943a1576",
 "site_copy": {
  "copied_at": "2026-11-22T02:07:44Z",
  "evidence_rows": 89,
  "names_not_shown": 24,
  "not_shown_paths": [
   ".items[0].evidence[1].undecided_with[1].name",
   ".items[0].evidence[2].undecided_with[1].name",
   ".items[0].evidence[3].source_owner",
   ".items[1].evidence[0].source_owner",
   ".items[2].evidence[1].superseded_by[1].name",
   ".items[2].evidence[2].superseded_by[1].name",
   ".items[2].evidence[3].source_owner",
   ".items[3].evidence[3].source_owner",
   ".items[6].evidence[0].source_owner",
   ".items[6].evidence[2].source_owner",
   ".items[7].evidence[1].source_owner",
   ".items[8].evidence[0].superseded_by[2].name",
   ".items[9].evidence[0].source_owner",
   ".items[9].evidence[0].undecided_with[8].name",
   ".items[9].evidence[1].source_owner",
   ".items[10].evidence[3].source_owner",
   ".items[13].superseded_on[0].by",
   ".items[13].verdict",
   ".items[13].evidence[0].superseded_by[0].name",
   ".items[13].evidence[0].undecided_with[3].name",
   ".items[14].superseded_on[0].by",
   ".items[14].verdict",
   ".items[14].evidence[0].superseded_by[0].name",
   ".items[14].evidence[0].undecided_with[1].name"
  ],
  "not_shown_text": "[name not shown]",
  "rule_text": {
   "superseded.cost_tolerance": {
    "provenance": "Hustle default, unruled ('<= cost per correct output'): B's cost per correct output must be <= A's x (1 + tolerance)",
    "text": "Hustle default, unruled ('<= cost per correct output'): B's cost per correct output must be <= A's x (1 + tolerance)",
    "type": "admin_user"
   },
   "superseded.eligible_kinds": {
    "provenance": "Hustle default, unruled: only a single model can supersede a model (Adam's rule is about models we list). Pipelines (two-call agreement cascades) are shown as 'beats the lineup' candidates in CURATION.md, never as the reason a model is marked superseded.",
    "text": "Hustle default, unruled: only a single model can supersede a model (Adam's rule is about models we list). Pipelines (two-call agreement cascades) are shown as 'beats the lineup' candidates in CURATION.md, never as the reason a model is marked superseded.",
    "type": "admin_user"
   },
   "superseded.known_cost_bases": {
    "provenance": "Hustle default, unruled; free_tier and owned_hardware are left OUT (unknown) following Fathom's arena rule (fathom/reports/arena-frontier-per-task-2026-10-19.md)",
    "text": "Hustle default, unruled; free_tier and owned_hardware are left OUT (unknown) following Fathom's arena rule (fathom/reports/arena-frontier-per-task-2026-10-19.md)",
    "type": "admin_user"
   },
   "superseded.min_n": {
    "provenance": "OPEN: no floor set (de-assume: no invented n). null = any n counts; intervals carry the noise.",
    "text": "OPEN: no floor set (de-assume: no invented n). null = any n counts; intervals carry the noise.",
    "type": "admin_user"
   },
   "superseded.missing_column": {
    "provenance": "Hustle default, unruled (never infer an unmeasured score)",
    "text": "if A was measured on a column B was not, B cannot supersede A on that task",
    "type": "admin_user"
   },
   "superseded.no_interval": {
    "provenance": "Hustle default, unruled",
    "text": "compare point scores",
    "type": "admin_user"
   },
   "superseded.quality_columns": {
    "provenance": "Hustle default, unruled ('>= quality on every shared task')",
    "text": "every quality column both rows carry (S1 precision AND recall; coding passes AND mergeable)",
    "type": "admin_user"
   },
   "superseded.quality_test": {
    "provenance": "Hustle default, unruled; Adam's words in VISION.md (Curation, 2026-11-19): 'superceded by a better model according to our tests'. Same test the site's go-to line uses (gaents site settings models.go_to_rule: point_within_reference_interval_all_columns).",
    "text": "B's score is at or above the LOWER end of A's 95% interval on that column (a tie within A's noise counts as >=). Where A has no interval, falls back to superseded.no_interval.",
    "type": "admin_user"
   },
   "superseded.reported_cost_as_upper_bound": {
    "provenance": "Hustle default, unruled. Pixel's Haiku placement cost is the CLI's figure at a Haiku price Forge later corrected DOWN (tech 2ffb1ec, $0.10/$0.50); it cannot be recomputed (no token counts) but it can only overstate. true = use it as an upper bound: a model surely more expensive than that bound is not cheaper; a model cheaper than the unknown true cost stays undecided.",
    "text": "Hustle default, unruled. Pixel's Haiku placement cost is the CLI's figure at a Haiku price Forge later corrected DOWN (tech 2ffb1ec, $0.10/$0.50); it cannot be recomputed (no token counts) but it can only overstate. true = use it as an upper bound: a model surely more expensive than that bound is not cheaper; a model cheaper than the unknown true cost stays undecided.",
    "type": "admin_user"
   },
   "superseded.require_strict_gain": {
    "provenance": "Hustle default, unruled: B must also be strictly cheaper per correct output OR strictly higher on some quality column, so two identical rows never supersede each other",
    "text": "Hustle default, unruled: B must also be strictly cheaper per correct output OR strictly higher on some quality column, so two identical rows never supersede each other",
    "type": "admin_user"
   },
   "superseded.row_pick": {
    "provenance": "Hustle default, unruled. Caveat: best-of-k configs is selection on the test set; CURATION.md names the picked config.",
    "text": "a subject measured several ways on one item set (batch size, effort, harness config) is represented by its best primary score (ties: cheaper)",
    "type": "admin_user"
   },
   "superseded.same_item_set_only": {
    "provenance": "Hustle default, unruled: rows are compared only when they scored the SAME items (e.g. Pixel's Grok arm ran 48 of 228 companies, so it is not compared with the 228-company arms)",
    "text": "Hustle default, unruled: rows are compared only when they scored the SAME items (e.g. Pixel's Grok arm ran 48 of 228 companies, so it is not compared with the 228-company arms)",
    "type": "admin_user"
   },
   "superseded.unknown_cost": {
    "provenance": "Hustle default, unruled",
    "text": "if either row's cost is unknown, no superseded verdict on that task (shown as 'undecided: cost unknown')",
    "type": "admin_user"
   },
   "usage.used_rule": {
    "provenance": "Hustle default, unruled; Adam 11-19: 'i don't suggest we stop having available models once they are on our site and have been used'. Today every call is ours (distinct_payers_external = 0 on every listing), so 'external_only' would make nothing 'used'; CURATION.md shows both.",
    "text": "origins=any: any successful call, including our own tests (ours, ours_test)",
    "type": "admin_user"
   }
  },
  "rule_text_note": null,
  "source": "services/marketplace/curation/curation.public.json",
  "source_sha256": "a44e688ca1b767392c3eef42d5b2ba2406c840d46cd5a16f2fe92cf57d70221b",
  "why": "show settings (show.defaults: exact model id, provider and build not shown); nothing else in the file is changed"
 },
 "usage_read_at": "2026-11-19T12:29:21Z",
 "used_rule": {
  "min_success": 1,
  "origins": "any"
 }
}