{
  "schema_version": "1.0",
  "id": "archive:https://openai.com/index/separating-signal-from-noise-coding-evaluations",
  "slug": "separating-signal-from-noise-coding-evaluations-17ha3r5",
  "url": "https://feed7.dev/p/separating-signal-from-noise-coding-evaluations-17ha3r5",
  "title": "Separating signal from noise in coding evaluations",
  "why_included": "OpenAI reports reliability and accuracy problems in SWE-Bench Pro, making its coding-model rankings unsafe to treat as settled evidence.",
  "summary": "OpenAI says its analysis found **reliability and accuracy problems** in **SWE-Bench Pro**. That puts the benchmark’s model rankings in doubt.",
  "practical_implication": "Builders comparing coding agents should avoid making model or routing decisions from this leaderboard alone until the evaluation issues are understood.",
  "agent_context": "OpenAI says its analysis found **reliability and accuracy problems** in **SWE-Bench Pro**. That puts the benchmark’s model rankings in doubt.\n\nBuilders comparing coding agents should avoid making model or routing decisions from this leaderboard alone until the evaluation issues are understood.\n\nThe supplied material does not describe the defects, their size, or corrected results, so it cannot show which rankings change or by how much.",
  "source": {
    "name": "OpenAI",
    "url": "https://openai.com/index/separating-signal-from-noise-coding-evaluations",
    "published_at": null
  },
  "source_class": "blog_post",
  "content_type": "Official Release",
  "layer": "benchmark",
  "domains": [
    "coding"
  ],
  "topics": [
    "agent-evals",
    "benchmark-integrity",
    "model-selection"
  ],
  "verification": {
    "status": "official_source",
    "label": "Official Source",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "The supplied material does not describe the defects, their size, or corrected results, so it cannot show which rankings change or by how much."
  ],
  "lifecycle": "Current",
  "published_at": null,
  "modified_at": null,
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/separating-signal-from-noise-coding-evaluations-17ha3r5",
    "json": "https://feed7.dev/p/separating-signal-from-noise-coding-evaluations-17ha3r5.json",
    "markdown": "https://feed7.dev/p/separating-signal-from-noise-coding-evaluations-17ha3r5.md"
  }
}