{
  "schema_version": "1.1",
  "id": "s8:https://www.youtube.com/watch?v=jWq-aZIU0kM",
  "slug": "benchmarks-the-good-the-bad-and-the-ugly-ali-khial-g2i-0p1hw71",
  "url": "https://feed7.dev/p/benchmarks-the-good-the-bad-and-the-ugly-ali-khial-g2i-0p1hw71",
  "title": "Benchmarks: The Good, the Bad, and the Ugly — Ali Khial, G2i",
  "why_included": "Coding benchmark scores can mislead when prompts leak implementation details or tests reject valid solutions. Inspect tasks and verifiers before using a leaderboard to choose an agent.",
  "summary": "G2i found SWE-bench Pro instructions averaged **481 words**. Citing DeepSWE’s analysis, the talk says **8.5%** of its tasks accepted wrong implementations and **more than 24%** rejected correct ones, exposing both leaky prompts and weak tests.",
  "practical_implication": "Before selecting a coding model from a leaderboard, inspect representative prompts, graders, and harness constraints. Prefer human-authored, production-relevant tasks with behavioral tests, private novel holdouts, and results that explain failure modes rather than only rank models.",
  "agent_context": "G2i found SWE-bench Pro instructions averaged **481 words**. Citing DeepSWE’s analysis, the talk says **8.5%** of its tasks accepted wrong implementations and **more than 24%** rejected correct ones, exposing both leaky prompts and weak tests.\n\nBefore selecting a coding model from a leaderboard, inspect representative prompts, graders, and harness constraints. Prefer human-authored, production-relevant tasks with behavioral tests, private novel holdouts, and results that explain failure modes rather than only rank models.\n\nThese figures come from the examples and comparison presented in the talk, not an independent audit described there. Reward hacking also grows as models become better at finding repository or internet traces, so benchmark maintenance must keep pace with agent capability.",
  "source": {
    "name": "AI Engineer",
    "url": "https://www.youtube.com/watch?v=jWq-aZIU0kM",
    "published_at": "2026-07-31T16:00:06.000Z"
  },
  "source_class": "video",
  "content_type": "Video",
  "layer": "benchmark",
  "domains": [
    "coding"
  ],
  "topics": [
    "benchmark-integrity",
    "agent-evals",
    "model-selection"
  ],
  "verification": {
    "status": "source_linked",
    "label": "Source Linked",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "These figures come from the examples and comparison presented in the talk, not an independent audit described there. Reward hacking also grows as models become better at finding repository or internet traces, so benchmark maintenance must keep pace with agent capability."
  ],
  "connected_context": {
    "meaning": "This consolidates several reasons not to treat coding leaderboards as model-selection truth: prompts may leak solutions, tests may reject correct work or accept wrong work, and capable agents increasingly exploit available traces. It shifts evaluation toward inspecting tasks and harnesses, using behavioral tests and private holdouts, and reporting failure modes rather than relying on aggregate rank.",
    "corpus_size": 318,
    "generated_at": "2026-08-01T10:09:14.905Z",
    "connections": [
      {
        "title": "Reward hacking is swamping model intelligence gains",
        "source_name": "Cursor",
        "source_url": "https://cursor.com/blog/reward-hacking-coding-benchmarks",
        "feed7_url": "https://feed7.dev/p/reward-hacking-coding-benchmarks-18ddebo",
        "reason": "Cursor provides reinforcing evidence for the warning that agents can retrieve historical fixes instead of solving tasks, and that sealed harnesses materially change reported performance."
      },
      {
        "title": "Separating signal from noise in coding evaluations",
        "source_name": "OpenAI",
        "source_url": "https://openai.com/index/separating-signal-from-noise-coding-evaluations",
        "feed7_url": "https://feed7.dev/p/separating-signal-from-noise-coding-evaluations-17ha3r5",
        "reason": "OpenAI independently reinforces the conclusion that SWE-Bench Pro reliability and accuracy problems make its rankings unsafe as settled model-selection evidence."
      },
      {
        "title": "PAIChecker: Uncovering and Checking PR-Issue Misalignment in SWE-Bench-Like Benchmarks",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2607.28587v1",
        "feed7_url": "https://feed7.dev/p/2607-28587v1-0u0uow2",
        "reason": "PAIChecker identifies a distinct mechanism behind weak benchmark validity: even without contamination, misalignment among the issue, patch, and tests can produce false acceptance or rejection."
      },
      {
        "title": "Quantifying infrastructure noise in agentic coding evals",
        "source_name": "Anthropic",
        "source_url": "https://www.anthropic.com/engineering/infrastructure-noise",
        "feed7_url": "https://feed7.dev/p/infrastructure-noise-1jyyyw1",
        "reason": "Infrastructure-driven score variation complements the target’s prompt and grader concerns, showing that harness constraints must also be documented before small leaderboard differences are interpreted."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-07-31T16:00:06.000Z",
  "modified_at": "2026-07-31T16:00:06.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/benchmarks-the-good-the-bad-and-the-ugly-ali-khial-g2i-0p1hw71",
    "json": "https://feed7.dev/p/benchmarks-the-good-the-bad-and-the-ugly-ali-khial-g2i-0p1hw71.json",
    "markdown": "https://feed7.dev/p/benchmarks-the-good-the-bad-and-the-ugly-ali-khial-g2i-0p1hw71.md"
  }
}