{
  "schema_version": "1.1",
  "id": "s13:https://arxiv.org/abs/2609.04167v1",
  "slug": "2609-04167v1-1vvofgw",
  "url": "https://feed7.dev/p/2609-04167v1-1vvofgw",
  "title": "SWE-Gate: Passing Functional Tests Is Not Enough for Software Engineering Agents",
  "why_included": "SWE-Gate shows why green tests are an incomplete agent-eval signal: 221 of 644 functionally passing repairs still violated constraints derived from code review.",
  "summary": "SWE-Gate contains **303 repair instances** from **75 Python repositories**, with separate tests for functionality and constraints derived from real pull-request reviews. Across four model backends, 644 repairs passed functional tests, but **221** still violated review constraints.",
  "practical_implication": "Treat green tests as one gate, not final acceptance, for coding-agent patches. Encode review expectations as executable checks where possible, and evaluate issue resolution separately from compliance with repository-specific requirements.",
  "agent_context": "SWE-Gate contains **303 repair instances** from **75 Python repositories**, with separate tests for functionality and constraints derived from real pull-request reviews. Across four model backends, 644 repairs passed functional tests, but **221** still violated review constraints.\n\nTreat green tests as one gate, not final acceptance, for coding-agent patches. Encode review expectations as executable checks where possible, and evaluate issue resolution separately from compliance with repository-specific requirements.\n\nThe benchmark uses synthesized repair instances and a common agent scaffold, so its failure rates may not transfer directly to your repos. It nevertheless exposes a concrete blind spot in functional-only coding-agent evaluations.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2609.04167v1",
    "published_at": "2026-09-03T17:53:34.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "benchmark",
  "domains": [
    "coding"
  ],
  "topics": [
    "agent-evals",
    "benchmark-integrity",
    "agent-reliability"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The benchmark uses synthesized repair instances and a common agent scaffold, so its failure rates may not transfer directly to your repos. It nevertheless exposes a concrete blind spot in functional-only coding-agent evaluations."
  ],
  "connected_context": {
    "meaning": "This converts a general warning about coding-agent benchmarks into a specific acceptance gap: functional success can coexist with violations of repository review constraints. It argues for separately executable functionality and compliance gates, while synthesized tasks and a shared scaffold limit direct extrapolation to production repositories.",
    "corpus_size": 691,
    "generated_at": "2026-09-05T10:08:08.498Z",
    "connections": [
      {
        "title": "DeepSWE: A Contamination-Resistant Coding Benchmark — James Shi, Datacurve",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=Yk87oUPVaxU",
        "feed7_url": "https://feed7.dev/p/deepswe-a-contamination-resistant-coding-benchmark-james-shi-datacurve-08p61c0",
        "reason": "Complements DeepSWE’s contamination and long-horizon focus by measuring a different hidden failure mode: patches that work but disregard review-derived constraints."
      },
      {
        "title": "Rethinking Environments for Long-Horizon Work — Rayan Garg, Theta Software",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=2aS7aKoXn64",
        "feed7_url": "https://feed7.dev/p/rethinking-environments-for-long-horizon-work-rayan-garg-theta-software-11r7wbx",
        "reason": "Implements the call for final-state inspection by splitting acceptance into functional state and repository-specific constraint compliance."
      },
      {
        "title": "Form, Not Content? A Preregistered, Placebo-Controlled Evaluation of Learned Error-Conditioned Self-Repair Through Prompts and Weights in Frozen Small Code Models",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2607.12962v1",
        "feed7_url": "https://feed7.dev/p/2607-12962v1-0q4i26c",
        "reason": "Together they weaken simplistic repair evidence: retries may not use error content, and passing functional tests may still leave the requested constraints unresolved."
      },
      {
        "title": "When Will The Benchmaxxing Plague End? — Nick Heiner, Surge AI",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=-npY6XjM8CQ",
        "feed7_url": "https://feed7.dev/p/when-will-the-benchmaxxing-plague-end-nick-heiner-surge-ai-178gqcg",
        "reason": "Provides concrete support for the claim that verifier design can inflate coding-agent success when tests omit requirements that human reviewers enforce."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-09-03T17:53:34.000Z",
  "modified_at": "2026-09-03T17:53:34.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2609-04167v1-1vvofgw",
    "json": "https://feed7.dev/p/2609-04167v1-1vvofgw.json",
    "markdown": "https://feed7.dev/p/2609-04167v1-1vvofgw.md"
  }
}