{
  "schema_version": "1.0",
  "id": "s13:https://arxiv.org/abs/2607.12963v1",
  "slug": "2607-12963v1-1oc0qmr",
  "url": "https://feed7.dev/p/2607-12963v1-1oc0qmr",
  "title": "The Illusion of Robustness: Aggregate Accuracy Hides Prediction Flips under Task-Irrelevant Context",
  "why_included": "Stable aggregate accuracy can hide individual answers flipping when irrelevant context is added. Agent evaluations should compare outputs per task and probe realistic context noise, not only average scores.",
  "summary": "Across multiple models and datasets, adding irrelevant context caused little aggregate accuracy change but flipped predictions on a subset of examples. Even **random-character pseudo-words** could improve some answers while degrading others.",
  "practical_implication": "Evaluate coding agents at the example level: rerun the same task with irrelevant files, longer context, or harmless textual noise, then track answer flips and regressions separately from average pass rates.",
  "agent_context": "Across multiple models and datasets, adding irrelevant context caused little aggregate accuracy change but flipped predictions on a subset of examples. Even **random-character pseudo-words** could improve some answers while degrading others.\n\nEvaluate coding agents at the example level: rerun the same task with irrelevant files, longer context, or harmless textual noise, then track answer flips and regressions separately from average pass rates.\n\nThe affected examples were **largely model-specific**, and instability varied with context type, length, test-time compute, and model development stage. The supplied abstract gives no effect sizes, so it does not establish how frequent the tail risk is in coding workloads.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2607.12963v1",
    "published_at": "2026-07-14T17:01:12.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "benchmark",
  "domains": [
    "coding"
  ],
  "topics": [
    "context-engineering",
    "agent-evals",
    "agent-reliability"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The affected examples were **largely model-specific**, and instability varied with context type, length, test-time compute, and model development stage. The supplied abstract gives no effect sizes, so it does not establish how frequent the tail risk is in coding workloads."
  ],
  "lifecycle": "Current",
  "published_at": "2026-07-14T17:01:12.000Z",
  "modified_at": "2026-07-14T17:01:12.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2607-12963v1-1oc0qmr",
    "json": "https://feed7.dev/p/2607-12963v1-1oc0qmr.json",
    "markdown": "https://feed7.dev/p/2607-12963v1-1oc0qmr.md"
  }
}