{
  "schema_version": "1.1",
  "id": "s13:https://arxiv.org/abs/2610.31593v1",
  "slug": "2610-31593v1-1lanp1p",
  "url": "https://feed7.dev/p/2610-31593v1-1lanp1p",
  "title": "Simulated Control Evals",
  "why_included": "Simulated tool responses predicted deployment-specific control safety more closely than a separate executed setting, offering a cheaper way to test agent safeguards before deployment.",
  "summary": "On LinuxArena, simulated control evaluations achieved **3.6 percentage-point RMSE** when predicting safety across five monitors at a 5% audit budget. BashArena, a separate executed control setting, reached **10.1 points**.",
  "practical_implication": "Builders testing high-risk agents should model deployment-specific tools and tasks instead of relying only on a generic control environment. Simulation can also surface attack strategies that improve monitors on held-out executed attacks.",
  "agent_context": "On LinuxArena, simulated control evaluations achieved **3.6 percentage-point RMSE** when predicting safety across five monitors at a 5% audit budget. BashArena, a separate executed control setting, reached **10.1 points**.\n\nBuilders testing high-risk agents should model deployment-specific tools and tasks instead of relying only on a generic control environment. Simulation can also surface attack strategies that improve monitors on held-out executed attacks.\n\nSCEs explained **78–85% of between-environment variance** and used fewer than 5% as many task-pair draws as the deployment set. The evidence comes from LinuxArena, so transfer to other tools and workflows remains open.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2610.31593v1",
    "published_at": "2026-10-22T17:26:42.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "benchmark",
  "domains": [
    "coding",
    "security"
  ],
  "topics": [
    "agent-evals",
    "agent-reliability",
    "tool-use"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "SCEs explained **78–85% of between-environment variance** and used fewer than 5% as many task-pair draws as the deployment set. The evidence comes from LinuxArena, so transfer to other tools and workflows remains open."
  ],
  "connected_context": {
    "meaning": "This makes simulated control evaluation a cheaper, deployment-specific forecasting layer rather than a substitute for executed safety tests. It extends prior calls for small eval loops and adversarial testing by showing that modeled tools and tasks can explain much of the safety variation and generate attacks that improve monitors, while the Linux-only evidence leaves transfer to other agent environments unresolved.",
    "corpus_size": 898,
    "generated_at": "2026-10-24T10:28:40.044Z",
    "connections": [
      {
        "title": "EpiSEM-Bench: Evaluating Safety and Evidence Recovery in LLM Agents under Missing Evidence",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2610.31650v1",
        "feed7_url": "https://feed7.dev/p/2610-31650v1-1wmxt69",
        "reason": "EpiSEM-Bench isolates safe behavior under missing evidence, while simulated control evals address whether such control performance will transfer to a particular deployment environment; the latter can therefore complement capability-specific safety tests with environment-fit estimates."
      },
      {
        "title": "RedEvoAgent: Automatic Red-Teaming Agent with Experience-Driven Skill Evolution",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2608.27439v1",
        "feed7_url": "https://feed7.dev/p/2608-27439v1-14ex5tu",
        "reason": "Both use evolving attack experience to strengthen safety evaluation: RedEvoAgent retains validated attack skills, while simulated control evals surface strategies that can improve monitors when tested on held-out executed attacks."
      },
      {
        "title": "How Evals and Prompts Shape Agent Behavior — Preetika Bhateja & Daniel Bump, YouTube Ads",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=xyL2Ltkh-SA",
        "feed7_url": "https://feed7.dev/p/how-evals-and-prompts-shape-agent-behavior-preetika-bhateja-daniel-bump-1cmecaw",
        "reason": "The production loop of small eval sets and trace review gains a scaling mechanism here: simulation can cheaply sample deployment-shaped conditions, but executed checks remain necessary to calibrate whether those predictions hold."
      },
      {
        "title": "Vulnerability Localization Benchmark: Measuring Agentic Security Analysis at Repository Scale",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2609.15939v1",
        "feed7_url": "https://feed7.dev/p/2609-15939v1-0w15mi5",
        "reason": "VLoc Bench shows that security evaluation must isolate specific capabilities; this work adds that control results also depend on the surrounding tool-and-task environment, so both capability decomposition and deployment matching matter."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-10-22T17:26:42.000Z",
  "modified_at": "2026-10-22T17:26:42.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2610-31593v1-1lanp1p",
    "json": "https://feed7.dev/p/2610-31593v1-1lanp1p.json",
    "markdown": "https://feed7.dev/p/2610-31593v1-1lanp1p.md"
  }
}