{
  "schema_version": "1.1",
  "id": "s13:https://arxiv.org/abs/2611.26613v1",
  "slug": "2611-26613v1-13zvj1v",
  "url": "https://feed7.dev/p/2611-26613v1-13zvj1v",
  "title": "AgentRoboCritic: Understanding the Capability of Frontier Agents on Direct Robot Control",
  "why_included": "AgentRoboCritic grades robot actions against each agent’s stated intent, exposing failures that episode-level success rates miss. Its released critic improves action failure-mode accuracy on a human-annotated set.",
  "summary": "The paper analyzes direct robot control across **12 manipulation benchmarks** with a critic that compares each action’s outcome to the policy agent’s stated local intent. It releases **12,844 annotated actions**; a fine-tuned open-source critic reaches **74.24%** failure-mode accuracy, up from 32.58%.",
  "practical_implication": "For builders evaluating agents that act through tools or devices, action-level critiques may help locate where execution breaks down instead of relying only on task completion. The paper also reports recovery from some local failures and zero-shot manipulation without task-specific adaptation.",
  "agent_context": "The paper analyzes direct robot control across **12 manipulation benchmarks** with a critic that compares each action’s outcome to the policy agent’s stated local intent. It releases **12,844 annotated actions**; a fine-tuned open-source critic reaches **74.24%** failure-mode accuracy, up from 32.58%.\n\nFor builders evaluating agents that act through tools or devices, action-level critiques may help locate where execution breaks down instead of relying only on task completion. The paper also reports recovery from some local failures and zero-shot manipulation without task-specific adaptation.\n\nThe evaluation is about robot manipulation, with physical limits around grasping and alignment and uneven recovery across failure modes. The critic’s accuracy was validated on a **400-action human-annotated benchmark**; broader applicability is not established here.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2611.26613v1",
    "published_at": "2026-11-19T18:57:25.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "benchmark",
  "domains": [
    "research"
  ],
  "topics": [
    "agent-evals",
    "agent-reliability"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The evaluation is about robot manipulation, with physical limits around grasping and alignment and uneven recovery across failure modes. The critic’s accuracy was validated on a **400-action human-annotated benchmark**; broader applicability is not established here."
  ],
  "connected_context": {
    "meaning": "AgentRoboCritic evaluates robot-control actions against the policy’s stated local intent, making action-level failure analysis a complement to task-level outcomes. Its annotated manipulation data and measured critic accuracy support that method in the tested robotics setting, but do not establish transfer to other tool-using agents.",
    "corpus_size": 1135,
    "generated_at": "2026-11-21T11:06:52.506Z",
    "connections": [
      {
        "title": "Mitigating Agentic Deception via Evaluation-Side Partially Observable Games",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2611.12835v1",
        "feed7_url": "https://feed7.dev/p/2611-12835v1-1i3l08c",
        "reason": "Both evaluate agent behavior across interaction traces rather than relying only on final outcomes; AgentRoboCritic compares each robot action with local intent, while the deception study monitors tool use and environmental effects."
      },
      {
        "title": "SocietyBench: Forecasting Counterfactual Social-World Evolution",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2608.04009v1",
        "feed7_url": "https://feed7.dev/p/2608-04009v1-05m5u8w",
        "reason": "SocietyBench shows that task-completion evaluation misses some forecasting failures; AgentRoboCritic supplies a different domain-specific example of inspecting intermediate behavior to locate where execution diverges."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-11-19T18:57:25.000Z",
  "modified_at": "2026-11-19T18:57:25.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2611-26613v1-13zvj1v",
    "json": "https://feed7.dev/p/2611-26613v1-13zvj1v.json",
    "markdown": "https://feed7.dev/p/2611-26613v1-13zvj1v.md"
  }
}