{
  "schema_version": "1.1",
  "id": "s13:https://arxiv.org/abs/2607.26041v1",
  "slug": "2607-26041v1-1x1gw81",
  "url": "https://feed7.dev/p/2607-26041v1-1x1gw81",
  "title": "Desktop-Delta Bench: Do Computer-Use Models Understand Desktop GUI Transitions?",
  "why_included": "Desktop-Delta Bench tests whether computer-use models can verify what changed after an action, exposing a reliability gap hidden by end-task and single-frame scores.",
  "summary": "Desktop-Delta Bench contains **2,013 human-verified instances** from multi-app Linux trajectories across about 15 applications and 50 task domains. It tests temporal ordering and action inference from before-and-after observations.",
  "practical_implication": "Use step-level transition checks when evaluating computer-use agents, especially around stale screenshots, delayed rendering, and recovery. The best ordering exact-match rates reached only **65.1% without decoys** and **65.7% with decoys**.",
  "agent_context": "Desktop-Delta Bench contains **2,013 human-verified instances** from multi-app Linux trajectories across about 15 applications and 50 task domains. It tests temporal ordering and action inference from before-and-after observations.\n\nUse step-level transition checks when evaluating computer-use agents, especially around stale screenshots, delayed rendering, and recovery. The best ordering exact-match rates reached only **65.1% without decoys** and **65.7% with decoys**.\n\nThis is an offline diagnostic rather than an end-to-end task benchmark. Action recognition also varies sharply: reported click and drag scores were **0.96 versus 0.76 F1**, so aggregate GUI grounding can hide operation-specific weaknesses.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2607.26041v1",
    "published_at": "2026-07-28T17:49:51.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "benchmark",
  "domains": [
    "coding"
  ],
  "topics": [
    "computer-use",
    "agent-evals",
    "agent-reliability"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "This is an offline diagnostic rather than an end-to-end task benchmark. Action recognition also varies sharply: reported click and drag scores were **0.96 versus 0.76 F1**, so aggregate GUI grounding can hide operation-specific weaknesses."
  ],
  "connected_context": {
    "meaning": "Desktop-Delta Bench narrows the evaluation gap identified by broader coding and rollout benchmarks to a specific failure surface: whether an agent understands the transition between GUI states. Its low ordering ceiling and operation-specific score spread support adding step-level temporal and action checks to reproducible trajectory evaluation, rather than treating end-to-end completion or aggregate grounding as sufficient.",
    "corpus_size": 262,
    "generated_at": "2026-07-29T10:06:19.862Z",
    "connections": [
      {
        "title": "Everything Is a Rollout — Alex Shaw + Ryan Marten, Terminal-Bench, Harbor, Laude Institute",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=jRCpXUjz4CI",
        "feed7_url": "https://feed7.dev/p/everything-is-a-rollout-alex-shaw-ryan-marten-terminal-bench-harbor-laud-0iz4rgx",
        "reason": "Harbor’s trajectory inspection provides the evaluation loop into which Desktop-Delta’s step-level transition checks can fit."
      },
      {
        "title": "From Agent Traces to Agent Simulations — Rustem Feyzkhanov, Snorkel AI",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=Ib5t2RLtxvM",
        "feed7_url": "https://feed7.dev/p/from-agent-traces-to-agent-simulations-rustem-feyzkhanov-snorkel-ai-0zwlzjq",
        "reason": "Replayable production traces could supply fixed GUI transitions for diagnosing the stale-state, rendering-delay, and recovery failures this benchmark isolates."
      },
      {
        "title": "DeepSWE: A Contamination-Resistant Coding Benchmark — James Shi, Datacurve",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=Yk87oUPVaxU",
        "feed7_url": "https://feed7.dev/p/deepswe-a-contamination-resistant-coding-benchmark-james-shi-datacurve-08p61c0",
        "reason": "Both show that end scores can conceal behavior-level weaknesses, but Desktop-Delta extends trajectory-sensitive evaluation from repository work to desktop state transitions."
      },
      {
        "title": "How Evals and Prompts Shape Agent Behavior — Preetika Bhateja & Daniel Bump, YouTube Ads",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=xyL2Ltkh-SA",
        "feed7_url": "https://feed7.dev/p/how-evals-and-prompts-shape-agent-behavior-preetika-bhateja-daniel-bump-1cmecaw",
        "reason": "Desktop-Delta supplies the small, behavior-specific transition cases that can make trace review and calibrated evaluation more diagnostic for computer-use agents."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-07-28T17:49:51.000Z",
  "modified_at": "2026-07-28T17:49:51.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2607-26041v1-1x1gw81",
    "json": "https://feed7.dev/p/2607-26041v1-1x1gw81.json",
    "markdown": "https://feed7.dev/p/2607-26041v1-1x1gw81.md"
  }
}