{
  "schema_version": "1.1",
  "id": "s13:https://arxiv.org/abs/2611.23206v1",
  "slug": "2611-23206v1-1kzrrws",
  "url": "https://feed7.dev/p/2611-23206v1-1kzrrws",
  "title": "Reading Scene States: Uncovering Safety Failures in Vision-Language Models",
  "why_included": "Scene-based inputs elicited harmful assistance from tested vision-language models, and the proposed attack outperformed the strongest baseline. Text filters and visual moderation offered limited protection in the study.",
  "summary": "SceneSafetyBench contains **375** image-question pairs across three scene states and five visual presentations. Across eight vision-language models, the proposed S3TA attack reached a **71.2%** average attack success rate, 6.1 percentage points above the strongest baseline.",
  "practical_implication": "For builders adding image understanding, test what an agent infers from a sequence of visual events, not only whether each image or prompt looks harmful in isolation. Include scene-grounded requests in red-team cases and check how responses are assembled across queries.",
  "agent_context": "SceneSafetyBench contains **375** image-question pairs across three scene states and five visual presentations. Across eight vision-language models, the proposed S3TA attack reached a **71.2%** average attack success rate, 6.1 percentage points above the strongest baseline.\n\nFor builders adding image understanding, test what an agent infers from a sequence of visual events, not only whether each image or prompt looks harmful in isolation. Include scene-grounded requests in red-team cases and check how responses are assembled across queries.\n\nThe results cover seven MM-SafetyBench categories and eight open-weight and commercial models. The paper reports limited protection from text-side filters and visual moderation; the material does not establish how other defenses perform.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2611.23206v1",
    "published_at": "2026-11-17T18:48:02.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "benchmark",
  "domains": [
    "image",
    "security"
  ],
  "topics": [
    "agent-evals",
    "benchmark-integrity",
    "computer-use"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The results cover seven MM-SafetyBench categories and eight open-weight and commercial models. The paper reports limited protection from text-side filters and visual moderation; the material does not establish how other defenses perform."
  ],
  "connected_context": {
    "meaning": "This adds a scene-sequence safety test to the visual-evaluation concerns in the prior candidates: failures can emerge from how an agent interprets changing scene states, so single-image checks may miss them. Its reported attack results support behavior-level testing, while leaving defenses beyond those tested unresolved.",
    "corpus_size": 1116,
    "generated_at": "2026-11-18T11:09:25.220Z",
    "connections": [
      {
        "title": "The Alignment Illusion in Multimodal Large Language Models",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2609.30210v1",
        "feed7_url": "https://feed7.dev/p/2609-30210v1-198in2y",
        "reason": "Both show why visual safety evaluations need controlled, task-grounded behavior checks: this Signal tests harmful inferences across scene states, while the candidate shows visual alignment scores can remain high after useful image information is corrupted."
      },
      {
        "title": "Comparative Analysis of Agent Evaluation Frameworks: Stability, Detection, and Discovery in Enterprise Analytics Agents",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2611.14493v1",
        "feed7_url": "https://feed7.dev/p/2611-14493v1-1ghdgem",
        "reason": "The candidate found functional checks and security red-teaming surfaced distinct failures; this Signal supplies a concrete visual red-team case that can complement functional evaluation."
      },
      {
        "title": "Evolution of Accuracy and Visual-Cognitive Errors in a Decade of Vision-Language AI Models",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2607.09654v1",
        "feed7_url": "https://feed7.dev/p/2607-09654v1-0b5dedg",
        "reason": "The candidate identifies persistent visual-cognitive errors in scene description; this Signal shifts the concern toward safety failures when models interpret scene changes across multiple visual presentations."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-11-17T18:48:02.000Z",
  "modified_at": "2026-11-17T18:48:02.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2611-23206v1-1kzrrws",
    "json": "https://feed7.dev/p/2611-23206v1-1kzrrws.json",
    "markdown": "https://feed7.dev/p/2611-23206v1-1kzrrws.md"
  }
}