{
  "schema_version": "1.1",
  "id": "s13:https://arxiv.org/abs/2609.21996v1",
  "slug": "2609-21996v1-16fmyv3",
  "url": "https://feed7.dev/p/2609-21996v1-16fmyv3",
  "title": "A Lie Detector Test for Language Models: Reading Knowledge a Model Won't Reveal",
  "why_included": "PIR reads internal model states to distinguish concealed knowledge from missing knowledge, offering a stronger audit for sandbagging and unlearning than output-only evals—when model internals are accessible.",
  "summary": "Probe of Internal Recognition inspects model states to identify which candidate answer the model recognizes. Across **eight models from five families**, it reached **0.70–0.87 balanced accuracy**, versus a 0.28–0.40 unknown-item baseline and 0.25 chance.",
  "practical_implication": "Builders evaluating agent models should separate refusal or deliberate underperformance from genuine inability. PIR is **reference-free**, so its audits do not require an honest comparison model or labeled truth corpus; it also extends beyond multiple choice to free-form generation.",
  "agent_context": "Probe of Internal Recognition inspects model states to identify which candidate answer the model recognizes. Across **eight models from five families**, it reached **0.70–0.87 balanced accuracy**, versus a 0.28–0.40 unknown-item baseline and 0.25 chance.\n\nBuilders evaluating agent models should separate refusal or deliberate underperformance from genuine inability. PIR is **reference-free**, so its audits do not require an honest comparison model or labeled truth corpus; it also extends beyond multiple choice to free-form generation.\n\nThe method requires access to internal states, which hosted model APIs may not expose. Its **0.85–0.93 recognition** under tested concealment methods is evidence across specific models and interventions, not proof that every hidden capability will remain detectable.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2609.21996v1",
    "published_at": "2026-09-18T16:57:00.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "benchmark",
  "domains": [],
  "topics": [
    "agent-evals",
    "benchmark-integrity"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The method requires access to internal states, which hosted model APIs may not expose. Its **0.85–0.93 recognition** under tested concealment methods is evidence across specific models and interventions, not proof that every hidden capability will remain detectable."
  ],
  "connected_context": {
    "meaning": "PIR adds a capability-audit distinction missing from ordinary output scoring: an incorrect, refused, or strategically weak answer need not mean the model lacks the answer internally. It reinforces mechanistic evaluation while narrowing what can be concluded from behavior alone. Unlike harness, judge, or task-quality audits, it targets concealed recognition inside accessible models, so it cannot directly audit opaque hosted APIs.",
    "corpus_size": 831,
    "generated_at": "2026-09-21T09:04:18.336Z",
    "connections": [
      {
        "title": "From Deceptive Outputs to Deceptive Mechanisms: A Causal Framework for Language-Model Deception Research",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2609.04166v1",
        "feed7_url": "https://feed7.dev/p/2609-04166v1-04gqa5r",
        "reason": "The causal deception framework warns that suspicious outputs do not identify their mechanism; PIR supplies internal evidence that can distinguish recognized-but-unrevealed answers from genuine lack of recognition, without by itself proving deceptive intent."
      },
      {
        "title": "Beyond Scores: Understanding LLM-as-a-Judge Mechanisms in Summarization Evaluation",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2609.01604v1",
        "feed7_url": "https://feed7.dev/p/2609-01604v1-02vljon",
        "reason": "Both move evaluation beneath scalar outputs by probing internal representations; the judge study localizes evidence routing and rating formation, while PIR tests whether an answer is internally recognized despite the emitted response."
      },
      {
        "title": "QuoteBench: How Matched Scores Can Hide Command-Path Failures",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2608.13547v1",
        "feed7_url": "https://feed7.dev/p/2608-13547v1-130h6xd",
        "reason": "QuoteBench separates model behavior from failures introduced by the execution path, while PIR separates expressed performance from internal recognition; together they show that an observed miss may not identify the underlying capability failure."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-09-18T16:57:00.000Z",
  "modified_at": "2026-09-18T16:57:00.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2609-21996v1-16fmyv3",
    "json": "https://feed7.dev/p/2609-21996v1-16fmyv3.json",
    "markdown": "https://feed7.dev/p/2609-21996v1-16fmyv3.md"
  }
}