{
  "schema_version": "1.0",
  "id": "archive:https://arxiv.org/abs/2607.11871v1",
  "slug": "2607-11871v1-17vejh0",
  "url": "https://feed7.dev/p/2607-11871v1-17vejh0",
  "title": "Inside the Unfair Judge: A Mechanistic Interpretability Account of LLM-as-Judge Bias",
  "why_included": "LLM judge bias appears as steerable hidden-state directions that predict failures on unseen benchmarks, so eval pipelines may need representation-level checks beyond prompt fixes.",
  "summary": "Across **7 judges, 7 bias types, and 9 benchmarks**, biased inputs displaced hidden states along low-dimensional, type-specific directions that became clearer with depth. Three estimator families recovered the structure consistently.",
  "practical_implication": "If model-based grading drives agent evals, test the judge itself under known bias perturbations. For models whose activations are accessible, projection onto learned bias directions could supplement score-delta tests and prompt-level mitigations.",
  "agent_context": "Across **7 judges, 7 bias types, and 9 benchmarks**, biased inputs displaced hidden states along low-dimensional, type-specific directions that became clearer with depth. Three estimator families recovered the structure consistently.\n\nIf model-based grading drives agent evals, test the judge itself under known bias perturbations. For models whose activations are accessible, projection onto learned bias directions could supplement score-delta tests and prompt-level mitigations.\n\nThe evidence comes from the studied judges and bias types, and activation access is unavailable for many hosted models. Prediction was tested on **3 unseen benchmarks**; broader transfer and practical intervention costs remain open.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2607.11871v1",
    "published_at": null
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "benchmark",
  "domains": [
    "research"
  ],
  "topics": [
    "benchmark-integrity",
    "agent-evals",
    "agent-reliability"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The evidence comes from the studied judges and bias types, and activation access is unavailable for many hosted models. Prediction was tested on **3 unseen benchmarks**; broader transfer and practical intervention costs remain open."
  ],
  "lifecycle": "Current",
  "published_at": null,
  "modified_at": null,
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2607-11871v1-17vejh0",
    "json": "https://feed7.dev/p/2607-11871v1-17vejh0.json",
    "markdown": "https://feed7.dev/p/2607-11871v1-17vejh0.md"
  }
}