{
  "schema_version": "1.1",
  "id": "archive:https://arxiv.org/abs/2611.14467v1",
  "slug": "2611-14467v1-1wobu6x",
  "url": "https://feed7.dev/p/2611-14467v1-1wobu6x",
  "title": "A Theory of Policy Optimization in Post-Training: Coverage Collapse and Reference Resets",
  "why_included": "A policy-optimization theory paper describes how REINFORCE-style training can lose access to good responses even as reward rises. It analyzes reference resets as a way to mitigate that failure under stated assumptions.",
  "summary": "The paper identifies **coverage collapse**: REINFORCE-style methods can make optimal responses increasingly unlikely even while reward improves. It proposes Approximate Policy Mirror Descent with repeated KL-reference resets to mitigate the problem.",
  "practical_implication": "For builders fine-tuning models, the work highlights a potential gap between rising reward and retaining access to strong responses. When evaluating post-training, consider tracking response coverage as well as reward.",
  "agent_context": "The paper identifies **coverage collapse**: REINFORCE-style methods can make optimal responses increasingly unlikely even while reward improves. It proposes Approximate Policy Mirror Descent with repeated KL-reference resets to mitigate the problem.\n\nFor builders fine-tuning models, the work highlights a potential gap between rising reward and retaining access to strong responses. When evaluating post-training, consider tracking response coverage as well as reward.\n\nThe convergence guarantees apply to softmax linear policies over pretrained features and specified algorithm variants. The paper is a theoretical analysis; the provided material does not establish that reference resets improve outcomes in arbitrary production training runs.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2611.14467v1",
    "published_at": "2026-11-10T18:31:36.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "model",
  "domains": [
    "research"
  ],
  "topics": [
    "reasoning",
    "open-models"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The convergence guarantees apply to softmax linear policies over pretrained features and specified algorithm variants. The paper is a theoretical analysis; the provided material does not establish that reference resets improve outcomes in arbitrary production training runs."
  ],
  "connected_context": {
    "meaning": "The theory warns that reward gains can coexist with losing access to optimal responses, so reward alone may hide a training failure. Coverage tracking is a useful evaluation implication, while the proposed mitigation remains bounded by the paper’s policy assumptions and is not established for production runs.",
    "corpus_size": 1059,
    "generated_at": "2026-11-11T11:09:47.282Z",
    "connections": [
      {
        "title": "$β$-OPSD: Deriving with Policy Optimization, Training with Self-Distillation",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2607.28582v1",
        "feed7_url": "https://feed7.dev/p/2607-28582v1-0egi1xh",
        "reason": "Both address post-training behavior through policy regularization: β-OPSD makes self-distillation’s balance tunable, while this paper analyzes reference resets as a way to mitigate coverage collapse. Their mechanisms and stated evidence differ."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-11-10T18:31:36.000Z",
  "modified_at": "2026-11-10T18:31:36.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2611-14467v1-1wobu6x",
    "json": "https://feed7.dev/p/2611-14467v1-1wobu6x.json",
    "markdown": "https://feed7.dev/p/2611-14467v1-1wobu6x.md"
  }
}