{
  "schema_version": "1.1",
  "id": "s13:https://arxiv.org/abs/2609.20807v1",
  "slug": "2609-20807v1-1tql3tf",
  "url": "https://feed7.dev/p/2609-20807v1-1tql3tf",
  "title": "Score Centering Stabilizes Off-policy Reinforcement Learning",
  "why_included": "Training–inference mismatch may destabilize LLM reinforcement learning through accumulating score drift. An additive centering correction matched or beat importance sampling in the reported tests.",
  "summary": "The paper attributes instability under training–inference mismatch to persistent score drift that accumulates across steps. Its additive **score-centering** correction cancels that drift across models from **0.6B to 30B parameters**.",
  "practical_implication": "If you train models with rollout and training engines that cannot be made identical, test score centering before paying the efficiency cost of eliminating the mismatch. The correction can also be combined with **importance sampling** rather than replacing it.",
  "agent_context": "The paper attributes instability under training–inference mismatch to persistent score drift that accumulates across steps. Its additive **score-centering** correction cancels that drift across models from **0.6B to 30B parameters**.\n\nIf you train models with rollout and training engines that cannot be made identical, test score centering before paying the efficiency cost of eliminating the mismatch. The correction can also be combined with **importance sampling** rather than replacing it.\n\nThe abstract reports matching or better results under quantization and stronger combined results in staleness experiments, but gives no absolute scores or broader workload evidence. This is primarily relevant to model-training infrastructure, not day-to-day agent use.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2609.20807v1",
    "published_at": "2026-09-17T17:58:17.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "model",
  "domains": [],
  "topics": [],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The abstract reports matching or better results under quantization and stronger combined results in staleness experiments, but gives no absolute scores or broader workload evidence. This is primarily relevant to model-training infrastructure, not day-to-day agent use."
  ],
  "connected_context": {
    "meaning": "This adds a targeted systems correction for off-policy RL instability when rollout and training engines differ: cancel accumulated score drift instead of assuming the mismatch must be eliminated. It complements importance sampling and appears compatible with quantized execution, but the missing absolute results and workload breadth leave its practical advantage over engine alignment unquantified.",
    "corpus_size": 807,
    "generated_at": "2026-09-18T10:06:53.960Z",
    "connections": [
      {
        "title": "UE5M3 FP4 Block Scaling for Stable Language Model Pretraining",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2609.02846v1",
        "feed7_url": "https://feed7.dev/p/2609-02846v1-014naa0",
        "reason": "Both address numerical stability under low precision, but at different stages: UE5M3 changes the FP4 pretraining recipe, while score centering corrects drift caused by mismatched rollout and training execution."
      },
      {
        "title": "The Structure of Quantization Damage in LLMs: Why the Next Bit Should Be Spent Globally",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2609.01587v1",
        "feed7_url": "https://feed7.dev/p/2609-01587v1-1kd5xo7",
        "reason": "The quantization-damage work concerns how reduced precision degrades model weights broadly; this Signal addresses a distinct consequence of quantized or stale execution—persistent score drift during off-policy training."
      },
      {
        "title": "RRC: Unlocking Generative Reward Models in LLM Reinforcement Learning via Ranking-Based Reward Construction",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2608.06310v1",
        "feed7_url": "https://feed7.dev/p/2608-06310v1-1e0x375",
        "reason": "RRC constructs a better reward from relative judgments, whereas score centering stabilizes how an off-policy training signal behaves across mismatched engines; the mechanisms address complementary parts of the RL pipeline."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-09-17T17:58:17.000Z",
  "modified_at": "2026-09-17T17:58:17.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2609-20807v1-1tql3tf",
    "json": "https://feed7.dev/p/2609-20807v1-1tql3tf.json",
    "markdown": "https://feed7.dev/p/2609-20807v1-1tql3tf.md"
  }
}