{
  "schema_version": "1.1",
  "id": "archive:https://arxiv.org/abs/2611.03633v1",
  "slug": "2611-03633v1-05w62ui",
  "url": "https://feed7.dev/p/2611-03633v1-05w62ui",
  "title": "Pretraining Data Scale Reverses the Muon-AdamW Fine-Tuning Gap",
  "why_included": "Muon’s pretraining advantage may not carry through fine-tuning: at larger data budgets, AdamW performs better downstream. Optimizer choice may depend on both training scale and model size.",
  "summary": "The study finds that **Muon** does better after fine-tuning at smaller pretraining data budgets, while **AdamW** takes the lead beyond a threshold. As model size grows, AdamW takes over at smaller pretraining token-to-parameter ratios.",
  "practical_implication": "If you train models, evaluate optimizers on downstream fine-tuning outcomes as well as pretraining loss, and account for both data budget and model size.",
  "agent_context": "The study finds that **Muon** does better after fine-tuning at smaller pretraining data budgets, while **AdamW** takes the lead beyond a threshold. As model size grows, AdamW takes over at smaller pretraining token-to-parameter ratios.\n\nIf you train models, evaluate optimizers on downstream fine-tuning outcomes as well as pretraining loss, and account for both data budget and model size.\n\nThe paper explains the shift using a margin-to-spread measure and proves a related result in a linear associative memory. The supplied material gives no threshold values for practical model sizes or training setups.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2611.03633v1",
    "published_at": "2026-11-02T18:56:43.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "model",
  "domains": [
    "research"
  ],
  "topics": [
    "model-selection",
    "open-models"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The paper explains the shift using a margin-to-spread measure and proves a related result in a linear associative memory. The supplied material gives no threshold values for practical model sizes or training setups."
  ],
  "connected_context": null,
  "lifecycle": "Current",
  "published_at": "2026-11-02T18:56:43.000Z",
  "modified_at": "2026-11-02T18:56:43.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2611-03633v1-05w62ui",
    "json": "https://feed7.dev/p/2611-03633v1-05w62ui.json",
    "markdown": "https://feed7.dev/p/2611-03633v1-05w62ui.md"
  }
}