{
  "schema_version": "1.1",
  "id": "archive:https://arxiv.org/abs/2610.37647v1",
  "slug": "2610-37647v1-0r1fv8r",
  "url": "https://feed7.dev/p/2610-37647v1-0r1fv8r",
  "title": "MuonGS: Pretraining LLMs with Relaxed Weight-Norm Control",
  "why_included": "MuonGS changes Muon pretraining by projecting out each update’s radial component without fixing weight norms. It reports lower long-run validation perplexity than MuonH, including an estimated 15% token-equivalent speedup in one dense-model comparison.",
  "summary": "MuonGS uses a Gram–Schmidt projection to remove the radial part of each Muon update without fixing the weight norm. The paper proves a **√t** upper bound on norm growth under stated conditions and reports up to **15%** estimated token-equivalent speedup at matched validation loss in a dense model.",
  "practical_implication": "For teams training language models, MuonGS is an optimizer candidate to compare when long training horizons matter. The reported experiments include dense models at **160M and 400M parameters** and a **135M-active / 1B-total MoE** model.",
  "agent_context": "MuonGS uses a Gram–Schmidt projection to remove the radial part of each Muon update without fixing the weight norm. The paper proves a **√t** upper bound on norm growth under stated conditions and reports up to **15%** estimated token-equivalent speedup at matched validation loss in a dense model.\n\nFor teams training language models, MuonGS is an optimizer candidate to compare when long training horizons matter. The reported experiments include dense models at **160M and 400M parameters** and a **135M-active / 1B-total MoE** model.\n\nThe speedup is an estimate from one dense-model comparison, and the theoretical guarantees depend on stated assumptions. The material does not establish that the result transfers to other architectures or training setups.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2610.37647v1",
    "published_at": "2026-10-27T17:56:40.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "model",
  "domains": [
    "research",
    "data"
  ],
  "topics": [
    "open-models",
    "model-selection"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The speedup is an estimate from one dense-model comparison, and the theoretical guarantees depend on stated assumptions. The material does not establish that the result transfers to other architectures or training setups."
  ],
  "connected_context": {
    "meaning": "MuonGS offers a way to limit optimizer-driven norm growth without fixing weight norms, with a reported token-equivalent speedup in a bounded set of model experiments. The result makes optimizer choice worth comparing for long training runs, but does not establish that the gain transfers across architectures or setups.",
    "corpus_size": 944,
    "generated_at": "2026-10-28T11:08:13.296Z",
    "connections": [
      {
        "title": "When Does Bigger Help? A Controlled Study of LLM Scale for Ontology Learning",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2608.31118v1",
        "feed7_url": "https://feed7.dev/p/2608-31118v1-1abyqwd",
        "reason": "The ontology study finds model-size rankings depend on the task and architecture; MuonGS likewise has bounded experimental evidence across specific model configurations. Together they support matched, workload-specific comparisons rather than assuming one training or model choice generalizes."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-10-27T17:56:40.000Z",
  "modified_at": "2026-10-27T17:56:40.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2610-37647v1-0r1fv8r",
    "json": "https://feed7.dev/p/2610-37647v1-0r1fv8r.json",
    "markdown": "https://feed7.dev/p/2610-37647v1-0r1fv8r.md"
  }
}