{
  "schema_version": "1.1",
  "id": "s14:https://engineering.fb.com/2026/08/03/ml-applications/training-gem-at-llm-scale-meta-ads-recommendation-foundation-model/",
  "slug": "training-gem-at-llm-scale-meta-ads-recommendation-foundation-model-0fsb4wa",
  "url": "https://feed7.dev/p/training-gem-at-llm-scale-meta-ads-recommendation-foundation-model-0fsb4wa",
  "title": "GEM Training: How Meta Doubled the Efficiency of Its LLM-Scale Ads Foundation Model",
  "why_included": "Meta doubled GEM’s training efficiency by co-designing recommendation-specific kernels, precision, parallelism, networking, and memory. The takeaway is to profile workload shape before reusing an LLM training stack.",
  "summary": "Meta reports GEM reached **20–25% end-to-end MFU** while training FLOPs grew **4× in 12 months** across several thousand GPUs. Custom kernels, MXFP8 training, topology-aware **5D parallelism**, and communication changes address jagged sequences and sparse-dense parameters.",
  "practical_implication": "Builders operating specialized model workloads should separate local compute efficiency from distributed scaling efficiency. Profile sequence shapes, padding, numerical sensitivity, communication overlap, memory pressure, and rank imbalance before assuming standard LLM kernels or parallelism recipes transfer.",
  "agent_context": "Meta reports GEM reached **20–25% end-to-end MFU** while training FLOPs grew **4× in 12 months** across several thousand GPUs. Custom kernels, MXFP8 training, topology-aware **5D parallelism**, and communication changes address jagged sequences and sparse-dense parameters.\n\nBuilders operating specialized model workloads should separate local compute efficiency from distributed scaling efficiency. Profile sequence shapes, padding, numerical sensitivity, communication overlap, memory pressure, and rank imbalance before assuming standard LLM kernels or parallelism recipes transfer.\n\nThese techniques target Meta’s ads model with trillions of sparse and billions of dense parameters. The material does not establish that the same kernels, precision choices, or topology will benefit smaller models or different recommendation workloads.",
  "source": {
    "name": "Meta AI",
    "url": "https://engineering.fb.com/2026/08/03/ml-applications/training-gem-at-llm-scale-meta-ads-recommendation-foundation-model/",
    "published_at": "2026-08-03T18:00:17.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Engineering Post",
  "layer": "infra",
  "domains": [
    "data"
  ],
  "topics": [
    "observability"
  ],
  "verification": {
    "status": "official_source",
    "label": "Official Source",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "These techniques target Meta’s ads model with trillions of sparse and billions of dense parameters. The material does not establish that the same kernels, precision choices, or topology will benefit smaller models or different recommendation workloads."
  ],
  "connected_context": {
    "meaning": "This extends observability from serving and agent traces into distributed training efficiency. It separates kernel-level utilization from scaling losses caused by communication, topology, memory pressure, and rank imbalance, making shape- and stage-specific profiling a prerequisite for optimization. Unlike the candidates’ broadly reusable operational controls, its reported techniques remain tied to Meta’s unusually large sparse-dense ads workload and should not be assumed to transfer.",
    "corpus_size": 340,
    "generated_at": "2026-08-04T10:05:18.022Z",
    "connections": [
      {
        "title": "Core dump epidemiology: fixing an 18-year-old bug",
        "source_name": "OpenAI",
        "source_url": "https://openai.com/index/core-dump-epidemiology-data-infrastructure-bug",
        "feed7_url": "https://feed7.dev/p/core-dump-epidemiology-data-infrastructure-bug-1vwhgy3",
        "reason": "Both use fleet-scale evidence to isolate failures that local reproduction or aggregate metrics can hide; GEM applies that discipline to utilization, communication, and rank imbalance rather than rare crashes."
      },
      {
        "title": "TokTier: Exact Stateful Tokenization for Agentic LLM Serving",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2607.29678v1",
        "feed7_url": "https://feed7.dev/p/2607-29678v1-1fuv1qw",
        "reason": "TokTier similarly identifies a performance cost hidden behind a healthier headline metric, reinforcing the need to instrument distinct pipeline stages instead of treating overall latency or utilization as one problem."
      },
      {
        "title": "AI Gateway logs now have a dedicated page",
        "source_name": "Vercel",
        "source_url": "https://vercel.com/changelog/ai-gateway-logs",
        "feed7_url": "https://feed7.dev/p/ai-gateway-logs-1272t5j",
        "reason": "Gateway logs provide request-level serving visibility, while GEM shows the analogous training requirement: expose component-level compute and communication behavior before choosing an optimization."
      },
      {
        "title": "How Forward Deployed Engineering is done at Factory — Eno Reyes",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=wpOA-UXynoM",
        "feed7_url": "https://feed7.dev/p/how-forward-deployed-engineering-is-done-at-factory-eno-reyes-0zgscmd",
        "reason": "Factory’s workflow instrumentation reinforces GEM’s broader implementation lesson that optimization and autonomy depend on measurable intermediate stages rather than only an end-to-end outcome."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-08-03T18:00:17.000Z",
  "modified_at": "2026-08-03T18:00:17.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/training-gem-at-llm-scale-meta-ads-recommendation-foundation-model-0fsb4wa",
    "json": "https://feed7.dev/p/training-gem-at-llm-scale-meta-ads-recommendation-foundation-model-0fsb4wa.json",
    "markdown": "https://feed7.dev/p/training-gem-at-llm-scale-meta-ads-recommendation-foundation-model-0fsb4wa.md"
  }
}