{
  "schema_version": "1.1",
  "id": "s8:https://www.youtube.com/watch?v=y2W4FNAuPEA",
  "slug": "deep-dive-on-llm-inference-at-scale-harshul-jain-audible-tanmay-sah-inde-04utjkh",
  "url": "https://feed7.dev/p/deep-dive-on-llm-inference-at-scale-harshul-jain-audible-tanmay-sah-inde-04utjkh",
  "title": "Deep dive on LLM Inference at Scale — Harshul Jain, Audible & Tanmay Sah, Independent AI Researcher",
  "why_included": "Long contexts can exhaust GPU memory before model weights do. Capacity planning for agent workloads must include KV cache, concurrency, and serving-engine behavior.",
  "summary": "For Mistral 7B, the workshop calculates **131 KB per cached token**. At **16,000 context tokens** and **80 concurrent users**, KV cache alone needs about **42 GB**, exceeding a 24 GB GPU before other memory costs are counted.",
  "practical_implication": "When self-hosting models for agents, size capacity from context length and concurrency, not weights alone. Evaluate paged attention, continuous batching, prefix caching, model quantization, and KV-cache quantization in a serving engine against your own latency and throughput targets.",
  "agent_context": "For Mistral 7B, the workshop calculates **131 KB per cached token**. At **16,000 context tokens** and **80 concurrent users**, KV cache alone needs about **42 GB**, exceeding a 24 GB GPU before other memory costs are counted.\n\nWhen self-hosting models for agents, size capacity from context length and concurrency, not weights alone. Evaluate paged attention, continuous batching, prefix caching, model quantization, and KV-cache quantization in a serving engine against your own latency and throughput targets.\n\nQuantization assumes an acceptable quality trade-off and needs external evaluation. The workshop is beginner-to-intermediate, and its examples do not establish one engine or optimization as best for every workload.",
  "source": {
    "name": "AI Engineer",
    "url": "https://www.youtube.com/watch?v=y2W4FNAuPEA",
    "published_at": "2026-09-08T15:00:06.000Z"
  },
  "source_class": "video",
  "content_type": "Video",
  "layer": "infra",
  "domains": [
    "coding"
  ],
  "topics": [
    "context-caching",
    "observability"
  ],
  "verification": {
    "status": "source_linked",
    "label": "Source Linked",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "Quantization assumes an acceptable quality trade-off and needs external evaluation. The workshop is beginner-to-intermediate, and its examples do not establish one engine or optimization as best for every workload."
  ],
  "connected_context": {
    "meaning": "This quantifies why model weights are an incomplete capacity estimate for long-running agents: context length and concurrency can make KV cache the binding GPU constraint. It supplies the memory premise behind cache-aware serving techniques, while narrowing optimization claims to workload-specific measurements and quality evaluation rather than a universally best engine.",
    "corpus_size": 732,
    "generated_at": "2026-09-10T10:09:53.895Z",
    "connections": [
      {
        "title": "KV Cache-Aware Routing and P/D Disaggregation on Kubernetes — Yuchen Fama & Ashish Kamra, Red Hat",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=YXowceUKYJI",
        "feed7_url": "https://feed7.dev/p/kv-cache-aware-routing-and-p-d-disaggregation-on-kubernetes-yuchen-fama-0u2jkju",
        "reason": "The calculated KV-cache pressure explains when cache-aware routing and split prefill/decode may become valuable; both require validation against actual concurrency, prompt shape, latency, and network costs."
      },
      {
        "title": "TokTier: Exact Stateful Tokenization for Agentic LLM Serving",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2607.29678v1",
        "feed7_url": "https://feed7.dev/p/2607-29678v1-1fuv1qw",
        "reason": "TokTier identifies tokenization as an additional long-session bottleneck, showing that solving GPU KV-cache capacity alone does not guarantee low time to first token."
      },
      {
        "title": "jundot/omlx",
        "source_name": "GitHub",
        "source_url": "https://github.com/jundot/omlx",
        "feed7_url": "https://feed7.dev/p/omlx-04mmjt8",
        "reason": "oMLX implements two techniques highlighted here—continuous batching and persistent KV caching—for local concurrent agents, but its limited hardware evidence reinforces the need for workload-specific evaluation."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-09-08T15:00:06.000Z",
  "modified_at": "2026-09-08T15:00:06.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/deep-dive-on-llm-inference-at-scale-harshul-jain-audible-tanmay-sah-inde-04utjkh",
    "json": "https://feed7.dev/p/deep-dive-on-llm-inference-at-scale-harshul-jain-audible-tanmay-sah-inde-04utjkh.json",
    "markdown": "https://feed7.dev/p/deep-dive-on-llm-inference-at-scale-harshul-jain-audible-tanmay-sah-inde-04utjkh.md"
  }
}