{
  "schema_version": "1.1",
  "id": "s8:https://www.youtube.com/watch?v=cQQbJqvZkpo",
  "slug": "vertical-mobility-inference-from-mvp-to-trillion-parameter-workloads-sit-11qvtzr",
  "url": "https://feed7.dev/p/vertical-mobility-inference-from-mvp-to-trillion-parameter-workloads-sit-11qvtzr",
  "title": "Vertical Mobility: Inference from MVP to Trillion-Parameter Workloads — Sitanshu Gupta, CoreWeave",
  "why_included": "Agent requests often repeat most of their input, making KV-cache-aware routing a direct latency and cost lever for hosted coding-agent workloads.",
  "summary": "CoreWeave describes one inference platform spanning serverless and dedicated deployments, heterogeneous GPUs, and several engines. In agentic traffic, **80–90% of input** may repeat across requests, so its router prioritizes KV-cache locality before falling back to the least-loaded target.",
  "practical_implication": "For a coding-agent backend, retain reusable prefixes across turns and make cache locality part of routing. Choose prefill/decode disaggregation only after measuring the workload, and schedule latency-tolerant batch work onto capacity left idle by real-time traffic.",
  "agent_context": "CoreWeave describes one inference platform spanning serverless and dedicated deployments, heterogeneous GPUs, and several engines. In agentic traffic, **80–90% of input** may repeat across requests, so its router prioritizes KV-cache locality before falling back to the least-loaded target.\n\nFor a coding-agent backend, retain reusable prefixes across turns and make cache locality part of routing. Choose prefill/decode disaggregation only after measuring the workload, and schedule latency-tolerant batch work onto capacity left idle by real-time traffic.\n\nThe talk presents platform design choices rather than controlled comparisons. Cache behavior varies by customer, provisioned throughput needs a known traffic profile, and disaggregation is explicitly not economical for every workload.",
  "source": {
    "name": "AI Engineer",
    "url": "https://www.youtube.com/watch?v=cQQbJqvZkpo",
    "published_at": "2026-09-19T16:30:34.000Z"
  },
  "source_class": "video",
  "content_type": "Video",
  "layer": "infra",
  "domains": [
    "coding"
  ],
  "topics": [
    "context-caching",
    "gateways",
    "cloud-agents"
  ],
  "verification": {
    "status": "source_linked",
    "label": "Source Linked",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "The talk presents platform design choices rather than controlled comparisons. Cache behavior varies by customer, provisioned throughput needs a known traffic profile, and disaggregation is explicitly not economical for every workload."
  ],
  "connected_context": {
    "meaning": "This confirms repeated prefixes as a concrete routing input for coding-agent infrastructure and places cache locality inside a broader platform spanning engines, GPUs, and deployment modes. It narrows the design choice by treating prefill/decode separation and provisioned capacity as traffic-dependent, while suggesting idle real-time capacity can absorb latency-tolerant batch work.",
    "corpus_size": 823,
    "generated_at": "2026-09-20T09:13:02.987Z",
    "connections": [
      {
        "title": "KV Cache-Aware Routing and P/D Disaggregation on Kubernetes — Yuchen Fama & Ashish Kamra, Red Hat",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=YXowceUKYJI",
        "feed7_url": "https://feed7.dev/p/kv-cache-aware-routing-and-p-d-disaggregation-on-kubernetes-yuchen-fama-0u2jkju",
        "reason": "Red Hat provides supporting case-study evidence for KV-aware routing and clarifies the concurrency and network conditions under which prefill/decode separation is worthwhile."
      },
      {
        "title": "The Frontier AI Inference Cloud for Agents — Byung-Gon (Gon) Chun, FriendliAI",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=Hvb2LfMH58c",
        "feed7_url": "https://feed7.dev/p/the-frontier-ai-inference-cloud-for-agents-byung-gon-gon-chun-friendliai-0b7vgoo",
        "reason": "FriendliAI extends the same locality principle from individual routing decisions to hierarchical KV storage and whole-task, agent-aware scheduling."
      },
      {
        "title": "AI Gateway logs now have a dedicated page",
        "source_name": "Vercel",
        "source_url": "https://vercel.com/changelog/ai-gateway-logs",
        "feed7_url": "https://feed7.dev/p/ai-gateway-logs-1272t5j",
        "reason": "Request-level routing, latency, token, and fallback logs provide part of the observability needed to validate whether locality policies and heterogeneous targets improve real workloads."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-09-19T16:30:34.000Z",
  "modified_at": "2026-09-19T16:30:34.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/vertical-mobility-inference-from-mvp-to-trillion-parameter-workloads-sit-11qvtzr",
    "json": "https://feed7.dev/p/vertical-mobility-inference-from-mvp-to-trillion-parameter-workloads-sit-11qvtzr.json",
    "markdown": "https://feed7.dev/p/vertical-mobility-inference-from-mvp-to-trillion-parameter-workloads-sit-11qvtzr.md"
  }
}