{
  "schema_version": "1.1",
  "id": "s13:https://arxiv.org/abs/2611.26619v1",
  "slug": "2611-26619v1-1cp2t0h",
  "url": "https://feed7.dev/p/2611-26619v1-1cp2t0h",
  "title": "TideCache: Efficient Hybrid LLM Serving with Transparent Cache Management and Layer-Wise Streaming",
  "why_included": "TideCache streams attention-cache data between host and GPU memory to free capacity for concurrent agent requests. Its reported gains are promising, but they come from specific SGLang and superchip configurations.",
  "summary": "TideCache combines a shared GPU cache pool, layer-wise prefetching, and request-aware scheduling. In tests on **GB300 and GH200** across **seven configurations**, it handled up to **2.04× as many agent turns** and cut median per-token latency by **20–59%** versus SGLang with HiCache.",
  "practical_implication": "If you operate hybrid-model serving, consider whether streaming attention-cache prefixes could trade GPU residency for more concurrent requests while overlapping transfers with compute. The implementation is integrated with SGLang and keeps generated KV resident.",
  "agent_context": "TideCache combines a shared GPU cache pool, layer-wise prefetching, and request-aware scheduling. In tests on **GB300 and GH200** across **seven configurations**, it handled up to **2.04× as many agent turns** and cut median per-token latency by **20–59%** versus SGLang with HiCache.\n\nIf you operate hybrid-model serving, consider whether streaming attention-cache prefixes could trade GPU residency for more concurrent requests while overlapping transfers with compute. The implementation is integrated with SGLang and keeps generated KV resident.\n\nThe reported gains are specific to the tested superchips and configurations. The material does not establish how the design performs on other hardware or serving stacks.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2611.26619v1",
    "published_at": "2026-11-19T18:58:31.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "infra",
  "domains": [
    "coding"
  ],
  "topics": [
    "cloud-agents",
    "context-caching"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The reported gains are specific to the tested superchips and configurations. The material does not establish how the design performs on other hardware or serving stacks."
  ],
  "connected_context": {
    "meaning": "TideCache reports higher agent-turn capacity and lower token latency from shared GPU cache management and layer-wise streaming on GB300 and GH200. It adds an implementation-level option for the cache-locality and GPU-capacity concerns in the supplied serving talks, but its results do not establish gains on other hardware or serving stacks.",
    "corpus_size": 1135,
    "generated_at": "2026-11-21T11:06:52.506Z",
    "connections": [
      {
        "title": "KV Cache-Aware Routing and P/D Disaggregation on Kubernetes — Yuchen Fama & Ashish Kamra, Red Hat",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=YXowceUKYJI",
        "feed7_url": "https://feed7.dev/p/kv-cache-aware-routing-and-p-d-disaggregation-on-kubernetes-yuchen-fama-0u2jkju",
        "reason": "Both address KV-cache pressure in agent serving; Red Hat focuses on routing and prefill/decode separation, while TideCache changes cache pooling, prefetching, and scheduling within a serving stack."
      },
      {
        "title": "The Frontier AI Inference Cloud for Agents — Byung-Gon (Gon) Chun, FriendliAI",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=Hvb2LfMH58c",
        "feed7_url": "https://feed7.dev/p/the-frontier-ai-inference-cloud-for-agents-byung-gon-gon-chun-friendliai-0b7vgoo",
        "reason": "FriendliAI argues that preserving shared prefixes and routing toward resident KV can improve whole-task latency; TideCache offers a distinct serving mechanism for managing cache residency and transfers, with results bounded to its tested hardware."
      },
      {
        "title": "Deep dive on LLM Inference at Scale — Harshul Jain, Audible & Tanmay Sah, Independent AI Researcher",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=y2W4FNAuPEA",
        "feed7_url": "https://feed7.dev/p/deep-dive-on-llm-inference-at-scale-harshul-jain-audible-tanmay-sah-inde-04utjkh",
        "reason": "The inference deep dive identifies KV cache as a potential GPU capacity constraint for long contexts and concurrency; TideCache measures a way to increase capacity under its specific tested configurations."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-11-19T18:58:31.000Z",
  "modified_at": "2026-11-19T18:58:31.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2611-26619v1-1cp2t0h",
    "json": "https://feed7.dev/p/2611-26619v1-1cp2t0h.json",
    "markdown": "https://feed7.dev/p/2611-26619v1-1cp2t0h.md"
  }
}