{
  "schema_version": "1.1",
  "id": "s13:https://arxiv.org/abs/2611.06858v1",
  "slug": "2611-06858v1-1vojzui",
  "url": "https://feed7.dev/p/2611-06858v1-1vojzui",
  "title": "CompressionBench: How can your coding agents compress LLMs?",
  "why_included": "CompressionBench tests whether coding agents can compress language models under storage and time limits; verification passed for 29 of 36 submissions, with generation tests catching failures perplexity missed.",
  "summary": "In **CompressionBench**, agents receive a model checkpoint, storage cap, inference environment, and time budget, then produce compressed weights and runnable inference code. **29 of 36 submissions** passed independent verification; the best per-task designs averaged a 3.4% perplexity increase and 77.33% downstream accuracy, versus 79.56% for BF16.",
  "practical_implication": "For agent-driven optimization, use end-to-end checks alongside local metrics: full-model feedback reversed 17 of 79 local rankings, and generation and inference tests caught failures that perplexity did not. The benchmark treats experimental judgment and capability validation as part of the task.",
  "agent_context": "In **CompressionBench**, agents receive a model checkpoint, storage cap, inference environment, and time budget, then produce compressed weights and runnable inference code. **29 of 36 submissions** passed independent verification; the best per-task designs averaged a 3.4% perplexity increase and 77.33% downstream accuracy, versus 79.56% for BF16.\n\nFor agent-driven optimization, use end-to-end checks alongside local metrics: full-model feedback reversed 17 of 79 local rankings, and generation and inference tests caught failures that perplexity did not. The benchmark treats experimental judgment and capability validation as part of the task.\n\nThe study covers **six agents and six model–size settings**. Its results describe this benchmark setup and do not establish how agents perform on other compression goals or environments.",
  "source": {
    "name": "arXiv",
    "url": "https://arxiv.org/abs/2611.06858v1",
    "published_at": "2026-11-04T18:14:50.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Paper",
  "layer": "benchmark",
  "domains": [
    "coding"
  ],
  "topics": [
    "coding-agents",
    "agent-evals",
    "agent-reliability"
  ],
  "verification": {
    "status": "needs_review",
    "label": "Needs Review",
    "method": "unverified",
    "verified_at": null
  },
  "uncertainty": [
    "The study covers **six agents and six model–size settings**. Its results describe this benchmark setup and do not establish how agents perform on other compression goals or environments."
  ],
  "connected_context": {
    "meaning": "CompressionBench evaluates agent work through runnable compressed models and downstream checks, showing that local metrics such as perplexity can misrank candidate designs. It adds measured evidence for end-to-end acceptance when agents optimize models, while keeping conclusions bounded to six agents and six model–size settings in this benchmark.",
    "corpus_size": 1009,
    "generated_at": "2026-11-05T11:04:31.498Z",
    "connections": [
      {
        "title": "SWE-Serve: Benchmarking Agentic Engineering For Production Inference Serving",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2609.26777v1",
        "feed7_url": "https://feed7.dev/p/2609-26777v1-1ylj2uv",
        "reason": "Both benchmarks find local checks can miss failures revealed by end-to-end verification; CompressionBench demonstrates this for model compression, while SWE-Serve tests production inference changes."
      },
      {
        "title": "Can Large Language Models Recover Semantic Optimization Opportunities That Compilers Miss?",
        "source_name": "arXiv",
        "source_url": "https://arxiv.org/abs/2608.03983v1",
        "feed7_url": "https://feed7.dev/p/2608-03983v1-07mv55y",
        "reason": "SeGaBench requires independent correctness and performance validation of agent-generated optimizations; CompressionBench similarly shows that local perplexity is insufficient without full-model and inference checks."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-11-04T18:14:50.000Z",
  "modified_at": "2026-11-04T18:14:50.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/2611-06858v1-1vojzui",
    "json": "https://feed7.dev/p/2611-06858v1-1vojzui.json",
    "markdown": "https://feed7.dev/p/2611-06858v1-1vojzui.md"
  }
}