{
  "schema_version": "1.0",
  "id": "s8:https://www.youtube.com/watch?v=0vphxNt4wyk",
  "slug": "don-t-ship-skills-without-evals-philipp-schmid-google-deepmind-0fuh3ko",
  "url": "https://feed7.dev/p/don-t-ship-skills-without-evals-philipp-schmid-google-deepmind-0fuh3ko",
  "title": "Don't Ship Skills Without Evals — Philipp Schmid, Google DeepMind",
  "why_included": "Agent skills need regression tests, not manual spot checks. Test triggering and output with and without each skill, across repeated trials and the harnesses your team actually uses.",
  "summary": "A cited scan indexed more than **50,000 skills** and found almost none had evals. DeepMind keeps tests beside each internal skill, runs them on every change, and checks triggers, commands, traces, and outputs with scripts or an LLM judge.",
  "practical_implication": "Start with positive and negative prompts, then add production traces. Run **2–6 trials per case**, test across the models and harnesses you support, and compare results with the skill enabled and removed; deterministic workflows may belong in scripts instead.",
  "agent_context": "A cited scan indexed more than **50,000 skills** and found almost none had evals. DeepMind keeps tests beside each internal skill, runs them on every change, and checks triggers, commands, traces, and outputs with scripts or an LLM judge.\n\nStart with positive and negative prompts, then add production traces. Run **2–6 trials per case**, test across the models and harnesses you support, and compare results with the skill enabled and removed; deterministic workflows may belong in scripts instead.\n\nAgent runs are nondeterministic, so a single pass proves little. Regex checks are cheap but narrow, while judge models add cost and judgment variance; retained evals remain useful after retiring a skill because they can expose later model regressions.",
  "source": {
    "name": "AI Engineer",
    "url": "https://www.youtube.com/watch?v=0vphxNt4wyk",
    "published_at": "2026-07-14T22:00:06.000Z"
  },
  "source_class": "video",
  "content_type": "Video",
  "layer": "benchmark",
  "domains": [
    "coding"
  ],
  "topics": [
    "skills",
    "agent-evals",
    "agent-reliability"
  ],
  "verification": {
    "status": "source_linked",
    "label": "Source Linked",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "Agent runs are nondeterministic, so a single pass proves little. Regex checks are cheap but narrow, while judge models add cost and judgment variance; retained evals remain useful after retiring a skill because they can expose later model regressions."
  ],
  "lifecycle": "Current",
  "published_at": "2026-07-14T22:00:06.000Z",
  "modified_at": "2026-07-14T22:00:06.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/don-t-ship-skills-without-evals-philipp-schmid-google-deepmind-0fuh3ko",
    "json": "https://feed7.dev/p/don-t-ship-skills-without-evals-philipp-schmid-google-deepmind-0fuh3ko.json",
    "markdown": "https://feed7.dev/p/don-t-ship-skills-without-evals-philipp-schmid-google-deepmind-0fuh3ko.md"
  }
}