{
  "schema_version": "1.0",
  "id": "s8:https://www.youtube.com/watch?v=Ib5t2RLtxvM",
  "slug": "from-agent-traces-to-agent-simulations-rustem-feyzkhanov-snorkel-ai-0zwlzjq",
  "url": "https://feed7.dev/p/from-agent-traces-to-agent-simulations-rustem-feyzkhanov-snorkel-ai-0zwlzjq",
  "title": "From Agent Traces to Agent Simulations — Rustem Feyzkhanov, Snorkel AI",
  "why_included": "Reconstructing production traces as replayable environments lets teams compare agent stacks under fixed conditions and gate releases on task outcomes, cost, latency, and retries.",
  "summary": "The proposed benchmark turns **production traces** into **offline simulations** by rebuilding the database state, tools, APIs, files, and policies an agent encountered. Verifiers inspect the final state, trace, and artifacts, while an Oracle confirms each task is solvable.",
  "practical_implication": "Treat this benchmark like integration-test software: pin dependencies, run it in CI, and use it as a **release gate**. Compare whole agent configurations—not only models—and place fixes in the harness, skills, prompts, or output schema according to the failure.",
  "agent_context": "The proposed benchmark turns **production traces** into **offline simulations** by rebuilding the database state, tools, APIs, files, and policies an agent encountered. Verifiers inspect the final state, trace, and artifacts, while an Oracle confirms each task is solvable.\n\nTreat this benchmark like integration-test software: pin dependencies, run it in CI, and use it as a **release gate**. Compare whole agent configurations—not only models—and place fixes in the harness, skills, prompts, or output schema according to the failure.\n\nFaithful reconstruction is substantial engineering work, and simulated users or mocked services may diverge from production. Keep a held-out set, cover routine and edge cases, and route verifier disagreements to subject-matter experts.",
  "source": {
    "name": "AI Engineer",
    "url": "https://www.youtube.com/watch?v=Ib5t2RLtxvM",
    "published_at": "2026-07-25T01:00:02.000Z"
  },
  "source_class": "video",
  "content_type": "Video",
  "layer": "benchmark",
  "domains": [
    "coding"
  ],
  "topics": [
    "agent-evals",
    "agent-reliability",
    "harness-engineering"
  ],
  "verification": {
    "status": "source_linked",
    "label": "Source Linked",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "Faithful reconstruction is substantial engineering work, and simulated users or mocked services may diverge from production. Keep a held-out set, cover routine and edge cases, and route verifier disagreements to subject-matter experts."
  ],
  "lifecycle": "Current",
  "published_at": "2026-07-25T01:00:02.000Z",
  "modified_at": "2026-07-25T01:00:02.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/from-agent-traces-to-agent-simulations-rustem-feyzkhanov-snorkel-ai-0zwlzjq",
    "json": "https://feed7.dev/p/from-agent-traces-to-agent-simulations-rustem-feyzkhanov-snorkel-ai-0zwlzjq.json",
    "markdown": "https://feed7.dev/p/from-agent-traces-to-agent-simulations-rustem-feyzkhanov-snorkel-ai-0zwlzjq.md"
  }
}