{
  "schema_version": "1.1",
  "id": "archive:https://www.youtube.com/watch?v=7vn4WpqNpck",
  "slug": "benchmarking-coding-agents-on-new-vs-legacy-codebases-denys-linkov-wised-0vhw2s2",
  "url": "https://feed7.dev/p/benchmarking-coding-agents-on-new-vs-legacy-codebases-denys-linkov-wised-0vhw2s2",
  "title": "Benchmarking Coding Agents on New vs Legacy Codebases — Denys Linkov, Wisedocs",
  "why_included": "A production refactor shows why coding-agent evaluations need acceptance criteria and end-to-end verification: fast output can still be incomplete scaffolding.",
  "summary": "An early O3-assisted task took **3 hours** and produced 10 major mistakes; newer Sonnet 4.6 solved it after one extra iteration and Opus 4.8 nearly one-shot it. A broader GPT-5.5 attempt finished in **10m 22s** but mostly wrote scaffolding.",
  "practical_implication": "Benchmark agents against explicit requirements, runnable end-to-end tests, deployment constraints, and hidden assumptions—not elapsed time or lines changed. Wisedocs also found a monorepo simpler for verification and sandbox setup across its former **10+ repositories**.",
  "agent_context": "An early O3-assisted task took **3 hours** and produced 10 major mistakes; newer Sonnet 4.6 solved it after one extra iteration and Opus 4.8 nearly one-shot it. A broader GPT-5.5 attempt finished in **10m 22s** but mostly wrote scaffolding.\n\nBenchmark agents against explicit requirements, runnable end-to-end tests, deployment constraints, and hidden assumptions—not elapsed time or lines changed. Wisedocs also found a monorepo simpler for verification and sandbox setup across its former **10+ repositories**.\n\nThese are task-specific observations from one refactor, not controlled cross-model results. Human review remained part of the project, and only **15 of 17 requirements** were met during the migration.",
  "source": {
    "name": "AI Engineer",
    "url": "https://www.youtube.com/watch?v=7vn4WpqNpck",
    "published_at": "2026-08-08T19:00:06.000Z"
  },
  "source_class": "video",
  "content_type": "Video",
  "layer": "benchmark",
  "domains": [
    "coding"
  ],
  "topics": [
    "agent-evals",
    "agent-reliability",
    "coding-agents"
  ],
  "verification": {
    "status": "source_linked",
    "label": "Source Linked",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "These are task-specific observations from one refactor, not controlled cross-model results. Human review remained part of the project, and only **15 of 17 requirements** were met during the migration."
  ],
  "connected_context": {
    "meaning": "This cautions that faster completion and plausible scaffolding are weak evidence of coding-agent success, especially in legacy migrations. It reinforces candidate benchmarks that prioritize behavioral correctness and reproducible verification, while adding deployment constraints and hidden assumptions to the acceptance surface. Its monorepo observation is useful for harness setup but remains task-specific.",
    "corpus_size": 400,
    "generated_at": "2026-08-09T18:05:11.619Z",
    "connections": [
      {
        "title": "ScarfBench: Benchmarking AI Agents for Enterprise Java Framework Migration",
        "source_name": "huggingface.co",
        "source_url": "https://huggingface.co/blog/ibm-research/scarfbench",
        "feed7_url": "https://feed7.dev/p/scarfbench-1u8lniy",
        "reason": "ScarfBench strongly reinforces the gap between apparent progress and behavioral success in legacy migrations, including agents claiming success on code that does not compile."
      },
      {
        "title": "Everything Is a Rollout — Alex Shaw + Ryan Marten, Terminal-Bench, Harbor, Laude Institute",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=jRCpXUjz4CI",
        "feed7_url": "https://feed7.dev/p/everything-is-a-rollout-alex-shaw-ryan-marten-terminal-bench-harbor-laud-0iz4rgx",
        "reason": "Harbor supplies the reproducible sandbox, outcome verification, and trajectory inspection loop needed to turn this talk’s requirement-based evaluation advice into repeatable practice."
      },
      {
        "title": "ReviewDebt: a practical framework for scoring every pull request — Sachin Gupta, Ebay",
        "source_name": "YouTube",
        "source_url": "https://www.youtube.com/watch?v=TJPInBjhE4Q",
        "feed7_url": "https://feed7.dev/p/reviewdebt-a-practical-framework-for-scoring-every-pull-request-sachin-g-0iyjtyk",
        "reason": "ReviewDebt adds a complementary operational measure: even requirement-passing output can become harmful if its verification burden grows faster than human review capacity."
      },
      {
        "title": "DeepSWE: A Contamination-Resistant Coding Benchmark — James Shi, Datacurve",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=Yk87oUPVaxU",
        "feed7_url": "https://feed7.dev/p/deepswe-a-contamination-resistant-coding-benchmark-james-shi-datacurve-08p61c0",
        "reason": "DeepSWE reinforces evaluation on original long-horizon repository tasks and behavioral traces, while this signal adds legacy-code, deployment, and hidden-assumption concerns that its task mix may not fully cover."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-08-08T19:00:06.000Z",
  "modified_at": "2026-08-08T19:00:06.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/benchmarking-coding-agents-on-new-vs-legacy-codebases-denys-linkov-wised-0vhw2s2",
    "json": "https://feed7.dev/p/benchmarking-coding-agents-on-new-vs-legacy-codebases-denys-linkov-wised-0vhw2s2.json",
    "markdown": "https://feed7.dev/p/benchmarking-coding-agents-on-new-vs-legacy-codebases-denys-linkov-wised-0vhw2s2.md"
  }
}