{
  "schema_version": "1.0",
  "id": "s8:https://www.youtube.com/watch?v=O3FEoMYvUf8",
  "slug": "stop-evaluating-models-like-it-s-the-50s-alejandro-vidal-mindmakers-0spx928",
  "url": "https://feed7.dev/p/stop-evaluating-models-like-it-s-the-50s-alejandro-vidal-mindmakers-0spx928",
  "title": "Stop Evaluating Models Like It's the 50s - Alejandro Vidal, Mindmakers",
  "why_included": "Item response theory can reveal weak eval questions, quantify uncertainty, and select smaller suites that preserve model rankings, making internal agent evals cheaper and more diagnostic.",
  "summary": "Raw accuracy weights every question equally. Item response theory estimates item difficulty and discrimination, produces uncertainty intervals, and can expose mislabeled, uninformative, or negatively correlated questions.",
  "practical_implication": "Calibrate internal agent evals before paying to run every case. In one example, **97 of 484 items** preserved a **99% correlation** with the full ranking when selected by discrimination, offering a concrete path to lower token and runtime costs.",
  "agent_context": "Raw accuracy weights every question equally. Item response theory estimates item difficulty and discrimination, produces uncertainty intervals, and can expose mislabeled, uninformative, or negatively correlated questions.\n\nCalibrate internal agent evals before paying to run every case. In one example, **97 of 484 items** preserved a **99% correlation** with the full ranking when selected by discrimination, offering a concrete path to lower token and runtime costs.\n\nThe reduction does not hold for every benchmark, and ranking retention is not the same as preserving every capability signal. Adaptive tests and fingerprint sets may help detect leakage, but the speaker explicitly describes that method as not bulletproof.",
  "source": {
    "name": "AI Engineer",
    "url": "https://www.youtube.com/watch?v=O3FEoMYvUf8",
    "published_at": "2026-07-13T17:56:55.000Z"
  },
  "source_class": "video",
  "content_type": "Video",
  "layer": "benchmark",
  "domains": [
    "data"
  ],
  "topics": [
    "agent-evals",
    "benchmark-integrity",
    "model-selection"
  ],
  "verification": {
    "status": "source_linked",
    "label": "Source Linked",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "The reduction does not hold for every benchmark, and ranking retention is not the same as preserving every capability signal. Adaptive tests and fingerprint sets may help detect leakage, but the speaker explicitly describes that method as not bulletproof."
  ],
  "lifecycle": "Current",
  "published_at": "2026-07-13T17:56:55.000Z",
  "modified_at": "2026-07-13T17:56:55.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/stop-evaluating-models-like-it-s-the-50s-alejandro-vidal-mindmakers-0spx928",
    "json": "https://feed7.dev/p/stop-evaluating-models-like-it-s-the-50s-alejandro-vidal-mindmakers-0spx928.json",
    "markdown": "https://feed7.dev/p/stop-evaluating-models-like-it-s-the-50s-alejandro-vidal-mindmakers-0spx928.md"
  }
}