{ "schema_version": "1.1", "id": "atlas-benchmark-integrity", "slug": "benchmark-integrity", "title": "Benchmark Integrity", "url": "https://feed7.dev/atlas/benchmark-integrity", "current_answer": null, "implementation_consequence": null, "agent_context": null, "confidence": "auto_collected", "last_verified": null, "last_updated": null, "evidence": [ { "schema_version": "1.1", "id": "auto-7bc753b969", "slug": "change2task-from-repository-changes-to-executable-coding-7bc753b969", "url": "https://feed7.dev/p/change2task-from-repository-changes-to-executable-coding-7bc753b969", "title": "Change2Task: From Repository Changes to Executable Coding Agent Tasks and Environments", "why_included": "Turn merged PR history into renewable eval tasks while preserving healthy, reconstructed, and restored repository states.", "summary": "Change2Task converts merged PR history into executable coding-agent tasks on current repository states. It could expand continuous eval sets while reducing stale environments and repeated setup.", "practical_implication": "Use repository history as renewable eval data, but preserve the full lifecycle: healthy base, reconstructed task state, and restored state. The five tested families include bug fixes, features, tests, API migrations, and security repairs.", "agent_context": "Change2Task turns merged pull requests into verified tasks on modern repository revisions using **Patch Reversal, Code Mapping, or Agent Reconstruction**. Across **1,130 eligible changes**, it constructed verified tasks at a 79.6% rate.\n\nUse repository history as renewable eval data, but preserve the full lifecycle: healthy base, reconstructed task state, and restored state. The five tested families include bug fixes, features, tests, API migrations, and security repairs.\n\nIt recovered **29.2% more verified tasks** than a PR-based baseline and reached up to 98.0% matched outcome agreement. These are construction results from the evaluated repositories, not proof that generated tasks represent every real maintenance workload.", "source": { "name": "arXiv", "url": "https://arxiv.org/abs/2607.28591v1", "published_at": "2026-07-30T00:00:00.000Z" }, "source_class": "blog_post", "content_type": "Paper", "layer": "benchmark", "domains": [ "coding", "security" ], "topics": [ "coding-agents", "agent-evals", "benchmark-integrity" ], "verification": { "status": "needs_review", "label": "Needs Review", "method": "unverified", "verified_at": null }, "uncertainty": [ "Automatically selected from source material; feed7 has not independently tested the claim." ], "connected_context": null, "lifecycle": "New", "published_at": "2026-07-30T00:00:00.000Z", "modified_at": "2026-07-30T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/change2task-from-repository-changes-to-executable-coding-7bc753b969", "json": "https://feed7.dev/p/change2task-from-repository-changes-to-executable-coding-7bc753b969.json", "markdown": "https://feed7.dev/p/change2task-from-repository-changes-to-executable-coding-7bc753b969.md" } }, { "schema_version": "1.1", "id": "auto-cc8e132c61", "slug": "how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61", "url": "https://feed7.dev/p/how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61", "title": "How enabling two settings tripled our scores on the ARC-AGI-3 benchmark", "why_included": "Record reasoning retention and compaction with the model name because runtime settings can materially alter agent eval results.", "summary": "Two API settings—reasoning retention and compaction—reportedly tripled GPT-5.6’s ARC-AGI-3 score. Agent evals should treat runtime configuration as part of the tested system.", "practical_implication": "Record these settings alongside the model name in agent evaluations. Configuration can materially affect results, so defaults and explicit settings should not be compared as equivalent systems.", "agent_context": "OpenAI says enabling **reasoning retention** and **compaction** produced **3× ARC-AGI-3 scores** for GPT-5.6 while also improving efficiency.\n\nRecord these settings alongside the model name in agent evaluations. Configuration can materially affect results, so defaults and explicit settings should not be compared as equivalent systems.\n\nThe supplied material provides no absolute scores, token usage, latency, or experimental detail, leaving the size and generality of the efficiency gain unclear.", "source": { "name": "OpenAI", "url": "https://openai.com/index/how-two-settings-tripled-our-arc-agi-3-scores", "published_at": "2026-07-29T00:00:00.000Z" }, "source_class": "blog_post", "content_type": "Official Release", "layer": "benchmark", "domains": [], "topics": [ "agent-evals", "benchmark-integrity", "context-caching" ], "verification": { "status": "official_source", "label": "Official Source", "method": "source_feed", "verified_at": null }, "uncertainty": [], "connected_context": null, "lifecycle": "New", "published_at": "2026-07-29T00:00:00.000Z", "modified_at": "2026-07-29T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61", "json": "https://feed7.dev/p/how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61.json", "markdown": "https://feed7.dev/p/how-enabling-two-settings-tripled-our-scores-on-the-arc--cc8e132c61.md" } }, { "schema_version": "1.1", "id": "auto-31ceb9467c", "slug": "quotebench-how-matched-scores-can-hide-command-path-fail-31ceb9467c", "url": "https://feed7.dev/p/quotebench-how-matched-scores-can-hide-command-path-fail-31ceb9467c", "title": "QuoteBench: How Matched Scores Can Hide Command-Path Failures", "why_included": "Test commands through the exact production transport and verify final state, since one parser cut task completion by 55.4–73.2 points.", "summary": "QuoteBench shows that shell-command scores can conceal failures introduced by serialization and reparsing. Agent evals should identify the execution path, not attribute every result to the model.", "practical_implication": "When the boundary was disclosed, six configurations recovered 30.4–60.7 points. Builders should test generated commands through the exact production transport and validate final state, especially where wrappers interpolate or reparse shell text.", "agent_context": "QuoteBench tests **56 one-shot tasks** from 14 incident-derived families across eight configurations. Adding one unescaped parser cut replayed-command success by **55.4–73.2 percentage points**.\n\nWhen the boundary was disclosed, six configurations recovered **30.4–60.7 points**. Builders should test generated commands through the exact production transport and validate final state, especially where wrappers interpolate or reparse shell text.\n\nAdaptation was uneven: two configurations recovered nothing or declined slightly. GPT-5.6-sol’s **−3.6-point matched gap** concealed −64.3 points of transport damage plus 60.7 points of compensation, so aggregate scores can misstate both model and harness quality.", "source": { "name": "arXiv", "url": "https://arxiv.org/abs/2608.13547v1", "published_at": "2026-08-13T00:00:00.000Z" }, "source_class": "blog_post", "content_type": "Paper", "layer": "benchmark", "domains": [ "coding", "security" ], "topics": [ "agent-evals", "benchmark-integrity", "harness-engineering" ], "verification": { "status": "needs_review", "label": "Needs Review", "method": "unverified", "verified_at": null }, "uncertainty": [ "Automatically selected from source material; feed7 has not independently tested the claim." ], "connected_context": null, "lifecycle": "New", "published_at": "2026-08-13T00:00:00.000Z", "modified_at": "2026-08-13T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/quotebench-how-matched-scores-can-hide-command-path-fail-31ceb9467c", "json": "https://feed7.dev/p/quotebench-how-matched-scores-can-hide-command-path-fail-31ceb9467c.json", "markdown": "https://feed7.dev/p/quotebench-how-matched-scores-can-hide-command-path-fail-31ceb9467c.md" } }, { "schema_version": "1.1", "id": "auto-a24f8f74a4", "slug": "computer-use-at-the-edge-of-the-statistical-precipice-pi-a24f8f74a4", "url": "https://feed7.dev/p/computer-use-at-the-edge-of-the-statistical-precipice-pi-a24f8f74a4", "title": "Computer Use at the Edge of the Statistical Precipice — Pierluca D'Oro, Programma Labs", "why_included": "Vary task data, appearance, and initial state so agent evals measure adaptation instead of rewarding replayed action scripts.", "summary": "Static computer-use benchmarks can reward memorized action scripts rather than adaptation. Vary task state, verify every generated case, and calculate uncertainty across both actions and environments.", "practical_implication": "For agent evals, vary data, appearance, and initial state; automatically reject invalid combinations; and use privileged verifiers inside a sandbox. DGWorld applies this design across 15 apps, 387 scenarios, and 3.2 million verified configurations.", "agent_context": "A replay agent stores one winning trajectory per task and blindly repeats it. On deterministic OSWorld and MobileWorld-style evaluations, a script **under 1 MB** can match or beat the frontier model that generated its traces, exposing benchmark replayability.\n\nFor agent evals, vary data, appearance, and initial state; automatically reject invalid combinations; and use privileged verifiers inside a sandbox. DGWorld applies this design across **15 apps, 387 scenarios, and 3.2 million verified configurations**.\n\nUncertainty must cover both model actions and environment variation. The talk reports that rollout-only intervals can provide roughly **17–20% coverage** where a correctly structured method approaches the intended 95%, but applying that method requires a benchmark with explicit variation structure.", "source": { "name": "AI Engineer", "url": "https://www.youtube.com/watch?v=CTLa_p6iOiY", "published_at": "2026-08-14T00:00:00.000Z" }, "source_class": "video", "content_type": "Video", "layer": "benchmark", "domains": [], "topics": [ "agent-evals", "benchmark-integrity", "agent-reliability" ], "verification": { "status": "source_linked", "label": "Source Linked", "method": "source_feed", "verified_at": null }, "uncertainty": [], "connected_context": null, "lifecycle": "New", "published_at": "2026-08-14T00:00:00.000Z", "modified_at": "2026-08-14T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/computer-use-at-the-edge-of-the-statistical-precipice-pi-a24f8f74a4", "json": "https://feed7.dev/p/computer-use-at-the-edge-of-the-statistical-precipice-pi-a24f8f74a4.json", "markdown": "https://feed7.dev/p/computer-use-at-the-edge-of-the-statistical-precipice-pi-a24f8f74a4.md" } }, { "schema_version": "1.1", "id": "auto-b3f2ae3bdf", "slug": "phantom-gains-auditing-self-improvement-against-a-measur-b3f2ae3bdf", "url": "https://feed7.dev/p/phantom-gains-auditing-self-improvement-against-a-measur-b3f2ae3bdf", "title": "Phantom Gains: Auditing Self-Improvement Against a Measured Null", "why_included": "Add frozen-baseline replicates and a measured null before treating one-decode gains or regressions as real.", "summary": "Per-problem self-improvement claims can arise from inference and evaluation noise. This audit argues every transition statistic needs a measured null from frozen baseline replicates.", "practical_implication": "For agent and model evaluations, do not treat problem-level gains and losses as ground truth from one decode. Measure each statistic’s null with baseline replicates, then use per-problem tests and false-discovery-rate control.", "agent_context": "The study ran three rounds of rank-32 LoRA self-training on Qwen3-8B alongside a frozen control using the same pipeline. It found **seven measurement failures** capable of reversing findings when the control was omitted.\n\nFor agent and model evaluations, do not treat problem-level gains and losses as ground truth from one decode. Measure each statistic’s null with baseline replicates, then use per-problem tests and false-discovery-rate control.\n\nA single greedy decode gave an untrained model an apparent expansion rate of **0.280**, while the replacement exact test detected **nothing on held-out replicates**. Results for problems the base model never solved remained inconclusive, and the method needs more baseline replicates than many studies possess.", "source": { "name": "arXiv", "url": "https://arxiv.org/abs/2608.20290v1", "published_at": "2026-08-20T00:00:00.000Z" }, "source_class": "blog_post", "content_type": "Paper", "layer": "benchmark", "domains": [ "research" ], "topics": [ "benchmark-integrity", "agent-evals", "agent-reliability" ], "verification": { "status": "needs_review", "label": "Needs Review", "method": "unverified", "verified_at": null }, "uncertainty": [ "Automatically selected from source material; feed7 has not independently tested the claim." ], "connected_context": null, "lifecycle": "New", "published_at": "2026-08-20T00:00:00.000Z", "modified_at": "2026-08-20T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/phantom-gains-auditing-self-improvement-against-a-measur-b3f2ae3bdf", "json": "https://feed7.dev/p/phantom-gains-auditing-self-improvement-against-a-measur-b3f2ae3bdf.json", "markdown": "https://feed7.dev/p/phantom-gains-auditing-self-improvement-against-a-measur-b3f2ae3bdf.md" } }, { "schema_version": "1.1", "id": "auto-7ebecc7b84", "slug": "swe-prime-fewer-trajectories-better-performance-7ebecc7b84", "url": "https://feed7.dev/p/swe-prime-fewer-trajectories-better-performance-7ebecc7b84", "title": "SWE-Prime: Fewer Trajectories, Better Performance", "why_included": "SWE-Prime's filtered 10% of coding traces beat the full resolved set, showing that passing outcomes are not automatically clean supervision.", "summary": "SWE-Prime finds that filtering coding-agent traces by process and segment quality can beat training on every resolved trajectory, reducing noisy imitation from redundant or risky steps.", "practical_implication": "The reported 10% trajectory subset beat training on the full resolved dataset, with relative gains up to 12.2% on SWE-Bench Pro and 24.2% on SWE-Bench Verified. Teams training coding models should evaluate how an issue was solved, not treat a passing outcome as clean supervision.", "agent_context": "SWE-Prime filters successful coding-agent traces in **two stages**: whole trajectories are screened for process quality, result quality, and representativeness, then semantic segments are judged for contribution, learnability, and risk. Only selected segments contribute to the training loss.\n\nThe reported **10% trajectory subset** beat training on the full resolved dataset, with relative gains up to **12.2% on SWE-Bench Pro** and **24.2% on SWE-Bench Verified**. Teams training coding models should evaluate how an issue was solved, not treat a passing outcome as clean supervision.\n\nAll segments remain in the input sequence to preserve context, so this is selective loss computation rather than simply deleting weak steps. The material reports benchmark results but does not establish whether the selection criteria transfer to other repositories, agents, or training setups.", "source": { "name": "arXiv", "url": "https://arxiv.org/abs/2608.27449v1", "published_at": "2026-08-27T00:00:00.000Z" }, "source_class": "blog_post", "content_type": "Paper", "layer": "benchmark", "domains": [ "coding", "data" ], "topics": [ "coding-agents", "agent-evals", "benchmark-integrity" ], "verification": { "status": "needs_review", "label": "Needs Review", "method": "unverified", "verified_at": null }, "uncertainty": [ "Automatically selected from source material; feed7 has not independently tested the claim." ], "connected_context": null, "lifecycle": "New", "published_at": "2026-08-27T00:00:00.000Z", "modified_at": "2026-08-27T00:00:00.000Z", "supersedes": [], "expires_at": null, "formats": { "html": "https://feed7.dev/p/swe-prime-fewer-trajectories-better-performance-7ebecc7b84", "json": "https://feed7.dev/p/swe-prime-fewer-trajectories-better-performance-7ebecc7b84.json", "markdown": "https://feed7.dev/p/swe-prime-fewer-trajectories-better-performance-7ebecc7b84.md" } } ], "conflicting_sources": [], "superseded_claims": [], "corpus_evidence": [ { "title": "User Feedback Provides a Unique Signal that LLMs Can not Detect", "url": "https://arxiv.org/abs/2609.02859v1", "source_name": "arXiv", "published_at": "2026-09-02T17:42:44+00:00", "summary": "User feedback helps models repair targeted faults, but LLM judges often miss those repairs. Agent evals should retain human feedback and avoid treating model preference as ground truth." }, { "title": "Beyond Scores: Understanding LLM-as-a-Judge Mechanisms in Summarization Evaluation", "url": "https://arxiv.org/abs/2609.01604v1", "source_name": "arXiv", "published_at": "2026-09-01T17:59:49+00:00", "summary": "Mechanistic tests suggest two open LLM judges route local errors early and form ratings late, giving eval builders concrete layers and components to probe for judge failures." }, { "title": "BLOOM-WILT: Logit Tilting for Behaviour Elicitation in Automated LLM Auditing", "url": "https://arxiv.org/abs/2608.31105v1", "source_name": "arXiv", "published_at": "2026-08-31T17:10:59+00:00", "summary": "BLOOM-WILT makes rare model behaviors easier to surface by adapting both the auditor’s strategy and target decoding. Its results also show that elicitation method can reverse safety rankings." }, { "title": "SOTA Generative Media Panel — Dumitru Erhan, Shane Gu & Nicole Brichtova, Google DeepMind", "url": "https://www.youtube.com/watch?v=KLDdXOw6jIc", "source_name": "AI Engineer", "published_at": "2026-08-30T14:00:06+00:00", "summary": "DeepMind’s panel shows why generative-media evals need task-specific human review: broad preferences can miss repeated artifacts, exact sizing, text errors, and brand consistency." }, { "title": "Can LLMs Write Fast Multi-GPU Kernels? — Simran Arora, Together AI", "url": "https://www.youtube.com/watch?v=pOvWgX7IJsc", "source_name": "AI Engineer", "published_at": "2026-08-27T17:00:39+00:00", "summary": "A multi-GPU kernel benchmark shows coding agents can improve with tools and iteration, but still struggle with communication scheduling, partitioning, and transfer choices." }, { "title": "Trace Integrity for LLM Data Agents: A Vision for Auditable Structured Reasoning in Real-World Systems", "url": "https://arxiv.org/abs/2608.26036v1", "source_name": "arXiv", "published_at": "2026-08-26T17:15:24+00:00", "summary": "Correct answers can hide invalid computation. Trace Integrity proposes executable, replayable contracts for data agents and reports 45.8–59.1% correct-answer/invalid-trace rates across three SQL modes." }, { "title": "Einstein Arena: Harnessing Collective Agent Intelligence for Open Science — James Zou, Together AI", "url": "https://www.youtube.com/watch?v=mMNkdYnIVC4", "source_name": "AI Engineer", "published_at": "2026-08-25T20:46:01+00:00", "summary": "Einstein Arena suggests multi-agent environments can outperform fixed workflows when they expose verifiers, shared solutions, forums, and incentives. Its results also show why benchmark shortcuts need active testing." }, { "title": "ConceptGuard: Benchmarking Context-Sensitive Unlearning in Large Language Models", "url": "https://arxiv.org/abs/2608.20338v1", "source_name": "arXiv", "published_at": "2026-08-20T17:59:57+00:00", "summary": "ConceptGuard tests whether model unlearning blocks harmful uses of a concept while preserving benign ones. Current methods show weak contextual control and sharp forgetting-versus-utility trade-offs." }, { "title": "TokEval: A Tokenizer Evaluation Suite", "url": "https://arxiv.org/abs/2608.18062v1", "source_name": "arXiv", "published_at": "2026-08-18T17:52:52+00:00", "summary": "TokEval links tokenizer properties to language, math, and code performance, offering cheaper screening signals before committing compute to pretraining sweeps." }, { "title": "Towards Computational Provenance: Carrying Causal-State Evidence in Generated Text", "url": "https://arxiv.org/abs/2608.16868v1", "source_name": "arXiv", "published_at": "2026-08-17T17:50:04+00:00", "summary": "A controlled study encoded authenticated internal-state evidence into unchanged answers, suggesting generated text could carry provenance signals, but not that current models reveal them naturally." }, { "title": "What Do Compliance Detectors Read? An Audit of Activation Probes and Guard Models", "url": "https://arxiv.org/abs/2608.16852v1", "source_name": "arXiv", "published_at": "2026-08-17T17:37:07+00:00", "summary": "Tested compliance guards often ignored the governing rule and classified from scenario cues. Builders should counterfactually swap policies before trusting a detector as an audit control." }, { "title": "LittleLearner: Language Models Under Pedagogically Controlled Knowledge Exposure", "url": "https://arxiv.org/abs/2608.13545v1", "source_name": "arXiv", "published_at": "2026-08-13T17:56:12+00:00", "summary": "LittleLearner offers a controlled model and corpus for studying knowledge acquisition without unknown prior exposure. Its initial results separate better use of known material from new capability." }, { "title": "Measuring Task-Agnostic Training Data Influence Across Language Model Pretraining", "url": "https://arxiv.org/abs/2608.13515v1", "source_name": "arXiv", "published_at": "2026-08-13T17:36:49+00:00", "summary": "A task-agnostic influence measure tracks which examples steer pretraining toward final parameters without choosing a downstream eval. It reveals a literature-to-STEM shift across training." }, { "title": "Cursor earns AIUC-1 certification for agent security and reliability", "url": "https://cursor.com/blog/aiuc-1", "source_name": "Cursor", "published_at": "2026-08-13T12:00:00+00:00", "summary": "Cursor’s AIUC-1 certification combines a controls audit with adversarial testing of live agents. For enterprise evaluation, it adds behavioral evidence beyond conventional data-security attestations." }, { "title": "Fisher-R1: Training LLM Agents for Reliable Hypothesis Testing", "url": "https://arxiv.org/abs/2608.07437v1", "source_name": "arXiv", "published_at": "2026-08-07T17:22:00+00:00", "summary": "P-Bench tests whether agents choose statistically valid methods, not merely execute code. Fisher-R1-14B improved single-trial success over DeepSeek-V4-Pro by 21% on average." }, { "title": "SABRE: Scalable and Automated Benchmarking of VLMs under Stress", "url": "https://arxiv.org/abs/2608.07435v1", "source_name": "arXiv", "published_at": "2026-08-07T17:21:04+00:00", "summary": "SABRE turns a Markdown test design into generated VLM stress tests, then filters and repairs candidates. It offers a repeatable pattern for refreshing evals as models improve." }, { "title": "The Low Frequency Trap: Video Language Models Fail at Simple Event Bookkeeping", "url": "https://arxiv.org/abs/2608.06361v1", "source_name": "arXiv", "published_at": "2026-08-06T17:57:06+00:00", "summary": "Controlled traces show video models can improve final counting scores without faithfully recovering events, so agents handling video need timestamp-level checks, not answer-only evals." }, { "title": "SocietyBench: Forecasting Counterfactual Social-World Evolution", "url": "https://arxiv.org/abs/2608.04009v1", "source_name": "arXiv", "published_at": "2026-08-04T17:59:56+00:00", "summary": "SocietyBench tests forecasting in anonymized social timelines, exposing gaps that task-completion evals miss and showing agent frameworks did not improve the shared base model." }, { "title": "WorldCup Arena: Prospective, Leakage-Free Evaluation of Frontier LLMs on a Live Tournament", "url": "https://arxiv.org/abs/2608.04008v1", "source_name": "arXiv", "published_at": "2026-08-04T17:59:55+00:00", "summary": "A live, pre-kickoff benchmark removes answer leakage by construction and finds six frontier models clustered near a bookmaker-favorite baseline, with no gain from majority voting." }, { "title": "Test-Time Scaling in Reasoning LLMs: Inference Regimes, Evaluation, and Reproducibility", "url": "https://arxiv.org/abs/2608.04001v1", "source_name": "arXiv", "published_at": "2026-08-04T17:57:20+00:00", "summary": "This paper separates three test-time scaling regimes and argues that model evals must report the full inference protocol, compute accounting, uncertainty, and replay artifacts." }, { "title": "onepot-Bench 0: towards lab-aware in silico chemistry benchmarks", "url": "https://arxiv.org/abs/2608.02595v1", "source_name": "arXiv", "published_at": "2026-08-03T17:58:27+00:00", "summary": "onepot-Bench 0 evaluates chemistry models with three lab-oriented tests, including private experimental data to reduce contamination risk. It is a useful eval design pattern, though results and reproducibility are absent here." }, { "title": "When Will The Benchmaxxing Plague End? — Nick Heiner, Surge AI", "url": "https://www.youtube.com/watch?v=-npY6XjM8CQ", "source_name": "AI Engineer", "published_at": "2026-08-02T16:30:06+00:00", "summary": "Nick Heiner argues that leaderboard gains can diverge from useful agent behavior through contamination, weak verifiers, reward hacking, and test conditions that users cannot inspect." }, { "title": "Teaching AI to Find Real Vulnerabilities — David Brumley, Bugcrowd", "url": "https://www.youtube.com/watch?v=ZFxh7sqbUZo", "source_name": "AI Engineer", "published_at": "2026-08-01T00:30:06+00:00", "summary": "Cybersecurity agents need deterministic exploit oracles and tasks that reward finding distinct vulnerabilities, not self-reported wins or one crash. A capability ladder makes partial progress measurable." }, { "title": "Rethinking Environments for Long-Horizon Work — Rayan Garg, Theta Software", "url": "https://www.youtube.com/watch?v=2aS7aKoXn64", "source_name": "AI Engineer", "published_at": "2026-08-01T00:00:06+00:00", "summary": "Human task duration is a weak proxy for agent difficulty when agents use different shortcuts and tools. Long-horizon evals need environment complexity, queryable trajectories, and judges that inspect final state." }, { "title": "Verifiable Environments for AI in Biology — Kenny Workman, LatchBio", "url": "https://www.youtube.com/watch?v=3ZMUiFaQ3qg", "source_name": "AI Engineer", "published_at": "2026-07-31T20:00:35+00:00", "summary": "Biology agents need evaluators that verify analysis of large experimental datasets, not recall. LatchBio found human review essential because valid scientific paths can defeat brittle graders." }, { "title": "FriendBench: Benchmarking Dyadic Familiarity Inference in Humans and Multimodal Large Language Models", "url": "https://arxiv.org/abs/2607.29602v1", "source_name": "arXiv", "published_at": "2026-07-31T16:33:39+00:00", "summary": "FriendBench shows why aggregate accuracy can hide behavioral bias: top multimodal models matched human panels overall but favored the “stranger” answer and gained less from video." }, { "title": "Benchmarks: The Good, the Bad, and the Ugly — Ali Khial, G2i", "url": "https://www.youtube.com/watch?v=jWq-aZIU0kM", "source_name": "AI Engineer", "published_at": "2026-07-31T16:00:06+00:00", "summary": "Coding benchmark scores can mislead when prompts leak implementation details or tests reject valid solutions. Inspect tasks and verifiers before using a leaderboard to choose an agent." }, { "title": "PAIChecker: Uncovering and Checking PR-Issue Misalignment in SWE-Bench-Like Benchmarks", "url": "https://arxiv.org/abs/2607.28587v1", "source_name": "arXiv", "published_at": "2026-07-30T17:42:44+00:00", "summary": "PAIChecker finds PR–issue mismatch in 13.6% of SWE-bench Verified instances. Benchmark maintainers should validate that the issue, patch, and tests describe the same task before trusting agent scores." }, { "title": "Sample More, Reflect Less: Self-Refine and Reflexion Lose to Repeated Sampling at Equal Token Cost, from 1.5B to 7B", "url": "https://arxiv.org/abs/2607.28576v1", "source_name": "arXiv", "published_at": "2026-07-30T17:38:23+00:00", "summary": "At equal generated-token cost, repeated sampling matched or beat seven reflection, critique, selection, and debate methods. Agent evals should budget every generated token, not compare against one-shot baselines." }, { "title": "Persona Engineering: A Field Guide to AI Synthetic Personas — Ishan Anand, InsightSciences.ai", "url": "https://www.youtube.com/watch?v=YnNF55QV0zs", "source_name": "AI Engineer", "published_at": "2026-07-29T20:15:35+00:00", "summary": "Synthetic personas can extend existing research, but they are forecasts, not extra respondents. Ground prompts richly and validate each setup against human data before using it." }, { "title": "Beyond Scale and Generation: Understanding Language Model-based Entity Matching", "url": "https://arxiv.org/abs/2607.24688v1", "source_name": "arXiv", "published_at": "2026-07-27T17:29:18+00:00", "summary": "A 1,215-run study finds entity-matching architecture and model variant matter more than scale alone; generative matchers mainly help under distribution shift." }, { "title": "Eviction as Estimation: A Fixed-Lag Smoothing View of Test-Time Memory, and When Measuring Beats Accumulating", "url": "https://arxiv.org/abs/2607.24667v1", "source_name": "arXiv", "published_at": "2026-07-27T17:08:27+00:00", "summary": "A delayed KV-eviction policy helps when memory reuse is sharp and observable, but loses its edge on natural-text benchmarks—a useful warning for agent-memory claims." }, { "title": "DeepSWE: A Contamination-Resistant Coding Benchmark — James Shi, Datacurve", "url": "https://www.youtube.com/watch?v=Yk87oUPVaxU", "source_name": "AI Engineer", "published_at": "2026-07-26T18:10:56+00:00", "summary": "DeepSWE uses original long-horizon tasks to reduce contamination and expose coding-agent behaviors hidden by saturated PR-mined suites. Its current task mix still underrepresents some everyday work." }, { "title": "State of Data — Sean Cai, Independent / State of Data", "url": "https://www.youtube.com/watch?v=ZyIoTOAbRfs", "source_name": "AI Engineer", "published_at": "2026-07-26T17:00:06+00:00", "summary": "Real workflow traces may teach agents more than manufactured tasks, while benchmark scores can shift with the harness. Build pipelines around live work and test across scaffolds." } ] }