{
  "schema_version": "1.1",
  "id": "s14:https://engineering.fb.com/2026/10/29/security/catching-bugs-before-they-land-jittestings-human-on-the-loop-approach-at-full-scale/",
  "slug": "catching-bugs-before-they-land-jittestings-human-on-the-loop-approach-at-0rua5z6",
  "url": "https://feed7.dev/p/catching-bugs-before-they-land-jittestings-human-on-the-loop-approach-at-0rua5z6",
  "title": "Catching Bugs Before They Land: JiTTesting’s Human-on-the-Loop Approach at Full Scale",
  "why_included": "Meta generates tests that expose behavioral changes in risky diffs, then combines deterministic rules, an LLM ensemble, and aggregate human review to filter likely false positives before code lands.",
  "summary": "Meta’s JiTTests target risky diffs and generate executable tests through two workflows: one infers change intent and risks; the other probes behavioral differences. A rules engine and LLM ensemble rank failures before engineers review selected results. In H1 2026, the system caught **74 privacy bugs** before production; **60%** of reported findings were true positives.",
  "practical_implication": "For coding agents, consider adding tests that demonstrate a diff’s behavioral change to the review pipeline. Meta’s setup also suggests routing only ranked findings to engineers, with people setting criteria and reviewing outcomes in aggregate.",
  "agent_context": "Meta’s JiTTests target risky diffs and generate executable tests through two workflows: one infers change intent and risks; the other probes behavioral differences. A rules engine and LLM ensemble rank failures before engineers review selected results. In H1 2026, the system caught **74 privacy bugs** before production; **60%** of reported findings were true positives.\n\nFor coding agents, consider adding tests that demonstrate a diff’s behavioral change to the review pipeline. Meta’s setup also suggests routing only ranked findings to engineers, with people setting criteria and reviewing outcomes in aggregate.\n\nThe system runs overnight on spare capacity and targets risky changes, so the material does not establish immediate coverage for every diff. The reported results come from Meta’s infrastructure and may depend on its codebase and review process.",
  "source": {
    "name": "Meta AI",
    "url": "https://engineering.fb.com/2026/10/29/security/catching-bugs-before-they-land-jittestings-human-on-the-loop-approach-at-full-scale/",
    "published_at": "2026-10-29T16:00:12.000Z"
  },
  "source_class": "blog_post",
  "content_type": "Engineering Post",
  "layer": "agent",
  "domains": [
    "coding",
    "security"
  ],
  "topics": [
    "harness-engineering",
    "agent-reliability"
  ],
  "verification": {
    "status": "official_source",
    "label": "Official Source",
    "method": "source_feed",
    "verified_at": null
  },
  "uncertainty": [
    "The system runs overnight on spare capacity and targets risky changes, so the material does not establish immediate coverage for every diff. The reported results come from Meta’s infrastructure and may depend on its codebase and review process."
  ],
  "connected_context": {
    "meaning": "JiTTesting provides a large-scale example of risk-ranked, human-reviewed tests that probe a diff's intended or observed behavior, with reported privacy-bug findings and a stated true-positive rate. It gives concrete support for integrating behavioral evidence into agent code review, while its overnight, risk-targeted setup leaves immediate universal coverage unproven.",
    "corpus_size": 966,
    "generated_at": "2026-10-30T11:05:04.018Z",
    "connections": [
      {
        "title": "Guide, Verify, Solve — Anirban Chatterjee, Sonar",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=03l29gJXpCE",
        "feed7_url": "https://feed7.dev/p/guide-verify-solve-anirban-chatterjee-sonar-1igfmbm",
        "reason": "Both argue that verification must be part of the coding loop; JiTTesting adds ranked, executable tests aimed at risky diffs, while Sonar emphasizes repeatable checks and warns that throughput can coexist with persistent quality issues."
      },
      {
        "title": "How to Kill the Code Review — Ankit Jain, Aviator",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=YgEv7IQzGdM",
        "feed7_url": "https://feed7.dev/p/how-to-kill-the-code-review-ankit-jain-aviator-0rku6kj",
        "reason": "JiTTesting supplies executable behavioral evidence for review, complementing the proposal to shift human review from line-by-line diffs toward verification evidence and retained intent."
      },
      {
        "title": "The AI bugpocalypse is here. Now what? - Jack Cable, Corridor",
        "source_name": "YouTube",
        "source_url": "https://www.youtube.com/watch?v=7JgIS42mz7U",
        "feed7_url": "https://feed7.dev/p/the-ai-bugpocalypse-is-here-now-what-jack-cable-corridor-1y20rna",
        "reason": "JiTTesting operationalizes security review within the development pipeline by generating and ranking tests for risky changes, aligning with the candidate's call to put security review inside coding-agent workflows."
      },
      {
        "title": "We let an AI agent execute Bash and lived to talk about it — Sarah Sanders, PostHog",
        "source_name": "AI Engineer",
        "source_url": "https://www.youtube.com/watch?v=4lXks428C9o",
        "feed7_url": "https://feed7.dev/p/we-let-an-ai-agent-execute-bash-and-lived-to-talk-about-it-sarah-sanders-0pgksvk",
        "reason": "Both use layered controls for agent-related security; JiTTesting ranks and reviews behavioral findings, while PostHog's candidate emphasizes deterministic policy enforcement and cautions against relying on detection alone."
      }
    ]
  },
  "lifecycle": "Current",
  "published_at": "2026-10-29T16:00:12.000Z",
  "modified_at": "2026-10-29T16:00:12.000Z",
  "supersedes": [],
  "expires_at": null,
  "formats": {
    "html": "https://feed7.dev/p/catching-bugs-before-they-land-jittestings-human-on-the-loop-approach-at-0rua5z6",
    "json": "https://feed7.dev/p/catching-bugs-before-they-land-jittestings-human-on-the-loop-approach-at-0rua5z6.json",
    "markdown": "https://feed7.dev/p/catching-bugs-before-they-land-jittestings-human-on-the-loop-approach-at-0rua5z6.md"
  }
}