{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.21257", "title": "Verify, Don't Trust: Agentic Model Development for Video Discovery Retrieval at Scale", "summary": "EvoPilot replaces ungated autoresearch loops, in which LLM agents propose, implement, and evaluate model changes autonomously, with role-specific agents whose rounds pass through versioned domain skills, typed adapters, durable records, and deterministic checks gated by human review. The cost is substantial supporting infrastructure: durable experiment state, replay and mutation testing of comparisons, deterministic enforcement of recorded lessons, and an hourly refreshed index over hundreds of millions of videos, sustained over a 37-day campaign. It can fail when the verification harness itself misses defects such as no-op code changes, data-window leakage, evaluator semantic drift, or divergent serving funnels, and its gating discipline adds human latency to every iteration.", "rationale": "The method's value was demonstrated only at the scale of a production video-discovery system with weeks-long campaigns and randomized online evaluation, so a small team on one GPU can adopt the principle of verifying agent-proposed experiments but cannot replicate the validated setup.", "technique": "Human-gated verification loops for online autoresearch agents", "family": "avaliacao_de_agente", "recommendation": "observar", "published": "2026-09-18", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.21257", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "EvoPilot replaces ungated autoresearch loops, in which LLM agents propose, implement, and evaluate model changes autonomously, with role-specific agents whose rounds pass through versioned domain skills, typed adapters, durable records, and deterministic checks gated by human review. The cost is substantial supporting infrastructure: durable experiment state, replay and mutation testing of comparisons, deterministic enforcement of recorded lessons, and an hourly refreshed index over hundreds of millions of videos, sustained over a 37-day campaign. It can fail when the verification harness itself misses defects such as no-op code changes, data-window leakage, evaluator semantic drift, or divergent serving funnels, and its gating discipline adds human latency to every iteration.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.21257", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Matched offline comparison measured a 3.20 percentage-point improvement; a seven-day randomized online evaluation estimated a 0.66% relative increase in the VDD slice of GSRR; artifact reuse avoided approximately five GPU-hours.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.21257", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }