{ "schema_version": 1, "builder_plan": { "scenario": "Your coding assistant makes changes that still need substantial repair before merging.", "change": "Try the technique on held-out issues from a repository with reliable tests.", "baseline": "The same coding agent, repository snapshot, tools, and token budget.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Correct patches, regression failures, human repair time, and total cost per accepted change.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.39304", "title": "Scale and Selection: What Makes Automatic Harness Evolution Work for Visual-Interface Robot Agents", "summary": "The method replaces hand-written agent harnesses (prompts, tools, control rules) with automatic revision by an optimizer coding agent, gated by Champion-Challenger selection on a fixed evaluation set. It costs substantial rollout compute, since trustworthy promotion decisions require on the order of 100 rollouts per round rather than 5, plus an evaluation harness and simulator infrastructure. It can fail through overfitting when few rollouts back each promotion (70% training vs 54% held-out), and through gradual performance drift if revisions are accepted without a selection safeguard.", "rationale": "The technique is implementable with API-based coding agents and a simulator, but its benefits depend on affording large rollout batches per round, so the reader should validate costs and gains on their own workload before adopting.", "technique": "Automated harness evolution via optimizer agent with Champion-Challenger selection", "family": "agentes_de_codigo", "recommendation": "testar", "published": "2026-09-30", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.39304", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The method replaces hand-written agent harnesses (prompts, tools, control rules) with automatic revision by an optimizer coding agent, gated by Champion-Challenger selection on a fixed evaluation set. It costs substantial rollout compute, since trustworthy promotion decisions require on the order of 100 rollouts per round rather than 5, plus an evaluation harness and simulator infrastructure. It can fail through overfitting when few rollouts back each promotion (70% training vs 54% held-out), and through gradual performance drift if revisions are accepted without a selection safeguard.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.39304", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Held-out success rises from 47% to 67% when training rollouts per round grow from 5 to 100, and from 51% to 67% over 30 rounds when unconditional acceptance is replaced with Champion-Challenger selection.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.39304", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }