{ "schema_version": 1, "builder_plan": { "scenario": "Your coding assistant makes changes that still need substantial repair before merging.", "change": "Try the technique on held-out issues from a repository with reliable tests.", "baseline": "The same coding agent, repository snapshot, tools, and token budget.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Correct patches, regression failures, human repair time, and total cost per accepted change.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.19526", "title": "Self Improvement via Fast Tree-search", "summary": "The method replaces the practice of re-running benchmark subsets to score every candidate self-modification of a coding agent with an LLM-as-a-judge pairwise comparison signal aggregated via a regularized Bradley-Terry model, reserving full task evaluations only for the most promising tree-search nodes. The cost is the added judge-model API calls and implementation complexity of a disaggregated tree search with rank-based parent sampling, though total reported resource use is lower than prior frameworks. What can fail: the judge's pairwise rankings may not correlate with true downstream task performance on a different codebase or task distribution, silently steering search toward patches the judge favors rather than patches that work, and gains are demonstrated only on the Polyglot benchmark with no reported replication.", "rationale": "The framework targets exactly the reader's constraint (API-based models, strict budgets) and reports cost reductions, but its reliance on judge-signal fidelity and a single benchmark warrants small-scale validation before production adoption.", "technique": "Judge-guided tree search for agent self-modification", "family": "agentes_de_codigo", "recommendation": "testar", "published": "2026-09-17", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.19526", "independent_implementations": 1, "total_implementations": 2, "stars": 9, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The method replaces the practice of re-running benchmark subsets to score every candidate self-modification of a coding agent with an LLM-as-a-judge pairwise comparison signal aggregated via a regularized Bradley-Terry model, reserving full task evaluations only for the most promising tree-search nodes. The cost is the added judge-model API calls and implementation complexity of a disaggregated tree search with rank-based parent sampling, though total reported resource use is lower than prior frameworks. What can fail: the judge's pairwise rankings may not correlate with true downstream task performance on a different codebase or task distribution, silently steering search toward patches the judge favors rather than patches that work, and gains are demonstrated only on the Polyglot benchmark with no reported replication.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.19526", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "SIFT outperforms existing tree-search-based self-evolution frameworks on the full Polyglot benchmark with significantly lower CPU hours, wall clock time, and API cost; the abstract reports no specific numeric factor.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.19526", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }