{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.04280", "title": "EVOHARNESSBENCH: Can Your Agents Keep Pace with an Evolving Harness?", "summary": "EvoHarnessBench replaces ad hoc or fixed-harness agent evaluation with a controlled benchmark of 17 multi-stage harness streams (802 tasks, 520 tools, 42 skills, 62 agents) that isolates retention and adaptation as tools, skills, and sub-agents are added. Adopting it costs evaluation compute and engineering to wire one's own agent stack into its verifier-based environments, and as a benchmark it delivers no direct capability gain. Its practical warning—that harness expansion alone can degrade previously solved tasks (harness-induced forgetting), and that self-evolving memory gains are inconsistent—can fail to transfer if the reader's harness, model, or task distribution differs from the benchmark's deterministic streams.", "rationale": "This is a diagnostic benchmark rather than a deployable method, but its documented finding of harness-induced forgetting justifies adding regression tests on previously solved tasks whenever the reader expands their agent's toolset.", "technique": "Benchmark for agents under evolving tool/skill/agent harnesses", "family": "avaliacao_de_agente", "recommendation": "observar", "published": "2026-09-03", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.04280", "independent_implementations": 2, "total_implementations": 3, "stars": 18, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "EvoHarnessBench replaces ad hoc or fixed-harness agent evaluation with a controlled benchmark of 17 multi-stage harness streams (802 tasks, 520 tools, 42 skills, 62 agents) that isolates retention and adaptation as tools, skills, and sub-agents are added. Adopting it costs evaluation compute and engineering to wire one's own agent stack into its verifier-based environments, and as a benchmark it delivers no direct capability gain. Its practical warning—that harness expansion alone can degrade previously solved tasks (harness-induced forgetting), and that self-evolving memory gains are inconsistent—can fail to transfer if the reader's harness, model, or task distribution differs from the benchmark's deterministic streams.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.04280", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [ { "anchor": "A3.E3", "label": "(3)", "section": "Appendix C Metrics", "role": "metric", "latex": "P_{t,\\tau}=\\frac{1}{|\\mathcal{D}_{\\tau,\\mathrm{eval}}|}\\sum_{(x_{i},y_{i})\\in\\mathcal{D}_{\\tau,\\mathrm{eval}}}V\\!\\left(\\mathcal{A}(x_{i};\\mathcal{H}_{t},z_{t}),\\;y_{i}\\right).", "mathml": "Pt,τ=1|𝒟τ,eval|​∑(xi,yi)∈𝒟τ,evalV⁡(𝒜⁡(xi,ℋt,zt),yi).", "context": "The core evaluation object is a lower-triangular performance matrix P∈ℝT×T, where entry Pt,τ (τ≤t) is the accuracy of the system on task cohort τ when evaluated under harness stage t:", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "A3.E4", "label": "(4)", "section": "Appendix C Metrics", "role": "metric", "latex": "\\mathrm{BWT}=\\frac{\\sum_{i=1}^{T-1}n_{i}(P_{T,i}-P_{i,i})}{\\sum_{i=1}^{T-1}n_{i}},", "mathml": "BWT=∑i=1T−1ni​(PT,i−Pi,i)∑i=1T−1ni,", "context": "Backward transfer (column delta).", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "A3.E5", "label": "(5)", "section": "Appendix C Metrics", "role": "metric", "latex": "\\mathrm{FWT}=\\frac{\\sum_{i=2}^{T}n_{i}(P_{i,i}-P_{i,i}^{\\mathrm{before}})}{\\sum_{i=2}^{T}n_{i}},", "mathml": "FWT=∑i=2Tni​(Pi,i−Pi,ibefore)∑i=2Tni,", "context": "Forward transfer (diagonal delta).", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" } ], "equations_status": "selected", "core_kind": "formula", "equations_fetched_at": "2026-09-25", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }