{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.00651", "title": "Agent Evaluation Reliability: More Tasks Won't (Always) Fix An Agent Leaderboard", "summary": "The framework replaces ad hoc agent leaderboard comparisons and the assumption that adding more tasks fixes unreliable rankings, substituting a Bayesian variance-decomposition that separates claim-relevant signal from scaffold- and task-induced noise. It costs statistical modeling effort on sparse, imbalanced leaderboard data and requires evaluators to specify the intended claim (fixed system vs. underlying model) before interpreting any score. It can fail when reliability is dominated by limited scaffold coverage, since even infinitely many similarly constructed tasks improve model-ranking reliability by at most 0.097, and pooled-benchmark gains depend on benchmark diversity rather than task count.", "rationale": "The reader makes production adoption decisions from agent benchmarks, and this paper gives directly actionable guidance: rank deployed model-scaffold systems rather than raw models, diversify scaffolds, and pool heterogeneous benchmarks instead of buying more tasks.", "technique": "Bayesian variance decomposition for agent evaluation design", "family": "avaliacao_de_agente", "recommendation": "adotar", "published": "2026-09-30", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.00651", "independent_implementations": 0, "total_implementations": 1, "stars": 10, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "The framework replaces ad hoc agent leaderboard comparisons and the assumption that adding more tasks fixes unreliable rankings, substituting a Bayesian variance-decomposition that separates claim-relevant signal from scaffold- and task-induced noise. It costs statistical modeling effort on sparse, imbalanced leaderboard data and requires evaluators to specify the intended claim (fixed system vs. underlying model) before interpreting any score. It can fail when reliability is dominated by limited scaffold coverage, since even infinitely many similarly constructed tasks improve model-ranking reliability by at most 0.097, and pooled-benchmark gains depend on benchmark diversity rather than task count.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.00651", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Pooling diverse benchmarks raises projected cross-task ranking reliability from 0.44 to 0.75 at the same task budget and can reduce projected evaluation cost by up to 83%.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.00651", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }