{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.37315", "title": "Do Agent Benchmarks Do What They Say? An Executable-Contract Audit of Tool-Using Agent Environments", "summary": "The method replaces trusting a tool's self-reported success messages and benchmark scores with executable contracts that check each mutating tool's implementation against its advertised interface and trace evaluator verdicts back to the state a defective tool should have written. It costs engineering effort to write per-tool contracts and probes plus a static/dynamic checking pass, but runs on ordinary development infrastructure with no GPU or training requirement. It can fail by missing defects the contracts nominally cover when no probe exposes them (29 of 33 scored misses), so a clean audit does not certify correctness.", "rationale": "For a small team shipping tool-using agents on constrained infrastructure, contract-testing their own tool surfaces and distrusting benchmark-reported action success is cheap, immediately applicable, and addresses a demonstrated failure mode where graders credit writes that never occurred.", "technique": "Executable-contract audit of agent tool environments", "family": "avaliacao_de_agente", "recommendation": "adotar", "published": "2026-09-29", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.37315", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "The method replaces trusting a tool's self-reported success messages and benchmark scores with executable contracts that check each mutating tool's implementation against its advertised interface and trace evaluator verdicts back to the state a defective tool should have written. It costs engineering effort to write per-tool contracts and probes plus a static/dynamic checking pass, but runs on ordinary development infrastructure with no GPU or training requirement. It can fail by missing defects the contracts nominally cover when no probe exposes them (29 of 33 scored misses), so a clean audit does not certify correctness.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.37315", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Across 34 audited mutating tools in four benchmarks, the audit confirmed seven tool defects and one evaluator property; on injected defects the checker raised no false positives in 25 flags but missed most injected defects (29 of 33 scored misses had a covering clause with no revealing probe), and its static half alone flagged 14 of 17 confirmed sites.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.37315", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }