{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.11678", "title": "TRACE: Diagnosing Verifier Brittleness in Agentic Evaluation", "summary": "TRACE replaces the practice of reading a verifier score change directly as a capability change: before accepting any score delta, it applies a targeted mutation to one evaluation component, compares paired runs, checks whether behavior changed, and rescores unchanged trajectories to isolate the scoring rule's contribution. The cost is modest engineering effort to implement mutations and rescoring plus the repeated runs needed for statistical power, which is feasible on a single GPU or via APIs since the paper itself ran only 88 tasks with public benchmarks. It can fail when run-to-run nondeterminism is ignored (identical reruns flip 15-36% of outcomes, invalidating single-run comparisons), when mutations are not behavior-preserving (misleading tool names are a genuine agent failure, not verifier brittleness), and when judges disagree because they grade different criteria (two frontier judges conflicted on 57% of records).", "rationale": "The protocol requires only API calls, small task suites, and repeated paired runs, making it directly adoptable under constrained infrastructure for any team that relies on verifier scores to judge agent changes.", "technique": "Paired-mutation diagnostic protocol for verifier robustness", "family": "avaliacao_de_agente", "recommendation": "adotar", "published": "2026-10-08", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.11678", "independent_implementations": 2, "total_implementations": 3, "stars": 136, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "TRACE replaces the practice of reading a verifier score change directly as a capability change: before accepting any score delta, it applies a targeted mutation to one evaluation component, compares paired runs, checks whether behavior changed, and rescores unchanged trajectories to isolate the scoring rule's contribution. The cost is modest engineering effort to implement mutations and rescoring plus the repeated runs needed for statistical power, which is feasible on a single GPU or via APIs since the paper itself ran only 88 tasks with public benchmarks. It can fail when run-to-run nondeterminism is ignored (identical reruns flip 15-36% of outcomes, invalidating single-run comparisons), when mutations are not behavior-preserving (misleading tool names are a genuine agent failure, not verifier brittleness), and when judges disagree because they grade different criteria (two frontier judges conflicted on 57% of records).", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.11678", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }