{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.18497", "title": "Judges Follow the Reference: The Derivation Gap in Verifying Tool-Call Arguments", "summary": "DeRef replaces giving the judge raw (possibly truncated or reference-free) conversation history by having a separate model read the full history without seeing the candidate argument and write an explicit reference stating what the value should be. It costs one additional model inference per judged argument, adding latency, API or GPU cost, and a new pipeline component. It can fail when the reference-writing model itself misderives the intended value, since judges are shown to follow the visible reference even when it is incorrect, so a wrong reference can systematically flip verdicts.", "rationale": "The pattern is deployable on constrained infrastructure (one extra small-model or API call) and addresses a documented judge failure mode, but the AUROC gains come from the paper's benchmarks and controlled test, so the reader should validate on their own tool-call workload before relying on it in production.", "technique": "Derived-reference generation for tool-call judges", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-13", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.18497", "independent_implementations": 0, "total_implementations": 1, "stars": 122, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "DeRef replaces giving the judge raw (possibly truncated or reference-free) conversation history by having a separate model read the full history without seeing the candidate argument and write an explicit reference stating what the value should be. It costs one additional model inference per judged argument, adding latency, API or GPU cost, and a new pipeline component. It can fail when the reference-writing model itself misderives the intended value, since judges are shown to follow the visible reference even when it is incorrect, so a wrong reference can systematically flip verdicts.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.18497", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "DeRef improves argument AUROC for all judges tested across three benchmarks, and raises AUROC from chance or below to between 98.40 and 99.59 on a controlled source-relocation test.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.18497", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }