{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.18005", "title": "A Calibrated Instrument for Measuring How Inference Optimizations Affect Output Quality", "summary": "The method replaces ad-hoc per-technique benchmark scores with a single calibrated LLM-as-judge protocol that validates itself against statistically identical runs and a provably null condition before comparing optimizations. It costs additional judged inference runs for calibration and null conditions, plus prompt-set curation per domain, and its resolution (about +/-0.3 points) limits detection of small degradations. It can fail if the judge drifts or is miscalibrated for a new domain, since perceived quality is strongly domain-dependent and model-specific, so magnitudes measured on one model do not transfer to another.", "rationale": "For a constrained team choosing between quantization, early exit, and speculative decoding, this calibrated-judge protocol offers a low-cost, workload-specific way to verify that an optimization does not degrade output quality before shipping it, but it requires local validation to trust.", "technique": "Calibrated LLM-judge instrument for optimization quality measurement", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-09-16", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.18005", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The method replaces ad-hoc per-technique benchmark scores with a single calibrated LLM-as-judge protocol that validates itself against statistically identical runs and a provably null condition before comparing optimizations. It costs additional judged inference runs for calibration and null conditions, plus prompt-set curation per domain, and its resolution (about +/-0.3 points) limits detection of small degradations. It can fail if the judge drifts or is miscalibrated for a new domain, since perceived quality is strongly domain-dependent and model-specific, so magnitudes measured on one model do not transfer to another.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.18005", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "4-bit quantization was indistinguishable from the 16-bit original within +/-0.3 judge points on English and Chinese prose; 3-bit cost 0.5-1.1 points depending on domain; early exit cost 0.7 points on prose but 2.5 on math (19/27 to 6/27 problems solved); identical quantizer cost 1.8 points on a Meta model versus 0.7 on an Alibaba model.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.18005", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }