{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.21135", "title": "Stronger Judges Do Not See More: Deployment-Integrity Blindness in LLM Evaluation", "summary": "The paper's proposed practice replaces output-only evaluation reporting with a graded deployment-identity descriptor plus a manifest hash of the runtime stack (model, weights, tokenizer, prompts, decoding config, tools) attached to evaluation reports. Cost is near-zero in compute and latency; the burden is added process complexity: generating, versioning, and verifying stack manifests in the CI/eval pipeline. It can fail because hashing proves identity but not correctness of the manifest itself, fingerprinting covers only the model component (one of six), trigger-conditional tampers can still evade any fixed evaluation suite, and an adversary controlling the reporting channel can forge descriptors unless hashes are anchored to a trusted source.", "rationale": "Manifest hashing and deployment-identity descriptors are cheap, infrastructure-agnostic process changes that close a demonstrated blind spot in output-only evaluation, so a small team using APIs or a single GPU can adopt them immediately without any compute cost.", "technique": "Deployment-identity manifest hashing for LLM evaluation", "family": "avaliacao_de_agente", "recommendation": "adotar", "published": "2026-10-15", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.21135", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "The paper's proposed practice replaces output-only evaluation reporting with a graded deployment-identity descriptor plus a manifest hash of the runtime stack (model, weights, tokenizer, prompts, decoding config, tools) attached to evaluation reports. Cost is near-zero in compute and latency; the burden is added process complexity: generating, versioning, and verifying stack manifests in the CI/eval pipeline. It can fail because hashing proves identity but not correctness of the manifest itself, fingerprinting covers only the model component (one of six), trigger-conditional tampers can still evade any fixed evaluation suite, and an adversary controlling the reporting channel can forge descriptors unless hashes are anchored to a trusted source.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.21135", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Stronger output-only judges accept more tampered deployments (88.6% rule-based vs 94.1% GPT-5-nano acceptance); zero-output-shift tampers are provably undetectable from outputs at any sample size, while a manifest-hash identity channel detects them with zero samples; behavioral fingerprinting identifies the model component at 76.7% but only one of six stack components.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.21135", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }