{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.40103", "title": "JuryFlow: Disagreement-Guided Human-in-the-Loop Multi-Agent Evaluation", "summary": "JuryFlow replaces single-judge or majority-vote LLM evaluation with a multi-judge panel that decomposes responses into atomic claims, flags high-entropy disagreements for targeted human or automatic re-evaluation, propagates corrections across similar claims and cases, and crystallizes them into reusable rubric entries. It costs a panel of heterogeneous judge calls per claim rather than one verdict per response, plus graph construction and rubric-maintenance infrastructure, with a human intervention loop in the full configuration. It can fail if claim decomposition or judge verdicts are noisy at the atomic level, if propagation spreads an incorrect correction to superficially similar claims, or if rubric accumulation drifts judge behavior over time; the human-in-the-loop benefits are not validated by human studies, only by an entropy-ranked automatic proxy.", "rationale": "The framework is implementable with third-party judge APIs and modest engineering, but the unquantified gains and absence of real human studies mean a small team should validate it on their own evaluation workload before relying on it in production.", "technique": "Disagreement-guided human-in-the-loop LLM judge panel", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-09-30", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.40103", "independent_implementations": 0, "total_implementations": 1, "stars": 10, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "JuryFlow replaces single-judge or majority-vote LLM evaluation with a multi-judge panel that decomposes responses into atomic claims, flags high-entropy disagreements for targeted human or automatic re-evaluation, propagates corrections across similar claims and cases, and crystallizes them into reusable rubric entries. It costs a panel of heterogeneous judge calls per claim rather than one verdict per response, plus graph construction and rubric-maintenance infrastructure, with a human intervention loop in the full configuration. It can fail if claim decomposition or judge verdicts are noisy at the atomic level, if propagation spreads an incorrect correction to superficially similar claims, or if rubric accumulation drifts judge behavior over time; the human-in-the-loop benefits are not validated by human studies, only by an entropy-ranked automatic proxy.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.40103", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "On MT-Bench and LLMBar, JuryFlow improves agreement with gold labels over single-judge and majority-vote panel baselines; no numeric effect size is given in the abstract.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.40103", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }