{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.39958", "title": "Better Deck or Different Judge? Evaluating Agentic Harness Gains in Corporate and Investment Banking", "summary": "The harness replaces direct single-prompt generation with a pipeline of financial calculations, narrative templates, and validation checks, developed and scored under LLM judges. It costs additional engineering complexity, per-run compute for calculation and validation stages, and dependence on judge-in-the-loop development using a 27B model that fits on constrained hardware only in quantized form. Judge scores shift on repeated grading of unchanged decks, judges disagree more on final-deck rankings than pooled scores, and near-zero margins against direct frontier-model generation mean the measured gains may partly reflect grading behavior rather than document quality.", "rationale": "The harness pattern is implementable on a single 24 GB GPU with a quantized 27B model, but the paper itself shows judge variability and weak margins against strong baselines, so workload-specific validation is required before production adoption.", "technique": "Agentic harness with LLM-judge-guided development", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-09-30", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.39958", "independent_implementations": 1, "total_implementations": 2, "stars": 131, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "The harness replaces direct single-prompt generation with a pipeline of financial calculations, narrative templates, and validation checks, developed and scored under LLM judges. It costs additional engineering complexity, per-run compute for calculation and validation stages, and dependence on judge-in-the-loop development using a 27B model that fits on constrained hardware only in quantized form. Judge scores shift on repeated grading of unchanged decks, judges disagree more on final-deck rankings than pooled scores, and near-zero margins against direct frontier-model generation mean the measured gains may partly reflect grading behavior rather than document quality.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.39958", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Five LLM judges score the full harness 20.4 to 33.6 points out of 95 above the same 27B model generating directly from a short prompt, and every judge ranks the system higher on all 17 deliverables; margins against direct Opus generation are only -4.7 to +0.8 points.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.39958", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }