{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.29167", "title": "IndicBankBench: Evaluating Safety and Reliability of Language Model Assistants in Indian Retail Banking", "summary": "The benchmark replaces final-response-only evaluation of banking assistants with four-stage scoring (safety, deterministic tool-use checks, response adequacy, advisory quality) and a strict pass^3 metric over repeated runs. It costs building or adapting a mock tool environment, running each case three times, and paying for an LLM judge on semantic adequacy, plus a narrow resolver for ambiguous confirmation cases. The domain is Indian retail banking with 799 cases, so adoption outside that domain requires porting the cases and harness; the LLM judge and resolver components can also introduce nondeterminism that the deterministic checks were designed to avoid.", "rationale": "The staged, deterministic, multi-trial evaluation methodology is directly reusable for anyone shipping tool-using assistants on constrained infrastructure, but the benchmark itself is domain-specific and must be validated against the reader's own workload before driving production decisions.", "technique": "Multi-stage benchmark for banking tool-use agents", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-09-24", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.29167", "independent_implementations": 0, "total_implementations": 2, "stars": 8, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The benchmark replaces final-response-only evaluation of banking assistants with four-stage scoring (safety, deterministic tool-use checks, response adequacy, advisory quality) and a strict pass^3 metric over repeated runs. It costs building or adapting a mock tool environment, running each case three times, and paying for an LLM judge on semantic adequacy, plus a narrow resolver for ambiguous confirmation cases. The domain is Indian retail banking with 799 cases, so adoption outside that domain requires porting the cases and harness; the LLM judge and resolver components can also introduce nondeterminism that the deterministic checks were designed to avoid.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.29167", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Across eleven models, strict pass^3 reliability is 43.7-58.2% while at-least-once success is 60-74%, showing single-trial or best-of-n metrics overstate dependable behavior.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.29167", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }