{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.16305", "title": "BLINDSPOT: A Benchmark for Safety and Refusal Calibration in Long-Horizon Tool-Using Agents", "summary": "Blindspot replaces single-turn or binary attack-success safety evaluation with trajectory-level adjudication of long-horizon tool-using agents, classifying each run as Safe Completion, Correct Refusal, Unsafe Completion, Over-Refusal, or Indeterminate across 22 attack families and 35 stateful scenarios. Adopting it costs multi-turn simulation infrastructure: stateful tool execution, adaptive adversarial interaction, execution-grounded adjudication, and API or GPU compute for trajectories averaging 14.7 turns, which at 2,500 trajectories implies a non-trivial evaluation budget. It can fail through adjudication errors in the Indeterminate category, scenario coverage gaps relative to the reader's actual tool surface, and results that depend heavily on which models and policies are instantiated, so benchmark rankings may not transfer to a specific production agent.", "rationale": "The benchmark is relevant for validating safety-utility calibration before deploying tool-using agents, but its cost and dependence on matching its scenarios to the reader's actual tools make a small pilot subset the appropriate first step rather than full adoption.", "technique": "Trajectory-level safety calibration benchmark for tool-using agents", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-09-14", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.16305", "independent_implementations": 0, "total_implementations": 1, "stars": 16, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "Blindspot replaces single-turn or binary attack-success safety evaluation with trajectory-level adjudication of long-horizon tool-using agents, classifying each run as Safe Completion, Correct Refusal, Unsafe Completion, Over-Refusal, or Indeterminate across 22 attack families and 35 stateful scenarios. Adopting it costs multi-turn simulation infrastructure: stateful tool execution, adaptive adversarial interaction, execution-grounded adjudication, and API or GPU compute for trajectories averaging 14.7 turns, which at 2,500 trajectories implies a non-trivial evaluation budget. It can fail through adjudication errors in the Indeterminate category, scenario coverage gaps relative to the reader's actual tool surface, and results that depend heavily on which models and policies are instantiated, so benchmark rankings may not transfer to a specific production agent.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.16305", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_formula", "core_kind": "evaluation_protocol", "equations_fetched_at": "2026-10-13", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }