{ "schema_version": 1, "builder_plan": { "scenario": "An agent reads external documents or tool responses that may contain malicious instructions.", "change": "Replay attacks and legitimate lookalikes in an isolated copy of your agent workflow.", "baseline": "Your existing checks with the same tasks, permissions, and tool responses.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Successful attacks missed, legitimate actions blocked, detection delay, and review cost.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.19630", "title": "From Intent to Action: Benchmarking LLM Safety in Vehicle Voice Command Authorization", "summary": "The paper contributes a 202-scenario benchmark and a seven-class pre-action authorization taxonomy that replaces ad hoc prompting of an LLM to decide whether a voice command may actuate vehicle functions. The structured policy prompt costs prompt tokens and inference latency comparable to a schema-only baseline, but even the best API model (89.1% alignment) cannot serve as the sole gate, so deployment additionally requires an independent deterministic enforcement layer that verifies tool permissions, speaker authentication, and vehicle-state constraints. The residual failure mode is False Execute decisions, plus persistent errors on confirmation and manual-control classes, which in a safety-critical setting are unacceptable without that external layer.", "rationale": "Any LLM issuing tool calls in a safety-critical product needs a verified enforcement layer regardless of model size, and this benchmark plus taxonomy gives a small team an off-the-shelf evaluation protocol and a clear architectural conclusion actionable on a single GPU or via APIs.", "technique": "Structured authorization policy for vehicle voice commands", "family": "seguranca_e_guardrails", "recommendation": "adotar", "published": "2026-09-17", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.19630", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The paper contributes a 202-scenario benchmark and a seven-class pre-action authorization taxonomy that replaces ad hoc prompting of an LLM to decide whether a voice command may actuate vehicle functions. The structured policy prompt costs prompt tokens and inference latency comparable to a schema-only baseline, but even the best API model (89.1% alignment) cannot serve as the sole gate, so deployment additionally requires an independent deterministic enforcement layer that verifies tool permissions, speaker authentication, and vehicle-state constraints. The residual failure mode is False Execute decisions, plus persistent errors on confirmation and manual-control classes, which in a safety-critical setting are unacceptable without that external layer.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.19630", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "A structured authorization policy raises Decision Alignment for Llama 3.2 3B from 28.2-29.2% (schema-only and generic-safety baselines) to 40.1%; API models reach 83.2-89.1% alignment but still produce two to three False Executes among 161 non-execution scenarios.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.19630", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }