{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent reads external documents or tool responses that may contain malicious instructions.",
    "change": "Replay attacks and legitimate lookalikes in an isolated copy of your agent workflow.",
    "baseline": "Your existing checks with the same tasks, permissions, and tool responses.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Successful attacks missed, legitimate actions blocked, detection delay, and review cost.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2610.03448",
  "title": "Passing the Test You Trained On: Re-evaluating Prompt-Injection Detectors for LLM Agents",
  "summary": "The method replaces selecting prompt-injection detectors by public benchmark scores with evaluation on the agent's own tool outputs, labeled by differential replay of ground-truth tool calls, reported at a low false-positive rate, plus an audit of detector training data. It costs building a replay harness for one's own agent traces and re-running candidate detectors, which is cheap inference-side work on a single GPU or via APIs; the detectors themselves are small and inexpensive. What can fail is that replay-generated benign outputs may diverge from live traffic, differential labeling may misclassify injections, and a detector chosen this way may still fail against novel attack styles absent from the replayed set.",
  "rationale": "Any team deploying an LLM agent with a prompt-injection detector should adopt this evaluation practice because the paper shows benchmark rankings are unreliable predictors of in-agent performance while the required replay evaluation is feasible on constrained infrastructure.",
  "technique": "Deployment-representative evaluation of prompt-injection detectors",
  "family": "seguranca_e_guardrails",
  "recommendation": "adotar",
  "published": "2026-10-02",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2610.03448",
  "independent_implementations": 1,
  "total_implementations": 2,
  "stars": 11,
  "citations": null,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "The method replaces selecting prompt-injection detectors by public benchmark scores with evaluation on the agent's own tool outputs, labeled by differential replay of ground-truth tool calls, reported at a low false-positive rate, plus an audit of detector training data. It costs building a replay harness for one's own agent traces and re-running candidate detectors, which is cheap inference-side work on a single GPU or via APIs; the detectors themselves are small and inexpensive. What can fail is that replay-generated benign outputs may diverge from live traffic, differential labeling may misclassify injections, and a detector chosen this way may still fail against novel attack styles absent from the replayed set.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.03448",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "Benchmark rankings transfer poorly across settings: the best BIPIA detector catches only 2% of AgentDojo injections at 1% FPR, and a detector catching 72% of AgentDojo injections catches 15% on tau-bench; false-positive rates on tool outputs do transfer between agent benchmarks.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.03448",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}