{ "schema_version": 1, "builder_plan": { "scenario": "Possible product use: an assistant that reads customer uploads or fetched web pages before calling your APIs.", "change": "Use the dataset to challenge your existing checks offline. Keep tools mocked so that a test cannot send messages, alter records, or expose real customer data.", "baseline": "Your current detector against a small human-reviewed set. Separate real attack success, resisted attacks, and legitimate content that looks suspicious.", "steps": [ "Fetch the official dataset linked by the paper, record its version and license, and inspect the label definitions. Start with a small sample of attacked and legitimate histories that resemble your workflow.", "Split by world identity so the same world cannot appear in training and evaluation. Keep product test examples separate from any detector tuning. Check for repeated templates and names.", "Replay the held-out cases through your existing checks with mocked tools. Record the exact step flagged, missed successful attacks, and legitimate actions blocked.", "Only then compare a candidate detection change on the same cases. Manually review disagreements and repeat with held-out examples from your own agent before enabling enforcement." ], "measure": "Attack recall, false-positive rate on legitimate tasks, delay until detection, review burden, and detector cost. Keep denominators and attack types visible instead of reporting one overall score.", "decision": "Keep a change only if it catches more consequential attacks within your preselected false-alarm and latency limits. Synthetic benchmark performance alone is not evidence that a production agent is protected." }, "builder_review": { "arxiv_id": "2609.06972", "source_url": "https://arxiv.org/pdf/2609.06972v1", "pdf_sha256": "8265100357767f3cb8b8563952dda96f3018811b31f49d87aa73766161d53433", "report_sha256": "1864b7dba77939c0afc3f74d93e6a97429cdfe569dacd7ae0f9d3f54d571e243", "reviewed_at": "2026-09-14", "intro": "An agent can follow an instruction hidden in a document instead of its user's request. AgentDrift provides labeled synthetic examples for studying where that happens. Its most useful lesson for a builder is how an impressive evaluation score can come from recognizing the test data.", "mechanism": "AgentDrift labels individual steps of synthetic tool-use histories. It supplies evaluation data, not a proven protective layer. Some examples contain attacks; others contain suspicious-looking but legitimate content.", "method_page": 3, "takeaway": "Treat this as an evaluation resource. The lookup result exposes a shortcut, while recall measures attacks caught by one simple detector. Those are different questions. One weak detector also does not prove that every non-sequential approach must fail.", "comparisons": [ { "title": "Accuracy that comes from a shortcut", "definition": "Binary accuracy counts correctly classified attacked and non-attacked histories.", "baseline_name": "Majority-class prediction", "method_name": "World-identity lookup", "before": 55.8, "after": 86.1, "unit": "percent", "scope": "Default 1,881-trajectory test split. The lookup reuses world identities seen in training.", "caveat": "This is leakage evidence, not a 30-point security improvement. A detector must be evaluated with identity shortcuts controlled.", "source_page": 14, "source_locator": "Table XI; explanation on page 13" }, { "title": "Attack recall is a different denominator", "definition": "Recall is the share of actual attacks detected. Here the baseline catches 460 of 831.", "baseline_name": "No alternative detector measured here", "method_name": "Simple feature classifier", "before": null, "after": 55.4, "unit": "percent", "scope": "Single held-out test evaluation of a six-feature logistic regression.", "caveat": "This is not overall accuracy. The partial-hijack subset has 8.2% recall, and benign histories have a 10.5% false-alarm rate.", "source_page": 14, "source_locator": "Table XIII" } ], "plan": { "scenario": "Possible product use: an assistant that reads customer uploads or fetched web pages before calling your APIs.", "change": "Use the dataset to challenge your existing checks offline. Keep tools mocked so that a test cannot send messages, alter records, or expose real customer data.", "baseline": "Your current detector against a small human-reviewed set. Separate real attack success, resisted attacks, and legitimate content that looks suspicious.", "steps": [ "Fetch the official dataset linked by the paper, record its version and license, and inspect the label definitions. Start with a small sample of attacked and legitimate histories that resemble your workflow.", "Split by world identity so the same world cannot appear in training and evaluation. Keep product test examples separate from any detector tuning. Check for repeated templates and names.", "Replay the held-out cases through your existing checks with mocked tools. Record the exact step flagged, missed successful attacks, and legitimate actions blocked.", "Only then compare a candidate detection change on the same cases. Manually review disagreements and repeat with held-out examples from your own agent before enabling enforcement." ], "measure": "Attack recall, false-positive rate on legitimate tasks, delay until detection, review burden, and detector cost. Keep denominators and attack types visible instead of reporting one overall score.", "decision": "Keep a change only if it catches more consequential attacks within your preselected false-alarm and latency limits. Synthetic benchmark performance alone is not evidence that a production agent is protected." } }, "arxiv_id": "2609.06972", "title": "AgentDrift: A Step-Labeled Benchmark of Injection-Hijacked LLM Agent Trajectories", "summary": "AgentDrift replaces whole-trace guard judgments and attack-success-only benchmarks with a step-labeled corpus (12,536 trajectories, 71,024 steps labeled benign, injection point, hijacked, or failed injection) for training and evaluating detectors of indirect prompt injection in tool-call trajectories. Adoption costs the integration of an offline evaluation harness against one's own agent traces plus detector development, since the paper provides data and a weak baseline rather than a deployable guard model. Key failure risks include domain mismatch with the five covered domains, synthetic-data artifacts the authors themselves document (template concentration, attack-goal-family collapse, world-identity leakage, an LLM judge fooled by hard negatives), and reduced validity against adaptive attackers once the corpus is public.", "rationale": "The corpus is a free, API-compatible resource for evaluating injection defenses on constrained infrastructure, but its documented synthetic-data limitations and lack of a trained detector mean it must be validated against the reader's own agent workloads before informing production guardrails.", "technique": "Step-labeled injection-hijack trajectory benchmark (AgentDrift)", "family": "seguranca_e_guardrails", "recommendation": "testar", "published": "2026-09-07", "as_of": "2026-10-18", "editorial_status": "source_mapped", "source_url": "https://arxiv.org/abs/2609.06972", "independent_implementations": 2, "total_implementations": 2, "stars": 3, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The corpus contains 12,536 trajectories and 71,024 step labels across four categories and five domains, with all labels conforming to a stated regular grammar.", "basis": "source_linked", "result": "12,536 trajectories; 71,024 steps; 4,000 benign / 5,536 attacked / 1,500 failed-attack / 1,500 hard-negative; 100% positional grammar adherence", "baseline": "Prior resources (InjecAgent, AgentDojo, TraceSafe, StepShield, etc.) lack the combination of per-step injection labels, a resisted class, and hard negatives (Table I)", "conditions": "Synthetic data from a single generator model; labels are protocol-specified, not independent human gold annotations", "source_url": "https://arxiv.org/pdf/2609.06972", "source_page": 1, "source_excerpt": "We present AgentDrift, a benchmark of 12,536 synthetic tool-call trajectories over five agent domains in which every one of the 71,024 steps carries one of four labels: benign, injection point, hijacked, or failed injection." }, { "claim_id": "claim-02", "statement": "A surface-feature logistic regression baseline recovers only 55.4% of attacks, establishing that the benchmark cannot be solved without sequence modeling.", "basis": "inferred", "result": "Precision 0.778, recall 0.554, F1 0.647, AUROC 0.734; recall 88.0% full hijack, 8.2% partial hijack, 23.1% delayed execution; failed attacks flagged at 0.187 vs 0.105 benign", "baseline": "Six-dimensional hand-crafted feature vector with logistic regression (the paper's own baseline, not an external method)", "conditions": "Evaluated once on the released 1,881-trajectory test split; baseline has access to world-grounded external-recipient features", "source_url": "https://arxiv.org/pdf/2609.06972", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-03", "statement": "World-identity leakage lets a trivial lookup predictor reach 86.1% binary accuracy on the default split, so uncontrolled results on this corpus are not evidence of injection detection.", "basis": "source_linked", "result": "86.1% binary accuracy overall (93.3%–98.2% in four domains) vs 55.8% majority-class rate; 47.7% in email", "baseline": "Majority-class baseline (55.8%)", "conditions": "Lookup of identical world object in training split; caused by concurrent SLURM jobs creating separate world pools per category in four domains", "source_url": "https://arxiv.org/pdf/2609.06972", "source_page": 13, "source_excerpt": "The predictor reaches 86.1% binary accuracy on the test split against a 55.8% majority-class rate, and between 93.3% and 98.2% in banking, web, coding, and medical" }, { "claim_id": "claim-04", "statement": "An LLM judge is an unreliable verifier of security labels: it rejected 99.6% of hard negatives as malicious, fooled by the surface cues the class exists to penalize.", "basis": "source_linked", "result": "Screening pass rates: benign 89.6%, full hijack 3.6%, delayed execution 1.4%, hard negatives 0.4%; manual audit found 99.6% label correctness on 1,200 trajectories", "baseline": "Programmatic grammar checks (100% adherence) and 1,200-trajectory manual audit (99.6% correct)", "conditions": "Same model (Llama-3.3-70B) at temperature 0.1 with category-specific checklists; screening verdicts not used to filter the release", "source_url": "https://arxiv.org/pdf/2609.06972", "source_page": 10, "source_excerpt": "the reviewer marked 1,494 of 1,500 as BAD because their observations “contain suspicious content” and “should be labeled as malicious”; that is, it was fooled by exactly the surface cues that the class exists to penalize" }, { "claim_id": "claim-05", "statement": "The corpus has measurable template concentration and attack-goal-family collapse that enable memorization shortcuts.", "basis": "inferred", "result": "28.9% of injected observations name an ext-audit.com address; ~75% of hard negatives mention $5,000, an in-domain audit@ address, or a transfer; external-recipient fraction is 0.47–0.57 across all six attack-goal families", "baseline": "", "conditions": "4-gram analysis over the released corpus; family tags released as metadata only, not certified classification targets", "source_url": "https://arxiv.org/pdf/2609.06972", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "inferred", "finding": "Minimum useful test: API or CPU. Published evidence: cluster.", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "inferred", "finding": "Reported setup: standard Python.", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "inferred", "finding": "Paperraft defines a 5-steps falsification test. No reproduction is recorded.", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [ { "risk_id": "risk-01", "statement": "All 12,536 trajectories come from a single generator model, so a detector may learn the generator's stylistic fingerprint rather than injection semantics; transfer to production agent traces is untested.", "basis": "inferred" }, { "risk_id": "risk-02", "statement": "World-identity leakage in four domains means results on the default split without anonymization or world-held-out splitting are not evidence of injection detection.", "basis": "inferred" }, { "risk_id": "risk-03", "statement": "Template concentration (28.9% of injections naming one fictitious domain; ~75% of hard negatives sharing three motifs) permits memorization unless target domains and amounts are masked at test time.", "basis": "inferred" }, { "risk_id": "risk-04", "statement": "Injections almost always fall at step 2 or 3 under fixed positional templates, enabling positional-prior shortcuts.", "basis": "inferred" }, { "risk_id": "risk-05", "statement": "Labels are protocol-specified and structurally validated rather than independently human-annotated; the 99.6% correctness figure is an estimate from a 1,200-trajectory sample.", "basis": "inferred" }, { "risk_id": "risk-06", "statement": "Short trajectories (3–11 steps), single agents, and no real agent traffic limit applicability to long-horizon or multi-agent settings.", "basis": "inferred" } ], "minimum_test": [ "Download the released corpus (CC BY 4.0, GitHub) and verify the counts and grammar claims directly: 12,536 trajectories, per-category label strings matching the stated regular languages, and the 70/15/15 split.", "Reproduce the world-identity lookup check (Table XI) on the test split; if a world-lookup predictor does not reach ~86% binary accuracy on your copy, the leakage claim fails; if it does, apply world anonymization or a world-held-out split before any further evaluation.", "Re-implement the six-feature logistic regression baseline (specified in Section IX-A, reproducible 'with a few lines of code') and confirm it recovers ~55% of attacks overall but under 25% on partial hijacks and delayed executions; if simple features solve those patterns, the paper's core difficulty claim fails.", "Train a small sequence model (e.g., a short transformer or BiLSTM over the documented step linearization) on CPU or a single small GPU under the world-anonymized protocol; if it does not beat the surface baseline on partial-hijack and delayed-execution recall, the claimed need for sequence modeling is not demonstrated on your workload.", "Spot-audit a random sample of attacked and hard-negative trajectories by hand against the category definitions to check whether the 99.6% label-correctness claim holds for your use case." ], "open_questions": [ "Does a sequence-aware detector trained on AgentDrift actually outperform the surface baseline under the world-anonymized and world-held-out protocols, and by how much on partial hijack and delayed execution?", "Does training on this synthetic distribution transfer to real agent traces or to trajectories from other generators?", "What inter-annotator agreement would human labeling of the injection index and hijacked span yield (listed as future work)?", "Are the released GitHub artifacts (data files, tool pools, loader) complete and consistent with the paper's reported counts?", "What compute budget is needed to train a competitive sequence model on the corpus (not reported; only the trivial baseline is evaluated)?" ], "independent_tests": [], "lab_reproductions": [], "report_available": true, "validation_tier": "api_or_cpu", "evidence_tier": "cluster", "infrastructure_basis": "explicit", "equations": [ { "anchor": "S9.E12", "label": "(12)", "section": "IX Benchmark Validation: A Surface Baseline", "role": "baseline", "latex": "\\phi(\\mathcal{T})=\\big(n,\\ \\textstyle\\sum_{t}c_{t},\\ \\max_{t}e_{t},\\ \\max_{t}u_{t},\\ \\textstyle\\sum_{t}s_{t},\\ k\\big),", "mathml": "ϕ⁡(𝒯)=(n,∑tct,maxt⁡et,maxt⁡ut,∑tst,k),", "context": "For each trajectory the baseline computes a six-dimensional feature vector with no sequence information and no semantic embedding,", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S9.E13", "label": "(13)", "section": "IX Benchmark Validation: A Surface Baseline", "role": "baseline", "latex": "\\hat{p}(\\mathcal{T})=\\sigma\\!\\big(\\mathbf{w}^{\\top}\\tilde{\\phi}(\\mathcal{T})+b\\big),\\qquad\\hat{y}=\\mathbb{1}[\\hat{p}\\geq 0.5],", "mathml": "p^(𝒯)=σ(𝐰⊤ϕ~(𝒯)+b),y^=𝟙[p^≥0.5],", "context": "where n is the step count, ct the number of external recipients in step t’s arguments, et the external-recipient indicator of Eq. (11), ut the external-URL indicator, st an indicator that the tool name contains a sink verb (delete, send, forward, execute, transfer, post, share, write_file, email), and k the number of distinct terms from the 26-term suspicious-keyword list that occur in the concatenated thoughts and observations. The vector of Eq. (12) is standardized on the training split and a logistic regression with balanced class weights is fit,", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S3.E10", "label": "(10)", "section": "III Problem Formulation", "role": "metric", "latex": "\\mathrm{Acc}_{\\tau}=\\mathbb{1}[\\hat{\\tau}=\\tau],\\qquad\\mathrm{IoU}_{\\mathsf{H}}=\\frac{|\\hat{S}_{\\mathsf{H}}\\cap S_{\\mathsf{H}}|}{|\\hat{S}_{\\mathsf{H}}\\cup S_{\\mathsf{H}}|},", "mathml": "Accτ=𝟙[τ^=τ],IoU𝖧=|S^𝖧∩S𝖧||S^𝖧∪S𝖧|,", "context": "The labels support three tasks of increasing difficulty. Detection predicts y⁡(𝒯) from the trajectory and is scored with precision, recall, and F1 on the attacked class, with benign, hard-negative, and failed-attack trajectories all counted as negatives; per-category false-positive rates should be reported separately because the three negative classes fail for different reasons. Localization predicts the injection index of Eq. (8) and the corrupted span. With predicted labels ℓ^, we define localization accuracy and hijacked-span overlap as", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" } ], "equations_status": "selected", "core_kind": "formula", "equations_fetched_at": "2026-09-27", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }