{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent reads external documents or tool responses that may contain malicious instructions.",
    "change": "Replay attacks and legitimate lookalikes in an isolated copy of your agent workflow.",
    "baseline": "Your existing checks with the same tasks, permissions, and tool responses.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Successful attacks missed, legitimate actions blocked, detection delay, and review cost.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2610.21309",
  "title": "Combining LLM and Dynamic Analysis for Universal Testing of Static Application Security Testing Tools",
  "summary": "It replaces manual or template-based metamorphic testing of static analysis tools with LLM-generated multi-language code transformations plus dynamic feedback and diff-based clustering. The cost is LLM inference for variant generation, execution infrastructure for dynamic feedback across languages, and the complexity of a report-centric differential oracle, all of which only pay off for teams that build or maintain SAST tools. False positives still reach roughly half of generated reports before clustering, the heuristics eliminate only 47.8% of them, and metamorphic oracles cannot catch faults where both seed and variant reports are identically wrong.",
  "rationale": "This is a toolsmithing technique for SAST vendors and security-tool QA teams, not a production practice for an ML engineer shipping models or applications on constrained infrastructure.",
  "technique": "LLM-driven metamorphic testing of SAST tools",
  "family": "seguranca_e_guardrails",
  "recommendation": "nao_aplica",
  "published": "2026-10-15",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2610.21309",
  "independent_implementations": 0,
  "total_implementations": 0,
  "stars": 0,
  "citations": 0,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "It replaces manual or template-based metamorphic testing of static analysis tools with LLM-generated multi-language code transformations plus dynamic feedback and diff-based clustering. The cost is LLM inference for variant generation, execution infrastructure for dynamic feedback across languages, and the complexity of a report-centric differential oracle, all of which only pay off for teams that build or maintain SAST tools. False positives still reach roughly half of generated reports before clustering, the heuristics eliminate only 47.8% of them, and metamorphic oracles cannot catch faults where both seed and variant reports are identically wrong.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.21309",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "Detects 3.4 times more unique faults in SAST tools than existing metamorphic-testing approaches while using only 39.8% of the generated variants; 66 unique faults found across six tools with 32 confirmed or fixed.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.21309",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}