{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2610.30491",
  "title": "What the Trace Shows and What the Label Says: Auditing Behavioral Detectors in Agent Evaluation",
  "summary": "The practice replaces unqualified trace-level metrics (final correctness, re-verification, fault propagation) with labels accompanied by the underlying trace evidence, applicability flags, execution-limit status, and human corrections, exposed in a trace viewer. It costs engineering effort to instrument detectors and a viewer, plus storage and review overhead, with negligible inference cost since it operates on existing traces. It can fail if detector parsers disagree with human judgment in unanticipated patterns (the paper found 14 verification disagreements in four recurring classes), or if reported comparisons ignore step/token budgets, which the ablation shows can swing results by up to 6 of 30 problems.",
  "rationale": "Any team running LLM-agent evaluations already produces these labels, and attaching trace evidence and execution-limit metadata is low-cost instrumentation that directly prevents the documented misinterpretation of verification and accuracy numbers.",
  "technique": "Trace-evidence auditing of agent evaluation labels",
  "family": "avaliacao_de_agente",
  "recommendation": "adotar",
  "published": "2026-10-21",
  "as_of": "2026-10-24",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2610.30491",
  "independent_implementations": 3,
  "total_implementations": 4,
  "stars": 14,
  "citations": 0,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "The practice replaces unqualified trace-level metrics (final correctness, re-verification, fault propagation) with labels accompanied by the underlying trace evidence, applicability flags, execution-limit status, and human corrections, exposed in a trace viewer. It costs engineering effort to instrument detectors and a viewer, plus storage and review overhead, with negligible inference cost since it operates on existing traces. It can fail if detector parsers disagree with human judgment in unanticipated patterns (the paper found 14 verification disagreements in four recurring classes), or if reported comparisons ignore step/token budgets, which the ablation shows can swing results by up to 6 of 30 problems.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.30491",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "Human review of 90 traces agreed with automated labels for final correctness (67/67), fault impact (19/19), and fault relevance (18/18), but only 67/81 for verification; execution-limit ablations shifted reported accuracy from 21/30 to 27/30 for one model.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.30491",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}