{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.06063", "title": "Explaining AI Agents Through Execution Traces", "summary": "The method replaces naive LLM-generated explanations and ad hoc manual log review by converting an agent's execution trace into a structured report with natural-language claims explicitly grounded in observable trace events. It costs an additional post-processing pass (typically another LLM call per trace), engineering effort to instrument and persist complete traces, and added latency only in offline analysis, not in the agent's inference path. It can fail when traces are incomplete or lack tool-call inputs and outputs, since grounding fidelity degrades to whatever the trace records, and its detection of unsupported claims depends on the evaluator model's reliability rather than formal verification.", "rationale": "Trace-grounded post-hoc explanation is implementable on constrained infrastructure and directly useful for auditing production agents, but its value depends on the team's trace instrumentation quality and must be validated on their specific agent workload before adoption.", "technique": "Post-hoc trace-grounded agent explanation reports", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-09-05", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.06063", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The method replaces naive LLM-generated explanations and ad hoc manual log review by converting an agent's execution trace into a structured report with natural-language claims explicitly grounded in observable trace events. It costs an additional post-processing pass (typically another LLM call per trace), engineering effort to instrument and persist complete traces, and added latency only in offline analysis, not in the agent's inference path. It can fail when traces are incomplete or lack tool-call inputs and outputs, since grounding fidelity degrades to whatever the trace records, and its detection of unsupported claims depends on the evaluator model's reliability rather than formal verification.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.06063", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [ { "anchor": "S4.E1", "label": "(1)", "section": "4 Experiments", "role": "metric", "latex": "\\mathrm{NFI}=\\frac{\\sum_{t}\\mathrm{matched}\\!\\left(R_{t}^{gen},R_{t}^{GT}\\right)}{\\sum_{t}|R_{t}^{GT}|},", "mathml": "NFI=∑tmatched⁡(Rtg​e​n,RtG​T)∑t|RtG​T|,", "context": "", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S3.Ex7", "label": "", "section": "3 Trace-Grounded XAI Methodology", "role": "proposed_method", "latex": "\\mathcal{X}(\\tau)\\;=\\;\\Bigl(\\,G,\\;\\bigl\\{\\bigl(a_{t},\\,\\mathcal{O}_{t},\\,\\Sigma_{t},\\,\\delta_{t},\\,\\Gamma_{t}\\bigr)\\bigr\\}_{t=0}^{T}\\,\\Bigr),", "mathml": "𝒳⁡(τ)=(G,{(at,𝒪t,Σt,δt,Γt)}t=0T),", "context": "Aggregating across the trajectory, the framework’s first XAI artifact is the structured report", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S3.Ex8", "label": "", "section": "3 Trace-Grounded XAI Methodology", "role": "proposed_method", "latex": "\\psi:\\;\\mathcal{X}(\\tau)\\;\\longrightarrow\\;\\bigl(E_{1},\\,E_{2},\\,\\ldots,\\,E_{T},\\,\\bar{E}\\bigr),", "mathml": "ψ:𝒳⁡(τ)⟶(E1,E2,…,ET,E¯),", "context": "The second XAI artifact is then produced by a constrained narrator", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" } ], "equations_status": "selected", "core_kind": "formula", "equations_fetched_at": "2026-09-27", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }