{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.01833", "title": "Continuous Process-Level Evaluation for Evolving Enterprise AI Agent Skills", "summary": "It replaces final-output-only evaluation of agent skills with continuous trajectory checks covering tool selection, arguments, execution order, and database integrity, plus a narrowly scoped LLM judge. The cost is engineering effort to compute per-run ground truth, materialize runtime-resolved template tests, and maintain programmatic checks per skill, plus modest judge inference cost. It can fail because templates and checks are skill-specific and may lag actual API evolution (longitudinal validation remains future work), the LLM judge can misclassify deviations, and specification sensitivity varies by model and harness, so results may not transfer across stacks.", "rationale": "The finding that over 92% of output-passing runs still exhibit process drift justifies adding trajectory checks to any production agent, but the per-skill template engineering and unvalidated behavior under real API evolution mean the reader should pilot it on one critical skill before committing.", "technique": "Continuous process-level evaluation of agent trajectories", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-01", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.01833", "independent_implementations": 0, "total_implementations": 1, "stars": 10, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "It replaces final-output-only evaluation of agent skills with continuous trajectory checks covering tool selection, arguments, execution order, and database integrity, plus a narrowly scoped LLM judge. The cost is engineering effort to compute per-run ground truth, materialize runtime-resolved template tests, and maintain programmatic checks per skill, plus modest judge inference cost. It can fail because templates and checks are skill-specific and may lag actual API evolution (longitudinal validation remains future work), the LLM judge can misclassify deviations, and specification sensitivity varies by model and harness, so results may not transfer across stacks.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.01833", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Of 175 trials passing all applicable final numerical checks, 162 (92.6%; Wilson 95% CI 87.7-95.6%) contained another evaluator-detected process deviation; dependency attribution reduced mean failed checks per run from 6.34 to 2.65 roots.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.01833", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }