{ "schema_version": 1, "builder_plan": { "scenario": "Tool errors or unavailable services cause your agent to abandon recoverable tasks.", "change": "Inject repeatable tool failures and change only the recovery policy.", "baseline": "Your current retry and fallback behavior under the same injected failures.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Recovered tasks, duplicate side effects, retries, time to recovery, and total cost.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.12375", "title": "OnTrack: Real-Time Monitoring and Intervention in LLM Agent Trajectories via Streaming Structure-Aware Optimal Transport", "summary": "OnTrack replaces per-step safeguard-agent calls (which add cost and latency to every action) and post-hoc log evaluation (which only reports after tokens are spent) with a streaming structure-aware optimal-transport comparison of live steps against reference trajectories or tool schemas, at roughly one millisecond per step. The cost is maintaining reference runs or tool schemas, integrating a blocking/alerting hook into the agent loop, and accepting degraded capability when access drops to raw logs only (loop, stall, and repeat detection rather than plan-violation detection). It can fail through false aborts on valid but atypical trajectories, missed failures when the reference set does not cover the current task distribution, and the approximately 17% incorrect-intervention rate observed in the reported abort policy.", "rationale": "The method runs in milliseconds on CPU alongside a single-GPU or API-based agent and directly addresses wasted compute on failing runs, but its 18% saving and AUROC gain come from one benchmark and must be validated on the reader's own trajectory distribution before production adoption.", "technique": "Streaming OT-based agent trajectory monitoring", "family": "recuperacao_de_falha", "recommendation": "testar", "published": "2026-10-08", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.12375", "independent_implementations": 0, "total_implementations": 1, "stars": 10, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "OnTrack replaces per-step safeguard-agent calls (which add cost and latency to every action) and post-hoc log evaluation (which only reports after tokens are spent) with a streaming structure-aware optimal-transport comparison of live steps against reference trajectories or tool schemas, at roughly one millisecond per step. The cost is maintaining reference runs or tool schemas, integrating a blocking/alerting hook into the agent loop, and accepting degraded capability when access drops to raw logs only (loop, stall, and repeat detection rather than plan-violation detection). It can fail through false aborts on valid but atypical trajectories, missed failures when the reference set does not cover the current task distribution, and the approximately 17% incorrect-intervention rate observed in the reported abort policy.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.12375", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Aborting runs flagged as failing saves about 18% of compute on SWE-bench trajectories, with 5 of 6 aborts (83%) being correct; ranking failing below succeeding trajectories improves by +0.057 AUROC over content-similarity baselines using the first 8 steps.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.12375", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }