{ "schema_version": 1, "builder_plan": { "scenario": "A support or data assistant repeats work across related customer requests.", "change": "Try the paper's memory change in a replay of past tasks, with separate memory per customer.", "baseline": "Your current memory pipeline plus a no-memory control using the same agent.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Task success, stale memories, retrieval misses, response latency, and all memory maintenance costs.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2608.22793", "title": "TRACE: A Self-Evolving Skill Bank for Consistent, Limit-Aware LLM Agents", "summary": "TRACE replaces static hand-written system prompts and agent policies with a Skill Bank of modular, retrievable tool-use rules that is iteratively refined by contrasting successful and failed trajectories, without modifying model weights. It costs an offline evolution loop of repeated evaluation rounds and trajectory analysis (API tokens and engineering effort), plus per-turn retrieval and state-conditioned skill orchestration that add latency and prompt complexity at deployment. It can fail when the contrastive refinement overfits to evaluation trajectories, when skills conflict or are mis-retrieved for out-of-distribution requests, and its gains are demonstrated only on a single in-car assistant benchmark with specific frontier models, so transfer to other domains is unverified.", "rationale": "The weight-free, API-compatible design fits constrained infrastructure and the consistency problem is real in production agents, but the evidence is single-domain and benchmark-specific, so a small-scale pilot on the reader's own tool-use workload is warranted before adoption.", "technique": "Self-evolving retrievable skill bank for agents", "family": "memoria_e_contexto", "recommendation": "testar", "published": "2026-08-24", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2608.22793", "independent_implementations": 0, "total_implementations": 1, "stars": 9, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "TRACE replaces static hand-written system prompts and agent policies with a Skill Bank of modular, retrievable tool-use rules that is iteratively refined by contrasting successful and failed trajectories, without modifying model weights. It costs an offline evolution loop of repeated evaluation rounds and trajectory analysis (API tokens and engineering effort), plus per-turn retrieval and state-conditioned skill orchestration that add latency and prompt complexity at deployment. It can fail when the contrastive refinement overfits to evaluation trajectories, when skills conflict or are mis-retrieved for out-of-distribution requests, and its gains are demonstrated only on a single in-car assistant benchmark with specific frontier models, so transfer to other domains is unverified.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2608.22793", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "On CAR-bench with GPT-5.5, TRACE raises Pass^3 consistency from 59.9% to 94.5% (+34.6 points); on the hidden set it reaches 70% Pass^3 with GPT-5.6-Sol, a 40% relative gain over baseline.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2608.22793", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [ { "anchor": "S3.E1", "label": "(1)", "section": "3 Experiments", "role": "metric", "latex": "\\text{Pass\\textasciicircum{}}k=\\frac{1}{|\\mathcal{C}|}\\sum_{c\\in\\mathcal{C}}\\frac{1}{T_{c}}\\sum_{t\\in c}\\mathbf{1}\\!\\left[\\sum_{j=1}^{k}r_{t}^{(j)}=k\\right],", "mathml": "Pass^k=1|𝒞|∑c∈𝒞1Tc∑t∈c[∑j=1krt(j)=k],", "context": "Metrics. To measure consistency directly rather than incidentally, each task is run k times to compute two metrics:", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S3.E2", "label": "(2)", "section": "3 Experiments", "role": "metric", "latex": "\\text{Pass}@k=\\frac{1}{|\\mathcal{C}|}\\sum_{c\\in\\mathcal{C}}\\frac{1}{T_{c}}\\sum_{t\\in c}\\mathbf{1}\\!\\left[\\sum_{j=1}^{k}r_{t}^{(j)}\\geq 1\\right],", "mathml": "Pass@k=1|𝒞|∑c∈𝒞1Tc∑t∈c[∑j=1krt(j)≥1],", "context": "Metrics. To measure consistency directly rather than incidentally, each task is run k times to compute two metrics:", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" } ], "equations_status": "selected", "core_kind": "formula", "equations_fetched_at": "2026-09-19", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }