{ "schema_version": 1, "builder_plan": { "scenario": "A support or data assistant repeats work across related customer requests.", "change": "Try the paper's memory change in a replay of past tasks, with separate memory per customer.", "baseline": "Your current memory pipeline plus a no-memory control using the same agent.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Task success, stale memories, retrieval misses, response latency, and all memory maintenance costs.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.24259", "title": "MemCalib: Benchmarking and Optimizing Memory Use in LLM Agents", "summary": "MemCalib-RL replaces generic post-training (GRPO, on-policy self-distillation) for memory-use behavior with an ordered bidirectional counterfactual credit-assignment algorithm that separates over-use and under-use signals and localizes credit via exact atom ablation. It costs a full RL training run on 8B to 35B-parameter models, which exceeds a single 24 GB GPU and requires counterfactual ablation infrastructure and training data; API-only users cannot apply it. After adoption it can fail if the calibrated behavior does not transfer to the deployed memory system or if the residual directional skew between over-use and under-use persists on out-of-distribution tasks.", "rationale": "The core contribution requires RL post-training of mid-scale models that is infeasible on the reader's constrained infrastructure, so the practical value lies in tracking the MemCalib benchmark as an evaluation tool rather than adopting the training method now.", "technique": "Bidirectional counterfactual RL for calibrated agent memory use", "family": "memoria_e_contexto", "recommendation": "observar", "published": "2026-09-21", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.24259", "independent_implementations": 0, "total_implementations": 1, "stars": 3, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "MemCalib-RL replaces generic post-training (GRPO, on-policy self-distillation) for memory-use behavior with an ordered bidirectional counterfactual credit-assignment algorithm that separates over-use and under-use signals and localizes credit via exact atom ablation. It costs a full RL training run on 8B to 35B-parameter models, which exceeds a single 24 GB GPU and requires counterfactual ablation infrastructure and training data; API-only users cannot apply it. After adoption it can fail if the calibrated behavior does not transfer to the deployed memory system or if the residual directional skew between over-use and under-use persists on out-of-distribution tasks.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.24259", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "MemCalib-RL achieves the best overall performance on MemCalib across Qwen3-8B, Ministral-3-8B, and Qwen3.5-35B-A3B while better balancing over-use and under-use of memory, with gains generalizing to external benchmarks; no multiplicative factor is reported in the abstract.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.24259", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }