{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.22209", "title": "Beyond Downstream QA: Rethinking How We Evaluate Agent Memory", "summary": "Replaces downstream QA accuracy as the sole proxy for agent memory quality with direct inspection of the memory store against known fact-evolution trajectories (FEB-30), generated synthetically with deterministic ground truth. Costs are modest: it is an evaluation methodology requiring integration with the target memory system's store inspection, not a model or infrastructure change, and fits constrained infrastructure. It can fail to transfer because 30 synthetic trajectories across 6 categories may not represent a production workload's fact distributions, and results obtained on one representative memory system may not generalize to other architectures.", "rationale": "Teams operating memory-augmented agents under constrained infrastructure need diagnostics that separate memory errors from retrieval and generation errors, and this small reproducible benchmark can be run cheaply, but its synthetic scope requires validation against the reader's own workload before adoption.", "technique": "Intrinsic fact-evolution benchmark for agent memory", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-15", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.22209", "independent_implementations": 1, "total_implementations": 2, "stars": 132, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "Replaces downstream QA accuracy as the sole proxy for agent memory quality with direct inspection of the memory store against known fact-evolution trajectories (FEB-30), generated synthetically with deterministic ground truth. Costs are modest: it is an evaluation methodology requiring integration with the target memory system's store inspection, not a model or infrastructure change, and fits constrained infrastructure. It can fail to transfer because 30 synthetic trajectories across 6 categories may not represent a production workload's fact distributions, and results obtained on one representative memory system may not generalize to other architectures.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.22209", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Intrinsic and extrinsic memory evaluation diverge by an average of 16 percentage points on a representative system; correct memory-operation classification occurs only 20% of the time, invisible to downstream QA accuracy.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.22209", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }