{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.03020", "title": "DyadMem: A Long-Term Memory Benchmark of How Agents Work with Users", "summary": "DyadMem replaces final-answer-only evaluation of long-term agent memory with a full-pipeline benchmark that separately scores Capture, Update, Recall, and QA over 3,065 episodes and 61,210 QA instances, including relationship-specific agent memory (URAM) absent from prior benchmarks. Adoption costs annotation-format alignment and evaluation compute rather than training; running the full pipeline against 50,961 sessions requires API budget or local inference time on the 24 GB GPU. It can fail as a decision tool if the reader's production memory workload differs from the benchmark's dyadic multi-session format, and its finding of low capture recall and unsafe deletion in frontier models means a passing score does not guarantee safe memory write/delete behavior in deployment.", "rationale": "The reader building or selecting a long-term memory layer can use this benchmark to validate Capture/Recall/Update behavior before production, but it is an evaluation asset rather than a deployable technique, so workload-specific testing is required.", "technique": "Dual-domain full-pipeline agent memory benchmark (URAM)", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-02", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.03020", "independent_implementations": 1, "total_implementations": 2, "stars": 131, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "DyadMem replaces final-answer-only evaluation of long-term agent memory with a full-pipeline benchmark that separately scores Capture, Update, Recall, and QA over 3,065 episodes and 61,210 QA instances, including relationship-specific agent memory (URAM) absent from prior benchmarks. Adoption costs annotation-format alignment and evaluation compute rather than training; running the full pipeline against 50,961 sessions requires API budget or local inference time on the 24 GB GPU. It can fail as a decision tool if the reader's production memory workload differs from the benchmark's dyadic multi-session format, and its finding of low capture recall and unsafe deletion in frontier models means a passing score does not guarantee safe memory write/delete behavior in deployment.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.03020", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Gold-Memory QA is consistently strong across 16 open-weight and 4 proprietary models while Full-Pipeline QA drops sharply; adding URAM annotations yields positive effects for all 20 models, with no multiplicative factor reported.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.03020", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }