{ "schema_version": 1, "builder_plan": { "scenario": "Profiling shows attention computation is a bottleneck in your self-hosted service.", "change": "Check the supported GPU, tensor shapes, and precision, then replace only the relevant kernel.", "baseline": "The existing kernel inside the same complete serving pipeline.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Numerical differences, end-to-end latency, throughput, and memory at realistic shapes.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.20734", "title": "On-Demand Attention: Language Models Know When to Recall", "summary": "ODA replaces full-attention decoding, where every generated token reads the entire KV history, with local-first decoding that invokes global attention only when a lightweight recall head predicts it will help. The cost is training a recall head per model, maintaining the full KV cache in memory (so no memory savings), and adopting a modified vLLM build with GPU-side conditional execution. It can fail when the recall head mispredicts the benefit of global reads on out-of-distribution workloads, silently degrading quality while the speedup depends on context length being long enough for skipped reads to dominate.", "rationale": "The method fits a single-GPU setup since pretrained weights stay frozen and only a small head is trained, but the gain is workload- and context-length-dependent and requires custom serving code, so it warrants validation on the reader's own long-context traffic before production adoption.", "technique": "Selective global attention via learned recall head", "family": "kernels_e_atencao", "recommendation": "testar", "published": "2026-09-17", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.20734", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "ODA replaces full-attention decoding, where every generated token reads the entire KV history, with local-first decoding that invokes global attention only when a lightweight recall head predicts it will help. The cost is training a recall head per model, maintaining the full KV cache in memory (so no memory savings), and adopting a modified vLLM build with GPU-side conditional execution. It can fail when the recall head mispredicts the benefit of global reads on out-of-distribution workloads, silently degrading quality while the speedup depends on context length being long enough for skipped reads to dominate.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.20734", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Recovers most of the performance lost under local attention while substantially reducing global reads, with practical decoding speedups over full attention at long context lengths; no specific factor reported in the abstract.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.20734", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }