{ "schema_version": 1, "builder_plan": { "scenario": "Profiling shows attention computation is a bottleneck in your self-hosted service.", "change": "Check the supported GPU, tensor shapes, and precision, then replace only the relevant kernel.", "baseline": "The existing kernel inside the same complete serving pipeline.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Numerical differences, end-to-end latency, throughput, and memory at realistic shapes.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.00097", "title": "Faster Than Flash: Exploiting Attention Sparsity for Efficient Long-Context Decoding", "summary": "It replaces metadata-indexed sparse-attention selection and the Flash Decoding kernel with a single fused kernel that performs content-aware block scanning via low-bit quantization (top-delta) and reuses the scan for computation. The cost is a custom fused-kernel integration and maintenance burden, extra low-bit scanning/block-filtering compute, and benefits that only materialize at genuinely long contexts that a 24 GB GPU can rarely host for larger models. It can fail when the quantized top-delta scan prunes blocks containing relevant tokens (degrading retrieval-heavy or sparsity-unfriendly inputs), when attention distributions are not sparse, or when the kernel does not support the target attention variant/hardware.", "rationale": "It is training-free, plug-and-play, and open-source, so it is testable on one GPU, but the 2.37x end-to-end gain is workload- and context-length-dependent and requires validating a custom kernel, justifying a scoped pilot rather than direct production adoption.", "technique": "Fused sparse-attention decoding kernel (FFD)", "family": "kernels_e_atencao", "recommendation": "testar", "published": "2026-08-31", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.00097", "independent_implementations": 0, "total_implementations": 1, "stars": 2, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "It replaces metadata-indexed sparse-attention selection and the Flash Decoding kernel with a single fused kernel that performs content-aware block scanning via low-bit quantization (top-delta) and reuses the scan for computation. The cost is a custom fused-kernel integration and maintenance burden, extra low-bit scanning/block-filtering compute, and benefits that only materialize at genuinely long contexts that a 24 GB GPU can rarely host for larger models. It can fail when the quantized top-delta scan prunes blocks containing relevant tokens (degrading retrieval-heavy or sparsity-unfriendly inputs), when attention distributions are not sparse, or when the kernel does not support the target attention variant/hardware.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.00097", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "The paper reports up to 11.6x kernel-level speedup and a 2.37x end-to-end throughput improvement at up to 256K context, with accuracy maintained on RULER and LongBench.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.00097", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [ { "anchor": "S3.E1", "label": "(1)", "section": "3 Method", "role": "proposed_method", "latex": "s_{ij}\\geq m_{i}-\\delta.", "mathml": "si​j≥mi−δ.", "context": "Let si​j=𝒒i⊤​𝒌jd be the pre-softmax attention score between query vector 𝒒i and key vector 𝒌j. Ideally, we aim to retain only those tokens whose attention weight contributes significantly to the distribution. We define a retention condition based on the relative magnitude δ compared to the maximum attention score mi=maxj⁡si​j, and thus only calculate the attention over the KV cache where", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S3.E3", "label": "(3)", "section": "3 Method", "role": "proposed_method", "latex": "\\tilde{m}_{i}=\\max{\\left(\\max_{t\\in S_{\\text{global}}}s_{it},\\max_{t\\in S_{\\text{local}}}s_{it}\\right)}.", "mathml": "m~i=max⁡(maxt∈Sglobal⁡si​t,maxt∈Slocal⁡si​t).", "context": "Implementing Eq. 1 strictly requires computing the global max mi, which incurs the same synchronization overhead as softmax. Fortunately, thanks to the phenomenon that attention distributions are typically dominated by either local context or initial sink tokens [16], we introduce the pseudo-max approximation m~i by estimating the global maximum using only these accessible subsets, sink tokens Sglobal and local context Slocal, as follows.", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S3.E4", "label": "(4)", "section": "3 Method", "role": "proposed_method", "latex": "s_{\\text{final}}=\\frac{\\bm{q}^{\\top}\\bm{k}^{\\text{quant}}+\\bm{q}^{\\top}\\bm{k}^{\\text{res}}}{\\sqrt{d}}", "mathml": "sfinal=𝒒⊤​𝒌quant+𝒒⊤​𝒌resd", "context": "The add-on computation in the refinement stage is shown as follows and ensures that for selected tokens, the attention score is computed with near-FP16 precision.", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" } ], "equations_status": "selected", "core_kind": "formula", "equations_fetched_at": "2026-09-24", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }