{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.16161", "title": "LLM Inference in a Flash!", "summary": "The method replaces floating-point inference and a conventional high-bandwidth-memory KV cache with integer-only execution and a sparse dictionary coding of KV vectors, targeted at Compute-in-Flash SSD-class devices. It costs model-specific integer quantization engineering, a learned or calibrated static dictionary per model, and some accuracy degradation, plus dependence on prototype near-memory hardware. For the reader it can fail outright because Compute-in-Flash devices are not commercially available on a single 24 GB GPU or through third-party APIs, and the compression gains may not transfer to standard GPU serving stacks.", "rationale": "The results depend on specialized Compute-in-Flash hardware that the reader cannot provision, so the work is worth monitoring but cannot be adopted or meaningfully tested on existing infrastructure.", "technique": "Integer-only inference with dictionary-based KV cache compression on Compute-in-Flash", "family": "cache_kv", "recommendation": "observar", "published": "2026-09-14", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.16161", "independent_implementations": 0, "total_implementations": 1, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The method replaces floating-point inference and a conventional high-bandwidth-memory KV cache with integer-only execution and a sparse dictionary coding of KV vectors, targeted at Compute-in-Flash SSD-class devices. It costs model-specific integer quantization engineering, a learned or calibrated static dictionary per model, and some accuracy degradation, plus dependence on prototype near-memory hardware. For the reader it can fail outright because Compute-in-Flash devices are not commercially available on a single 24 GB GPU or through third-party APIs, and the compression gains may not transfer to standard GPU serving stacks.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.16161", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Reduces dynamic KV cache traffic by 15x with limited accuracy degradation on Llama-3.1-8B and Qwen-2.5-7B, in an end-to-end integer-only formulation.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.16161", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [ { "anchor": "S3.E1", "label": "(1)", "section": "3 Algorithm", "role": "proposed_method", "latex": "\\mathrm{Score}(i)=Q\\cdot\\Big(\\sum_{j=1}^{K_{\\mathrm{low}}}\\gamma_{i,j}\\,\\mathbf{d}_{\\mathrm{idx}_{i,j}}\\Big).", "mathml": "Score⁡(i)=Q⋅(∑j=1Klowγi,j​𝐝idxi,j).", "context": "Concretely, we form a coarse key approximation for each token i using only its first Klow dictionary atoms and compute a proxy attention logit:", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" } ], "equations_status": "selected", "core_kind": "formula", "equations_fetched_at": "2026-10-11", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }