{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.11245", "title": "Read What Matters: Query-Adaptive Quantization for KV Caches", "summary": "ReadKV replaces uniform per-token KV-cache quantization (fixed-bit stored-and-read schemes such as 4-bit caches or TurboQuant-style codecs) with a progressive code: more bits are retained than are fetched per query, and each decoding query adaptively selects which key-channel and value-token prefixes to read. The cost is retained cache capacity roughly double the payload actually read (eight stored bits versus four read), plus added per-query allocation logic and calibration of distortion objectives, which increases implementation complexity over a static codec. It can fail if the per-query prefix-selection overhead outweighs read savings at small batch or short context, if real kernels do not expose the benchmarked latency gains outside the tested single-layer A10G setup, or if attention-guided allocation misjudges important entries on out-of-distribution long-context workloads.", "rationale": "The method directly targets the dominant cost in long-context decoding on a single GPU—cache bytes moved per step—with negligible perplexity impact, but the latency evidence is narrow (one workload, batch one, single layer), so the reader should validate it on their own serving stack before committing to production.", "technique": "Query-adaptive progressive KV-cache quantization", "family": "cache_kv", "recommendation": "testar", "published": "2026-10-08", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.11245", "independent_implementations": 0, "total_implementations": 1, "stars": 4, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "ReadKV replaces uniform per-token KV-cache quantization (fixed-bit stored-and-read schemes such as 4-bit caches or TurboQuant-style codecs) with a progressive code: more bits are retained than are fetched per query, and each decoding query adaptively selects which key-channel and value-token prefixes to read. The cost is retained cache capacity roughly double the payload actually read (eight stored bits versus four read), plus added per-query allocation logic and calibration of distortion objectives, which increases implementation complexity over a static codec. It can fail if the per-query prefix-selection overhead outweighs read savings at small batch or short context, if real kernels do not expose the benchmarked latency gains outside the tested single-layer A10G setup, or if attention-guided allocation misjudges important entries on out-of-distribution long-context workloads.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.11245", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "On an 8K-token, batch-one, single-layer A10G workload, an eight-bit ReadKV reader with a two-bit mean payload-read budget has 39% lower latency than the tested TurboQuant codec; reading four bits on average from an eight-bit cache raises C4 perplexity by at most 0.66% across six models.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.11245", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }