{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.17966", "title": "BELT : BEhavior-Aligned Low-Rank KV Cache Compression with Learned Top-k Rank Allocation", "summary": "BELT replaces the full-rank KV cache in a pretrained LLM with a low-rank factorized cache, where a learned allocator distributes a fixed component budget across layers and heads, and where the compression objective matches post-softmax attention distributions rather than raw keys. Adoption costs an offline training/calibration phase for the low-rank factors and rank allocator, integration of custom Triton kernels (or a custom attention path) into the inference stack, and additional engineering complexity around combining low-rank storage with quantization. It can fail when the learned rank allocation does not transfer to domains or sequence lengths outside its training distribution, when real long-context workloads lose the reported LongBench quality margin at aggressive budgets, or when kernel and serving-stack integration overhead erodes the reported throughput gains on the reader's single-GPU setup.", "rationale": "KV cache compression directly addresses the dominant memory and decoding-throughput bottleneck on a single 24 GB GPU, but the method requires model-specific training, kernel integration, and workload-specific quality validation before production adoption.", "technique": "Low-rank KV cache compression with learned per-component rank allocation", "family": "cache_kv", "recommendation": "testar", "published": "2026-10-13", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.17966", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "BELT replaces the full-rank KV cache in a pretrained LLM with a low-rank factorized cache, where a learned allocator distributes a fixed component budget across layers and heads, and where the compression objective matches post-softmax attention distributions rather than raw keys. Adoption costs an offline training/calibration phase for the low-rank factors and rank allocator, integration of custom Triton kernels (or a custom attention path) into the inference stack, and additional engineering complexity around combining low-rank storage with quantization. It can fail when the learned rank allocation does not transfer to domains or sequence lengths outside its training distribution, when real long-context workloads lose the reported LongBench quality margin at aggressive budgets, or when kernel and serving-stack integration overhead erodes the reported throughput gains on the reader's single-GPU setup.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.17966", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Up to 70% low-rank KV cache compression (approximately 3.3x) and up to 17.8x total KV cache reduction when combined with quantization, while outperforming the strongest baseline by up to 23.3 points on LongBench; with Triton kernels, up to 2.7x attention-module speedup and 3.3x end-to-end decoding throughput.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.17966", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }