{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.03027", "title": "Tailoring the Quantization Space for 1-Bit KV Cache Compression", "summary": "TaSQ replaces BF16 KV cache storage with 1-bit vector quantization applied to a transformed space (query-guided channel weighting, cross-head normalization, covariance-aware grouping) that is folded into projection weights and codebooks, keeping the standard VQ lookup path. The cost is an offline transformation and codebook construction step plus dependence on a specific SGLang implementation; reported serving overhead is negligible and gains were demonstrated on one RTX 6000 Ada. Failure modes include quality degradation on workloads whose activation statistics differ from the calibration assumptions, benchmark results that may not transfer to other models or serving stacks, and integration risk if the reader relies on a different inference engine.", "rationale": "The memory and throughput gains directly address the reader's single-GPU constraint, but the method requires validating calibration quality and SGLang compatibility on their specific models before production adoption.", "technique": "Query-guided 1-bit VQ for KV cache", "family": "cache_kv", "recommendation": "testar", "published": "2026-10-02", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.03027", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "TaSQ replaces BF16 KV cache storage with 1-bit vector quantization applied to a transformed space (query-guided channel weighting, cross-head normalization, covariance-aware grouping) that is folded into projection weights and codebooks, keeping the standard VQ lookup path. The cost is an offline transformation and codebook construction step plus dependence on a specific SGLang implementation; reported serving overhead is negligible and gains were demonstrated on one RTX 6000 Ada. Failure modes include quality degradation on workloads whose activation statistics differ from the calibration assumptions, benchmark results that may not transfer to other models or serving stacks, and integration risk if the reader relies on a different inference engine.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.03027", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "On a single RTX 6000 Ada, the SGLang implementation supports up to 14x larger batch sizes and achieves 1.87x higher peak throughput versus the BF16 baseline, while outperforming prior low-bit VQ baselines on reasoning and long-context benchmarks.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.03027", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }