{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2608.08097", "title": "OasisKV: Scaling In-Decode KV Cache Beyond HBM with Lookahead Sparse Prefetching", "summary": "OasisKV replaces storing the full KV cache in HBM with a sparse KV budget (e.g., 2,048 tokens) whose blocks are prefetched from host or remote memory, using speculative-decoding lookahead tokens to predict which blocks matter next. The cost is integration with a modified vLLM serving stack, dependence on speculative decoding as a predictor, added prefetch pipeline complexity, and a small accuracy loss (under 0.7 points at the reported budget). It can fail when lookahead prediction degrades on workloads unlike the evaluated reasoning tasks, when PCIe or network bandwidth cannot hide prefetch latency, or when the team lacks capacity to maintain a custom vLLM fork.", "rationale": "The 24 GB GPU memory ceiling is exactly the bottleneck this addresses, but the gain depends on workload-specific sparsity and requires a modified serving stack, so it warrants a controlled pilot rather than immediate adoption.", "technique": "KV cache offloading with lookahead prefetch", "family": "cache_kv", "recommendation": "testar", "published": "2026-08-08", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2608.08097", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "OasisKV replaces storing the full KV cache in HBM with a sparse KV budget (e.g., 2,048 tokens) whose blocks are prefetched from host or remote memory, using speculative-decoding lookahead tokens to predict which blocks matter next. The cost is integration with a modified vLLM serving stack, dependence on speculative decoding as a predictor, added prefetch pipeline complexity, and a small accuracy loss (under 0.7 points at the reported budget). It can fail when lookahead prediction degrades on workloads unlike the evaluated reasoning tasks, when PCIe or network bandwidth cannot hide prefetch latency, or when the team lacks capacity to maintain a custom vLLM fork.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2608.08097", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "1.69x throughput over dense vLLM on reasoning workloads with 0.1 accuracy points loss; up to 2.1x on multi-GPU long-context serving; about 2x dense throughput under prefill-decode disaggregation with 6.5-9.7x less KV admitted per request.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2608.08097", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "selector_failed", "core_kind": "", "equations_fetched_at": "2026-09-17", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }