{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.18140", "title": "SparPR: Runtime KV-Cache Recovery for Resource-Constrained LLM Inference on Consumer Hardware", "summary": "SparPR replaces static KV-cache eviction policies (e.g., StreamingLLM, H2O) and full offloading/reload in FlexGen-style constrained inference with runtime query-aware block selection, dynamic cache budgeting, and on-demand local recomputation with a cost-aware fallback to full recovery, requiring no weight changes or offline training. Its cost is runtime overhead for scoring, cost estimation, and selective recomputation, plus implementation complexity tied to the FlexGen/FlexLLMGen stack. It can fail if the runtime cost model mispredicts copy-versus-recompute tradeoffs on different hardware, if query-aware selection drops blocks needed by later tokens in workloads unlike the Needle benchmark, or if gains measured on an 8GB laptop do not transfer to a 24 GB production GPU serving different models and request patterns.", "rationale": "The method directly targets the reader's single-GPU constraint with quantified latency and throughput gains and no retraining prerequisite, but its validation on one consumer laptop and one serving framework means production value must be confirmed on the reader's own hardware, models, and workloads before adoption.", "technique": "Runtime KV-cache recovery via sparse partial recomputation", "family": "cache_kv", "recommendation": "testar", "published": "2026-10-13", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.18140", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "SparPR replaces static KV-cache eviction policies (e.g., StreamingLLM, H2O) and full offloading/reload in FlexGen-style constrained inference with runtime query-aware block selection, dynamic cache budgeting, and on-demand local recomputation with a cost-aware fallback to full recovery, requiring no weight changes or offline training. Its cost is runtime overhead for scoring, cost estimation, and selective recomputation, plus implementation complexity tied to the FlexGen/FlexLLMGen stack. It can fail if the runtime cost model mispredicts copy-versus-recompute tradeoffs on different hardware, if query-aware selection drops blocks needed by later tokens in workloads unlike the Needle benchmark, or if gains measured on an 8GB laptop do not transfer to a 24 GB production GPU serving different models and request patterns.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.18140", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Cuts end-to-end latency by 31%, 52%, and 63% at 1k, 2k, and 4k prompt tokens (63% latency reduction corresponds to ~2.7x speedup), with 1.4x-2.7x throughput, on a single 8GB consumer GPU versus a FlexGen baseline.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.18140", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }