{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.13060", "title": "KV-Repurpose: A Testbed for Evaluating Non-Prefix KV-Cache Reuse", "summary": "Replaces exact-prefix KV-cache reuse with segment-wise reuse, allowing cached document blocks to be reused across different system prompts and conversation histories in a vLLM-based serving stack. Costs include adopting and maintaining a custom scheduling, prompt-construction, and capacity-management layer on top of vLLM, plus engineering effort to segment workloads correctly. Naive segment reuse largely preserves quality on single-document QA and summarization, but the paper reports clear degradation in cross-document reasoning, and the reported gains are demonstrated only under a specific Zipfian RAG workload, so real traffic may benefit far less.", "rationale": "The approach is directly relevant to RAG serving on constrained infrastructure and shows large TTFT gains, but it is an early testbed whose quality regressions and workload dependence require validation on the reader's own traffic before production adoption.", "technique": "Segment-wise KV-cache reuse beyond prefixes", "family": "cache_kv", "recommendation": "testar", "published": "2026-10-09", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.13060", "independent_implementations": 0, "total_implementations": 1, "stars": 122, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "Replaces exact-prefix KV-cache reuse with segment-wise reuse, allowing cached document blocks to be reused across different system prompts and conversation histories in a vLLM-based serving stack. Costs include adopting and maintaining a custom scheduling, prompt-construction, and capacity-management layer on top of vLLM, plus engineering effort to segment workloads correctly. Naive segment reuse largely preserves quality on single-document QA and summarization, but the paper reports clear degradation in cross-document reasoning, and the reported gains are demonstrated only under a specific Zipfian RAG workload, so real traffic may benefit far less.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.13060", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Under a Zipfian RAG workload with 20% of requests sharing a fixed prefix, token-level cache hit rate rises to 63.9% versus 3.3% for prefix caching at the same 30 GB capacity (prefix caching reaches only 17.9% even with 1 TB), resulting in lower TTFT.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.13060", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }