{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.27746", "title": "The KV Cache Working Set: Online Capacity Planning for LLM Inference Systems", "summary": "KVSET replaces brute-force capacity-by-capacity simulation of KV prefix caches with a single-pass Mattson stack analysis that computes hit rates across all candidate capacities from LRU stack distances. The cost is modest: it runs as an online analyzer over request traces with low compute and memory overhead, and an open-source implementation supports both live traffic and offline replay, but it presumes an existing serving stack with paged KV caching and trace instrumentation. It can fail when production traffic drifts from observed traces, when workloads lack the prefix reuse patterns the analysis assumes, or when eviction policy in the deployed cache deviates from the LRU model the stack algorithm encodes.", "rationale": "Working-set capacity planning is a real concern for multi-tenant or agentic serving, but on a single 24 GB GPU the cache capacity decision is trivial and the tool only pays off at serving scales the reader does not operate.", "technique": "Online KV cache working-set estimation via Mattson stack algorithm", "family": "cache_kv", "recommendation": "observar", "published": "2026-09-23", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.27746", "independent_implementations": 0, "total_implementations": 1, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "KVSET replaces brute-force capacity-by-capacity simulation of KV prefix caches with a single-pass Mattson stack analysis that computes hit rates across all candidate capacities from LRU stack distances. The cost is modest: it runs as an online analyzer over request traces with low compute and memory overhead, and an open-source implementation supports both live traffic and offline replay, but it presumes an existing serving stack with paged KV caching and trace instrumentation. It can fail when production traffic drifts from observed traces, when workloads lack the prefix reuse patterns the analysis assumes, or when eviction policy in the deployed cache deviates from the LRU model the stack algorithm encodes.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.27746", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "The paper claims KVSET estimates closely match measurements from real cache deployments and that it substantially reduces the computational and memory overhead of capacity-by-capacity simulation, but reports no quantified multiplicative improvement factor in the abstract.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.27746", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }