{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.07966", "title": "MetaKV: Adaptive KV Cache Compression for Constrained LLM Inference", "summary": "MetaKV replaces a single fixed KV cache compression configuration with per-prompt selection among existing methods (KVQuant, H2O, RocketKV, FP16) guided by lightweight predictors of latency, peak memory, and correctness. The cost is training and maintaining three prediction models plus a selection step per prompt, adding engineering complexity and a small inference-time overhead on top of the underlying compressors. It can fail if the accuracy or latency predictors mispredict on out-of-distribution prompts or on hardware and models different from the evaluation setup, causing constraint violations or suboptimal choices.", "rationale": "The framework sits on top of compression methods the reader can already run on a 24 GB GPU, but its benefit depends on predictor accuracy for the reader's specific model, hardware, and workload, so it warrants a small-scale validation before production adoption.", "technique": "Per-prompt adaptive KV cache compression selection", "family": "cache_kv", "recommendation": "testar", "published": "2026-09-07", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.07966", "independent_implementations": 2, "total_implementations": 2, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "MetaKV replaces a single fixed KV cache compression configuration with per-prompt selection among existing methods (KVQuant, H2O, RocketKV, FP16) guided by lightweight predictors of latency, peak memory, and correctness. The cost is training and maintaining three prediction models plus a selection step per prompt, adding engineering complexity and a small inference-time overhead on top of the underlying compressors. It can fail if the accuracy or latency predictors mispredict on out-of-distribution prompts or on hardware and models different from the evaluation setup, causing constraint violations or suboptimal choices.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.07966", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "MetaKV improves constrained success rate (fraction of prompts answered correctly within latency and memory budgets) by approximately 0.07 on average and up to 0.135 over the best static configuration, across four datasets and ten configurations drawn from KVQuant, H2O, RocketKV, and FP16.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.07966", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "no_equations", "core_kind": "", "equations_fetched_at": "2026-09-27", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }