{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.29913", "title": "MILO: Efficient Many-shot In-Context Learning with Block-wise Low-rank Compression", "summary": "MILO replaces storing the full many-shot KV cache with block-wise low-rank compressed representations, where rank budgets are allocated per block by information entropy. It costs implementation complexity for block-level compression and dynamic rank allocation, plus some approximation error in retained blocks, and it applies only when the workload uses long many-shot prompts. It can fail if the workload does not use thousands of demonstrations (making the compression overhead pointless), if entropy-based rank allocation misidentifies critical blocks and degrades accuracy, or if results on Qwen2.5 do not transfer to the reader's chosen model or to API-served models where the KV cache is not user-controllable.", "rationale": "The 2x memory and 1.8x throughput gains directly address a 24 GB GPU constraint, but the benefit exists only for many-shot ICL workloads and requires custom inference-stack modifications that are impossible with third-party APIs, so it warrants a targeted pilot rather than outright adoption.", "technique": "Block-wise low-rank KV cache compression", "family": "cache_kv", "recommendation": "testar", "published": "2026-09-24", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.29913", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "MILO replaces storing the full many-shot KV cache with block-wise low-rank compressed representations, where rank budgets are allocated per block by information entropy. It costs implementation complexity for block-level compression and dynamic rank allocation, plus some approximation error in retained blocks, and it applies only when the workload uses long many-shot prompts. It can fail if the workload does not use thousands of demonstrations (making the compression overhead pointless), if entropy-based rank allocation misidentifies critical blocks and degrades accuracy, or if results on Qwen2.5 do not transfer to the reader's chosen model or to API-served models where the KV cache is not user-controllable.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.29913", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Up to 50% reduction in KV cache memory and 1.8x throughput improvement on Qwen2.5 models, with negligible degradation on classification and reasoning benchmarks.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.29913", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }