{ "schema_version": 1, "builder_plan": { "scenario": "Long customer conversations exhaust memory or limit concurrent requests.", "change": "Change only the attention cache policy, after checking model and runtime support.", "baseline": "The same model with its ordinary attention cache.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak cache memory, long-context answer quality, first-token latency, and requests completed.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.25431", "title": "MOSAIC: Multi-Objective Search for Allocating Inference Cache Across LLM Decoder Layers", "summary": "MOSAIC replaces uniform per-layer KV-cache budgets (as used by SnapKV, H2O, AdaKV, L2Norm-style eviction) with a per-layer budget allocation found by one multi-objective search whose best allocation transfers across target budgets, optionally refined by a budget-constrained search. The cost is a one-time offline search per model and eviction method (reported as cheaper than EvolKV, but still multi-evaluation GPU work on a single 24 GB card for 7-8B models), plus serving-side plumbing to track and enforce per-layer budgets, with no runtime quality loss claimed at matched budgets. It can fail if the searched allocation does not transfer to models, tasks, context lengths, or eviction methods outside the validated Llama-3-8B / Mistral-7B and RULER/LongBench settings, or if the serving stack cannot apply heterogeneous per-layer budgets without kernel or scheduler changes.", "rationale": "The 4x memory reduction directly addresses the dominant constraint of single-GPU long-context serving, but the one-time search cost and unverified transfer to the reader's own model, eviction method, and workload make workload-specific validation necessary before production adoption.", "technique": "Per-layer KV-cache budget allocation via multi-objective search", "family": "cache_kv", "recommendation": "testar", "published": "2026-10-18", "as_of": "2026-10-25", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.25431", "independent_implementations": 0, "total_implementations": 1, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "MOSAIC replaces uniform per-layer KV-cache budgets (as used by SnapKV, H2O, AdaKV, L2Norm-style eviction) with a per-layer budget allocation found by one multi-objective search whose best allocation transfers across target budgets, optionally refined by a budget-constrained search. The cost is a one-time offline search per model and eviction method (reported as cheaper than EvolKV, but still multi-evaluation GPU work on a single 24 GB card for 7-8B models), plus serving-side plumbing to track and enforce per-layer budgets, with no runtime quality loss claimed at matched budgets. It can fail if the searched allocation does not transfer to models, tasks, context lengths, or eviction methods outside the validated Llama-3-8B / Mistral-7B and RULER/LongBench settings, or if the serving stack cannot apply heterogeneous per-layer budgets without kernel or scheduler changes.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.25431", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Reaches the score of uniform KV-cache allocation with up to 4x less KV cache (75% smaller); beats uniform allocation in 146 of 154 settings, by up to +48 points on RULER and +11.9 on LongBench (LCC); against EvolKV, 4x fewer evaluations and 18% fewer GPU-hours per budget.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.25431", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }