{ "schema_version": 1, "builder_plan": { "scenario": "A self-hosted assistant is too expensive because its model uses most of the GPU memory.", "change": "Try the paper's supported lower-precision weights on a copy of your serving setup.", "baseline": "The same model at its original precision, on the same GPU.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak memory, task pass rate, latency, and total serving cost at the same context and load.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2608.30076", "title": "Budget-Aware Compression Pipeline for Single-GPU LLM Inference: Methods, Trade-offs, and Coupling Effects", "summary": "Replaces ad-hoc, independently chosen compression methods with a budget-aware pipeline that exploits measured couplings: layer-wise pruning applied before weight quantization, KV-cache sparsification combined with INT8 KV quantization, and avoidance of static vector quantizers with dynamic caching. Costs include a multi-stage compression and calibration effort, up to roughly 5% absolute accuracy degradation, and a 33 GB footprint that still exceeds a 24 GB GPU, so the exact pipeline cannot be reproduced on the reader's hardware. Static vector quantizers can conflict with dynamic KV caching, and the reported coupling rules are validated on one model and one GPU, so they may not transfer to other architectures or workloads.", "rationale": "The design rules are directly applicable to compressing smaller models that fit 24 GB, but the specific 70B-to-33 GB result requires more memory than the reader has, so the coupling guidelines should be validated on the reader's own models before production adoption.", "technique": "Coupled pruning, quantization, and KV-cache compression pipeline", "family": "quantizacao", "recommendation": "testar", "published": "2026-08-30", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2608.30076", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "Replaces ad-hoc, independently chosen compression methods with a budget-aware pipeline that exploits measured couplings: layer-wise pruning applied before weight quantization, KV-cache sparsification combined with INT8 KV quantization, and avoidance of static vector quantizers with dynamic caching. Costs include a multi-stage compression and calibration effort, up to roughly 5% absolute accuracy degradation, and a 33 GB footprint that still exceeds a 24 GB GPU, so the exact pipeline cannot be reproduced on the reader's hardware. Static vector quantizers can conflict with dynamic KV caching, and the reported coupling rules are validated on one model and one GPU, so they may not transfer to other architectures or workloads.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2608.30076", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "70B model compressed to ~33 GB (versus ~140 GB in FP16), sustaining ~57 tokens/s on 10k-token prompts on a single A40 with absolute accuracy within 5% on common and reasoning benchmarks", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2608.30076", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "unavailable", "core_kind": "", "equations_fetched_at": "2026-09-23", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }