{ "schema_version": 1, "builder_plan": { "scenario": "A self-hosted assistant is too expensive because its model uses most of the GPU memory.", "change": "Try the paper's supported lower-precision weights on a copy of your serving setup.", "baseline": "The same model at its original precision, on the same GPU.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak memory, task pass rate, latency, and total serving cost at the same context and load.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.26823", "title": "Text Scores Can Miss Waveform Use: A Qwen2-Audio Quantization Case Study", "summary": "The protocol replaces the practice of validating post-training quantization with text-output scores and nominal bit widths alone, adding transcript-insensitive endpoints (e.g., emotion recognition) and measured packed-memory implementation. It costs additional evaluation runs per allocation on frozen, speaker-disjoint test sets plus implementation of a genuinely packed kernel rather than a dequantized simulation. Adopting low-bit speech quantization without this evaluation can silently degrade waveform-dependent behavior: at 4.08 bits every tested allocation lost roughly 10 emotion points, and the average-6-bit dequantized simulation retained FP16 peak memory.", "rationale": "For a constrained-infrastructure team deploying quantized audio models, this cheap evaluation discipline directly prevents shipping a model whose lexical metrics hide regressions in paralinguistic tasks and whose nominal bit width overstates actual memory savings.", "technique": "Multi-endpoint evaluation protocol for speech model quantization", "family": "quantizacao", "recommendation": "adotar", "published": "2026-09-20", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.26823", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The protocol replaces the practice of validating post-training quantization with text-output scores and nominal bit widths alone, adding transcript-insensitive endpoints (e.g., emotion recognition) and measured packed-memory implementation. It costs additional evaluation runs per allocation on frozen, speaker-disjoint test sets plus implementation of a genuinely packed kernel rather than a dequantized simulation. Adopting low-bit speech quantization without this evaluation can silently degrade waveform-dependent behavior: at 4.08 bits every tested allocation lost roughly 10 emotion points, and the average-6-bit dequantized simulation retained FP16 peak memory.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.26823", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "A translation-selected 6-bit allocation improves chrF by 2.36 (95% bootstrap CI [1.04, 3.62]) but loses 3.91 percentage points on speaker-disjoint emotion recognition versus FP16; a same-budget uniform control achieves higher emotion accuracy.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.26823", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }