{ "schema_version": 1, "builder_plan": { "scenario": "A self-hosted assistant is too expensive because its model uses most of the GPU memory.", "change": "Try the paper's supported lower-precision weights on a copy of your serving setup.", "baseline": "The same model at its original precision, on the same GPU.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Peak memory, task pass rate, latency, and total serving cost at the same context and load.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.16391", "title": "Where Post-Training Quantization Breaks Text Embedders: A Measured Map Across Four Embedder Families", "summary": "For retrieval embedders below INT4, a small task-distilled student quantized to INT3 replaces aggressive quantization of the larger teacher, rather than applying LLM-carried heuristics (embedding-table protection, module-wise bit allocation, ranking-aware objectives), which the paper shows do not transfer to embedders. The cost is a distillation pipeline and task-representative training data, plus the engineering of validating each (family, width, group-size) point instead of relying on a reusable sensitivity ordering; reconstruction-error proxies remain usable for screening uniform widths but not for selecting protected tensors. It can fail when the deployment task drifts from the distillation task, where the student's advantage over the quantized teacher does not hold, and joint damage at INT2-INT3 is not predictable from isolated module measurements.", "rationale": "An engineer on a 24 GB GPU running production retrieval can apply INT4 PTQ safely, but at extreme widths the paper's measured evidence favors distilling a small student over heuristically quantizing the teacher, provided the distillation task matches production traffic.", "technique": "Weight-only post-training quantization for retrieval embedders, with task-specific distillation as the alternative at extreme widths", "family": "quantizacao", "recommendation": "testar", "published": "2026-09-14", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.16391", "independent_implementations": 0, "total_implementations": 1, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "For retrieval embedders below INT4, a small task-distilled student quantized to INT3 replaces aggressive quantization of the larger teacher, rather than applying LLM-carried heuristics (embedding-table protection, module-wise bit allocation, ranking-aware objectives), which the paper shows do not transfer to embedders. The cost is a distillation pipeline and task-representative training data, plus the engineering of validating each (family, width, group-size) point instead of relying on a reusable sensitivity ordering; reconstruction-error proxies remain usable for screening uniform widths but not for selecting protected tensors. It can fail when the deployment task drifts from the distillation task, where the student's advantage over the quantized teacher does not hold, and joint damage at INT2-INT3 is not predictable from isolated module measurements.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.16391", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "A distilled 109M student at INT3 holds 78.04 NDCG@10 in 68.4 MB, versus its 0.6B teacher under extreme PTQ at 64.46 NDCG@10 in 297.9 MB, dominating on both size and quality within the distillation task.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.16391", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "no_equations", "core_kind": "", "equations_fetched_at": "2026-10-13", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }