{ "schema_version": 1, "builder_plan": { "scenario": "Possible product use: a self-hosted writing or coding assistant where output generation is a measured bottleneck.", "change": "Begin with an artifact check. Verify a usable drafter for your exact target and serving version. If none exists, record the adaptation work and budget before booking GPUs; a new target is an experiment of its own.", "baseline": "The same target with speculation disabled, your current drafter if present, and the candidate drafter. Keep the GPU, precision, sampling, context, and request trace fixed.", "steps": [ "Inspect the official repository linked by the paper for weights, license, target support, and pinned runtime instructions. Confirm whether you can reuse pretraining and what adaptation remains. Do not infer compatibility from model size.", "Save 50 held-out product prompts with realistic input and output lengths. Include an unusual-domain slice and define quality, response-time, and cost limits before running. This is a proposed pilot size.", "Once the artifact and budget checks pass, replay the same trace for each condition at low load and your expected concurrency. Repeat paired runs after warm-up, then inspect outputs for regressions.", "Record both accepted draft length and end-to-end performance. Keep preparation, adaptation, idle GPU time, and serving costs in the result sheet. Preserve a configuration that disables the drafter." ], "measure": "First-token and completion latency, tokens per second, completed requests, output correctness, peak memory, and total cost per successful request. Report low-load and concurrent results separately.", "decision": "Keep the drafter only if it improves the workload that matters while preserving quality and meeting latency limits. Divide setup cost by measured savings per request to estimate break-even traffic; if savings are zero or negative, there is no cost break-even. A pilot on another model does not validate GLM support." }, "builder_review": { "arxiv_id": "2609.09338", "source_url": "https://arxiv.org/pdf/2609.09338v1", "pdf_sha256": "1f3254967cf7b2c5412ec3712681c257814eed1065d46692a91498a6402619ec", "report_sha256": "46754036f9c84e36ecadd9979a249d5df4889551fe34663caadd840d2da8a903", "reviewed_at": "2026-09-14", "intro": "A small draft model can suggest several tokens for a larger model to check together. Osprey studies how to make that drafter more useful across workloads. The practical question is whether a compatible checkpoint saves enough serving time to cover adaptation and integration.", "mechanism": "Osprey pretrains a shallow draft model, then adapts it to a target model. The target still verifies proposed tokens. Reusing a compatible checkpoint matters because the preparation stage requires substantial compute.", "method_page": 4, "takeaway": "Try this only after checking target and runtime compatibility. The paper evaluates Qwen, Llama, and MiniMax targets; these results do not establish support or speedups for GLM-4.7-Flash. Table 10 on page 19 reports 576–3,040 H100-hours for pretraining, before target adaptation. Hardware-hours are not a rental quote.", "comparisons": [ { "title": "Accepted length: a diagnostic, not app speed", "definition": "Mean draft tokens accepted per verification step.", "baseline_name": "Reproduced EAGLE-3", "method_name": "Osprey", "before": 2.45, "after": 2.97, "unit": "tokens/step", "scope": "Llama-3.3-70B-Instruct, five domains; SGLang, batch 1.", "caveat": "Longer acceptance can help, but the draft model also takes time to run. Measure throughput separately.", "source_page": 7, "source_locator": "Table 1" }, { "title": "Throughput: tokens generated each second", "definition": "Decoding throughput measures output rate in the tested serving configuration.", "baseline_name": "Reproduced EAGLE-3", "method_name": "Osprey", "before": 155.8, "after": 183.7, "unit": "tokens/s", "scope": "Same Llama comparison and serving setup.", "caveat": "A throughput increase is not the same percentage reduction in response time or cost. Queueing, prompt processing, and utilization also matter.", "source_page": 7, "source_locator": "Table 1" }, { "title": "A regression hidden by the average", "definition": "MT-Bench is the conversational slice of the MiniMax comparison.", "baseline_name": "Public EAGLE-3", "method_name": "Osprey", "before": 214.4, "after": 201.6, "unit": "tokens/s", "scope": "MiniMax-M2.5, FP8, batch 1; 64 prompts in this benchmark.", "caveat": "Osprey is slower on this slice despite a higher overall mean. Use your own traffic mix.", "source_page": 7, "source_locator": "Table 2" } ], "plan": { "scenario": "Possible product use: a self-hosted writing or coding assistant where output generation is a measured bottleneck.", "change": "Begin with an artifact check. Verify a usable drafter for your exact target and serving version. If none exists, record the adaptation work and budget before booking GPUs; a new target is an experiment of its own.", "baseline": "The same target with speculation disabled, your current drafter if present, and the candidate drafter. Keep the GPU, precision, sampling, context, and request trace fixed.", "steps": [ "Inspect the official repository linked by the paper for weights, license, target support, and pinned runtime instructions. Confirm whether you can reuse pretraining and what adaptation remains. Do not infer compatibility from model size.", "Save 50 held-out product prompts with realistic input and output lengths. Include an unusual-domain slice and define quality, response-time, and cost limits before running. This is a proposed pilot size.", "Once the artifact and budget checks pass, replay the same trace for each condition at low load and your expected concurrency. Repeat paired runs after warm-up, then inspect outputs for regressions.", "Record both accepted draft length and end-to-end performance. Keep preparation, adaptation, idle GPU time, and serving costs in the result sheet. Preserve a configuration that disables the drafter." ], "measure": "First-token and completion latency, tokens per second, completed requests, output correctness, peak memory, and total cost per successful request. Report low-load and concurrent results separately.", "decision": "Keep the drafter only if it improves the workload that matters while preserving quality and meeting latency limits. Divide setup cost by measured savings per request to estimate break-even traffic; if savings are zero or negative, there is no cost break-even. A pilot on another model does not validate GLM support." } }, "arxiv_id": "2609.09338", "title": "Osprey: Target-agnostic Pre-training Makes Stronger Drafters in Speculative Decoding", "summary": "Osprey replaces target-specific drafter pretraining that consumes target hidden states and is distilled per target with a reusable pruned small-LM backbone plus lightweight per-target adaptation. It costs access to target hidden states and output distributions, pruning and restoration pretraining, vocabulary alignment and QKV expansion engineering, and extra drafter memory and serving complexity. It can fail where target internals are API-inaccessible, adaptation data mismatches production traffic, vocabulary or hidden-state alignment degrades acceptance, or domain shift again reduces acceptance length.", "rationale": "The method is directly relevant to lowering LLM serving latency on a single-GPU self-hosted stack, but its benefit depends on hidden-state access, target model choice, and workload-specific acceptance rates, so it warrants a controlled pilot rather than immediate production adoption.", "technique": "Target-agnostic pretrained drafters for speculative decoding", "family": "decodificacao_especulativa", "recommendation": "testar", "published": "2026-09-08", "as_of": "2026-10-18", "editorial_status": "source_mapped", "source_url": "https://arxiv.org/abs/2609.09338", "independent_implementations": 0, "total_implementations": 1, "stars": 7, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "Osprey improves mean accepted length over EAGLE-3 on Qwen3-8B across a 5x5 train/eval domain matrix", "basis": "inferred", "result": "Osprey mean AL 3.686 vs EAGLE-3 3.176, +16.1%; higher AL for every domain pair", "baseline": "EAGLE-3 one-layer drafter trained from scratch, matched adaptation budget", "conditions": "Qwen3-8B target, five domains (chat, code, commonsense, finance, math), 65k prompts/domain, 512 held-out eval prompts/domain, SGLang batch=1, steps=5, topk=1, draft_tokens=6", "source_url": "https://arxiv.org/pdf/2609.09338", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Gains are larger out-of-domain than in-domain on Qwen3-8B", "basis": "inferred", "result": "OOD mean AL 3.493 vs 2.977 (+17.3%); ID 4.465 vs 3.971 (+12.4%); OOD/ID ratio 0.782 vs 0.750", "baseline": "EAGLE-3 one-layer drafter", "conditions": "Qwen3-8B cross-domain 5x5 evaluation, single runs per domain pair", "source_url": "https://arxiv.org/pdf/2609.09338", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-03", "statement": "Same backbone transfers to Llama-3.3-70B-Instruct with higher AL and throughput", "basis": "source_linked", "result": "Mean AL 2.97 vs 2.45 (+21.2%); throughput 183.7 vs 155.8 tok/s (+17.9%); +27.5% AL vs official EAGLE-3 checkpoint", "baseline": "Reproduced EAGLE-3 trained on same target-regenerated Open-PerfectBlend data; also official EAGLE-3 checkpoint", "conditions": "Llama-3.3-70B-Instruct target, 37.5k adaptation steps, five evaluation domains", "source_url": "https://arxiv.org/pdf/2609.09338", "source_page": 7, "source_excerpt": "Its mean AL and throughput are respectively 21.2% and 17.9% higher than those of our reproduced EAGLE-3 baseline, and 27.5% and 22.2% higher than those of the official EAGLE-3 checkpoint." }, { "claim_id": "claim-04", "statement": "On MiniMax-M2.5 (229B MoE, FP8), Osprey beats the public EAGLE-3 draft on mean AL and throughput, but roughly ties on chat/commonsense", "basis": "inferred", "result": "Mean AL 3.232 vs 2.634 (+22.7%); throughput 219.8 vs 187.1 TPS (+17.5%); MT-BENCH AL 2.805 vs 2.868 (Osprey lower)", "baseline": "Publicly available one-layer EAGLE-3 checkpoint (Lu et al., 2025); cost-parity not claimed", "conditions": "Code-only adaptation (70k prompts), five benchmarks with 64 prompts each, 4xB200 FP8 serving", "source_url": "https://arxiv.org/pdf/2609.09338", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-05", "statement": "The advantage comes primarily from the pretrained prior, not from the extra layer, and most of Stage-2 benefit is captured by 10k pretraining steps", "basis": "inferred", "result": "Osprey AL 3.790 vs parameter-matched 2-layer EAGLE-3 3.385 (+0.405); 10k vs 55k pretraining steps differ by only 0.026 overall AL", "baseline": "Two-layer EAGLE-3 trained from scratch, same adaptation data/optimizer/LR; 10k-step pretrained checkpoint", "conditions": "Qwen3-8B, ablation over chat/code/finance/math (16 pairs, commonsense excluded); lr=10^-5 for the budget comparison", "source_url": "https://arxiv.org/pdf/2609.09338", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "inferred", "finding": "Minimum useful test: 1 GPU, 48 to 80 GB. Published evidence: cluster.", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "inferred", "finding": "Reported setup: custom runtime, distributed stack.", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "inferred", "finding": "Training requirement: fine-tuning.", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "inferred", "finding": "Paperraft defines a 5-steps falsification test. No reproduction is recorded.", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [ { "risk_id": "risk-01", "statement": "Most results are single runs, not multiple seeds (stated in Limitations); small differences such as MT-BENCH ties are not statistically established.", "basis": "inferred" }, { "risk_id": "risk-02", "statement": "Osprey is not uniformly better: on MiniMax-M2.5 it is slightly worse than EAGLE-3 on MT-BENCH (2.805 vs 2.868 AL, 201.6 vs 214.4 T/s) and nearly tied on COMMONSENSE-EVAL.", "basis": "inferred" }, { "risk_id": "risk-03", "statement": "Throughput advantage narrows at larger batch sizes (24% at batch 1 down to 13% at batch 16) and degrades with sampling temperature (35% at T=1.0 vs 41% at T=0).", "basis": "inferred" }, { "risk_id": "risk-04", "statement": "Upfront pretraining cost (576-3,040 H100-hours) is only amortized under high-volume serving or multi-target reuse; the paper itself states it may not be recovered in low-traffic deployments.", "basis": "inferred" }, { "risk_id": "risk-05", "statement": "Gains depend on adapting with target-regenerated data; MiniMax-M2.5 results use code-only adaptation data, so non-code targets/domains are untested at that scale.", "basis": "inferred" }, { "risk_id": "risk-06", "statement": "Only three targets and one serving stack (SGLang) were evaluated; behavior on other targets, hardware, or verification configurations is unknown.", "basis": "inferred" } ], "minimum_test": [ "Skip the 462B-token pretraining: obtain or build a shallow pruned backbone (e.g., 2-layer Qwen3-4B) and use the authors' Stage-2 ablation insight (10k steps ≈ 576 H100-hours claimed, or reuse their released checkpoint from github.com/LeanModels/Osprey).", "Pick your actual serving target model and one workload that is out-of-domain relative to the authors' five domains (e.g., your production traffic), generate ~65k target-regenerated prompts for adaptation.", "Adapt the Osprey drafter and a from-scratch EAGLE-3 baseline on identical data and steps (authors used 2xH100 for Qwen3-8B-scale targets; an 8B target fits on 1-2 80GB GPUs).", "Serve both drafters in SGLang under your real configuration (batch size, draft tokens, temperature) on one 80GB GPU for an 8B-class target; the paper itself shows Osprey's throughput edge shrinks from 24% at batch 1 to 13% at batch 16, so test at your operating batch size.", "Measure mean accepted length and tokens/s on held-out prompts from your workload, including an OOD slice; the claim is disproven for your setting if Osprey does not beat the matched from-scratch EAGLE-3 on your OOD traffic or if the gain is smaller than the cost of adaptation." ], "open_questions": [ "Whether the released code/checkpoints are sufficient to skip Stage-2 pretraining entirely and only run per-target adaptation (per-target cost ~48 H100-hours for 8B targets).", "Statistical significance of the gains, given single-run reporting; multi-seed replication needed before adoption.", "Performance at the reader's actual batch size, temperature, and draft-token count, since gains are sensitive to all three.", "Whether a much shorter pretraining (or an existing shallow model) suffices, since 10k steps already captures nearly all of the benefit.", "Adaptation cost and drafter size for the reader's target family, especially cross-tokenizer alignment quality for targets far from Qwen/Llama tokenizers.", "For the MiniMax-M2.5 comparison, the public EAGLE-3 baseline's training cost is unknown, so cost parity is unverified there." ], "independent_tests": [], "lab_reproductions": [], "report_available": true, "validation_tier": "single_gpu_48_80gb", "evidence_tier": "cluster", "infrastructure_basis": "explicit", "equations": [ { "anchor": "S3.E5", "label": "(5)", "section": "3 Method", "role": "loss", "latex": "\\mathcal{L}_{\\mathrm{adapt}}=\\sum_{k=0}^{K_{\\mathrm{train}}-1}\\lambda^{k}\\operatorname{CE}\\left(p_{\\theta}^{(k)},q_{\\phi}^{(k)}\\right),\\quad\\lambda=0.8.", "mathml": "ℒadapt=∑k=0Ktrain−1λk​CE⁡(pθ(k),qϕ(k)),λ=0.8.", "context": "Let pθ(k) and qϕ(k) denote the target’s and drafter’s next-token distributions at TTT step k. We train with the following discounted distillation loss, which emphasizes earlier drafting steps:", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S3.E4", "label": "(4)", "section": "3 Method", "role": "proposed_method", "latex": "\\widetilde{W}[v]=\\begin{cases}W_{S}[v],&v\\in\\mathcal{V}_{S},\\\\[3.0pt]\n\\displaystyle\\frac{1}{m_{v}}\\sum_{i=1}^{m_{v}}W_{S}[s_{i}],&v\\notin\\mathcal{V}_{S}.\\end{cases}", "mathml": "W~​[v]={WS​[v],v∈𝒱S,1mv​∑i=1mvWS​[si],v∉𝒱S.", "context": "Formally, let WS denote either pretrained source matrix and let W~ denote its resized counterpart. For each target token v∈𝒱T∖𝒱S, let (s1,…,smv)=τS​(str⁡(v)) be the sequence obtained by tokenizing its string representation with the source tokenizer. We initialize each row as", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" } ], "equations_status": "selected", "core_kind": "formula", "equations_fetched_at": "2026-09-28", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }