{ "schema_version": 1, "builder_plan": { "scenario": "A narrow, repeated product task may not need your largest model.", "change": "Check whether a released student model fits the task before generating training data.", "baseline": "Your current model and an off-the-shelf small model on the same held-out tasks.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Student quality, serving cost, teacher calls, training cost, and failures on unusual inputs.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.20174", "title": "RM-OPD: On-Policy Distillation for Generative Reward Models", "summary": "RM-OPD replaces mixed-domain GRPO training of generative reward models with a two-stage pipeline: single-domain GRPO teacher training followed by routed on-policy distillation with domain reweighting into one student. The cost is substantial: multiple domain-specialist RL fine-tuning runs plus a distillation stage, requiring multi-GPU training capacity and per-domain preference data that a single 24 GB GPU setup cannot realistically supply. It can fail through residual cross-domain imbalance if reweighting is miscalibrated, teacher quality bottlenecks if any specialist undertrains, and reward hacking inherited from the teachers' supervision signal.", "rationale": "The method targets teams training generalist reward models with RL fine-tuning infrastructure, which exceeds the reader's single-GPU budget and is not a standard production adoption decision.", "technique": "On-policy distillation for generative reward models", "family": "destilacao", "recommendation": "observar", "published": "2026-10-14", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.20174", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "RM-OPD replaces mixed-domain GRPO training of generative reward models with a two-stage pipeline: single-domain GRPO teacher training followed by routed on-policy distillation with domain reweighting into one student. The cost is substantial: multiple domain-specialist RL fine-tuning runs plus a distillation stage, requiring multi-GPU training capacity and per-domain preference data that a single 24 GB GPU setup cannot realistically supply. It can fail through residual cross-domain imbalance if reweighting is miscalibrated, teacher quality bottlenecks if any specialist undertrains, and reward hacking inherited from the teachers' supervision signal.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.20174", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "RM-OPD consistently outperforms vanilla Mix-RL and other baselines across multiple RM benchmarks and two base models, with an emergent teacher-surpassing effect; no multiplicative factor is reported in the abstract.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.20174", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }