{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.13356", "title": "SWE-Critix: Post-Training Generalist Verifiers for Assessing Coding-Agent Trajectories", "summary": "SWE-Critix replaces prompting an off-the-shelf general-purpose LLM to judge coding-agent trajectories with a dedicated 30B verifier (Qwen3-30B-A3B) post-trained via SFT plus RL to predict completion labels and produce natural-language explanations. The reader cannot reproduce the training pipeline, but the released 30B-A3B MoE model can run quantized on a 24 GB GPU or via API, at the cost of serving a large model and adding a verification step to every agent run. It can fail because 78% accuracy still means roughly one in five judgments is wrong, the verifier is trained on CodeForge-Preview trajectories and may degrade on other agents, repositories, or task distributions, and verifier errors can silently propagate if used for automated filtering or reward signals.", "rationale": "The released model is deployable on constrained hardware and offers a measurable gain over LLM-as-judge baselines, but its 78% accuracy and dependence on the CodeForge-Preview distribution require validation on the reader's own agent workloads before production reliance.", "technique": "Post-trained generalist verifier for coding-agent trajectories", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-09", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.13356", "independent_implementations": 1, "total_implementations": 2, "stars": 132, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "SWE-Critix replaces prompting an off-the-shelf general-purpose LLM to judge coding-agent trajectories with a dedicated 30B verifier (Qwen3-30B-A3B) post-trained via SFT plus RL to predict completion labels and produce natural-language explanations. The reader cannot reproduce the training pipeline, but the released 30B-A3B MoE model can run quantized on a 24 GB GPU or via API, at the cost of serving a large model and adding a verification step to every agent run. It can fail because 78% accuracy still means roughly one in five judgments is wrong, the verifier is trained on CodeForge-Preview trajectories and may degrade on other agents, repositories, or task distributions, and verifier errors can silently propagate if used for automated filtering or reward signals.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.13356", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Held-out classification accuracy of the verifier improves from 57% (base Qwen3-30B-A3B) to 66% after SFT and 78% after RL; these are absolute point gains, not a multiplicative factor.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.13356", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }