{ "schema_version": 1, "builder_plan": { "scenario": "A narrow, repeated product task may not need your largest model.", "change": "Check whether a released student model fits the task before generating training data.", "baseline": "Your current model and an off-the-shelf small model on the same held-out tasks.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Student quality, serving cost, teacher calls, training cost, and failures on unusual inputs.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.27717", "title": "SkillGym: Internalizing Human Skills into LLMs for Real-World Problem Solving", "summary": "The method replaces inference-time prompting of human-written skill instructions with supervised fine-tuning (and optionally RL) on verified skill-execution trajectories, internalizing workflows into model weights. It costs an environment-construction and verification pipeline, trajectory collection averaging 49 tool calls and over 60k tokens each, and fine-tuning of a 35B-class model, which exceeds a single 24 GB GPU without aggressive quantization or rented compute. It can fail if the released trajectories do not transfer to the reader's domain, if code-based checkers verify spurious success signals, or if benchmark gains (GDPval, Terminal-Bench, SkillsBench) do not materialize on the reader's actual workloads.", "rationale": "The gains are quantified but require 35B-scale fine-tuning infrastructure and trajectory pipelines beyond a 24 GB setup, though the released 2,756 environments and 8,364 trajectories make small-scale replication worth monitoring.", "technique": "Skill trajectory SFT for LLM agents", "family": "destilacao", "recommendation": "observar", "published": "2026-09-23", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.27717", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The method replaces inference-time prompting of human-written skill instructions with supervised fine-tuning (and optionally RL) on verified skill-execution trajectories, internalizing workflows into model weights. It costs an environment-construction and verification pipeline, trajectory collection averaging 49 tool calls and over 60k tokens each, and fine-tuning of a 35B-class model, which exceeds a single 24 GB GPU without aggressive quantization or rented compute. It can fail if the released trajectories do not transfer to the reader's domain, if code-based checkers verify spurious success signals, or if benchmark gains (GDPval, Terminal-Bench, SkillsBench) do not materialize on the reader's actual workloads.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.27717", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "SFT on SkillGym trajectories improves Qwen3.5-35B-A3B by 199 Elo on GDPval-AA v2, 19.10 percentage points on Terminal-Bench 2.1, and 28.13/12.38 points on SkillsBench v1.1 with/without skills; the 35B agent reaches 51.47% on skill-assisted SkillsBench.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.27717", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }