{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "Start with one repeated product task that has a measurable failure or cost.",
    "change": "Read the methods and identify one change you can isolate in that workflow.",
    "baseline": "Your current implementation on the same held-out tasks.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Successful outcomes, response time, human rework, and total cost per successful task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2609.17435",
  "title": "Right Tool, Right Job: Native-Language Evaluation, Tokenizer Sensitivity, and Methodological Findings from a French-Only BabyLM",
  "summary": "The method replaces reliance on single-token zero-shot scoring and translated English benchmarks with native-language minimal-pair evaluation, tokenizer-swap sensitivity checks, and placebo-controlled prompting when assessing small language models. It costs little: no additional training or hardware, only extra evaluation passes and the effort of constructing or adopting native minimal-pair benchmarks, plus rank-16 LoRA fine-tuning for task evaluation. It can fail if native benchmarks do not exist for the target language or domain, if results are overgeneralized from the 125M child scale to production-size models, or if the diagnostics are treated as a substitute for task-relevant evaluation rather than a complement to it.",
  "rationale": "These diagnostics are free to run on constrained infrastructure and directly prevent a concrete, demonstrated failure mode where tokenizer and template artifacts dominate zero-shot scores, so teams evaluating or fine-tuning small or multilingual models should adopt them as standard practice.",
  "technique": "Tokenizer-swap and placebo-controlled LM evaluation diagnostics",
  "family": "outro",
  "recommendation": "adotar",
  "published": "2026-09-15",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2609.17435",
  "independent_implementations": 0,
  "total_implementations": 1,
  "stars": 1,
  "citations": null,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "The method replaces reliance on single-token zero-shot scoring and translated English benchmarks with native-language minimal-pair evaluation, tokenizer-swap sensitivity checks, and placebo-controlled prompting when assessing small language models. It costs little: no additional training or hardware, only extra evaluation passes and the effort of constructing or adopting native minimal-pair benchmarks, plus rank-16 LoRA fine-tuning for task evaluation. It can fail if native benchmarks do not exist for the target language or domain, if results are overgeneralized from the 125M child scale to production-size models, or if the diagnostics are treated as a substitute for task-relevant evaluation rather than a complement to it.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.17435",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}