{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2609.14795",
  "title": "Mind Which Bird You Favour: Parameterizing Adequacy-Fluency Balance in Meta-Evaluation of Machine Translation",
  "summary": "The method replaces uniform weighting of translation systems in MT meta-evaluation datasets with optimized per-system weights that achieve a chosen adequacy-fluency balance while staying close to uniform. It costs an exact optimization step with pruning over the system set, plus a scorer-augmentation validation framework, adding methodological complexity but negligible compute. It can fail if the available system pool is too small or skewed to support the target balance without distorting weights, and its conclusions do not transfer directly to non-MT evaluation settings.",
  "rationale": "This is a meta-evaluation methodology for machine translation metrics, relevant only if the reader builds or benchmarks MT scorers; otherwise it does not address their production inference or training constraints.",
  "technique": "System reweighting for adequacy-fluency balance in MT meta-evaluation",
  "family": "avaliacao_de_agente",
  "recommendation": "observar",
  "published": "2026-09-13",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2609.14795",
  "independent_implementations": 0,
  "total_implementations": 1,
  "stars": 0,
  "citations": 0,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "The method replaces uniform weighting of translation systems in MT meta-evaluation datasets with optimized per-system weights that achieve a chosen adequacy-fluency balance while staying close to uniform. It costs an exact optimization step with pruning over the system set, plus a scorer-augmentation validation framework, adding methodological complexity but negligible compute. It can fail if the available system pool is too small or skewed to support the target balance without distorting weights, and its conclusions do not transfer directly to non-MT evaluation settings.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.14795",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_formula",
  "core_kind": "algorithm",
  "equations_fetched_at": "2026-10-02",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}