{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2610.10239",
  "title": "ProtocolMatch: Protocol-Dependent Model Selection for Scientific Dynamics Forecasting",
  "summary": "It replaces fixed-architecture benchmarking and winner-take-all leaderboards with a compute-matched, validation-selected evaluation that reports accuracy, physical validity, and shifted-distribution reliability separately. The cost is additional evaluation engineering: multiple protocols (refreshed history vs. closed loop), failure-preserving logging, and matched-compute baselines, rather than any extra model capacity or training budget. It can fail if adopted conclusions are over-generalized, since rankings reverse with dataset size, history length, system, and distribution shift, and closed-loop rollouts can still produce finite explosive errors that accuracy-only reporting hides.",
  "rationale": "The framework is cheap to run on constrained infrastructure and directly applicable to selecting small forecasting models in production, but its central finding is that no ranking transfers across protocols, so the reader must validate it against their own workload before committing.",
  "technique": "Protocol-dependent model selection for dynamics forecasting",
  "family": "avaliacao_de_agente",
  "recommendation": "testar",
  "published": "2026-10-07",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2610.10239",
  "independent_implementations": 0,
  "total_implementations": 0,
  "stars": 0,
  "citations": null,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "It replaces fixed-architecture benchmarking and winner-take-all leaderboards with a compute-matched, validation-selected evaluation that reports accuracy, physical validity, and shifted-distribution reliability separately. The cost is additional evaluation engineering: multiple protocols (refreshed history vs. closed loop), failure-preserving logging, and matched-compute baselines, rather than any extra model capacity or training budget. It can fail if adopted conclusions are over-generalized, since rankings reverse with dataset size, history length, system, and distribution shift, and closed-loop rollouts can still produce finite explosive errors that accuracy-only reporting hides.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.10239",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "No single quantified improvement: the paper shows the causal-attention vs. recurrent ordering reverses with training-set size, a low-rank linear predictor has the lowest mean error in the six-spin local-observable comparison, a latest-state MLP beats persistence on all five refreshed-history cells, and in-distribution intervals lose most coverage under a driving-frequency shift.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.10239",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}