{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2609.26693",
  "title": "Measuring the Serving Stack Instead of the Model: Hidden Confounds in Local Tool-Use Evaluation",
  "summary": "The technique replaces naive tool-call fidelity measurement, which treats serving-stack rejections and parse failures as model errors, with a protocol that logs structured failure metadata, probes per-model serving behavior (Ollama template flags, llama.cpp, vLLM, SGLang), and reports per-instance rather than turn-pooled estimates. It costs additional harness instrumentation, per-model serving configuration checks, and more verbose failure logging, but requires no extra compute or training. It can fail if constrained decoding is applied blindly, since it eliminates parse failures but can induce non-termination, and if teams assume one protocol fits all models, since uniform text protocols degrade models with native tool-call support.",
  "rationale": "The reader evaluates or deploys local tool-using agents on constrained infrastructure, and this checklist-level protocol prevents misattributing serving-stack artifacts to model behavior at negligible cost.",
  "technique": "Serving-stack-aware tool-call evaluation protocol",
  "family": "avaliacao_de_agente",
  "recommendation": "adotar",
  "published": "2026-09-22",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2609.26693",
  "independent_implementations": 2,
  "total_implementations": 3,
  "stars": 3,
  "citations": 0,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "The technique replaces naive tool-call fidelity measurement, which treats serving-stack rejections and parse failures as model errors, with a protocol that logs structured failure metadata, probes per-model serving behavior (Ollama template flags, llama.cpp, vLLM, SGLang), and reports per-instance rather than turn-pooled estimates. It costs additional harness instrumentation, per-model serving configuration checks, and more verbose failure logging, but requires no extra compute or training. It can fail if constrained decoding is applied blindly, since it eliminates parse failures but can induce non-termination, and if teams assume one protocol fits all models, since uniform text protocols degrade models with native tool-call support.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.26693",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "Turn-pooled versus per-instance fidelity estimates differ by up to about 55 percentage points; adding a text tool list alongside the native channel recovers much of measured fidelity for accepted models, while a uniform text protocol reduces fidelity for natively tool-capable Llama-3.2.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.26693",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}