{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2609.18453",
  "title": "The Mirage of Calibrated Confidence: Trajectory-Independence of Verbalized Confidence in Vision-Language Models",
  "summary": "The method replaces sole reliance on ECE/AUROC-style calibration metrics with TGS-self and TGS-pair probes that test whether a VLM's verbalized confidence actually depends on its reasoning trajectory. It costs an additional evaluation pass: running confidence queries with and without the trajectory and scoring controlled good/bad trajectory pairs, which roughly doubles inference for the evaluated samples plus benchmark construction effort. It can fail if adopted as a training signal, since the paper itself shows calibration training can increase trajectory-independence, and verbalized-confidence fixes may not transfer across model families without revalidation.",
  "rationale": "For a team deploying VLMs via APIs or a single GPU, this is a low-cost evaluation add-on that exposes a production-relevant failure mode (confident wrong answers after self-correction) that standard metrics miss, but its diagnostic value for the reader's specific models must be validated before acting on it.",
  "technique": "Trajectory-grounded confidence evaluation (TGS/TGS-Bench)",
  "family": "avaliacao_de_agente",
  "recommendation": "testar",
  "published": "2026-09-16",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2609.18453",
  "independent_implementations": 0,
  "total_implementations": 0,
  "stars": 0,
  "citations": 0,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "The method replaces sole reliance on ECE/AUROC-style calibration metrics with TGS-self and TGS-pair probes that test whether a VLM's verbalized confidence actually depends on its reasoning trajectory. It costs an additional evaluation pass: running confidence queries with and without the trajectory and scoring controlled good/bad trajectory pairs, which roughly doubles inference for the evaluated samples plus benchmark construction effort. It can fail if adopted as a training signal, since the paper itself shows calibration training can increase trajectory-independence, and verbalized-confidence fixes may not transfer across model families without revalidation.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.18453",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "No single quantified factor; the paper shows conventional calibration rankings (ECE, AUROC) diverge from trajectory-grounding rankings on TGS-Bench across 10 benchmarks, and that calibration training can worsen the disconnect between verbalized confidence and reasoning content.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.18453",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}