{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2610.21561",
  "title": "DialFork: Going Beyond Factuality to Evaluate Procedural Hallucination in Conversational Agents",
  "summary": "DialFork replaces semantic-similarity and factuality-based evaluation of conversational agents with flow-aware metrics (FlowKL, FlowScore) plus a forked-dialogue benchmark that detects procedural deviation at three granularities. Cost is limited to running encoder inference and computing flow metrics, feasible on a single 24 GB GPU or via APIs, though using the benchmark may require mapping one's own dialogue flows onto its task structure. It can fail when deployed dialogue processes differ structurally from the benchmark's 24 tasks, since flow-aware similarity may not transfer to domain-specific procedures, and turn-level guardrail accuracy is not established as strongly as dialogue-level scoring.",
  "rationale": "Teams deploying conversational agents need a low-cost way to detect procedural drift in production, but the metrics should be validated against the team's own dialogue flows before adoption.",
  "technique": "Dialog-flow metrics for procedural hallucination evaluation",
  "family": "avaliacao_de_agente",
  "recommendation": "testar",
  "published": "2026-10-15",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2610.21561",
  "independent_implementations": 3,
  "total_implementations": 3,
  "stars": 1632,
  "citations": 0,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "DialFork replaces semantic-similarity and factuality-based evaluation of conversational agents with flow-aware metrics (FlowKL, FlowScore) plus a forked-dialogue benchmark that detects procedural deviation at three granularities. Cost is limited to running encoder inference and computing flow metrics, feasible on a single 24 GB GPU or via APIs, though using the benchmark may require mapping one's own dialogue flows onto its task structure. It can fail when deployed dialogue processes differ structurally from the benchmark's 24 tasks, since flow-aware similarity may not transfer to domain-specific procedures, and turn-level guardrail accuracy is not established as strongly as dialogue-level scoring.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.21561",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "At the dialogue level, FlowScore significantly outperforms all baselines across seven encoders for detecting procedural deviation; no multiplicative factor is reported in the abstract.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.21561",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}