{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2609.20625",
  "title": "Chronicle: Cut-Point Replay for Regression Testing of LLM Agents",
  "summary": "The method replaces ad-hoc manual reproduction of agent failures and full live re-runs with recorded envelopes at model and tool boundaries, replayed selectively so that only the code under test executes live, turning each incident into a deterministic CI regression test. The cost is instrumentation of all non-deterministic boundaries, storage of immutable run records, and 23 microseconds of recording overhead per crossing, plus the engineering effort to maintain replay-compatible boundaries as the agent evolves. It can fail when trajectories drift so that the recorded suffix no longer matches the new code's live prefix, when tools or models change their interfaces, and because the evidence base is only six incidents with simulated model boundaries, leaving real-model fidelity unproven.",
  "rationale": "Deterministic replay of recorded failures is directly useful to a small production team, but the six-incident simulated-boundary benchmark requires validation on the reader's own agent stack before adoption.",
  "technique": "Cut-point record-and-replay regression tests for agents",
  "family": "avaliacao_de_agente",
  "recommendation": "testar",
  "published": "2026-09-17",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2609.20625",
  "independent_implementations": 1,
  "total_implementations": 2,
  "stars": 25,
  "citations": 0,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "The method replaces ad-hoc manual reproduction of agent failures and full live re-runs with recorded envelopes at model and tool boundaries, replayed selectively so that only the code under test executes live, turning each incident into a deterministic CI regression test. The cost is instrumentation of all non-deterministic boundaries, storage of immutable run records, and 23 microseconds of recording overhead per crossing, plus the engineering effort to maintain replay-compatible boundaries as the agent evolves. It can fail when trajectories drift so that the recorded suffix no longer matches the new code's live prefix, when tools or models change their interfaces, and because the evidence base is only six incidents with simulated model boundaries, leaving real-model fidelity unproven.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.20625",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "Recording adds 23 microseconds per non-deterministic boundary crossing (0.008% of an assumed 300 ms model call); full replay issues zero model calls, eliminating inference cost in CI regression runs.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.20625",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}