{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2610.31118",
  "title": "ProductWebBench: Evaluating Website Continuity Beyond Task Completion",
  "summary": "ProductWebBench replaces single-target task-completion grading of frontend coding agents with evaluation against living repositories, scoring both staged requirement completion and continuity of previously working states via deterministic browser checks. It costs the infrastructure to snapshot repositories, run browser-based checks continuously throughout each run, and maintain the reference-validation pipeline, plus the engineering effort to integrate it into the team's CI or agent-selection workflow. It can fail as an adoption decision if the reader's agents do not modify existing frontend codebases, since the continuity signal is specific to Change tasks on live repositories, and benchmark scores on its 400 tasks may not transfer to a proprietary codebase without constructing equivalent staged checks.",
  "rationale": "The benchmark addresses a real production risk for teams deploying frontend coding agents, but its value depends on the reader actually modifying existing web codebases and investing in browser-check infrastructure, so workload-specific validation is required before adoption.",
  "technique": "Continuity-aware benchmark for frontend coding agents",
  "family": "avaliacao_de_agente",
  "recommendation": "testar",
  "published": "2026-10-22",
  "as_of": "2026-10-23",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2610.31118",
  "independent_implementations": 0,
  "total_implementations": 1,
  "stars": 0,
  "citations": null,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "ProductWebBench replaces single-target task-completion grading of frontend coding agents with evaluation against living repositories, scoring both staged requirement completion and continuity of previously working states via deterministic browser checks. It costs the infrastructure to snapshot repositories, run browser-based checks continuously throughout each run, and maintain the reference-validation pipeline, plus the engineering effort to integrate it into the team's CI or agent-selection workflow. It can fail as an adoption decision if the reader's agents do not modify existing frontend codebases, since the continuity signal is specific to Change tasks on live repositories, and benchmark scores on its 400 tasks may not transfer to a proprietary codebase without constructing equivalent staged checks.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.31118",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "Across 8,737 runs from 13 models, 28.2% of Change runs passing all requested-work checks still fail a continuity check, 10.8% regress a previously passed browser state, and interaction checks fail in 29.5% of runs whose static DOM passes; these are diagnostic findings, not a method improvement factor.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.31118",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}