{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2609.18805",
  "title": "ProgramDistill: From Interactive Web Apps to Verifiable Reference-Guided SWE Tasks",
  "summary": "ProgramDistill replaces issue- or instruction-specified coding benchmarks (e.g., SWE-bench-style tasks) with 4,063 automatically constructed tasks in which agents must infer behavior by interacting with a working reference application, at controlled difficulty via restoration depth. It costs evaluation infrastructure: replayable application environments, agent runs against the benchmark, and the associated API or compute budget, but requires no model training. It can fail as a selection signal if the 26 covered web applications do not represent the reader's production stack, if agent scores are confounded by environment and replay fragility rather than capability, or if task contamination emerges as the benchmark becomes public.",
  "rationale": "The paper offers an evaluation benchmark rather than a deployable method, so it is only useful to the reader if they need to select or diagnose coding agents, and its predictive value for their specific workloads requires small-scale validation first.",
  "technique": "Reference-guided benchmark for coding agents",
  "family": "avaliacao_de_agente",
  "recommendation": "testar",
  "published": "2026-09-16",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2609.18805",
  "independent_implementations": 0,
  "total_implementations": 0,
  "stars": 0,
  "citations": 0,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "ProgramDistill replaces issue- or instruction-specified coding benchmarks (e.g., SWE-bench-style tasks) with 4,063 automatically constructed tasks in which agents must infer behavior by interacting with a working reference application, at controlled difficulty via restoration depth. It costs evaluation infrastructure: replayable application environments, agent runs against the benchmark, and the associated API or compute budget, but requires no model training. It can fail as a selection signal if the 26 covered web applications do not represent the reader's production stack, if agent scores are confounded by environment and replay fragility rather than capability, or if task contamination emerges as the benchmark becomes public.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.18805",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "Nine frontier coding agents scored at most 49.2% (GPT-6 Astra) and 28.8% (Claude Opus 5) success on cumulative full-application reconstruction; partial-reconstruction success falls from 100% to 64% as restoration depth increases from 1 to 8.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2609.18805",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}