{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2610.26861",
  "title": "Can It Actually Work? Synthesizing Functional CAD Mechanisms",
  "summary": "FunCAD-Harness replaces visual plausibility checks and generic simulation passes with candidate-specific physical tests that translate abstract functional requirements into executable verification of LLM-generated CAD mechanisms. Adoption costs a physics simulation stack (CAD modeling, kinematic/dynamic simulators), test-generation logic per candidate design, and iterative agent-simulation loops that increase latency and API token spend per design. It can fail where requirements are too underspecified to operationalize as tests, where simulators diverge from real manufacturing tolerances, and it transfers only partially to domains outside mechanical CAD.",
  "rationale": "The verification-in-the-loop pattern is sound, but production adoption requires CAD simulation infrastructure and applies to a narrow mechanical-design workload that most constrained-infrastructure ML teams do not operate, so it warrants monitoring rather than immediate use.",
  "technique": "Simulation-feedback harness for functional CAD agents",
  "family": "avaliacao_de_agente",
  "recommendation": "observar",
  "published": "2026-10-19",
  "as_of": "2026-10-20",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2610.26861",
  "independent_implementations": 2,
  "total_implementations": 2,
  "stars": 7,
  "citations": null,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "FunCAD-Harness replaces visual plausibility checks and generic simulation passes with candidate-specific physical tests that translate abstract functional requirements into executable verification of LLM-generated CAD mechanisms. Adoption costs a physics simulation stack (CAD modeling, kinematic/dynamic simulators), test-generation logic per candidate design, and iterative agent-simulation loops that increase latency and API token spend per design. It can fail where requirements are too underspecified to operationalize as tests, where simulators diverge from real manufacturing tolerances, and it transfers only partially to domains outside mechanical CAD.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.26861",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "The paper reports that FunCAD-Harness substantially improves functional success rates of frontier LLM agents over standard simulation-augmented baselines on FunCADBench, but the abstract provides no quantitative figure.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.26861",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}