{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.37686", "title": "EngiWorld: What Can Frontier Agents Deliver in Professional Engineering Environments?", "summary": "EngiWorld replaces ad-hoc or GUI-only evaluation of agents with an artifact-centric benchmark: 1,301 tasks across CAD, CAE, CAM, BIM, EDA, and visualization platforms, verified programmatically for geometric validity, physical feasibility, and rule compliance, with continuous scoring by specification attainment. For a reader with constrained infrastructure, adopting it costs access to 26 commercial engineering platforms, GUI-capable agent harnesses, and per-task API or compute spend on frontier models, since local 24 GB models are unlikely to score meaningfully. It can fail as an adoption guide because results are specific to professional engineering workflows and do not transfer to general coding, RAG, or business-automation agents the reader is more likely to deploy.", "rationale": "This is a measurement tool for a specialized domain with no deployable technique, so it informs expectations about agent reliability in engineering software but warrants no production action unless the reader builds agents for CAD/CAE/EDA environments.", "technique": "EngiWorld: professional engineering software agent benchmark", "family": "avaliacao_de_agente", "recommendation": "observar", "published": "2026-09-29", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.37686", "independent_implementations": 3, "total_implementations": 5, "stars": 164, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "EngiWorld replaces ad-hoc or GUI-only evaluation of agents with an artifact-centric benchmark: 1,301 tasks across CAD, CAE, CAM, BIM, EDA, and visualization platforms, verified programmatically for geometric validity, physical feasibility, and rule compliance, with continuous scoring by specification attainment. For a reader with constrained infrastructure, adopting it costs access to 26 commercial engineering platforms, GUI-capable agent harnesses, and per-task API or compute spend on frontier models, since local 24 GB models are unlikely to score meaningfully. It can fail as an adoption guide because results are specific to professional engineering workflows and do not transfer to general coding, RAG, or business-automation agents the reader is more likely to deploy.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.37686", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Best frontier model scores EngiScore 44.3; only 3.6% of multi-software task attempts succeed.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.37686", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }