{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.24890", "title": "OSWorld-Pro: Process-based Evaluation for Computer Use Agents", "summary": "It replaces binary end-state functional verification (as in OSWorld) with LLM-judge evaluation of over 2800 intermediate subgoals grounded in human annotations, revealing where and why an agent fails during execution. The cost is the evaluation infrastructure itself: a 300+ task environment with sequential subgoal checkpoints plus LLM-judge inference per subgoal, which raises evaluation latency and API spend relative to a single end-state check. The LLM judges, despite human-alignment validation, can misclassify subgoal fulfillment, and a benchmark-specific diagnosis may not transfer to the reader's own task distribution.", "rationale": "The reader operating computer-use agents on constrained infrastructure can adopt the process-evaluation methodology (instrumenting subgoal checkpoints and LLM judges on their own tasks) cheaply to target mitigations, but should validate judge reliability and task coverage before trusting the diagnostics.", "technique": "Process-based subgoal evaluation for computer-use agents", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-09-21", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.24890", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "It replaces binary end-state functional verification (as in OSWorld) with LLM-judge evaluation of over 2800 intermediate subgoals grounded in human annotations, revealing where and why an agent fails during execution. The cost is the evaluation infrastructure itself: a 300+ task environment with sequential subgoal checkpoints plus LLM-judge inference per subgoal, which raises evaluation latency and API spend relative to a single end-state check. The LLM judges, despite human-alignment validation, can misclassify subgoal fulfillment, and a benchmark-specific diagnosis may not transfer to the reader's own task distribution.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.24890", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "OSWorld-Pro exposes process-level failure modes (e.g., click-based vs. keyboard-input errors) that end-state evaluation hides; top models score 75.7% on OSWorld-Pro vs. 83.4% on OSWorld, indicating the benchmark is more discriminative, though no method-level improvement factor is claimed.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.24890", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }