{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.11050", "title": "AgentHorizon: Evaluating Agentic Judges for Long-Horizon Computer-Use Tasks", "summary": "The benchmark replaces ad hoc or human-only inspection of computer-use agent trajectories with a paired instruction-swap evaluation that measures whether automatic judges can distinguish a successful trajectory from a similar but incompatible one. Adopting it costs API spend or GPU inference to run judges over trajectories of up to 300 screenshots, plus engineering effort to integrate either direct trajectory passing or agent-harness-based judging into the evaluation pipeline. The main failure modes are that even the best judge reaches only 80.9% balanced accuracy, that tool-use degrades open-weight judge performance, and that individual judges show strong asymmetric biases toward accepting valid or rejecting failed trajectories, so a single judge should not be trusted as an unsupervised success signal.", "rationale": "Reliable automatic judging is directly relevant to production evaluation of computer-use agents on constrained infrastructure, but the demonstrated error rates and model-dependent tool-use effects require workload-specific validation before replacing human review.", "technique": "AgentHorizon benchmark for evaluating computer-use agent judges", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-08", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.11050", "independent_implementations": 1, "total_implementations": 2, "stars": 132, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "The benchmark replaces ad hoc or human-only inspection of computer-use agent trajectories with a paired instruction-swap evaluation that measures whether automatic judges can distinguish a successful trajectory from a similar but incompatible one. Adopting it costs API spend or GPU inference to run judges over trajectories of up to 300 screenshots, plus engineering effort to integrate either direct trajectory passing or agent-harness-based judging into the evaluation pipeline. The main failure modes are that even the best judge reaches only 80.9% balanced accuracy, that tool-use degrades open-weight judge performance, and that individual judges show strong asymmetric biases toward accepting valid or rejecting failed trajectories, so a single judge should not be trusted as an unsupervised success signal.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.11050", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Best agentic judge (GPT-5.5) achieves 80.9% balanced accuracy on the AH subset; judges differ drastically in accepting valid versus rejecting failed trajectories.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.11050", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }