{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.20673", "title": "HPCAgent-Bench: A Framework for Sound Benchmarking of Code-Optimizing Agents, Harnesses, and Skills", "summary": "HPCAgent-Bench replaces ad hoc, single-accelerator evaluations of code-optimizing agents with a hardened framework of 689 kernels, hidden fuzzed-input judging, and paired-run intervention metrics. Adopting it costs engineering effort to wrap submissions through its C ABI or Python interface and to run the harness, plus agent token spend, which the paper shows can grow up to 3.0x without corresponding speedup gains. It can fail to inform production decisions if the reader's workloads differ from its HPC- and ML-kernel mix, and its cost conclusions depend on the token weights chosen.", "rationale": "For a small team deciding which agent harnesses or skills to pay for, the benchmark's central finding—that most interventions raise token cost without improving speedup—directly supports a low-cost, evidence-based adoption decision before committing production budget.", "technique": "Hardened benchmark for code-optimizing agents", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-14", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.20673", "independent_implementations": 0, "total_implementations": 1, "stars": 4, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "HPCAgent-Bench replaces ad hoc, single-accelerator evaluations of code-optimizing agents with a hardened framework of 689 kernels, hidden fuzzed-input judging, and paired-run intervention metrics. Adopting it costs engineering effort to wrap submissions through its C ABI or Python interface and to run the harness, plus agent token spend, which the paper shows can grow up to 3.0x without corresponding speedup gains. It can fail to inform production decisions if the reader's workloads differ from its HPC- and ML-kernel mix, and its cost conclusions depend on the token weights chosen.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.20673", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Agents reach up to 4.2x speedup over automatic parallelizers on the benchmark's kernels; across 40 matched setups, added tools or skills significantly changed speedup only twice while increasing token cost by up to 3.0x.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.20673", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }