{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.29695", "title": "Benchmarking Coding Agents for System-Level Performance Optimization", "summary": "This benchmark replaces ad hoc or function-level evaluation of coding agents with 65 reproducible system-level optimization instances across seven microservice systems, each with executable environments, functional tests, and mixed-workload measurement. The cost is infrastructure effort: each instance requires a runnable multi-service environment and workload harness, which is heavier than typical function-level benchmarks though feasible on a single machine or modest cloud budget. It can fail to predict production behavior because the anti-patterns are synthetically injected rather than observed in real incidents, so agent scores may not transfer to organically occurring bottlenecks.", "rationale": "The reader adopting coding agents in production needs a realistic validation tool before trusting agents with performance work, and this benchmark directly measures the failure mode (correct but ineffective changes) that matters for that decision.", "technique": "MicroPerf-Bench: system-level coding agent benchmark", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-21", "as_of": "2026-10-22", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.29695", "independent_implementations": 0, "total_implementations": 1, "stars": 0, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "This benchmark replaces ad hoc or function-level evaluation of coding agents with 65 reproducible system-level optimization instances across seven microservice systems, each with executable environments, functional tests, and mixed-workload measurement. The cost is infrastructure effort: each instance requires a runnable multi-service environment and workload harness, which is heavier than typical function-level benchmarks though feasible on a single machine or modest cloud budget. It can fail to predict production behavior because the anti-patterns are synthetically injected rather than observed in real incidents, so agent scores may not transfer to organically occurring bottlenecks.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.29695", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Across 1,755 runs and nine agent-model configurations, the best configuration recovers only 42.0% of injected performance degradation; the dominant failure is functionally correct changes that do not improve end-to-end performance.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.29695", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }