{
  "schema_version": 1,
  "builder_plan": {
    "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.",
    "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.",
    "baseline": "Your current evaluator and a manually reviewed reference set.",
    "steps": [
      "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.",
      "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.",
      "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.",
      "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test."
    ],
    "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.",
    "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation."
  },
  "builder_review": null,
  "arxiv_id": "2610.12183",
  "title": "A Closer Look at Agentic BBO: Benchmarking LLM Agents for Black-Box Optimization",
  "summary": "The paper introduces AgenticBBO-Bench, replacing ad-hoc, non-comparable evaluations of LLM agents for black-box optimization (synthetic functions, HPO, database tuning, chip and molecular design) with a unified finite-budget protocol, and shows agentic BBO generally beats both direct LLM prompting and classical numerical optimizers. Costs are LLM API calls per optimization step plus orchestration complexity, and the authors find that adding numerical tools does not consistently help while specific priors are unreliable. Failure modes include wasted evaluation budget on expensive objectives, gains that depend on the underlying LLM and harness, and limited transfer to domains outside the five benchmarked.",
  "rationale": "For a constrained team that occasionally needs hyperparameter or configuration tuning, an agentic BBO loop via third-party APIs is plausible, but the absence of a quantified factor and the mixed results on tools and priors mean the gain must be validated on their own workloads before production adoption.",
  "technique": "Benchmark of LLM agents for black-box optimization",
  "family": "avaliacao_de_agente",
  "recommendation": "testar",
  "published": "2026-10-08",
  "as_of": "2026-10-18",
  "editorial_status": "indexed",
  "source_url": "https://arxiv.org/abs/2610.12183",
  "independent_implementations": 2,
  "total_implementations": 3,
  "stars": 12,
  "citations": null,
  "claims": [
    {
      "claim_id": "claim-01",
      "statement": "The paper introduces AgenticBBO-Bench, replacing ad-hoc, non-comparable evaluations of LLM agents for black-box optimization (synthetic functions, HPO, database tuning, chip and molecular design) with a unified finite-budget protocol, and shows agentic BBO generally beats both direct LLM prompting and classical numerical optimizers. Costs are LLM API calls per optimization step plus orchestration complexity, and the authors find that adding numerical tools does not consistently help while specific priors are unreliable. Failure modes include wasted evaluation budget on expensive objectives, gains that depend on the underlying LLM and harness, and limited transfer to domains outside the five benchmarked.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.12183",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "claim_id": "claim-02",
      "statement": "Agentic BBO achieves higher family-averaged scores than direct LLM-based methods in all five domains and outperforms the best numerical optimizers in four of five; no multiplicative factor is reported.",
      "basis": "inferred",
      "result": "",
      "baseline": "",
      "conditions": "",
      "source_url": "https://arxiv.org/abs/2610.12183",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "exposure_map": [
    {
      "dimension": "quality",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compute",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "latency",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "operations",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "compatibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "security",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "data_and_training",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    },
    {
      "dimension": "reproducibility",
      "basis": "not_evaluated",
      "finding": "",
      "interpretation": "",
      "limitation": "",
      "next_check": "",
      "source_url": "",
      "source_page": null,
      "source_excerpt": ""
    }
  ],
  "risks": [],
  "minimum_test": [],
  "open_questions": [],
  "independent_tests": [],
  "lab_reproductions": [],
  "report_available": false,
  "validation_tier": "unknown",
  "evidence_tier": "unknown",
  "infrastructure_basis": "unknown",
  "equations": [],
  "equations_status": "not_fetched",
  "core_kind": "",
  "equations_fetched_at": "",
  "equations_source_url": "",
  "equations_source_sha256": "",
  "exposures_reviewed_at": "",
  "exposures_source_sha256": ""
}