{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.26714", "title": "HoloCosm: Beyond Single-Metric Benchmark Compression Across Multiple Metrics", "summary": "HoloCosm replaces full-benchmark evaluation and single-metric sample-selection methods by selecting a small subset of samples that jointly minimizes score-reconstruction error across multiple metrics while constraining ranking distortion. It costs an offline optimization pass over existing model scores on the full benchmark, meaning it requires score data from a population of source models and adds selection complexity before it saves any compute. It can fail when applied to models outside the source population used for selection, when metric correlations shift, or when too few historical model scores exist to calibrate the compressed subset, in which case random sampling may match or beat it.", "rationale": "The method credibly reduces evaluation cost by roughly 5x while preserving multi-metric rankings, but its gains depend on having a corpus of prior model scores and workload-specific validation before production adoption.", "technique": "Multi-metric benchmark compression (HoloCosm)", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-19", "as_of": "2026-10-20", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.26714", "independent_implementations": 1, "total_implementations": 2, "stars": 122, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "HoloCosm replaces full-benchmark evaluation and single-metric sample-selection methods by selecting a small subset of samples that jointly minimizes score-reconstruction error across multiple metrics while constraining ranking distortion. It costs an offline optimization pass over existing model scores on the full benchmark, meaning it requires score data from a population of source models and adds selection complexity before it saves any compute. It can fail when applied to models outside the source population used for selection, when metric correlations shift, or when too few historical model scores exist to calibrate the compressed subset, in which case random sampling may match or beat it.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.26714", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "At 80% compression, average NMAE drops 35.3% to 0.236, worst-component NMAE drops 39.5% to 0.330, and average Spearman correlation rises from 0.884 to 0.915 versus random sampling, across 15 benchmarks with 85 component metrics.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.26714", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }