{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.24928", "title": "Trajectory-Aware Benchmark Subset Selection for Cost-Efficient Software Engineering Agent Regression Testing", "summary": "It replaces random or pass/fail-stratified random benchmark sampling with deterministic selection of instances closest to the centroid of each outcome group in a trajectory-embedding space, preserving the historical pass/fail rate. The cost is one recent full benchmark run to obtain trajectories and outcome labels, plus embedding computation and selection infrastructure, and the selected subset is fixed rather than freshly sampled each run. It can fail when agent changes alter behavior in ways not represented by the reference trajectories (e.g., a new framework whose errors concentrate on unselected instances), and the method presumes access to logged trajectories, which third-party API-only agent stacks may not expose.", "rationale": "The 90% token-cost reduction directly addresses a real production bottleneck for teams iterating on SWE-agents, but the error guarantees are validated only against the paper's three regression scenarios and require existing trajectory logs, so workload-specific validation on the reader's own agent and benchmark is warranted before relying on it for release decisions.", "technique": "Trajectory-aware benchmark subset selection", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-09-21", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.24928", "independent_implementations": 0, "total_implementations": 1, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "It replaces random or pass/fail-stratified random benchmark sampling with deterministic selection of instances closest to the centroid of each outcome group in a trajectory-embedding space, preserving the historical pass/fail rate. The cost is one recent full benchmark run to obtain trajectories and outcome labels, plus embedding computation and selection infrastructure, and the selected subset is fixed rather than freshly sampled each run. It can fail when agent changes alter behavior in ways not represented by the reference trajectories (e.g., a new framework whose errors concentrate on unselected instances), and the method presumes access to logged trajectories, which third-party API-only agent stacks may not expose.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.24928", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "A 10% trajectory-aware subset keeps median pass-rate estimation error below 5% while cutting token cost by roughly 90%, reducing average estimation error by 3-11% and worst-case error by up to 46% relative to the strongest stratified-random baseline.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.24928", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }