{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.06193", "title": "Correct Code, Broken Contributions? SWE-CC: Benchmarking Repository Policy Compliance for Coding Agents", "summary": "The paper replaces test-pass-only evaluation of coding agents with deterministic per-policy checkers plus runtime auditing of agent behavior and final deliverables. For a small team the cost is authoring machine-checkable versions of their own contribution policies and wiring the checks into CI or the agent scaffold; the benchmark itself requires no GPU, but reproducing it at scale does not apply directly. What can fail is incomplete or noisy policy extraction from documentation, checkers that flag superficial patterns while missing intent, and a false sense of governance coverage if only easy-to-check policies are encoded.", "rationale": "The finding that correct patches routinely violate project policy is directly actionable for anyone deploying coding agents in production, and adding lightweight deterministic policy checks to an agent pipeline is feasible on constrained infrastructure, but the value depends on the reader's own repositories and policies, so it warrants a small pilot rather than full adoption.", "technique": "Repository policy compliance auditing for coding agents", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-05", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.06193", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "The paper replaces test-pass-only evaluation of coding agents with deterministic per-policy checkers plus runtime auditing of agent behavior and final deliverables. For a small team the cost is authoring machine-checkable versions of their own contribution policies and wiring the checks into CI or the agent scaffold; the benchmark itself requires no GPU, but reproducing it at scale does not apply directly. What can fail is incomplete or noisy policy extraction from documentation, checkers that flag superficial patterns while missing intent, and a false sense of governance coverage if only easy-to-check policies are encoded.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.06193", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Agents produce functionally correct patches yet violate 43.1% of applicable repository policies, with nearly half of violations occurring in intermediate execution steps; this is a measured gap, not an improvement factor.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.06193", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }