{ "schema_version": 1, "builder_plan": { "scenario": "An assistant chooses the wrong API or sends arguments your product cannot use.", "change": "Apply the proposed tool-selection or argument-generation change in a sandbox.", "baseline": "Your current agent with the same tool schemas and task budget.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Correct completed actions, invalid calls, retries, latency, and cost per completed task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.02206", "title": "KaliBench: A Fine-Grained Benchmark for Cybersecurity Tool Use on Kali Linux with Runtime-Free Verifiable Rewards", "summary": "KaliBench replaces knowledge-based or end-to-end agentic cybersecurity evaluation with fine-grained natural-language-to-CLI pairs (8,504 queries, 1,642 tools) and runtime-free verifiable rewards usable for SFT and RLVR. Adopting it costs building or reusing the dataset, canonicalization and alias-aware scoring infrastructure, and fine-tuning runs, though an 8B model with parameter-efficient methods fits a single 24 GB GPU; teams outside the cybersecurity domain gain nothing directly. It can fail through reward hacking of exact-match signals, overfitting to the benchmark's 1,642-tool distribution while real environments use different tool versions and flags, and commands that score correct but are unsafe or non-executable in a live sandbox.", "rationale": "The approach is reproducible at 8B scale on a single GPU and demonstrates a large quality gain, but its value is confined to cybersecurity CLI workloads, so a small-scale pilot on the reader's actual tasks is warranted before production adoption.", "technique": "Verifiable-reward training for cybersecurity CLI generation", "family": "uso_de_ferramenta", "recommendation": "testar", "published": "2026-10-01", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.02206", "independent_implementations": 2, "total_implementations": 2, "stars": 1, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "KaliBench replaces knowledge-based or end-to-end agentic cybersecurity evaluation with fine-grained natural-language-to-CLI pairs (8,504 queries, 1,642 tools) and runtime-free verifiable rewards usable for SFT and RLVR. Adopting it costs building or reusing the dataset, canonicalization and alias-aware scoring infrastructure, and fine-tuning runs, though an 8B model with parameter-efficient methods fits a single 24 GB GPU; teams outside the cybersecurity domain gain nothing directly. It can fail through reward hacking of exact-match signals, overfitting to the benchmark's 1,642-tool distribution while real environments use different tool versions and flags, and commands that score correct but are unsafe or non-executable in a live sandbox.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.02206", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "No open-weight model exceeds 42% exact-command accuracy in the unrestricted setting; SFT plus RL with verifiable rewards from KaliBench raises an 8B model to performance comparable to a 685B MoE model.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.02206", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }