{ "schema_version": 1, "builder_plan": { "scenario": "A product assistant retrieves plausible documents but gives unsupported answers.", "change": "Change one retrieval or ranking step while keeping the document snapshot fixed.", "baseline": "Your current retrieval pipeline with identical questions and answer model.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Supported answers, retrieval misses, correct abstentions, index cost, and response latency.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.14762", "title": "TriCalRAG: A Three-Strategy, Retrieval-Augmented Benchmark for On-Premise LLM-Based Root Cause Analysis in AIOps", "summary": "Replaces zero-shot and few-shot prompting of cloud or local LLMs for log-based root cause analysis with retrieval over a labeled incident history served locally via vLLM. Costs a retrieval index and labeled incident data, plus the VRAM and roughly half throughput of the better-calibrated model; 4-bit quantization mitigates latency without accuracy loss. Can fail through calibration breakdown in some model/dataset configurations (Mistral-Small failed in 7 of 12), dependence on the quality and coverage of the incident history, and evaluation limited to four public log datasets that may not match production distributions.", "rationale": "RAG over a labeled incident history is feasible on a single 24 GB GPU with 4-bit quantization, but the accuracy-versus-calibration trade-off and dataset dependence require validation on the reader's own logs before production adoption.", "technique": "RAG over incident history for LLM log root cause analysis", "family": "recuperacao_e_rag", "recommendation": "testar", "published": "2026-09-13", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.14762", "independent_implementations": 2, "total_implementations": 3, "stars": 57, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "Replaces zero-shot and few-shot prompting of cloud or local LLMs for log-based root cause analysis with retrieval over a labeled incident history served locally via vLLM. Costs a retrieval index and labeled incident data, plus the VRAM and roughly half throughput of the better-calibrated model; 4-bit quantization mitigates latency without accuracy loss. Can fail through calibration breakdown in some model/dataset configurations (Mistral-Small failed in 7 of 12), dependence on the quality and coverage of the incident history, and evaluation limited to four public log datasets that may not match production distributions.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.14762", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "RAG improves mean F1 by 0.10-0.27 over zero-shot prompting and keeps predicted-positive rates near the true class balance; separately, batching scales throughput 41x on a single card and 4-bit quantization cuts latency 20% with no measurable accuracy loss.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.14762", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "no_equations", "core_kind": "", "equations_fetched_at": "2026-10-01", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }