{ "schema_version": 1, "builder_plan": { "scenario": "Your inference service slows down when several customers send requests together.", "change": "Replay a recorded request trace with only the proposed serving policy changed.", "baseline": "Your current scheduler on the same model and hardware.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Requests meeting your latency target, throughput, failures, and cost per successful request.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.17475", "title": "JustFit: 200K-Token LLM Serving on a 24 GiB Laptop with Just-in-Time State Management", "summary": "JustFit replaces the standard mlx-vlm runtime's eager KV allocation and full-component residency with just-in-time materialization, compressed KV execution, and state-preserving transitions, independent of weight quantization. The cost is a specialized MLX-based runtime restricted to Apple Silicon, added systems complexity in state lifecycle management, and reliance on MXFP4-class compressed execution whose latency at long contexts (19.11 tokens/s at 32K input) is modest. Failure modes include reconstruction or swap errors corrupting long reasoning chains, degraded throughput under multi-request contention, and fragility outside the tested Qwen/MLX stack, with the AIME result serving as anecdotal rather than systematic quality evidence.", "rationale": "The 6.93x context gain on exactly the reader's hardware class is compelling, but the runtime is MLX/Apple-Silicon-specific, early-stage, and lacks independent validation, so it warrants a workload-specific pilot rather than production adoption.", "technique": "Just-in-time KV and state management for long-context local serving", "family": "serving_e_batching", "recommendation": "testar", "published": "2026-09-15", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.17475", "independent_implementations": 0, "total_implementations": 1, "stars": 2, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "JustFit replaces the standard mlx-vlm runtime's eager KV allocation and full-component residency with just-in-time materialization, compressed KV execution, and state-preserving transitions, independent of weight quantization. The cost is a specialized MLX-based runtime restricted to Apple Silicon, added systems complexity in state lifecycle management, and reliance on MXFP4-class compressed execution whose latency at long contexts (19.11 tokens/s at 32K input) is modest. Failure modes include reconstruction or swap errors corrupting long reasoning chains, degraded throughput under multi-request contention, and fragility outside the tested Qwen/MLX stack, with the AIME result serving as anecdotal rather than systematic quality evidence.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.17475", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Completed single-request context rises from 30,720 to 212,992 positions (6.93x) on a 24 GiB M4 Pro running Qwen3.8-27B MXFP4, with 19.11 tokens/s on a 32K-input probe and a median peak footprint of 16,374 MiB.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.17475", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }