{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2610.06744", "title": "ufakzeka-karar: An Open Turkish Typed-Decision Model with Order-Invariant Option Scoring", "summary": "Replaces generative multiple-choice or Likert/yes-no answering (prompting an LLM to emit an option) with a 182M-parameter scoring head that returns temperature-scaled probabilities per option plus an expected-error 'not sure' signal in one CPU forward pass for up to ten options. Costs: a fixed answer-type task format, an additional small model to host and calibrate, and quality below top leaderboard systems; calibration via temperature scaling degraded on held-out support questions (smooth ECE 0.027 to 0.045), so recalibration is needed per domain. Failure modes: Turkish-only scope, benchmark numbers contaminated by iterative test-set reading on guardrail, moderation, and customer-support tracks, and expected-error abstention that may not transfer to new question distributions.", "rationale": "For Turkish fixed-type decision tasks it offers cheap CPU inference and order-invariant scoring that generative prompting lacks, but its non-blind evaluation and calibration regression on unseen questions require workload-specific validation before production use.", "technique": "Order-invariant typed-decision classification head (no generation)", "family": "avaliacao_de_agente", "recommendation": "testar", "published": "2026-10-05", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2610.06744", "independent_implementations": 1, "total_implementations": 2, "stars": 122, "citations": null, "claims": [ { "claim_id": "claim-01", "statement": "Replaces generative multiple-choice or Likert/yes-no answering (prompting an LLM to emit an option) with a 182M-parameter scoring head that returns temperature-scaled probabilities per option plus an expected-error 'not sure' signal in one CPU forward pass for up to ten options. Costs: a fixed answer-type task format, an additional small model to host and calibrate, and quality below top leaderboard systems; calibration via temperature scaling degraded on held-out support questions (smooth ECE 0.027 to 0.045), so recalibration is needed per domain. Failure modes: Turkish-only scope, benchmark numbers contaminated by iterative test-set reading on guardrail, moderation, and customer-support tracks, and expected-error abstention that may not transfer to new question distributions.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.06744", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Composite 0.660 (95% CI 0.642-0.677), 7th of 16 on HakemBench v1.0; a sequential head changed 2.3-2.8% of answers under option reordering, and REINFORCE lost 10.2 macro F1 points to cross-entropy. Benchmark numbers are not blind and three tracks are flagged as shaped by reading test results.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2610.06744", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }