{ "schema_version": 1, "builder_plan": { "scenario": "Start with one repeated product task that has a measurable failure or cost.", "change": "Read the methods and identify one change you can isolate in that workflow.", "baseline": "Your current implementation on the same held-out tasks.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Successful outcomes, response time, human rework, and total cost per successful task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.15624", "title": "Beyond AI Literacy: A Structured Review and Exploratory Meta-Analysis of Measures for Competent Generative-AI Use", "summary": "This proposes a four-domain assessment battery (knowledge, epistemic oversight, reliance calibration, operational control of agents) to replace informal self-reported AI literacy when evaluating workers or applicants, but its thresholds and decision rules are untested. The cost is implementing and maintaining multi-domain objective tests rather than cheap self-ratings, and no validated instrument yet covers the full agent-oversight combination the authors describe. The key failure mode is documented in the paper itself: pooled self-report versus objective performance correlations are near zero (r = .055, CI includes negative values), so any adoption that falls back on self-assessment would yield invalid competence judgments.", "rationale": "The paper is a measurement review for assessing human users, not an engineering technique, and its proposed battery is explicitly unvalidated, so the reader should track it rather than adopt it in production.", "technique": "Four-layer workplace GenAI competence assessment battery", "family": "outro", "recommendation": "observar", "published": "2026-09-14", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.15624", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "This proposes a four-domain assessment battery (knowledge, epistemic oversight, reliance calibration, operational control of agents) to replace informal self-reported AI literacy when evaluating workers or applicants, but its thresholds and decision rules are untested. The cost is implementing and maintaining multi-domain objective tests rather than cheap self-ratings, and no validated instrument yet covers the full agent-oversight combination the authors describe. The key failure mode is documented in the paper itself: pooled self-report versus objective performance correlations are near zero (r = .055, CI includes negative values), so any adoption that falls back on self-assessment would yield invalid competence judgments.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.15624", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [ { "anchor": "S3.E1", "label": "(1)", "section": "3 Methods", "role": "baseline", "latex": "z_{i}=\\operatorname{atanh}(r_{i}),\\qquad v_{i}=\\frac{1}{N_{i}-3}.", "mathml": "zi=atanh⁡(ri),vi=1Ni−3.", "context": "Each direct correlation was transformed as", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S3.E2", "label": "(2)", "section": "3 Methods", "role": "proposed_method", "latex": "q=\\frac{\\sum_{i}w_{i}(z_{i}-\\hat{\\mu})^{2}}{k-1},\\qquad\\hat{\\mu}\\;\\pm\\;t_{k-1,.975}\\sqrt{\\frac{q}{\\sum_{i}w_{i}}}.", "mathml": "q=∑iwi​(zi−μ^)2k−1,μ^±tk−1,.975​q∑iwi.", "context": "We fitted a random-effects model in Fisher-z space using restricted maximum likelihood (REML) to estimate between-study variance. With wi=1/(vi+τ^2) and μ^=∑iwi​zi/∑iwi, we used the unmodified Hartung-Knapp interval as the primary interval (Knapp and Hartung, 2003; Röver et al., 2015):", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" }, { "anchor": "S3.E3", "label": "(3)", "section": "3 Methods", "role": "metric", "latex": "v_{\\text{comp}}=\\frac{1}{N-3}\\cdot\\frac{1+(m-1)\\rho}{m},", "mathml": "vcomp=1N−3⋅1+(m−1)​ρm,", "context": "The 12 coefficients share participants and factors. Because their covariance matrix is unavailable, we approximated the sampling variance of the composite as", "headline": "", "explanation": "", "objective": "", "symbols": {}, "evidence": "", "evidence_url": "", "source_url": "" } ], "equations_status": "selected", "core_kind": "formula", "equations_fetched_at": "2026-10-08", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }