{ "schema_version": 1, "builder_plan": { "scenario": "An agent looks good in demos, but you cannot tell which product tasks it fails.", "change": "Use the study's evaluation idea to inspect a small set of tasks with human-checked answers.", "baseline": "Your current evaluator and a manually reviewed reference set.", "steps": [ "Locate the paper's implementation and check its license, model support, data needs, and hardware requirements. Stop here if you cannot obtain the required artifacts.", "Save a small set of representative product tasks and expected outcomes. Keep evaluation tasks out of any training or tuning, and choose quality and cost limits before running.", "Run the baseline and the proposed change on identical inputs. Keep other settings fixed and repeat paired runs to expose variation.", "Save per-task outputs, failures, timings, and all usage costs. Review regressions before expanding the test." ], "measure": "Agreement with human review, missed failures, false alarms, and evaluation cost per task.", "decision": "Keep the change only if it meets your preselected quality and latency limits and improves the metric you care about after setup and operating costs. Otherwise retain the baseline. This is an exploration plan, not a validated recommendation." }, "builder_review": null, "arxiv_id": "2609.27408", "title": "What Looks Like a Capability Limit in Vision-Language Models Is a Readout Limit", "summary": "This replaces the assumption that benchmark answer conventions (letters, names, pixel coordinates, hue angles) are neutral, requiring practitioners to audit answer formats and scorer parsing when evaluating VLMs rather than adopting new models or infrastructure. The cost is additional evaluation engineering: testing each model under multiple answer conventions, verifying that scoring matches the model's output format (the authors measured capable models as incapable five times due to scorer disagreement), and avoiding coordinate-format options when the underlying task does not require them. What can fail is that convention sensitivity is model-specific (GPT-4o shows no penalty), conventions a model demonstrably parses did not predict accuracy on untried formats, and a fixed answer vocabulary can itself disadvantage particular models, so findings must be revalidated per model and per benchmark.", "rationale": "For a small team selecting VLMs via third-party APIs, benchmark rankings can reverse purely due to answer formatting, so auditing readout conventions before trusting comparative scores is a zero-infrastructure change that directly affects adoption decisions.", "technique": "Answer-format-aware VLM evaluation (readout vs capability)", "family": "avaliacao_de_agente", "recommendation": "adotar", "published": "2026-09-23", "as_of": "2026-10-18", "editorial_status": "indexed", "source_url": "https://arxiv.org/abs/2609.27408", "independent_implementations": 0, "total_implementations": 0, "stars": 0, "citations": 0, "claims": [ { "claim_id": "claim-01", "statement": "This replaces the assumption that benchmark answer conventions (letters, names, pixel coordinates, hue angles) are neutral, requiring practitioners to audit answer formats and scorer parsing when evaluating VLMs rather than adopting new models or infrastructure. The cost is additional evaluation engineering: testing each model under multiple answer conventions, verifying that scoring matches the model's output format (the authors measured capable models as incapable five times due to scorer disagreement), and avoiding coordinate-format options when the underlying task does not require them. What can fail is that convention sensitivity is model-specific (GPT-4o shows no penalty), conventions a model demonstrably parses did not predict accuracy on untried formats, and a fixed answer vocabulary can itself disadvantage particular models, so findings must be revalidated per model and per benchmark.", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.27408", "source_page": null, "source_excerpt": "" }, { "claim_id": "claim-02", "statement": "Qwen3-VL-4B answers correctly 68.5% of the time with English location names versus 20.0% with pixel-coordinate options (chance 11.1%); tied models differ by 39-54 points depending on coordinate convention, reversing the winner; Gemini shows an 11.1-point penalty on parseable answers (p = 1e-4).", "basis": "inferred", "result": "", "baseline": "", "conditions": "", "source_url": "https://arxiv.org/abs/2609.27408", "source_page": null, "source_excerpt": "" } ], "exposure_map": [ { "dimension": "quality", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compute", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "latency", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "operations", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "compatibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "security", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "data_and_training", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" }, { "dimension": "reproducibility", "basis": "not_evaluated", "finding": "", "interpretation": "", "limitation": "", "next_check": "", "source_url": "", "source_page": null, "source_excerpt": "" } ], "risks": [], "minimum_test": [], "open_questions": [], "independent_tests": [], "lab_reproductions": [], "report_available": false, "validation_tier": "unknown", "evidence_tier": "unknown", "infrastructure_basis": "unknown", "equations": [], "equations_status": "not_fetched", "core_kind": "", "equations_fetched_at": "", "equations_source_url": "", "equations_source_sha256": "", "exposures_reviewed_at": "", "exposures_source_sha256": "" }