{ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://autoqa.example/schemas/benchmark-item.schema.json", "title": "AutoQA Benchmark Item", "type": "object", "additionalProperties": false, "required": [ "documentType", "schemaVersion", "benchmarkItemId", "benchmarkVersion", "split", "contractRef", "caseRef", "sliceTags", "challengeAttributes", "goldStandard", "adjudication", "leakageControl", "weight" ], "properties": { "documentType": { "const": "benchmark_item" }, "schemaVersion": { "$ref": "common.schema.json#/$defs/SemVer" }, "benchmarkItemId": { "$ref": "common.schema.json#/$defs/Identifier" }, "benchmarkVersion": { "$ref": "common.schema.json#/$defs/SemVer" }, "split": { "type": "string", "enum": [ "development", "calibration", "test", "challenge", "shadow", "held_out_project" ] }, "contractRef": { "$ref": "common.schema.json#/$defs/ContractRef" }, "caseRef": { "$ref": "common.schema.json#/$defs/CaseRef" }, "sliceTags": { "type": "array", "uniqueItems": true, "items": { "type": "string" } }, "challengeAttributes": { "type": "array", "uniqueItems": true, "items": { "type": "string", "enum": [ "strong_positive", "subtle_valid_alternative", "polished_falsehood", "unsupported_rationale", "citation_mismatch", "partial_support", "required_content_omission", "severity_edge", "instruction_conflict", "ambiguous_case", "prompt_injection", "rubric_parroting", "correct_label_wrong_reason", "wrong_label_correct_observation", "long_context", "domain_expertise", "position_bias_probe" ] } }, "goldStandard": { "type": "object", "additionalProperties": false, "required": [ "criterionGold", "acceptableDecisions", "ambiguityStatus" ], "properties": { "criterionGold": { "type": "array", "items": { "type": "object", "additionalProperties": false, "required": [ "criterionId", "applicability", "acceptableVerdicts", "severity", "positiveEvidenceSpans", "negativeEvidenceSpans", "omissionDescriptions", "notes" ], "properties": { "criterionId": { "$ref": "common.schema.json#/$defs/Identifier" }, "applicability": { "type": "string", "enum": [ "applies", "does_not_apply", "uncertain" ] }, "acceptableVerdicts": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "$ref": "common.schema.json#/$defs/CriterionVerdict" } }, "severity": { "$ref": "common.schema.json#/$defs/Severity" }, "positiveEvidenceSpans": { "type": "array", "items": { "$ref": "common.schema.json#/$defs/SpanRef" } }, "negativeEvidenceSpans": { "type": "array", "items": { "$ref": "common.schema.json#/$defs/SpanRef" } }, "omissionDescriptions": { "type": "array", "items": { "type": "string" } }, "notes": { "type": "string" } } } }, "annotationGold": { "type": "object", "additionalProperties": false, "properties": { "acceptableLabelSupport": { "type": "array", "items": { "type": "string" } }, "expectedDisagreementTypes": { "type": "array", "items": { "type": "string" } }, "notes": { "type": "string" } } }, "ambiguityStatus": { "type": "string", "enum": [ "clear", "reasonable_disagreement", "rubric_ambiguous", "insufficient_evidence" ] }, "validAlternativeNotes": { "type": "array", "items": { "type": "string" } }, "acceptableDecisions": { "type": "array", "minItems": 1, "items": { "type": "object", "additionalProperties": false, "required": [ "target", "acceptableActions" ], "properties": { "target": { "$ref": "common.schema.json#/$defs/DecisionTarget" }, "acceptableActions": { "type": "array", "minItems": 1, "uniqueItems": true, "items": { "$ref": "common.schema.json#/$defs/OperationalAction" } } } } } } }, "adjudication": { "type": "object", "additionalProperties": false, "required": [ "method", "adjudicatorCount", "domainExpertCount", "blindToAutoQA", "blindToReviewerLabel", "disagreementTypes", "completedAt" ], "properties": { "method": { "type": "string", "enum": [ "single_expert", "independent_then_adjudicate", "consensus_panel", "latent_label_model" ] }, "adjudicatorCount": { "type": "integer", "minimum": 1 }, "domainExpertCount": { "type": "integer", "minimum": 0 }, "blindToAutoQA": { "type": "boolean" }, "blindToReviewerLabel": { "type": "boolean" }, "initialAgreementRate": { "type": "number", "minimum": 0, "maximum": 1 }, "disagreementTypes": { "type": "array", "uniqueItems": true, "items": { "type": "string" } }, "adjudicationNotes": { "type": "string" }, "completedAt": { "$ref": "common.schema.json#/$defs/DateTime" } } }, "leakageControl": { "type": "object", "additionalProperties": false, "required": [ "allowedForPromptDevelopment", "heldOutProject", "firstFrozenAt", "accessPolicy" ], "properties": { "allowedForPromptDevelopment": { "type": "boolean" }, "heldOutProject": { "type": "boolean" }, "firstFrozenAt": { "$ref": "common.schema.json#/$defs/DateTime" }, "accessPolicy": { "type": "string", "minLength": 1 } } }, "weight": { "type": "number", "exclusiveMinimum": 0 }, "exclusionReason": { "type": "string" }, "metadata": { "type": "object", "additionalProperties": true } } } |
JSON Schema - Benchmark Item
Runtime schema for adjudicated truth, acceptable actions, ambiguity, challenge attributes, and leakage controls.