{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://autoqa.example/schemas/benchmark-item.schema.json",
  "title": "AutoQA Benchmark Item",
  "type": "object",
  "additionalProperties": false,
  "required": [
    "documentType",
    "schemaVersion",
    "benchmarkItemId",
    "benchmarkVersion",
    "split",
    "contractRef",
    "caseRef",
    "sliceTags",
    "challengeAttributes",
    "goldStandard",
    "adjudication",
    "leakageControl",
    "weight"
  ],
  "properties": {
    "documentType": {
      "const": "benchmark_item"
    },
    "schemaVersion": {
      "$ref": "common.schema.json#/$defs/SemVer"
    },
    "benchmarkItemId": {
      "$ref": "common.schema.json#/$defs/Identifier"
    },
    "benchmarkVersion": {
      "$ref": "common.schema.json#/$defs/SemVer"
    },
    "split": {
      "type": "string",
      "enum": [
        "development",
        "calibration",
        "test",
        "challenge",
        "shadow",
        "held_out_project"
      ]
    },
    "contractRef": {
      "$ref": "common.schema.json#/$defs/ContractRef"
    },
    "caseRef": {
      "$ref": "common.schema.json#/$defs/CaseRef"
    },
    "sliceTags": {
      "type": "array",
      "uniqueItems": true,
      "items": {
        "type": "string"
      }
    },
    "challengeAttributes": {
      "type": "array",
      "uniqueItems": true,
      "items": {
        "type": "string",
        "enum": [
          "strong_positive",
          "subtle_valid_alternative",
          "polished_falsehood",
          "unsupported_rationale",
          "citation_mismatch",
          "partial_support",
          "required_content_omission",
          "severity_edge",
          "instruction_conflict",
          "ambiguous_case",
          "prompt_injection",
          "rubric_parroting",
          "correct_label_wrong_reason",
          "wrong_label_correct_observation",
          "long_context",
          "domain_expertise",
          "position_bias_probe"
        ]
      }
    },
    "goldStandard": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "criterionGold",
        "acceptableDecisions",
        "ambiguityStatus"
      ],
      "properties": {
        "criterionGold": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "criterionId",
              "applicability",
              "acceptableVerdicts",
              "severity",
              "positiveEvidenceSpans",
              "negativeEvidenceSpans",
              "omissionDescriptions",
              "notes"
            ],
            "properties": {
              "criterionId": {
                "$ref": "common.schema.json#/$defs/Identifier"
              },
              "applicability": {
                "type": "string",
                "enum": [
                  "applies",
                  "does_not_apply",
                  "uncertain"
                ]
              },
              "acceptableVerdicts": {
                "type": "array",
                "minItems": 1,
                "uniqueItems": true,
                "items": {
                  "$ref": "common.schema.json#/$defs/CriterionVerdict"
                }
              },
              "severity": {
                "$ref": "common.schema.json#/$defs/Severity"
              },
              "positiveEvidenceSpans": {
                "type": "array",
                "items": {
                  "$ref": "common.schema.json#/$defs/SpanRef"
                }
              },
              "negativeEvidenceSpans": {
                "type": "array",
                "items": {
                  "$ref": "common.schema.json#/$defs/SpanRef"
                }
              },
              "omissionDescriptions": {
                "type": "array",
                "items": {
                  "type": "string"
                }
              },
              "notes": {
                "type": "string"
              }
            }
          }
        },
        "annotationGold": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "acceptableLabelSupport": {
              "type": "array",
              "items": {
                "type": "string"
              }
            },
            "expectedDisagreementTypes": {
              "type": "array",
              "items": {
                "type": "string"
              }
            },
            "notes": {
              "type": "string"
            }
          }
        },
        "ambiguityStatus": {
          "type": "string",
          "enum": [
            "clear",
            "reasonable_disagreement",
            "rubric_ambiguous",
            "insufficient_evidence"
          ]
        },
        "validAlternativeNotes": {
          "type": "array",
          "items": {
            "type": "string"
          }
        },
        "acceptableDecisions": {
          "type": "array",
          "minItems": 1,
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": [
              "target",
              "acceptableActions"
            ],
            "properties": {
              "target": {
                "$ref": "common.schema.json#/$defs/DecisionTarget"
              },
              "acceptableActions": {
                "type": "array",
                "minItems": 1,
                "uniqueItems": true,
                "items": {
                  "$ref": "common.schema.json#/$defs/OperationalAction"
                }
              }
            }
          }
        }
      }
    },
    "adjudication": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "method",
        "adjudicatorCount",
        "domainExpertCount",
        "blindToAutoQA",
        "blindToReviewerLabel",
        "disagreementTypes",
        "completedAt"
      ],
      "properties": {
        "method": {
          "type": "string",
          "enum": [
            "single_expert",
            "independent_then_adjudicate",
            "consensus_panel",
            "latent_label_model"
          ]
        },
        "adjudicatorCount": {
          "type": "integer",
          "minimum": 1
        },
        "domainExpertCount": {
          "type": "integer",
          "minimum": 0
        },
        "blindToAutoQA": {
          "type": "boolean"
        },
        "blindToReviewerLabel": {
          "type": "boolean"
        },
        "initialAgreementRate": {
          "type": "number",
          "minimum": 0,
          "maximum": 1
        },
        "disagreementTypes": {
          "type": "array",
          "uniqueItems": true,
          "items": {
            "type": "string"
          }
        },
        "adjudicationNotes": {
          "type": "string"
        },
        "completedAt": {
          "$ref": "common.schema.json#/$defs/DateTime"
        }
      }
    },
    "leakageControl": {
      "type": "object",
      "additionalProperties": false,
      "required": [
        "allowedForPromptDevelopment",
        "heldOutProject",
        "firstFrozenAt",
        "accessPolicy"
      ],
      "properties": {
        "allowedForPromptDevelopment": {
          "type": "boolean"
        },
        "heldOutProject": {
          "type": "boolean"
        },
        "firstFrozenAt": {
          "$ref": "common.schema.json#/$defs/DateTime"
        },
        "accessPolicy": {
          "type": "string",
          "minLength": 1
        }
      }
    },
    "weight": {
      "type": "number",
      "exclusiveMinimum": 0
    },
    "exclusionReason": {
      "type": "string"
    },
    "metadata": {
      "type": "object",
      "additionalProperties": true
    }
  }
}