{
  "title": "Jev on German legal text: engine test",
  "publisher": "Lexbeam Software",
  "url": "https://normlotse.de/messung",
  "run_date": "2026-09-20",
  "model": {
    "name": "Jev 1.13",
    "vendor": "TypeSafe",
    "kind": "System One (typed-decision) model",
    "access": "OpenRouter System One endpoint, early access"
  },
  "corpus": {
    "items_collected": 200,
    "items_analysed": 185,
    "feeds": 11,
    "sources": [
      "three state data protection authorities (Baden-Württemberg, Bavaria, Saxony)",
      "the DSK",
      "the EDPB",
      "the BSI",
      "four federal courts (BGH, BAG, BVerwG, BVerfG)",
      "the European Commission"
    ],
    "questions_per_item": 15,
    "question_language": "German"
  },
  "reference": {
    "kind": "majority of three chat models",
    "jurors": [
      "GPT-5.6 Luna",
      "GLM 5.3 Flash",
      "gpt-oss-120b"
    ],
    "contested_left_out": 59,
    "judgments_before_exclusion": 2655
  },
  "results": {
    "judgments_compared": 2596,
    "agreement": 0.974,
    "precision": 0.897,
    "recall": 0.902,
    "confusion": {
      "tp": 304,
      "fp": 35,
      "fn": 33,
      "tn": 2224,
      "positives": 337
    },
    "brier": 0.0228,
    "brier_base_rate": 0.113,
    "latency_s": {
      "median": 0.43,
      "p95": 0.65
    },
    "usd_per_1000_judgments": 0.0077,
    "by_language": {
      "german_items": {
        "n": 2375,
        "agreement": 0.975
      },
      "english_items": {
        "n": 221,
        "agreement": 0.959
      }
    },
    "policy_bands": {
      "fire_at_or_above": 0.8,
      "silent_at_or_below": 0.3,
      "fired": 248,
      "fired_against_jury": 10,
      "silent": 2190,
      "silent_on_jury_positive": 14,
      "review": 158,
      "decided_share_pct": 93.9,
      "review_share_pct": 6.1
    },
    "calibration_bins": [
      {
        "from": 0.0,
        "to": 0.1,
        "judgments": 1927,
        "jury_true": 1
      },
      {
        "from": 0.1,
        "to": 0.2,
        "judgments": 185,
        "jury_true": 4
      },
      {
        "from": 0.2,
        "to": 0.3,
        "judgments": 78,
        "jury_true": 9
      },
      {
        "from": 0.3,
        "to": 0.4,
        "judgments": 40,
        "jury_true": 9
      },
      {
        "from": 0.4,
        "to": 0.5,
        "judgments": 28,
        "jury_true": 10
      },
      {
        "from": 0.5,
        "to": 0.6,
        "judgments": 34,
        "jury_true": 21
      },
      {
        "from": 0.6,
        "to": 0.7,
        "judgments": 30,
        "jury_true": 22
      },
      {
        "from": 0.7,
        "to": 0.8,
        "judgments": 27,
        "jury_true": 23
      },
      {
        "from": 0.8,
        "to": 0.9,
        "judgments": 48,
        "jury_true": 40
      },
      {
        "from": 0.9,
        "to": 1.0,
        "judgments": 200,
        "jury_true": 198
      }
    ],
    "questions": [
      {
        "question_de": "Gerichtsentscheidung",
        "gloss_en": "court decision",
        "jury_positives": 63,
        "caught_at_0_5": 61,
        "false_alarms_at_0_5": 0
      },
      {
        "question_de": "Datenschutz",
        "gloss_en": "data protection",
        "jury_positives": 57,
        "caught_at_0_5": 56,
        "false_alarms_at_0_5": 4
      },
      {
        "question_de": "Datenpanne / Sicherheit",
        "gloss_en": "breach or security",
        "jury_positives": 38,
        "caught_at_0_5": 29,
        "false_alarms_at_0_5": 2
      },
      {
        "question_de": "Handlungsbedarf",
        "gloss_en": "action required",
        "jury_positives": 37,
        "caught_at_0_5": 30,
        "false_alarms_at_0_5": 3
      },
      {
        "question_de": "KI",
        "gloss_en": "artificial intelligence",
        "jury_positives": 34,
        "caught_at_0_5": 33,
        "false_alarms_at_0_5": 1
      },
      {
        "question_de": "Termin / Personalie",
        "gloss_en": "event or appointment",
        "jury_positives": 25,
        "caught_at_0_5": 21,
        "false_alarms_at_0_5": 0
      },
      {
        "question_de": "EU-Digitalrecht",
        "gloss_en": "EU digital law",
        "jury_positives": 20,
        "caught_at_0_5": 17,
        "false_alarms_at_0_5": 3
      },
      {
        "question_de": "Öffentliche Stellen",
        "gloss_en": "public bodies",
        "jury_positives": 18,
        "caught_at_0_5": 15,
        "false_alarms_at_0_5": 6
      },
      {
        "question_de": "Betroffenenrechte",
        "gloss_en": "data subject rights",
        "jury_positives": 15,
        "caught_at_0_5": 13,
        "false_alarms_at_0_5": 2
      },
      {
        "question_de": "Gesundheit",
        "gloss_en": "health",
        "jury_positives": 9,
        "caught_at_0_5": 8,
        "false_alarms_at_0_5": 0
      },
      {
        "question_de": "Video",
        "gloss_en": "video",
        "jury_positives": 6,
        "caught_at_0_5": 6,
        "false_alarms_at_0_5": 6
      },
      {
        "question_de": "Bußgeld / Sanktion",
        "gloss_en": "fine or sanction",
        "jury_positives": 5,
        "caught_at_0_5": 5,
        "false_alarms_at_0_5": 4
      },
      {
        "question_de": "Werbung / Tracking",
        "gloss_en": "advertising or tracking",
        "jury_positives": 5,
        "caught_at_0_5": 5,
        "false_alarms_at_0_5": 1
      },
      {
        "question_de": "Beschäftigte",
        "gloss_en": "employees",
        "jury_positives": 3,
        "caught_at_0_5": 3,
        "false_alarms_at_0_5": 0
      },
      {
        "question_de": "Drittland",
        "gloss_en": "third country",
        "jury_positives": 2,
        "caught_at_0_5": 2,
        "false_alarms_at_0_5": 3
      }
    ],
    "jurors": [
      {
        "name": "GPT-5.6 Luna",
        "median_s_per_item": 2.49,
        "agreement_with_other_two": 0.966,
        "usd_per_1000_judgments": 0.0549
      },
      {
        "name": "GLM 5.3 Flash",
        "median_s_per_item": 13.67,
        "agreement_with_other_two": 0.985,
        "usd_per_1000_judgments": 0.0863
      },
      {
        "name": "gpt-oss-120b",
        "median_s_per_item": 14.89,
        "agreement_with_other_two": 0.968,
        "usd_per_1000_judgments": 0.0238
      }
    ]
  },
  "limits": [
    "The reference is a model jury, not human ground truth; a human audit is next.",
    "One run on one day; 15 PDF-only items skipped; one item refused by a content filter.",
    "Rare questions rest on few positives (two for third countries, three for employees).",
    "Aggregate results only; per-item model outputs are not published."
  ],
  "not_legal_advice": true,
  "contact": "https://lexbeam.com/kontakt"
}