{
  "id": "enwiki-001362608766",
  "slug": "mmlu",
  "title": "MMLU",
  "qid": "",
  "lead": "Measuring Massive Multitask Language Understanding (MMLU) is a popular benchmark for evaluating the capabilities of large language models. It inspired several other versions and spin-offs, such as MMLU-Pro , MMMLU and MMLU-Redux .",
  "body": "Measuring Massive Multitask Language Understanding (MMLU) is a popular benchmark for evaluating the capabilities of large language models. It inspired several other versions and spin-offs, such as MMLU-Pro , MMMLU and MMLU-Redux . ==Overview== MMLU consists of 15,908 multiple-choice questions, with 1,540 of them being used to select and assess optimal settings for models – temperature, batch size and learning rate. The questions span across 57 subjects, from highly complex STEM fields and international law to nutrition and religion. It was one of the most commonly used benchmarks for comparing the capabilities of large language models, with over 100&nbsp;million downloads as of July 2024. The benchmark was released by Dan Hendrycks and a team of researchers on 7&nbsp;September 2020. It was purpose-made to be more challenging than existing benchmarks at the time, such as General Language Understanding Evaluation (GLUE), as models began outperforming humans in easier tests. When MMLU was released, most existing language models scored near the level of random chance (25%). The best-performing model, GPT-3 175B, achieved 43.9% accuracy. The creators of the MMLU estimated that human domain-experts achieve around 89.8% accuracy. By mid-2024, the majority of powerful language models such as Claude 3.5 Sonnet, GPT-4o and Llama 3.1 405B consistently achieved 88%. As of 2025, MMLU has been partially phased out in favor of more difficult alternatives. ==References==",
  "categories": [
    "23118434",
    "23603337",
    "27077925",
    "52180832"
  ],
  "license": "CC BY-SA 4.0",
  "source_uri": "https://en.wikipedia.org/w/index.php?oldid=1362608766",
  "content_hash": "",
  "created_at": null,
  "updated_at": 1783213319.0,
  "claims": [
    {
      "id": 33767056,
      "subject_qid": "",
      "subject_slug": "mmlu",
      "predicate": "release date",
      "object_text": "2020",
      "object_num": 20200000.0,
      "object_unit": null,
      "polarity": 1,
      "qualifier": "",
      "source_uri": "https://en.wikipedia.org/w/index.php?oldid=1362608766",
      "source_name": "Wikipedia",
      "source_tier": 3,
      "retrieved_at": 1792538881.3121438,
      "confidence": 0.95,
      "article_slug": "mmlu",
      "span_start": 18,
      "span_end": 66
    }
  ],
  "contradictions": [],
  "confidence": 0.95,
  "csl": {
    "id": "enwiki-001362608766",
    "type": "webpage",
    "title": "MMLU",
    "author": [
      {
        "family": "Anticloud FZ LLE",
        "literal": "Anticloud FZ LLE"
      }
    ],
    "container-title": "encyclopegia.org",
    "publisher": "Anticloud FZ LLE",
    "publisher-place": "Dubai, United Arab Emirates",
    "URL": "https://encyclopegia.org/a/mmlu",
    "issued": {
      "date-parts": [
        [
          2026,
          11,
          21
        ]
      ]
    },
    "accessed": {
      "date-parts": [
        [
          2026,
          11,
          21
        ]
      ]
    },
    "license": "CC BY-SA 4.0",
    "DOI": "eo1:enwiki-001362608766",
    "isBasedOn": "https://en.wikipedia.org/w/index.php?oldid=1362608766",
    "note": "Immutable snapshot enwiki-001362608766"
  }
}