{
 "@context": {
  "@language": "en",
  "@vocab": "https://schema.org/",
  "citeAs": "cr:citeAs",
  "column": "cr:column",
  "conformsTo": "dct:conformsTo",
  "cr": "http://mlcommons.org/croissant/",
  "rai": "http://mlcommons.org/croissant/RAI/",
  "data": {
   "@id": "cr:data",
   "@type": "@json"
  },
  "dataType": {
   "@id": "cr:dataType",
   "@type": "@vocab"
  },
  "dct": "http://purl.org/dc/terms/",
  "equivalentProperty": "cr:equivalentProperty",
  "examples": {
   "@id": "cr:examples",
   "@type": "@json"
  },
  "extract": "cr:extract",
  "field": "cr:field",
  "fileProperty": "cr:fileProperty",
  "fileObject": "cr:fileObject",
  "fileSet": "cr:fileSet",
  "format": "cr:format",
  "includes": "cr:includes",
  "isLiveDataset": "cr:isLiveDataset",
  "jsonPath": "cr:jsonPath",
  "key": "cr:key",
  "md5": "cr:md5",
  "parentField": "cr:parentField",
  "path": "cr:path",
  "recordSet": "cr:recordSet",
  "references": "cr:references",
  "regex": "cr:regex",
  "repeated": "cr:repeated",
  "replace": "cr:replace",
  "samplingRate": "cr:samplingRate",
  "sc": "https://schema.org/",
  "separator": "cr:separator",
  "source": "cr:source",
  "subField": "cr:subField",
  "transform": "cr:transform"
 },
 "@type": "sc:Dataset",
 "conformsTo": "http://mlcommons.org/croissant/1.0",
 "name": "Tokenizer efficiency across languages: 16 tokenizers from DeepSeek, GLM, Qwen, Gemma, OpenAI, Meta, Moonshot, MiniMax, Microsoft, Mistral and NVIDIA",
 "description": "Token counts for 2,009 aligned FLORES-200 sentences per language (Modern Standard, Egyptian, Levantine, Najdi and Mesopotamian Arabic, Hindi, Urdu, Chinese, French, Spanish and English) and openly licensed texts (Gulf Arabic and Arabizi sentences from Tatoeba, Wikipedia prose, Python, JavaScript, SQL and JSON) under 16 official tokenizers pinned to exact commits (DeepSeek V4.1 Flash, GLM 5.3, Qwen3.8 27B, Gemma 4 VL 26B, GPT (o200k), gpt-oss (o200k_harmony), Llama 3.3 70B, Llama 4 Maverick, Kimi K3, MiniMax M3, Gemma 3 27B, Phi-4, Mistral Small 3.1, Ministral 3 14B, Nemotron 3 Ultra, Qwen3 32B), with parallel-text ratios against English and 95% bootstrap intervals, tokens per word, tokens per character and input cost per 1M characters on BroadNet models. FLORES-200 text is not included, only counts.",
 "url": "https://broadrouter.com/data/tokenizer-efficiency/",
 "version": "4.0",
 "datePublished": "2026-11-14",
 "dateModified": "2026-11-15",
 "license": "https://creativecommons.org/licenses/by/4.0/",
 "creator": {
  "@type": "Organization",
  "@id": "https://broadrouter.com/#org",
  "name": "BroadRouter",
  "url": "https://broadrouter.com"
 },
 "publisher": {
  "@type": "Organization",
  "@id": "https://broadrouter.com/#org",
  "name": "BroadRouter",
  "url": "https://broadrouter.com"
 },
 "citeAs": "@misc{broadrouter_tokenizer_efficiency,\n  author       = {BroadRouter},\n  title        = {Tokenizer efficiency across languages: 16 tokenizers from DeepSeek, GLM, Qwen, Gemma, OpenAI, Meta, Moonshot, MiniMax, Microsoft, Mistral and NVIDIA},\n  year         = {2026},\n  version      = {4.0},\n  publisher    = {BroadNet Technologies},\n  howpublished = {\\url{https://broadrouter.com/data/tokenizer-efficiency/}},\n  note         = {Data set, data as of 2026-11-15},\n}",
 "isAccessibleForFree": true,
 "keywords": [
  "tokenizer",
  "FLORES-200",
  "Arabic tokens",
  "Arabizi",
  "token tax",
  "tokens per word",
  "LLM token count",
  "DeepSeek tokenizer",
  "Qwen tokenizer",
  "GLM tokenizer",
  "Gemma tokenizer",
  "Llama tokenizer",
  "Kimi tokenizer",
  "MiniMax tokenizer",
  "Mistral Tekken",
  "o200k_base",
  "o200k_harmony"
 ],
 "inLanguage": [
  "en",
  "ar",
  "fr",
  "es",
  "hi",
  "ur",
  "zh"
 ],
 "distribution": [
  {
   "@type": "cr:FileObject",
   "@id": "tokenizer-efficiency.csv",
   "name": "tokenizer-efficiency.csv",
   "description": "CSV, one row per text and tokenizer (long format); the CSV has counts and labels only, no text. 832 rows, 21 columns, UTF-8, header row. Licence: Token counts, ratios, cost figures, tables and method: CC BY 4.0. The source texts in the JSON download are not relicensed: each keeps its source licence, given per text. Wikipedia texts are CC BY-SA 4.0, so a copy or adaptation of those texts must stay under CC BY-SA 4.0 and credit Wikipedia contributors (the revision link lists them). Tatoeba sentences are CC BY 2.0 FR: credit the authors listed per text (each sentence ID links to its author). Wikidata JSON is CC0; the Project Gutenberg excerpts are public domain. FLORES-200 sentences (CC BY-SA 4.0, NLLB Team, Meta) are not in the download at all: only our counts are, with the corpus pinned and linked. FLORES-200 sentences are CC BY-SA 4.0 (NLLB Team et al., No Language Left Behind: Scaling Human-Centered Machine Translation (2022), arXiv:2207.04672; FLORES-200 evaluation benchmark.). Our CC BY 4.0 files contain no FLORES-200 text: only our own token, word and character counts for each language, with the corpus identified by its pinned tarball (SHA-256 b8b0b76783024b85797e5cc75064eb83fc5288b41e9654dabc7be6ae944011f6) and splits (dev, devtest). Get the sentences from https://dl.fbaipublicfiles.com/nllb/flores200_dataset.tar.gz.",
   "contentUrl": "https://broadrouter.com/data/tokenizer-efficiency.csv",
   "encodingFormat": "text/csv",
   "contentSize": "191107 B",
   "sha256": "ce2a0ada3c79c15cbff9a86d230bcf86aad9d15e33ab2274548a2c32e67c8246"
  },
  {
   "@type": "cr:FileObject",
   "@id": "tokenizer-efficiency.json",
   "name": "tokenizer-efficiency.json",
   "description": "JSON with metadata, method, column dictionary, tokenizers and pinned commits, per-group summaries, bootstrap intervals and the full openly licensed texts (Tatoeba, Wikipedia, Project Gutenberg, Wikidata) with source, licence and revision per text. No FLORES-200 sentences. Licence: Token counts, ratios, cost figures, tables and method: CC BY 4.0. The source texts in the JSON download are not relicensed: each keeps its source licence, given per text. Wikipedia texts are CC BY-SA 4.0, so a copy or adaptation of those texts must stay under CC BY-SA 4.0 and credit Wikipedia contributors (the revision link lists them). Tatoeba sentences are CC BY 2.0 FR: credit the authors listed per text (each sentence ID links to its author). Wikidata JSON is CC0; the Project Gutenberg excerpts are public domain. FLORES-200 sentences (CC BY-SA 4.0, NLLB Team, Meta) are not in the download at all: only our counts are, with the corpus pinned and linked. FLORES-200 sentences are CC BY-SA 4.0 (NLLB Team et al., No Language Left Behind: Scaling Human-Centered Machine Translation (2022), arXiv:2207.04672; FLORES-200 evaluation benchmark.). Our CC BY 4.0 files contain no FLORES-200 text: only our own token, word and character counts for each language, with the corpus identified by its pinned tarball (SHA-256 b8b0b76783024b85797e5cc75064eb83fc5288b41e9654dabc7be6ae944011f6) and splits (dev, devtest). Get the sentences from https://dl.fbaipublicfiles.com/nllb/flores200_dataset.tar.gz.",
   "contentUrl": "https://broadrouter.com/data/tokenizer-efficiency.json",
   "encodingFormat": "application/json",
   "contentSize": "352027 B",
   "sha256": "c808dd8188c127155fd2cd53ac580d5b5bb3c3f030298857d92a79749cbb553f"
  }
 ],
 "recordSet": [
  {
   "@type": "cr:RecordSet",
   "@id": "tokenizer-efficiency",
   "name": "tokenizer-efficiency",
   "description": "The CSV file: one record per text and tokenizer (long format); the CSV has counts and labels only, no text.",
   "key": [
    {
     "@id": "tokenizer-efficiency/text_id"
    },
    {
     "@id": "tokenizer-efficiency/tokenizer"
    }
   ],
   "field": [
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/text_id",
     "name": "text_id",
     "description": "Text ID; the JSON \"corpus\" holds the full text, source, authors and sentence or revision IDs.",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "text_id"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/text",
     "name": "text",
     "description": "Short label for the text (the text itself is only in the JSON).",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "text"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/group",
     "name": "group",
     "description": "Language or text kind, e.g. ar-msa, arabizi, code-python, json.",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "group"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/kind",
     "name": "kind",
     "description": "parallel (FLORES-200 or Tatoeba sentences), parallel-en (their English side), prose, code or json.",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "kind"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/source",
     "name": "source",
     "description": "Where the text comes from: FLORES-200, Tatoeba, Wikipedia, Project Gutenberg or Wikidata.",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "source"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/license",
     "name": "license",
     "description": "Licence of the source text (it is not relicensed). FLORES-200 text is not in our files at all; only our counts are.",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "license"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/url",
     "name": "url",
     "description": "Source link: the pinned FLORES-200 page, first Tatoeba sentence ID, or the exact page revision.",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "url"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/sentences",
     "name": "sentences",
     "description": "Parallel texts: number of sentences (FLORES-200: 997 dev + 1,012 devtest).",
     "dataType": "sc:Integer",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "sentences"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/chars",
     "name": "chars",
     "description": "Unicode code points.",
     "dataType": "sc:Integer",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "chars"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/words",
     "name": "words",
     "description": "Word-like segments from Intl.Segmenter.",
     "dataType": "sc:Integer",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "words"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/tokenizer",
     "name": "tokenizer",
     "description": "Tokenizer ID: deepseek, glm, qwen, gemma, gpt (o200k_base), gptoss (o200k_harmony), llama3, llama4, kimi, minimax, gemma3, phi4, mistral, ministral, nemotron or qwen3. The JSON \"tokenizers\" list gives each repo and pinned commit.",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "tokenizer"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/core",
     "name": "core",
     "description": "yes for the five v2 tokenizers behind the site's headline averages and token tools; no for the 11 added in v3.",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "core"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/tokens",
     "name": "tokens",
     "description": "Token count, no special or chat-template tokens.",
     "dataType": "sc:Integer",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "tokens"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/tokens_per_word",
     "name": "tokens_per_word",
     "description": "tokens ÷ words.",
     "dataType": "sc:Float",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "tokens_per_word"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/tokens_per_char",
     "name": "tokens_per_char",
     "description": "tokens ÷ chars.",
     "dataType": "sc:Float",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "tokens_per_char"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/vs_english_per_word",
     "name": "vs_english_per_word",
     "description": "tokens_per_word ÷ English prose tokens per word, same tokenizer.",
     "dataType": "sc:Float",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "vs_english_per_word"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/vs_english_parallel",
     "name": "vs_english_parallel",
     "description": "Parallel texts only: tokens ÷ tokens of the English translation of the same sentences.",
     "dataType": "sc:Float",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "vs_english_parallel"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/vs_english_parallel_ci95_low",
     "name": "vs_english_parallel_ci95_low",
     "description": "Lower end of the 95% bootstrap interval for vs_english_parallel (10,000 seeded resamples of sentences).",
     "dataType": "sc:Float",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "vs_english_parallel_ci95_low"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/vs_english_parallel_ci95_high",
     "name": "vs_english_parallel_ci95_high",
     "description": "Upper end of that interval.",
     "dataType": "sc:Float",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "vs_english_parallel_ci95_high"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/broadnet_model",
     "name": "broadnet_model",
     "description": "BroadNet model that uses this tokenizer, if any.",
     "dataType": "sc:Text",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "broadnet_model"
      }
     }
    },
    {
     "@type": "cr:Field",
     "@id": "tokenizer-efficiency/input_usd_per_1m_chars",
     "name": "input_usd_per_1m_chars",
     "description": "tokens_per_char × that model's input price per 1M tokens (BroadNet pricing.json at build time).",
     "dataType": "sc:Float",
     "source": {
      "fileObject": {
       "@id": "tokenizer-efficiency.csv"
      },
      "extract": {
       "column": "input_usd_per_1m_chars"
      }
     }
    }
   ]
  }
 ]
}
