Super-squash branch 'main' using huggingface_hub

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,32 @@
+---
+license: mit
+library_name: transformers
+---
+
+# tiktoken `cl100k_base`: as HF MLM tokenizer
+
+based on `RobertaTokenizerFast`
+
+
+```py
+from pathlib import Path
+from transformers import RobertaTokenizerFast, AutoTokenizer
+
+repo_id = "BEE-spoke-data/cl100k_base-mlm"
+tk = AutoTokenizer.from_pretrained(repo_id)
+len(tk)
+# 100266
+```
+
+testing that it does what it should:
+
+```py
+input_text = "i love memes"
+tokenized_ids = tk.encode(input_text)
+decoded_tokens = tk.convert_ids_to_tokens(tokenized_ids)
+
+print(f"for input '{input_text}' -> {tokenized_ids} -> {decoded_tokens}")
+# for input 'i love memes' -> [100277, 72, 3021, 62277, 100278] -> ['<s>', 'i', 'Ġlove', 'Ġmemes', '</s>']
+```
+
+---

added_tokens.json

@@ -0,0 +1,7 @@
+{
+  "</s>": 100278,
+  "<mask>": 100281,
+  "<pad>": 100280,
+  "<s>": 100277,
+  "<unk>": 100279
+}

special_tokens_map.json

@@ -0,0 +1,51 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "cls_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "mask_token": {
+    "content": "<mask>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tiktoken-to-hf-masked.ipynb

@@ -0,0 +1,430 @@
+{
+  "cells": [
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "ZypJVeIMFQGQ"
+      },
+      "outputs": [],
+      "source": [
+        "import tiktoken\n",
+        "from transformers.models.roberta.tokenization_roberta import bytes_to_unicode\n",
+        "from typing import Dict, Optional\n",
+        "\n",
+        "byte_encoder = bytes_to_unicode()\n",
+        "\n",
+        "\n",
+        "def token_bytes_to_string(b):\n",
+        "    return \"\".join([byte_encoder[ord(char)] for char in b.decode(\"latin-1\")])\n",
+        "\n",
+        "\n",
+        "# Adapted from https://github.com/openai/tiktoken/issues/60#issuecomment-1499977960\n",
+        "def bpe(\n",
+        "    mergeable_ranks: dict[bytes, int], token: bytes, max_rank: Optional[int] = None\n",
+        ") -> list[bytes]:\n",
+        "    parts = [bytes([b]) for b in token]\n",
+        "    while True:\n",
+        "        min_idx = None\n",
+        "        min_rank = None\n",
+        "        for i, pair in enumerate(zip(parts[:-1], parts[1:])):\n",
+        "            rank = mergeable_ranks.get(pair[0] + pair[1])\n",
+        "            if rank is not None and (min_rank is None or rank < min_rank):\n",
+        "                min_idx = i\n",
+        "                min_rank = rank\n",
+        "        if min_rank is None or (max_rank is not None and min_rank >= max_rank):\n",
+        "            break\n",
+        "        assert min_idx is not None\n",
+        "        parts = (\n",
+        "            parts[:min_idx]\n",
+        "            + [parts[min_idx] + parts[min_idx + 1]]\n",
+        "            + parts[min_idx + 2 :]\n",
+        "        )\n",
+        "    return parts\n",
+        "\n",
+        "\n",
+        "def generate_vocab_and_merges(encoder):\n",
+        "    mergeable_ranks = encoder._mergeable_ranks\n",
+        "\n",
+        "    merges = []\n",
+        "    vocab = {}\n",
+        "    for token, rank in mergeable_ranks.items():\n",
+        "        vocab[token_bytes_to_string(token)] = rank\n",
+        "\n",
+        "        if len(token) == 1:\n",
+        "            continue\n",
+        "        merged = tuple(bpe(mergeable_ranks, token, max_rank=rank))\n",
+        "        assert len(merged) == 2\n",
+        "\n",
+        "        merges.append(\" \".join(map(token_bytes_to_string, merged)))\n",
+        "\n",
+        "    # Also add special tokens\n",
+        "    vocab.update(encoder._special_tokens)\n",
+        "\n",
+        "    return vocab, merges"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "enc = tiktoken.get_encoding(\"cl100k_base\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "O87Zz6Vzhb5C"
+      },
+      "outputs": [],
+      "source": [
+        "import json\n",
+        "import os\n",
+        "from transformers import GPT2TokenizerFast, AutoTokenizer\n",
+        "\n",
+        "\n",
+        "# https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb\n",
+        "\n",
+        "MODEL_INFO = {\n",
+        "    # GPT-2 and GPT-3 models (r50k_base)\n",
+        "    \"gpt2\": {\n",
+        "        \"tokenizer_class\": \"GPT2Tokenizer\",\n",
+        "        \"model_max_length\": 1024,\n",
+        "    },\n",
+        "    \"davinci\": {  # (gpt-3)\n",
+        "        \"tokenizer_class\": \"GPT3Tokenizer\",\n",
+        "        \"model_max_length\": 2048,\n",
+        "    },\n",
+        "    # GPT-3.5 and GPT-4 models (cl100k_base)\n",
+        "    \"gpt-3.5-turbo\": {\n",
+        "        \"tokenizer_class\": \"GPT3_5Tokenizer\",\n",
+        "        \"model_max_length\": 4096,\n",
+        "    },\n",
+        "    \"gpt-3.5-turbo-16k\": {\n",
+        "        \"tokenizer_class\": \"GPT3_5Tokenizer\",\n",
+        "        \"model_max_length\": 16384,\n",
+        "    },\n",
+        "    \"gpt-4\": {\n",
+        "        \"tokenizer_class\": \"GPT4Tokenizer\",\n",
+        "        \"model_max_length\": 8192,\n",
+        "    },\n",
+        "    \"cl100k_base\": {\n",
+        "        \"tokenizer_class\": \"RobertaTokenizer\",\n",
+        "        \"model_max_length\": 8192,\n",
+        "    },\n",
+        "    \"text-embedding-ada-002\": {\n",
+        "        \"tokenizer_class\": \"GPT4Tokenizer\",\n",
+        "        \"model_max_length\": 8192,\n",
+        "    },\n",
+        "    # Codex models (p50k_base)\n",
+        "    \"text-davinci-002\": {\n",
+        "        \"tokenizer_class\": \"CodexTokenizer\",\n",
+        "        \"model_max_length\": 4096,\n",
+        "    },\n",
+        "    \"text-davinci-003\": {\n",
+        "        \"tokenizer_class\": \"CodexTokenizer\",\n",
+        "        \"model_max_length\": 4096,\n",
+        "    },\n",
+        "}\n",
+        "\n",
+        "\n",
+        "def convert_tiktoken(model_name, output_dir=None):\n",
+        "    if output_dir is None:\n",
+        "        output_dir = model_name\n",
+        "\n",
+        "    encoder = tiktoken.get_encoding(\"cl100k_base\")\n",
+        "\n",
+        "    vocab, merges = generate_vocab_and_merges(encoder)\n",
+        "\n",
+        "    added_tokens = [\n",
+        "        {\n",
+        "            \"id\": id,\n",
+        "            \"content\": content,\n",
+        "            \"single_word\": False,\n",
+        "            \"lstrip\": False,\n",
+        "            \"rstrip\": False,\n",
+        "            \"normalized\": False,\n",
+        "            \"special\": True,\n",
+        "        }\n",
+        "        for content, id in encoder._special_tokens.items()\n",
+        "    ]\n",
+        "    # original_mlm_specials = [\n",
+        "    #     {\n",
+        "    #         \"id\": 0,\n",
+        "    #         \"content\": \"<s>\",\n",
+        "    #         \"single_word\": False,\n",
+        "    #         \"lstrip\": False,\n",
+        "    #         \"rstrip\": False,\n",
+        "    #         \"normalized\": True,\n",
+        "    #         \"special\": True,\n",
+        "    #     },\n",
+        "    #     {\n",
+        "    #         \"id\": 1,\n",
+        "    #         \"content\": \"<pad>\",\n",
+        "    #         \"single_word\": False,\n",
+        "    #         \"lstrip\": False,\n",
+        "    #         \"rstrip\": False,\n",
+        "    #         \"normalized\": True,\n",
+        "    #         \"special\": True,\n",
+        "    #     },\n",
+        "    #     {\n",
+        "    #         \"id\": 2,\n",
+        "    #         \"content\": \"</s>\",\n",
+        "    #         \"single_word\": False,\n",
+        "    #         \"lstrip\": False,\n",
+        "    #         \"rstrip\": False,\n",
+        "    #         \"normalized\": True,\n",
+        "    #         \"special\": True,\n",
+        "    #     },\n",
+        "    #     {\n",
+        "    #         \"id\": 3,\n",
+        "    #         \"content\": \"<unk>\",\n",
+        "    #         \"single_word\": False,\n",
+        "    #         \"lstrip\": False,\n",
+        "    #         \"rstrip\": False,\n",
+        "    #         \"normalized\": True,\n",
+        "    #         \"special\": True,\n",
+        "    #     },\n",
+        "    #     {\n",
+        "    #         \"id\": 50264,\n",
+        "    #         \"content\": \"<mask>\",\n",
+        "    #         \"single_word\": False,\n",
+        "    #         \"lstrip\": True,\n",
+        "    #         \"rstrip\": False,\n",
+        "    #         \"normalized\": False,\n",
+        "    #         \"special\": True,\n",
+        "    #     },\n",
+        "    # ]\n",
+        "    # added_tokens.extend(original_mlm_specials)\n",
+        "\n",
+        "    # https://huggingface.co/Xenova/gpt2/raw/main/tokenizer.json\n",
+        "    tokenizer_template = {\n",
+        "        \"version\": \"1.0\",\n",
+        "        \"truncation\": None,\n",
+        "        \"padding\": None,\n",
+        "        \"added_tokens\": added_tokens,\n",
+        "        \"normalizer\": None,\n",
+        "        \"pre_tokenizer\": {\n",
+        "            \"type\": \"ByteLevel\",\n",
+        "            \"add_prefix_space\": False,\n",
+        "            \"trim_offsets\": True,\n",
+        "            \"use_regex\": True,\n",
+        "        },\n",
+        "        \"post_processor\": {\n",
+        "            \"type\": \"RobertaProcessing\",\n",
+        "            \"sep\": [\"</s>\", 2],  # YOU HAVE TO UPDATE THIS TO THE ACTUAL ID!!\n",
+        "            \"cls\": [\"<s>\", 0],  # TODO: add auto-fixing for this\n",
+        "            \"trim_offsets\": True,\n",
+        "            \"add_prefix_space\": False,\n",
+        "        },\n",
+        "        \"decoder\": {\n",
+        "            \"type\": \"ByteLevel\",\n",
+        "            \"add_prefix_space\": True,\n",
+        "            \"trim_offsets\": True,\n",
+        "            \"use_regex\": True,\n",
+        "        },\n",
+        "        \"model\": {\n",
+        "            \"type\": \"BPE\",\n",
+        "            \"dropout\": None,\n",
+        "            \"unk_token\": None,\n",
+        "            \"continuing_subword_prefix\": \"\",\n",
+        "            \"end_of_word_suffix\": \"\",\n",
+        "            \"fuse_unk\": False,\n",
+        "            \"byte_fallback\": False,\n",
+        "            \"vocab\": vocab,\n",
+        "            \"merges\": merges,\n",
+        "        },\n",
+        "    }\n",
+        "\n",
+        "    # https://huggingface.co/Xenova/gpt2/raw/main/tokenizer_config.json\n",
+        "\n",
+        "    tokenizer_config_template = {\n",
+        "        \"add_prefix_space\": False,\n",
+        "        \"bos_token\": \"<s>\",\n",
+        "        \"clean_up_tokenization_spaces\": True,\n",
+        "        \"cls_token\": \"<s>\",\n",
+        "        \"eos_token\": \"</s>\",\n",
+        "        \"errors\": \"replace\",\n",
+        "        \"mask_token\": \"<mask>\",\n",
+        "        \"pad_token\": \"<pad>\",\n",
+        "        \"sep_token\": \"</s>\",\n",
+        "        \"trim_offsets\": True,\n",
+        "        \"unk_token\": \"<unk>\",\n",
+        "    }\n",
+        "    tokenizer_config_template.update(\n",
+        "        MODEL_INFO[model_name]\n",
+        "    )  # Adds `model_max_length` and `tokenizer_class`\n",
+        "    tokenizer_config_template = dict(\n",
+        "        sorted(tokenizer_config_template.items(), key=lambda x: x[0])\n",
+        "    )\n",
+        "\n",
+        "    # special tokens map\n",
+        "    MLM_specials = {\n",
+        "        \"bos_token\": \"<s>\",\n",
+        "        \"cls_token\": \"<s>\",\n",
+        "        \"eos_token\": \"</s>\",\n",
+        "        \"mask_token\": {\n",
+        "            \"content\": \"<mask>\",\n",
+        "            \"lstrip\": True,\n",
+        "            \"normalized\": False,\n",
+        "            \"rstrip\": False,\n",
+        "            \"single_word\": False,\n",
+        "        },\n",
+        "        \"pad_token\": \"<pad>\",\n",
+        "        \"sep_token\": \"</s>\",\n",
+        "        \"unk_token\": \"<unk>\",\n",
+        "    }\n",
+        "    os.makedirs(output_dir, exist_ok=True)\n",
+        "\n",
+        "    # Save to files\n",
+        "    with open(os.path.join(output_dir, \"vocab.json\"), \"w\", encoding=\"utf-8\") as fp:\n",
+        "        json.dump(vocab, fp, indent=2, ensure_ascii=False)\n",
+        "\n",
+        "    with open(os.path.join(output_dir, \"tokenizer.json\"), \"w\", encoding=\"utf-8\") as fp:\n",
+        "        json.dump(tokenizer_template, fp, indent=2, ensure_ascii=False)\n",
+        "\n",
+        "    with open(\n",
+        "        os.path.join(output_dir, \"tokenizer_config.json\"), \"w\", encoding=\"utf-8\"\n",
+        "    ) as fp:\n",
+        "        json.dump(tokenizer_config_template, fp, indent=2, ensure_ascii=False)\n",
+        "\n",
+        "    with open(\n",
+        "        os.path.join(output_dir, \"special_tokens_map.json\"), \"w\", encoding=\"utf-8\"\n",
+        "    ) as fp:\n",
+        "        json.dump(\n",
+        "            MLM_specials,\n",
+        "            fp,\n",
+        "            indent=2,\n",
+        "            ensure_ascii=False,\n",
+        "        )\n",
+        "\n",
+        "    with open(os.path.join(output_dir, \"merges.txt\"), \"w\", encoding=\"utf-8\") as fp:\n",
+        "        fp.write(\"#version: 0.2\\n\")\n",
+        "        fp.write(\"\\n\".join(merges))\n",
+        "\n",
+        "    # load with autotokenizer and rewrite\n",
+        "    tk = AutoTokenizer.from_pretrained(output_dir)\n",
+        "    tk.save_pretrained(output_dir)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from pathlib import Path\n",
+        "\n",
+        "output_dir = \"tiktoken-tokenizers\"\n",
+        "outpath = Path.cwd() / output_dir / \"cl100k_base-as-roberta\"\n",
+        "convert_tiktoken(\"cl100k_base\", outpath)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 11,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stderr",
+          "output_type": "stream",
+          "text": [
+            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
+          ]
+        },
+        {
+          "data": {
+            "text/plain": [
+              "100266"
+            ]
+          },
+          "execution_count": 11,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "from pathlib import Path\n",
+        "from transformers import GPT2TokenizerFast, AutoTokenizer\n",
+        "\n",
+        "\n",
+        "tk = AutoTokenizer.from_pretrained(outpath)\n",
+        "len(tk)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 12,
+      "metadata": {},
+      "outputs": [
+        {
+          "name": "stdout",
+          "output_type": "stream",
+          "text": [
+            "[100277, 72, 3021, 62277, 100278] ['<s>', 'i', 'Ġlove', 'Ġmemes', '</s>']\n"
+          ]
+        }
+      ],
+      "source": [
+        "input_text = \"i love memes\"\n",
+        "tokenized_ids = tk.encode(input_text)\n",
+        "decoded_tokens = tk.convert_ids_to_tokens(tokenized_ids)\n",
+        "\n",
+        "print(tokenized_ids, decoded_tokens)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": 10,
+      "metadata": {},
+      "outputs": [
+        {
+          "data": {
+            "text/plain": [
+              "100277"
+            ]
+          },
+          "execution_count": 10,
+          "metadata": {},
+          "output_type": "execute_result"
+        }
+      ],
+      "source": [
+        "tk.cls_token_id"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "---\n"
+      ]
+    }
+  ],
+  "metadata": {
+    "colab": {
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.13"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}

tokenizer_config.json

@@ -0,0 +1,97 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100278": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100279": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100280": {
+      "content": "<pad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100281": {
+      "content": "<mask>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "<s>",
+  "eos_token": "</s>",
+  "errors": "replace",
+  "mask_token": "<mask>",
+  "model_max_length": 8192,
+  "pad_token": "<pad>",
+  "sep_token": "</s>",
+  "tokenizer_class": "RobertaTokenizer",
+  "trim_offsets": true,
+  "unk_token": "<unk>"
+}