Improved tokenizer

added_tokens.json

@@ -4,5 +4,6 @@
   "[MASK]": 103,
   "[PAD]": 0,
   "[SEP]": 102,
-  "[UNK]": 100
+  "[UNK]": 100,
+  "[URL]": 30523
 }

config.json

@@ -32,5 +32,5 @@
   "transformers_version": "4.34.0",
   "type_vocab_size": 2,
   "use_cache": true,
-  "vocab_size": 30523
+  "vocab_size": 30524
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:11e80d3faac321eaade102956c75dfb3d4b50950f95ed9a773f31ee2746ce04d
-size 438129650
+oid sha256:81ad53d92c2e6e7974b64abf67be4dfcd32c80702ab9b205be5d544d8c139ccd
+size 438132722

tokenizer.json

@@ -70,6 +70,15 @@
       "rstrip": false,
       "normalized": true,
       "special": false
+    },
+    {
+      "id": 30523,
+      "content": "[URL]",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -47,6 +47,14 @@
       "rstrip": false,
       "single_word": false,
       "special": false
+    },
+    "30523": {
+      "content": "[URL]",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "additional_special_tokens": [],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:62d0097c98d05139df21066c9233196a833baec3c003dd35e8dfdd8cf61fc111
+oid sha256:b9acb4e58e8ed6bf0bd605a87d385ad8d18dd39d69684d4df4033eb2f0e53238
 size 4472