Step... (186000/250000 | Loss: 1.7381113767623901, Acc: 0.6502522826194763): 74%|█████████████████ | 186090/250000 [2:39:06<32:58:04, 1.86s/it]

flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:393c37966461709fe51a3b3f84befb7fa7e5030025856d171308efd40dbbc7da
+oid sha256:3362fadd5306539e1775600df59349acf0fd260751129abaa45b997091d089ce
 size 249750019

mc4/mc4.py

@@ -1,11 +1,11 @@
-"""mC4 dataset based on Common Crawl."""
+"""Perplexity Sampled mC4 dataset based on Common Crawl."""
 
 
 import gzip
 import json
 
 import datasets
-import kenlm
+import kenlm  # pip install https://github.com/kpu/kenlm/archive/master.zip
 import numpy as np
 from numpy.random import default_rng
 
@@ -289,6 +289,7 @@ class Mc4(datasets.GeneratorBasedBuilder):
         self.sampling_factor = kwargs.pop("sampling_factor", None)
         self.boundaries = kwargs.pop("boundaries", None)
         self.seed = kwargs.pop("seed", None)
+        self.kwargs = kwargs
         if self.sampling_method:
             if self.seed is not None:
                 self.rng = default_rng(self.seed)
@@ -316,7 +317,7 @@ class Mc4(datasets.GeneratorBasedBuilder):
             doc_length += length
         return 10.0 ** (-doc_log_score / doc_length)
 
-    def _should_keep_doc_step(self, doc, factor=1.5e5, boundaries=None):
+    def _should_keep_doc_step(self, doc, factor=1.5e5, boundaries=None, **kwargs):
         perplexity = self.get_perplexity(doc)
         if boundaries is None:
             boundaries = [536394.99320948, 662247.50212365, 919250.87225178]
@@ -331,17 +332,18 @@ class Mc4(datasets.GeneratorBasedBuilder):
         probability = factor / quartile_range
         return self.rng.uniform() < probability
 
-    def _should_keep_doc_gaussian(self, doc, factor=0.78, boundaries=None):
+    def _should_keep_doc_gaussian(self, doc, factor=0.78, boundaries=None, **kwargs):
+        width = kwargs.get("width", 9 / 2)  # width (spread) of the exponential curve
         perplexity = self.get_perplexity(doc)
         if boundaries is not None:
             m = boundaries[1]
         else:
             m = 662247.50212365
-        exponential = np.exp(-9/2 * ((perplexity - m) / m) ** 2)
+        exponential = np.exp((-1 / width) * ((perplexity - m) / m) ** 2)
         weighted_perplexity = factor * exponential
         return self.rng.uniform() < weighted_perplexity
 
-    def _should_keep_doc_random(self, doc, factor=None, boundaries=None):
+    def _should_keep_doc_random(self, doc, factor=None, boundaries=None, **kwargs):
         if factor is None:
             factor = 0.5
         return self.rng.uniform() <= factor
@@ -374,13 +376,13 @@ class Mc4(datasets.GeneratorBasedBuilder):
                 for lang in self.config.languages
                 for index in range(_N_SHARDS_PER_SPLIT[lang][split])
             ]
-        if "train" in self.data_files:
+        if self.data_files and "train" in self.data_files:
             train_downloaded_files = self.data_files["train"]
             if not isinstance(train_downloaded_files, (tuple, list)):
                 train_downloaded_files = [train_downloaded_files]
         else:
             train_downloaded_files = dl_manager.download(data_urls["train"])
-        if "validation" in self.data_files:
+        if self.data_files and "validation" in self.data_files:
             validation_downloaded_files = self.data_files["validation"]
             if not isinstance(validation_downloaded_files, (tuple, list)):
                 validation_downloaded_files = [validation_downloaded_files]
@@ -415,7 +417,8 @@ class Mc4(datasets.GeneratorBasedBuilder):
                                 if self.should_keep_doc(
                                     example["text"],
                                     factor=self.sampling_factor,
-                                    boundaries=self.boundaries):
+                                    boundaries=self.boundaries,
+                                    **self.kwargs):
                                     yield id_, example
                                     id_ += 1
                     else:

outputs/checkpoints/checkpoint-140001/training_state.json

@@ -1 +0,0 @@
-{"step": 140001}

outputs/checkpoints/checkpoint-150001/training_state.json

@@ -1 +0,0 @@
-{"step": 150001}

outputs/checkpoints/checkpoint-160001/training_state.json

@@ -1 +0,0 @@
-{"step": 160001}

outputs/checkpoints/checkpoint-170001/training_state.json

@@ -1 +0,0 @@
-{"step": 170001}

outputs/checkpoints/checkpoint-180001/data_collator.joblib

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e02a6e9cfa63cb321cac9402efd29841b652999fcbf787800ae050e747b161ee
-size 1471394

outputs/checkpoints/checkpoint-180001/flax_model.msgpack

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:393c37966461709fe51a3b3f84befb7fa7e5030025856d171308efd40dbbc7da
-size 249750019

outputs/checkpoints/checkpoint-180001/optimizer_state.msgpack

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3a33cad417a7e78eaafc1c041f93fd54ad9f63869d01e1351bac4abcd58e4eeb
-size 499500278

outputs/checkpoints/checkpoint-180001/training_args.joblib

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:8bc14fe16573d318dd510c7cfb42ebb7cc87b4dcf77e99247a2d1605cffd772b
-size 1876

outputs/checkpoints/checkpoint-180001/training_state.json

@@ -1 +0,0 @@
-{"step": 180001}

outputs/checkpoints/{checkpoint-140001 → checkpoint-182000}/data_collator.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e02a6e9cfa63cb321cac9402efd29841b652999fcbf787800ae050e747b161ee
-size 1471394
+oid sha256:0321b1a9629e1be122045cd72470365a63c8496fec109fdeec34827f01ffbb9e
+size 1471424

outputs/checkpoints/{checkpoint-150001 → checkpoint-182000}/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9f2a38ac6c111d01809dd28ae9078aab932064126a7de753ce0d88bd60421e4
+oid sha256:73b385f39d585a15aeee595103523252872a4277da8c21c9bc9bbfd0232a72c9
 size 249750019

outputs/checkpoints/{checkpoint-150001 → checkpoint-182000}/optimizer_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:84f53f9b574ccfb97696f637d71903b9762ef2718c656bea201e5aeb9078c328
+oid sha256:a6c074f7bc1330f514d95eb6277f0232abc85b0ec2f8c1c5972fd46af01985d4
 size 499500278

outputs/checkpoints/{checkpoint-160001 → checkpoint-182000}/training_args.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bc14fe16573d318dd510c7cfb42ebb7cc87b4dcf77e99247a2d1605cffd772b
-size 1876
+oid sha256:79196d5f797a7527e60287e48f716fd4626c5d57186661d6a214a2027998c86d
+size 1873

outputs/checkpoints/checkpoint-182000/training_state.json

@@ -0,0 +1 @@
+{"step": 182001}

outputs/checkpoints/{checkpoint-150001 → checkpoint-183000}/data_collator.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e02a6e9cfa63cb321cac9402efd29841b652999fcbf787800ae050e747b161ee
-size 1471394
+oid sha256:0321b1a9629e1be122045cd72470365a63c8496fec109fdeec34827f01ffbb9e
+size 1471424

outputs/checkpoints/{checkpoint-140001 → checkpoint-183000}/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bb3b6443b0b4e0fd6b95f7409525ddde51fb73dd99318041f2fecda9f547f5a6
+oid sha256:0059f1c021d4be233f41fc29384c42e8844d69c44e1d31806620a38d76d6aa00
 size 249750019

outputs/checkpoints/{checkpoint-160001 → checkpoint-183000}/optimizer_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea3a8f65ea9c3c6c3606f1167c4e54049784fa8b2a5ee3f4936563ecd4f811b6
+oid sha256:e99296d552d436c722721d611be2ed7cf4862c4b6a40abbc3464f7a75ca25c58
 size 499500278

outputs/checkpoints/{checkpoint-170001 → checkpoint-183000}/training_args.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bc14fe16573d318dd510c7cfb42ebb7cc87b4dcf77e99247a2d1605cffd772b
-size 1876
+oid sha256:79196d5f797a7527e60287e48f716fd4626c5d57186661d6a214a2027998c86d
+size 1873

outputs/checkpoints/checkpoint-183000/training_state.json

@@ -0,0 +1 @@
+{"step": 183001}

outputs/checkpoints/{checkpoint-160001 → checkpoint-184000}/data_collator.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e02a6e9cfa63cb321cac9402efd29841b652999fcbf787800ae050e747b161ee
-size 1471394
+oid sha256:0321b1a9629e1be122045cd72470365a63c8496fec109fdeec34827f01ffbb9e
+size 1471424

outputs/checkpoints/{checkpoint-160001 → checkpoint-184000}/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b86d26169d8fb7bb58ae7fecd67ca557a0affc93bf2d5b5947af0070ee894ab9
+oid sha256:5063b47bd0228e701c1cbc12f4ce305c577868d34589323f88eb3ca2057b322d
 size 249750019

outputs/checkpoints/{checkpoint-140001 → checkpoint-184000}/optimizer_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:73ce4d1287008fdfac801ca7df44a0debe3e41f901970f3132f0cd49d2ad6bd0
+oid sha256:03b1a223244cdeec59f4a501edcc3e29d75fa5a7467d514c3c30d9a2fcb49401
 size 499500278

outputs/checkpoints/{checkpoint-150001 → checkpoint-184000}/training_args.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bc14fe16573d318dd510c7cfb42ebb7cc87b4dcf77e99247a2d1605cffd772b
-size 1876
+oid sha256:79196d5f797a7527e60287e48f716fd4626c5d57186661d6a214a2027998c86d
+size 1873

outputs/checkpoints/checkpoint-184000/training_state.json

@@ -0,0 +1 @@
+{"step": 184001}

outputs/checkpoints/{checkpoint-170001 → checkpoint-185000}/data_collator.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e02a6e9cfa63cb321cac9402efd29841b652999fcbf787800ae050e747b161ee
-size 1471394
+oid sha256:0321b1a9629e1be122045cd72470365a63c8496fec109fdeec34827f01ffbb9e
+size 1471424

outputs/checkpoints/{checkpoint-170001 → checkpoint-185000}/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c40291527e2cf6e418cf78bb9cd4eec53ac716230987ad7a0a447bf0ce041d4c
+oid sha256:53e61bbf07baa76be1c67dcb57fd59f87d0171d7ce5ebdff16e054e5e4c547fe
 size 249750019

outputs/checkpoints/{checkpoint-170001 → checkpoint-185000}/optimizer_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:90dbe4fe7d7694dd86d13e9b075953620aa4dabb4fdc2023b6ede17aa720848e
+oid sha256:1a2a3d946f859ac106a7ede9530e200364d0850a98d8c958672d441c57ab89bd
 size 499500278

outputs/checkpoints/{checkpoint-140001 → checkpoint-185000}/training_args.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bc14fe16573d318dd510c7cfb42ebb7cc87b4dcf77e99247a2d1605cffd772b
-size 1876
+oid sha256:79196d5f797a7527e60287e48f716fd4626c5d57186661d6a214a2027998c86d
+size 1873

outputs/checkpoints/checkpoint-185000/training_state.json

@@ -0,0 +1 @@
+{"step": 185001}

outputs/checkpoints/checkpoint-186000/data_collator.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0321b1a9629e1be122045cd72470365a63c8496fec109fdeec34827f01ffbb9e
+size 1471424

outputs/checkpoints/checkpoint-186000/flax_model.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3362fadd5306539e1775600df59349acf0fd260751129abaa45b997091d089ce
+size 249750019

outputs/checkpoints/checkpoint-186000/optimizer_state.msgpack

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e08084e7879de2059bc4fd38ff876b0bd1e987a646397956dd4bef19bb2dd1bf
+size 499500278

outputs/checkpoints/checkpoint-186000/training_args.joblib

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79196d5f797a7527e60287e48f716fd4626c5d57186661d6a214a2027998c86d
+size 1873

outputs/checkpoints/checkpoint-186000/training_state.json

@@ -0,0 +1 @@
+{"step": 186001}

outputs/data_collator.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e02a6e9cfa63cb321cac9402efd29841b652999fcbf787800ae050e747b161ee
-size 1471394
+oid sha256:0321b1a9629e1be122045cd72470365a63c8496fec109fdeec34827f01ffbb9e
+size 1471424

outputs/events.out.tfevents.1627122688.tablespoon.2185269.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96e9ae0a574bba9629f3ffc65a3b0714cef7fb3ee796573b1d132a3c80f25fc3
+size 40

outputs/events.out.tfevents.1627122817.tablespoon.2191003.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8351632f70e7465eea0642599fff77709273a7fc9a50b607543f49754c6d694e
+size 149322

outputs/events.out.tfevents.1627125745.tablespoon.2266135.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca4fd9200f1731b7b75feb068806b3d69d179f8e5d467282752c6a8efac760c0
+size 149322

outputs/events.out.tfevents.1627128247.tablespoon.2330108.3.v2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8afd683918d2323382c5b27888561ecb6d879ebe3f6360a1c6ac02435d2f2122
+size 746450

outputs/flax_model.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:393c37966461709fe51a3b3f84befb7fa7e5030025856d171308efd40dbbc7da
+oid sha256:3362fadd5306539e1775600df59349acf0fd260751129abaa45b997091d089ce
 size 249750019

outputs/optimizer_state.msgpack

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3a33cad417a7e78eaafc1c041f93fd54ad9f63869d01e1351bac4abcd58e4eeb
+oid sha256:e08084e7879de2059bc4fd38ff876b0bd1e987a646397956dd4bef19bb2dd1bf
 size 499500278

outputs/training_args.joblib

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8bc14fe16573d318dd510c7cfb42ebb7cc87b4dcf77e99247a2d1605cffd772b
-size 1876
+oid sha256:79196d5f797a7527e60287e48f716fd4626c5d57186661d6a214a2027998c86d
+size 1873

outputs/training_state.json

@@ -1 +1 @@
-{"step": 180001}
+{"step": 186001}