pretrain dataset

scripts/prepare_pretrain_dataset.py

@@ -87,13 +87,13 @@ datasets_configs = [
  #
  # 2.89 GB, 430,000, English September of 2017
  *[
- {'path': 'jordiclive/wikipedia-summary-dataset', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['summary']}
- for i in range(0, 100, 10)
+ {'path': 'jordiclive/wikipedia-summary-dataset', 'split': f'train[{i}%:{i + 20}%]', 'format': lambda n: n['summary']}
+ for i in range(0, 100, 20)
  ],
  # 3.18 GB, 1,010,500
  *[
- {'path': 'JeanKaddour/minipile', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['text']}
- for i in range(0, 100, 10)
+ {'path': 'JeanKaddour/minipile', 'split': f'train[{i}%:{i + 20}%]', 'format': lambda n: n['text']}
+ for i in range(0, 100, 20)
  ],
 
  #
@@ -106,18 +106,18 @@ datasets_configs = [
  #
  # 12.2 MB, 500,000
  [
- {'path': 'fblgit/simple-math', 'revision': 'refs/convert/parquet', 'split': f'train[{i}%:{i + 10}%]+test', 'format': '{instruction} = {output}'}
- for i in range(0, 100, 10)
+ {'path': 'fblgit/simple-math', 'revision': 'refs/convert/parquet', 'split': f'train[{i}%:{i + 20}%]+test', 'format': '{instruction} = {output}'}
+ for i in range(0, 100, 20)
  ],
  # 125 MB, 1,000,000
  [
- {'path': 'Gusarich/math-expressions-1m', 'revision': 'refs/convert/parquet', 'split': f'train[{i}%:{i + 10}%]', 'format': '{expression} = {result}'}
- for i in range(0, 100, 10)
+ {'path': 'Gusarich/math-expressions-1m', 'revision': 'refs/convert/parquet', 'split': f'train[{i}%:{i + 20}%]', 'format': '{expression} = {result}'}
+ for i in range(0, 100, 20)
  ],
  # 3.49 GB, 22,259,474
  [
- {'path': 'AtlasUnified/atlas-math-sets', 'split': f'train[{i}%:{i + 10}%]+validation+test', 'format': '{instruction} . {output}'}
- for i in range(0, 100, 10)
+ {'path': 'AtlasUnified/atlas-math-sets', 'split': f'train[{i}%:{i + 20}%]+validation+test', 'format': '{instruction} . {output}'}
+ for i in range(0, 100, 20)
  ],
  # 9.05 GB, 2,583,257
  [
@@ -130,8 +130,8 @@ datasets_configs = [
  #
  # 1.52 GB, 2,101,279
  [
- {'path': 'milkshake721/2.1M-wiki-STEM', 'split': f'train[{i}%:{i + 10}%]', 'format': lambda n: n['text']}
- for i in range(0, 100, 10)
+ {'path': 'milkshake721/2.1M-wiki-STEM', 'split': f'train[{i}%:{i + 20}%]', 'format': lambda n: n['text']}
+ for i in range(0, 100, 20)
  ],
  
  #