import json roles_map = { 'system': 'system', 'user': 'user', 'human': 'user', 'assistant': 'assistant', 'gpt': 'assistant', 'AI': 'assistant', } pretrain_instruct_datasets = [ # # general instructs # # 138 MB, 205,568 {'kind': 'instruct', 'path': 'CohereForAI/aya_dataset', 'transform': lambda r: [ {'role': 'user', 'content': r['inputs']}, {'role': 'assistant', 'content': r['targets']}, ]}, # ~3 GB, 4,976,850 *[ {'kind': 'instruct', 'path': 'saillab/taco-datasets', 'data_dir': name, 'split': 'train', 'transform': lambda r: [ {'role': 'system', 'content': r['instruction']}, {'role': 'user', 'content': r['input']}, {'role': 'assistant', 'content': r['output']}, ]} for name in [ 'multilingual-instruction-tuning-dataset /multilingual-alpaca-52k-gpt-4', 'multilingual-instruction-tuning-dataset /multilinugal-dolly-15k', ] ], # 1.48 GB, 1,420,909 # mlabonne/open-perfectblend # meta-math/MetaMathQA 395,000 # openbmb/UltraInteract_sft 288,579 # HuggingFaceH4/ultrachat_200k 207,865 # microsoft/orca-math-word-problems-200k 200,035 # HuggingFaceH4/ultrafeedback_binarized 187,405 # theblackcat102/evol-codealpaca-v1 111,272 # Post-training-Data-Flywheel/AutoIF-instruct-61k 61,492 # mlabonne/lmsys-arena-human-preference-55k-sharegpt 57,362 *[ {'kind': 'instruct', 'path': 'mlabonne/open-perfectblend', 'split': f'train[{i}%:{i + 10}%]', 'field': 'conversations', 'transform': lambda msgs: [ {'role': roles_map[m['from']], 'content': m['value']} for m in msgs ]} for i in range(0, 100, 10) ], # 4.58 GB, 1,752,473 # arcee-ai/The-Tome # - arcee-ai/infini-instruct-top-500k (BAAI/Infinity-Instruct) # - TIGER-Lab/WebInstructSub (top-500k) - IGNORE # - jondurbin/airoboros-3.2 # - gardner/glaive-function-calling-v2-sharegpt # - arcee-ai/reasoning-sharegpt (SkunkworksAI/reasoning-0.01) # - arcee-ai/self-instruct-sharegpt (bigcode/self-oss-instruct-sc2-exec-filter-50k) # - cognitivecomputations/ultrainteract_trajectories_sharegpt # - cognitivecomputations/SystemChat-2.0 # - arcee-ai/qwen2-72b-magpie-en *[ {'kind': 'instruct', 'path': 'arcee-ai/The-Tome', 'split': f'train[{i}%:{i + 10}%]', 'field': 'conversations', 'transform': lambda msgs: [ {'role': roles_map[m['from']], 'content': m['value']} for m in msgs ]} for i in range(0, 100, 10) ], # 2.48 GB, 5,808,694 # rombodawg/Everything_Instruct_Multilingual # Science: # antiven0m/physical-reasoning-dpoScience # LawalAfeez/science-dataset # Social media: # Kyle1668/AG-Tweets # euclaise/reddit-instruct-curated # General Knowledge: # NousResearch/CharacterCodex_Characters # jstet/quotes-500k_Famous_Quotes # FronkonGames/steam-games-dataset_Video_Games # totuta_youtube_subs_howto100M_HowTo # Multi-lingual: # Amani27/massive_translation_dataset # udmurtNLP/udmurt-russian-english-labse # grosenthal/latin_english # msarmi9/korean-english-multitarget-ted-talks-task # HaiderSultanArc/MT-Urdu-English_Translate # Garsa3112/ChineseEnglishTranslationDataset # Cooking: # andrewsiah/se_cooking_preference_sft # Hieu-Phamkaggle/food_recipes # Writing: # shahules786/PoetryFoundationData # euclaise/writingprompts # qwedsacf/ivypanda-essaysEssay # Medicine: # keivalya/MedQuad-MedicalQnADataset # nuvocare/MSD # History: # ambrosfitz10k/history_data_v4 # Law: # dzunggg/legal-qa-v1 # Role-Play: # roleplay4/fun_CoupleRP # Undi95andrijdavid/roleplay-conversation-sharegpt # News: # RealTimeData/bbc_news_alltime # Coding: (rombodawg/code_bagel) # layoric/tiny-codes-alpaca # glaiveai/glaive-code-assistant-v3 # ajibawa-2023/Code-290k-ShareGPT # chargoddard/commitpack-ft-instruct-rated # iamtarun/code_instructions_120k_alpaca # ise-uiuc/Magicoder-Evol-Instruct-110K # cognitivecomputations/dolphin-coder # nickrosh/Evol-Instruct-Code-80k-v1 # coseal/CodeUltraFeedback_binarized # CyberNative/Code_Vulnerability_Security_DPO # Math: (rombodawg/code_bagel) # TIGER-Lab/MathInstruct # Function calling: (rombodawg/code_bagel) # glaiveai/glaive-function-calling-v2 # General Instruct: (rombodawg/OpenHermes-2.5-Uncensored) # teknium/OpenHermes-2.5 *[ {'kind': 'instruct', 'path': 'rombodawg/Everything_Instruct_Multilingual', 'split': f'train[{i}%:{i + 10}%]', 'transform': lambda r: [ {'role': 'system', 'content': r['instruction']}, {'role': 'user', 'content': r['input']}, {'role': 'assistant', 'content': r['output']}, ]} for i in range(0, 100, 10) ], # 1.41 GB, 939,343 # allenai/tulu-3-sft-mixture # CoCoNot (ODC-BY-1.0), 10,983 prompts (Brahman et al., 2024) # FLAN v2 via ai2-adapt-dev/flan_v2_converted, 89,982 prompts (Longpre et al., 2023) # No Robots (CC-BY-NC-4.0), 9,500 prompts (Rajani et al. 2023) # OpenAssistant Guanaco (Apache 2.0), 7,132 prompts (Kopf et al., 2024) # Tulu 3 Persona MATH (ODC-BY-1.0), 149,960 prompts # Tulu 3 Persona GSM (ODC-BY-1.0), 49,980 prompts # Tulu 3 Persona Python (ODC-BY-1.0), 34,999 prompts # Tulu 3 Persona Algebra (ODC-BY-1.0), 20,000 prompts # Tulu 3 Persona IF (ODC-BY-1.0), 29,980 prompts # NuminaMath-TIR (Apache 2.0), 64,312 prompts (Beeching et al. 2024) # Tulu 3 WildGuardMix (Apache 2.0), 50,000 prompts (Han et al., 2024) # Tulu 3 WildJailbreak (ODC-BY-1.0), 50,000 prompts (Wildteaming, 2024) # Tulu 3 Hardcoded (CC-BY-4.0), 240 prompts # Aya (Apache 2.0), 100,000 prompts (Singh et al., 2024) # WildChat GPT-4 (ODC-BY-1.0), 100,000 prompts (Zhao et al., 2024) # TableGPT (MIT), 5,000 prompts (Zha et al., 2023) # SciRIFF (ODC-BY-1.0), 10,000 prompts (Wadden et al., 2024) # Evol CodeAlpaca (Apache 2.0), 107,276 prompts (Luo et al., 2023) *[ {'kind': 'instruct', 'path': 'allenai/tulu-3-sft-mixture', 'split': f'train[{i}%:{i + 10}%]', 'field': 'messages'} for i in range(0, 100, 10) ], # # tool/function calling # # 65.7 MB, 11,578 {'kind': 'instruct', 'path': 'NousResearch/hermes-function-calling-v1', 'field': 'conversations', 'transform': lambda msgs: [ {'role': roles_map[m['from']], 'content': m['value']} for m in msgs ]}, # # agent # # 1.51 GB, 485,874 *[ {'kind': 'instruct', 'path': 'arcee-ai/agent-data', 'split': f'train[{i}%:{i + 10}%]', 'field': 'conversations', 'transform': lambda msgs: [ {'role': roles_map[m['from']], 'content': m['value']} for m in msgs ]} for i in range(0, 100, 10) ], # 2.21 GB, 1,046,410 *[ {'kind': 'instruct', 'path': 'microsoft/orca-agentinstruct-1M-v1', 'split': split, 'field': 'messages', 'transform': lambda msgs: json.loads(msgs)} for split in [ 'creative_content', 'text_modification', 'struct2text_flow', 'rc', 'rag', 'text_extraction', 'mcq', 'follow_up', 'analytical_reasoning', 'fermi', 'fs_cot_flow', 'code_', 'brain_teaser', 'text_classification', 'open_domain_qa', ] ], # # general instructs # # 1.52 GB, 214k (3.98 GB, 814,334) {'kind': 'instruct', 'path': 'cognitivecomputations/dolphin-r1', 'data_files': 'dolphin-r1-nonreasoning.jsonl', 'split': 'train', 'field': 'messages'}, # 4.15 GB, 2,197,730 {'kind': 'instruct', 'path': 'HuggingFaceTB/smoltalk', 'name': 'all', 'field': 'messages'}, ]