| @inproceedings{barbaresi-2021-trafilatura, | |
| title = {Trafilatura: A Web Scraping Library and Command-Line Tool for Text Discovery and Extraction}, | |
| author = "Barbaresi, Adrien", | |
| booktitle = "Proceedings of the Joint Conference of the 59th Annual Meeting of the Association for Computational Linguistics and the 11th International Joint Conference on Natural Language Processing: System Demonstrations", | |
| pages = "122--131", | |
| publisher = "Association for Computational Linguistics", | |
| url = "https://aclanthology.org/2021.acl-demo.15", | |
| year = 2021, | |
| } | |
| @misc{penedo2023refinedweb, | |
| title={The RefinedWeb Dataset for Falcon LLM: Outperforming Curated Corpora with Web Data, and Web Data Only}, | |
| author={Guilherme Penedo and Quentin Malartic and Daniel Hesslow and Ruxandra Cojocaru and Alessandro Cappelli and Hamza Alobeidli and Baptiste Pannier and Ebtesam Almazrouei and Julien Launay}, | |
| year={2023}, | |
| eprint={2306.01116}, | |
| archivePrefix={arXiv}, | |
| primaryClass={cs.CL} | |
| } | |
| @article{joulin2016fasttext, | |
| title={FastText.zip: Compressing text classification models}, | |
| author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Douze, Matthijs and J{\'e}gou, H{\'e}rve and Mikolov, Tomas}, | |
| journal={arXiv preprint arXiv:1612.03651}, | |
| year={2016} | |
| } | |
| @article{joulin2016bag, | |
| title={Bag of Tricks for Efficient Text Classification}, | |
| author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Mikolov, Tomas}, | |
| journal={arXiv preprint arXiv:1607.01759}, | |
| year={2016} | |
| } | |
| @misc{penedo2024datatrove, | |
| author = {Penedo, Guilherme and Kydlíček, Hynek and Cappelli, Alessandro and Wolf, Thomas and Sasko, Mario}, | |
| title = {DataTrove: large scale data processing}, | |
| year = {2024}, | |
| publisher = {GitHub}, | |
| journal = {GitHub repository}, | |
| url = {https://github.com/huggingface/datatrove} | |
| } | |
| @misc{chiang2024chatbot, | |
| title={Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference}, | |
| author={Wei-Lin Chiang and Lianmin Zheng and Ying Sheng and Anastasios Nikolas Angelopoulos and Tianle Li and Dacheng Li and Hao Zhang and Banghua Zhu and Michael Jordan and Joseph E. Gonzalez and Ion Stoica}, | |
| year={2024}, | |
| eprint={2403.04132}, | |
| archivePrefix={arXiv}, | |
| primaryClass={cs.AI} | |
| } | |
| @misc{rae2022scaling, | |
| title={Scaling Language Models: Methods, Analysis & Insights from Training Gopher}, | |
| author={Jack W. Rae and Sebastian Borgeaud and Trevor Cai and Katie Millican and Jordan Hoffmann and Francis Song and John Aslanides and Sarah Henderson and Roman Ring and Susannah Young and Eliza Rutherford and Tom Hennigan and Jacob Menick and Albin Cassirer and Richard Powell and George van den Driessche and Lisa Anne Hendricks and Maribeth Rauh and Po-Sen Huang and Amelia Glaese and Johannes Welbl and Sumanth Dathathri and Saffron Huang and Jonathan Uesato and John Mellor and Irina Higgins and Antonia Creswell and Nat McAleese and Amy Wu and Erich Elsen and Siddhant Jayakumar and Elena Buchatskaya and David Budden and Esme Sutherland and Karen Simonyan and Michela Paganini and Laurent Sifre and Lena Martens and Xiang Lorraine Li and Adhiguna Kuncoro and Aida Nematzadeh and Elena Gribovskaya and Domenic Donato and Angeliki Lazaridou and Arthur Mensch and Jean-Baptiste Lespiau and Maria Tsimpoukelli and Nikolai Grigorev and Doug Fritz and Thibault Sottiaux and Mantas Pajarskas and Toby Pohlen and Zhitao Gong and Daniel Toyama and Cyprien de Masson d'Autume and Yujia Li and Tayfun Terzi and Vladimir Mikulik and Igor Babuschkin and Aidan Clark and Diego de Las Casas and Aurelia Guy and Chris Jones and James Bradbury and Matthew Johnson and Blake Hechtman and Laura Weidinger and Iason Gabriel and William Isaac and Ed Lockhart and Simon Osindero and Laura Rimell and Chris Dyer and Oriol Vinyals and Kareem Ayoub and Jeff Stanway and Lorrayne Bennett and Demis Hassabis and Koray Kavukcuoglu and Geoffrey Irving}, | |
| year={2022}, | |
| eprint={2112.11446}, | |
| archivePrefix={arXiv}, | |
| primaryClass={cs.CL} | |
| } | |
| @misc{lee2022deduplicating, | |
| title={Deduplicating Training Data Makes Language Models Better}, | |
| author={Katherine Lee and Daphne Ippolito and Andrew Nystrom and Chiyuan Zhang and Douglas Eck and Chris Callison-Burch and Nicholas Carlini}, | |
| year={2022}, | |
| eprint={2107.06499}, | |
| archivePrefix={arXiv}, | |
| primaryClass={cs.CL} | |
| } | |
| @misc{carlini2023quantifying, | |
| title={Quantifying Memorization Across Neural Language Models}, | |
| author={Nicholas Carlini and Daphne Ippolito and Matthew Jagielski and Katherine Lee and Florian Tramer and Chiyuan Zhang}, | |
| year={2023}, | |
| eprint={2202.07646}, | |
| archivePrefix={arXiv}, | |
| primaryClass={cs.LG} | |
| } | |
| @misc{raffel2023exploring, | |
| title={Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer}, | |
| author={Colin Raffel and Noam Shazeer and Adam Roberts and Katherine Lee and Sharan Narang and Michael Matena and Yanqi Zhou and Wei Li and Peter J. Liu}, | |
| year={2023}, | |
| eprint={1910.10683}, | |
| archivePrefix={arXiv}, | |
| primaryClass={cs.LG} | |
| } | |
| @misc{touvron2023llama, | |
| title={LLaMA: Open and Efficient Foundation Language Models}, | |
| author={Hugo Touvron and Thibaut Lavril and Gautier Izacard and Xavier Martinet and Marie-Anne Lachaux and Timothée Lacroix and Baptiste Rozière and Naman Goyal and Eric Hambro and Faisal Azhar and Aurelien Rodriguez and Armand Joulin and Edouard Grave and Guillaume Lample}, | |
| year={2023}, | |
| eprint={2302.13971}, | |
| archivePrefix={arXiv}, | |
| primaryClass={cs.CL} | |
| } | |
| @article{dolma, | |
| title = {Dolma: an Open Corpus of Three Trillion Tokens for Language Model Pretraining Research}, | |
| author={ | |
| Luca Soldaini and Rodney Kinney and Akshita Bhagia and Dustin Schwenk and David Atkinson and | |
| Russell Authur and Ben Bogin and Khyathi Chandu and Jennifer Dumas and Yanai Elazar and | |
| Valentin Hofmann and Ananya Harsh Jha and Sachin Kumar and Li Lucy and Xinxi Lyu and | |
| Nathan Lambert and Ian Magnusson and Jacob Morrison and Niklas Muennighoff and Aakanksha Naik and | |
| Crystal Nam and Matthew E. Peters and Abhilasha Ravichander and Kyle Richardson and Zejiang Shen and | |
| Emma Strubell and Nishant Subramani and Oyvind Tafjord and Pete Walsh and Luke Zettlemoyer and | |
| Noah A. Smith and Hannaneh Hajishirzi and Iz Beltagy and Dirk Groeneveld and Jesse Dodge and Kyle Lo | |
| }, | |
| year = {2024}, | |
| journal={arXiv preprint}, | |
| } | |
| @article{gao2020pile, | |
| title={The {P}ile: An 800{GB} dataset of diverse text for language modeling}, | |
| author={Gao, Leo and Biderman, Stella and Black, Sid and Golding, Laurence and Hoppe, Travis and Foster, Charles and Phang, Jason and He, Horace and Thite, Anish and Nabeshima, Noa and others}, | |
| journal={arXiv preprint arXiv:2101.00027}, | |
| year={2020} | |
| } | |
| @misc{cerebras2023slimpajama, | |
| author = {Soboleva, Daria and Al-Khateeb, Faisal and Myers, Robert and Steeves, Jacob R and Hestness, Joel and Dey, Nolan}, | |
| title = {SlimPajama: A 627B token cleaned and deduplicated version of RedPajama}, | |
| month = {June}, | |
| year = 2023, | |
| url = {https://huggingface.co/datasets/cerebras/SlimPajama-627B}, | |
| } | |
| @software{together2023redpajama, | |
| author = {Together Computer}, | |
| title = {RedPajama: an Open Dataset for Training Large Language Models}, | |
| month = {October}, | |
| year = 2023, | |
| url = {https://github.com/togethercomputer/RedPajama-Data} | |
| } | |
| @article{jaccard1912distribution, | |
| title={The distribution of the flora in the alpine zone. 1}, | |
| author={Jaccard, Paul}, | |
| journal={New phytologist}, | |
| volume={11}, | |
| number={2}, | |
| pages={37--50}, | |
| year={1912}, | |
| publisher={Wiley Online Library} | |
| } | |