Script to download train and test datasets

category_classification/datasets/en/download_train_test.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+import os
+from pathlib import Path
+
+from kagglehub import dataset_download
+
+dataset = "hibiscus4000/arxiv-dataset"
+
+test_dataset = "arxiv_test.json"
+train_dataset = "arxiv_train.json"
+
+dataset_path = Path(dataset_download(dataset))
+os.symlink(dataset_path / test_dataset, test_dataset)
+os.symlink(dataset_path / train_dataset, train_dataset)