WIP

script.py

@@ -59,8 +59,8 @@ if __name__ == "__main__":
 
     # dataset = load_dataset(params['dataset'], trust_remote_code=True, use_auth_token=params['token'])
     data_files = {
-        "validation": [str(p) for p in [*data_path.rglob('*validation*.arrow')]+[*data_path.rglob('*public*.tar')]],
-        "test": [str(p) for p in [*data_path.rglob('*test*.arrow')]+[*data_path.rglob('*private*.tar')]],
+        "validation": [str(p) for p in [*data_path.rglob('*validation*.arrow')]+[*data_path.rglob('*public*/**/*.tar')]],
+        "test": [str(p) for p in [*data_path.rglob('*test*.arrow')]+[*data_path.rglob('*private*/**/*.tar')]],
     }
     print(data_files)