File size: 872 Bytes
ae81e0f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 |
"""
Load dataloaders
"""
import importlib
def load_data(dataset_config: dict, dataloader_config: dict):
"""Return dataloaders from dataset_config"""
try:
dataset_module = importlib.import_module(f'dataloaders.{dataset_config["name"]}')
except Exception:
try:
dataset_module = importlib.import_module(f'src.dataloaders.{dataset_config["name"]}')
except Exception as e2:
print(e2)
try: # e.g., tasks like GLUE where name is benchmark and path specifies the dataset / task
dataset_module = importlib.import_module(f'dataloaders.{dataset_config["path"]}')
except Exception as e3:
print(f'Error from {dataset_config}')
raise e3
_load_data = getattr(dataset_module, 'load_data')
return _load_data(**dataset_config, **dataloader_config) |