:lady_beetle: App crashed cz it failed to load the entire dataset at a time

.gitignore

@@ -1,3 +1,5 @@
 lightning_logs/
 *.pth
-my-awesome-model/
+my-awesome-model/
+my-awesome-model-200/
+my-awesome-model-4000/

app.py

@@ -376,16 +376,19 @@ def start(classifier_model, model_save_path, is_attention_model=False, m_optimiz
   if is_binned:
     file_suffix = "_binned"
 
-  dataset_map = load_dataset("fahimfarhan/mqtl-classification-dataset-binned-200")
+  dataset_map = load_dataset("fahimfarhan/mqtl-classification-datasets")
 
-  train_dataset = MQTLDataset(dataset_map["train"], check_if_pipeline_is_ok_by_inserting_debug_motif=is_debug)
-  val_dataset = MQTLDataset(dataset_map["validate"], check_if_pipeline_is_ok_by_inserting_debug_motif=is_debug)
-  test_dataset = MQTLDataset(dataset_map["test"], check_if_pipeline_is_ok_by_inserting_debug_motif=is_debug)
+  train_dataset = MQTLDataset(dataset_map[f"train_binned_{WINDOW}"], check_if_pipeline_is_ok_by_inserting_debug_motif=is_debug)
+  val_dataset = MQTLDataset(dataset_map[f"validate_binned_{WINDOW}"], check_if_pipeline_is_ok_by_inserting_debug_motif=is_debug)
+  test_dataset = MQTLDataset(dataset_map[f"test_binned_{WINDOW}"], check_if_pipeline_is_ok_by_inserting_debug_motif=is_debug)
 
   data_module = MqtlDataModule(train_ds=train_dataset, val_ds=val_dataset, test_ds=test_dataset)
 
   classifier_model = classifier_model  #.to(DEVICE)
-  classifier_model = classifier_model.from_pretrained("my-awesome-model")
+  try:
+    classifier_model = classifier_model.from_pretrained(f"my-awesome-model-{WINDOW}")
+  except Exception as x:
+    print(x)
 
   classifier_module = MQtlClassifierLightningModule(classifier=classifier_model, regularization=2,
                                                     m_optimizer=m_optimizer)
@@ -403,13 +406,13 @@ def start(classifier_model, model_save_path, is_attention_model=False, m_optimiz
   torch.save(classifier_module.state_dict(), model_save_path)
 
   #  save locally
-  classifier_model.save_pretrained("my-awesome-model")
+  classifier_model.save_pretrained(f"my-awesome-model-{WINDOW}")
 
   # push to the hub
   classifier_model.push_to_hub(repo_id="fahimfarhan/mqtl-classifier-model", commit_message=":tada: Push model using huggingface_hub")
 
   # reload
-  model = classifier_model.from_pretrained("my-awesome-model")
+  model = classifier_model.from_pretrained(f"my-awesome-model-{WINDOW}")
   # repo_url = "https://huggingface.co/fahimfarhan/mqtl-classifier-model"
   #
   # push_to_hub(
@@ -426,7 +429,7 @@ def start(classifier_model, model_save_path, is_attention_model=False, m_optimiz
 
 
 if __name__ == '__main__':
-  WINDOW = 200
+  WINDOW = 4000
   simple_cnn = Cnn1dClassifier(seq_len=WINDOW)
   simple_cnn.enable_logging = True