backup

app.py

@@ -26,7 +26,7 @@ FINAL_REPO = "taesiri/DatasetOfHardQuestions5"
 # Download existing data from hub
 def sync_with_hub():
     """
-    Synchronize local data with the hub by cloning the dataset repo
+    Synchronize local data with the hub by downloading latest dataset
     """
     print("Starting sync with hub...")
     data_dir = Path("./data")
@@ -37,37 +37,27 @@ def sync_with_hub():
             shutil.rmtree(backup_dir)
         shutil.copytree(data_dir, backup_dir)
 
-    # Clone/pull latest data from hub
-    repo_url = f"https://huggingface.co/datasets/{DATASET_REPO}"
-    hub_data_dir = Path("hub_data")
-
-    if hub_data_dir.exists():
-        # If repo exists, do a git pull
-        print("Pulling latest changes...")
-        repo = git.Repo(hub_data_dir)
-        origin = repo.remotes.origin
-        origin.pull()
-    else:
-        # Clone the repo
-        print("Cloning repository...")
-        git.Repo.clone_from(repo_url, hub_data_dir)
+    # Download latest data from hub
+    repo_path = snapshot_download(
+        repo_id="taesiri/zb_dataset_storage", repo_type="dataset", local_dir="hub_data"
+    )
 
     # Merge hub data with local data
-    hub_data_source = hub_data_dir / "data"
-    if hub_data_source.exists():
+    hub_data_dir = Path(repo_path) / "data"
+    if hub_data_dir.exists():
         # Create data dir if it doesn't exist
         data_dir.mkdir(exist_ok=True)
 
         # Copy files from hub
-        for item in hub_data_source.glob("*"):
+        for item in hub_data_dir.glob("*"):
             if item.is_dir():
                 dest = data_dir / item.name
                 if not dest.exists():  # Only copy if doesn't exist locally
                     shutil.copytree(item, dest)
 
-    # Clean up cloned repo
-    if hub_data_dir.exists():
-        shutil.rmtree(hub_data_dir)
+    # Clean up downloaded repo
+    if Path("hub_data").exists():
+        shutil.rmtree("hub_data")
     print("Finished syncing with hub!")