update

hf_scrapper.py

@@ -2,6 +2,7 @@ import os
 import requests
 import json
 import urllib.request
+import time
 from requests.exceptions import RequestException
 from tqdm import tqdm
 
@@ -49,7 +50,7 @@ def download_file(file_url, token, cache_path, proxies, film_id, title, chunk_si
         response.raise_for_status()
         
         total_size = int(response.headers.get('content-length', 0))
-        download_progress[film_id] = {"total": total_size, "downloaded": 0, "status": "Downloading"}
+        download_progress[film_id] = {"total": total_size, "downloaded": 0, "status": "Downloading", "start_time": time.time()}
 
         os.makedirs(os.path.dirname(cache_path), exist_ok=True)
         with open(cache_path, 'wb') as file, tqdm(total=total_size, unit='B', unit_scale=True, desc=cache_path) as pbar:
@@ -68,9 +69,8 @@ def download_file(file_url, token, cache_path, proxies, film_id, title, chunk_si
         print(f"Error writing file {cache_path}: {e}")
         download_progress[film_id]["status"] = "Failed"
     finally:
-        # Instead of deleting the progress, we mark it as complete or failed
         if download_progress[film_id]["status"] != "Downloading":
-            del download_progress[film_id]
+            download_progress[film_id]["end_time"] = time.time()
 
 def get_download_progress(film_id):
     """
@@ -80,15 +80,24 @@ def get_download_progress(film_id):
         film_id (str): The unique identifier for the film download.
 
     Returns:
-        dict: A dictionary containing the total size, downloaded size, progress percentage, and status.
+        dict: A dictionary containing the total size, downloaded size, progress percentage, status, and ETA.
     """
     if film_id in download_progress:
         total = download_progress[film_id]["total"]
         downloaded = download_progress[film_id]["downloaded"]
         status = download_progress[film_id].get("status", "In Progress")
         progress = (downloaded / total) * 100 if total > 0 else 0
-        return {"total": total, "downloaded": downloaded, "progress": progress, "status": status}
-    return {"total": 0, "downloaded": 0, "progress": 0, "status": "Not Found"}
+
+        eta = None
+        if status == "Downloading" and downloaded > 0:
+            elapsed_time = time.time() - download_progress[film_id]["start_time"]
+            estimated_total_time = elapsed_time * (total / downloaded)
+            eta = estimated_total_time - elapsed_time
+        elif status == "Completed":
+            eta = 0
+
+        return {"total": total, "downloaded": downloaded, "progress": progress, "status": status, "eta": eta}
+    return {"total": 0, "downloaded": 0, "progress": 0, "status": "Not Found", "eta": None}
 
 def update_film_store_json(title, cache_path):
     """