fix dl civitai files with params in url

files_cells/notebooks/en/downloading_en.ipynb

@@ -333,7 +333,7 @@
         "    ]\n",
         "}\n",
         "\n",
-        "extension_repo = []\n",
+        "url = \"\"\n",
         "prefixes = {\n",
         "    \"model\": models_dir,\n",
         "    \"vae\": vaes_dir,\n",
@@ -344,11 +344,10 @@
         "    \"adetailer\": adetailer_dir\n",
         "}\n",
         "\n",
-        "# !mkdir -p {models_dir} {vaes_dir} {loras_dir} {embeddings_dir} {extensions_dir} {control_dir} {adetailer_dir}\n",
+        "extension_repo = []\n",
         "directories = [value for key, value in prefixes.items()] # for unpucking zip files\n",
         "!mkdir -p {\" \".join(directories)}\n",
         "\n",
-        "url = \"\"\n",
         "hf_token = optional_huggingface_token if optional_huggingface_token else \"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\"\n",
         "user_header = f\"\\\"Authorization: Bearer {hf_token}\\\"\"\n",
         "\n",
@@ -361,9 +360,6 @@
         "        response = requests.get(base_url, headers=headers)\n",
         "        if response.status_code == 200:\n",
         "            return response.json()\n",
-        "        else:\n",
-        "            print(f\"Failed to retrieve data. Status code: {response.status_code}\")\n",
-        "            return None\n",
         "    except requests.exceptions.RequestException as e:\n",
         "        print(f\"An error occurred: {e}\")\n",
         "        return None\n",
@@ -373,9 +369,11 @@
         "    image_url = data['images'][0]['url'] # get preview: first image\n",
         "    return model_name, image_url\n",
         "\n",
-        "def generate_preview_filename(model_name):\n",
-        "    file_parts = model_name.split('.')\n",
-        "    return f\"{file_parts[0]}.preview.jpeg\" # assigning the original image format\n",
+        "def generate_preview_filename(model_name, image_url):\n",
+        "    name = model_name.split('.')\n",
+        "    img_ext = image_url.split('.')\n",
+        "    print(f\"\\n\\033[31m[Preview DL]\\033[0m: {name[0]}.preview.{img_ext[-1]}\")\n",
+        "    return f\"{name[0]}.preview.{img_ext[-1]}\" # assigning the original image format\n",
         "\n",
         "''' main download code '''\n",
         "\n",
@@ -399,27 +397,26 @@
         "    basename = url.split(\"/\")[-1] if file_name is None else file_name\n",
         "    aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'\n",
         "\n",
-        "    print(\"\\033[32m---\"*45 + f\"\\n\\033[33mURL: \\033[34m{url}\\n\\033[33mSAVE DIR: \\033[34m{dst_dir}\\n\\033[33mFILE NAME: \\033[34m{file_name}\\033[32m\\n~~~\\033[0m\")\n",
+        "    print(f\"\\n\\033[32m{'---'*45}\\n\\033[33mURL: \\033[34m{url}\\n\\033[33mSAVE DIR: \\033[34m{dst_dir}\\n\\033[33mFILE NAME: \\033[34m{file_name}033[0m\")\n",
         "    # print(url, dst_dir, file_name)\n",
         "\n",
         "    # === CivitAi API ===\n",
         "    civitai_token = \"62c0c5956b2f9defbd844d754000180b\"\n",
-        "    if 'civitai' in url and civitai_token:\n",
+        "    if 'civitai' in url:\n",
         "        model_id = url.split('/')[-1]\n",
-        "        data = get_data_from_api(model_id, civitai_token)\n",
-        "        url = f\"{url}?token={civitai_token}\"\n",
-        "\n",
-        "        if data:\n",
-        "            model_name, image_url = extract_file_and_image_info(data)\n",
-        "            if model_name and image_url:\n",
-        "                image_file_name = generate_preview_filename(model_name if not file_name else file_name)\n",
-        "                with capture.capture_output() as cap: # clear shit\n",
-        "                    !aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}\n",
-        "                del cap\n",
+        "        url = f\"{url}&token={civitai_token}\" if '?' in url else f\"{url}?token={civitai_token}\"\n",
+        "\n",
+        "        if dst_dir not in [vaes_dir, control_dir, adetailer_dir]: # filter\n",
+        "            data = get_data_from_api(model_id, civitai_token)\n",
+        "            if data:\n",
+        "                model_name, image_url = extract_file_and_image_info(data)\n",
+        "                if model_name and image_url:\n",
+        "                    image_file_name = generate_preview_filename(model_name if not file_name else file_name, image_url)\n",
+        "                    with capture.capture_output() as cap: # clear shit\n",
+        "                        !aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}\n",
+        "                    del cap\n",
         "            else:\n",
-        "                print(\"File name or image URL missing.\")\n",
-        "        else:\n",
-        "            print(\"Failed to retrieve data from the API.\")\n",
+        "                print(\"\\033[31m[Preview DL]\\033[0m: Failed to retrieve data from the API.\")\n",
         "\n",
         "    # -- GDrive --\n",
         "    if 'drive.google' in url:\n",
@@ -463,7 +460,7 @@
         "\n",
         "## unpucking zip files\n",
         "def unpucking_zip_files():\n",
-        "    # directories = [models_dir, vaes_dir, embeddings_dir, loras_dir , extensions_dir, control_dir , adetailer_dir]\n",
+        "    # directories - above\n",
         "    for directory in directories:\n",
         "        for root, dirs, files in os.walk(directory):\n",
         "            for file in files:\n",

files_cells/notebooks/ru/downloading_ru.ipynb

@@ -333,7 +333,7 @@
         "    ]\n",
         "}\n",
         "\n",
-        "extension_repo = []\n",
+        "url = \"\"\n",
         "prefixes = {\n",
         "    \"model\": models_dir,\n",
         "    \"vae\": vaes_dir,\n",
@@ -344,11 +344,10 @@
         "    \"adetailer\": adetailer_dir\n",
         "}\n",
         "\n",
-        "# !mkdir -p {models_dir} {vaes_dir} {loras_dir} {embeddings_dir} {extensions_dir} {control_dir} {adetailer_dir}\n",
+        "extension_repo = []\n",
         "directories = [value for key, value in prefixes.items()] # for unpucking zip files\n",
         "!mkdir -p {\" \".join(directories)}\n",
         "\n",
-        "url = \"\"\n",
         "hf_token = optional_huggingface_token if optional_huggingface_token else \"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\"\n",
         "user_header = f\"\\\"Authorization: Bearer {hf_token}\\\"\"\n",
         "\n",
@@ -361,9 +360,6 @@
         "        response = requests.get(base_url, headers=headers)\n",
         "        if response.status_code == 200:\n",
         "            return response.json()\n",
-        "        else:\n",
-        "            print(f\"Failed to retrieve data. Status code: {response.status_code}\")\n",
-        "            return None\n",
         "    except requests.exceptions.RequestException as e:\n",
         "        print(f\"An error occurred: {e}\")\n",
         "        return None\n",
@@ -373,9 +369,11 @@
         "    image_url = data['images'][0]['url'] # get preview: first image\n",
         "    return model_name, image_url\n",
         "\n",
-        "def generate_preview_filename(model_name):\n",
-        "    file_parts = model_name.split('.')\n",
-        "    return f\"{file_parts[0]}.preview.jpeg\" # assigning the original image format\n",
+        "def generate_preview_filename(model_name, image_url):\n",
+        "    name = model_name.split('.')\n",
+        "    img_ext = image_url.split('.')\n",
+        "    print(f\"\\n\\033[31m[Preview DL]\\033[0m: {name[0]}.preview.{img_ext[-1]}\")\n",
+        "    return f\"{name[0]}.preview.{img_ext[-1]}\" # assigning the original image format\n",
         "\n",
         "''' main download code '''\n",
         "\n",
@@ -399,27 +397,26 @@
         "    basename = url.split(\"/\")[-1] if file_name is None else file_name\n",
         "    aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'\n",
         "\n",
-        "    print(\"\\033[32m---\"*45 + f\"\\n\\033[33mURL: \\033[34m{url}\\n\\033[33mSAVE DIR: \\033[34m{dst_dir}\\n\\033[33mFILE NAME: \\033[34m{file_name}\\033[32m\\n~~~\\033[0m\")\n",
+        "    print(f\"\\n\\033[32m{'---'*45}\\n\\033[33mURL: \\033[34m{url}\\n\\033[33mSAVE DIR: \\033[34m{dst_dir}\\n\\033[33mFILE NAME: \\033[34m{file_name}033[0m\")\n",
         "    # print(url, dst_dir, file_name)\n",
         "\n",
         "    # === CivitAi API ===\n",
         "    civitai_token = \"62c0c5956b2f9defbd844d754000180b\"\n",
-        "    if 'civitai' in url and civitai_token:\n",
+        "    if 'civitai' in url:\n",
         "        model_id = url.split('/')[-1]\n",
-        "        data = get_data_from_api(model_id, civitai_token)\n",
-        "        url = f\"{url}?token={civitai_token}\"\n",
-        "\n",
-        "        if data:\n",
-        "            model_name, image_url = extract_file_and_image_info(data)\n",
-        "            if model_name and image_url:\n",
-        "                image_file_name = generate_preview_filename(model_name if not file_name else file_name)\n",
-        "                with capture.capture_output() as cap: # clear shit\n",
-        "                    !aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}\n",
-        "                del cap\n",
+        "        url = f\"{url}&token={civitai_token}\" if '?' in url else f\"{url}?token={civitai_token}\"\n",
+        "\n",
+        "        if dst_dir not in [vaes_dir, control_dir, adetailer_dir]: # filter\n",
+        "            data = get_data_from_api(model_id, civitai_token)\n",
+        "            if data:\n",
+        "                model_name, image_url = extract_file_and_image_info(data)\n",
+        "                if model_name and image_url:\n",
+        "                    image_file_name = generate_preview_filename(model_name if not file_name else file_name, image_url)\n",
+        "                    with capture.capture_output() as cap: # clear shit\n",
+        "                        !aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}\n",
+        "                    del cap\n",
         "            else:\n",
-        "                print(\"File name or image URL missing.\")\n",
-        "        else:\n",
-        "            print(\"Failed to retrieve data from the API.\")\n",
+        "                print(\"\\033[31m[Preview DL]\\033[0m: Failed to retrieve data from the API.\")\n",
         "\n",
         "    # -- GDrive --\n",
         "    if 'drive.google' in url:\n",
@@ -463,7 +460,7 @@
         "\n",
         "## unpucking zip files\n",
         "def unpucking_zip_files():\n",
-        "    # directories = [models_dir, vaes_dir, embeddings_dir, loras_dir , extensions_dir, control_dir , adetailer_dir]\n",
+        "    # directories - above\n",
         "    for directory in directories:\n",
         "        for root, dirs, files in os.walk(directory):\n",
         "            for file in files:\n",

files_cells/python/en/downloading_en.py

@@ -314,7 +314,7 @@ controlnet_list = {
     ]
 }
 
-extension_repo = []
+url = ""
 prefixes = {
     "model": models_dir,
     "vae": vaes_dir,
@@ -325,11 +325,10 @@ prefixes = {
     "adetailer": adetailer_dir
 }
 
-# !mkdir -p {models_dir} {vaes_dir} {loras_dir} {embeddings_dir} {extensions_dir} {control_dir} {adetailer_dir}
+extension_repo = []
 directories = [value for key, value in prefixes.items()] # for unpucking zip files
 get_ipython().system('mkdir -p {" ".join(directories)}')
 
-url = ""
 hf_token = optional_huggingface_token if optional_huggingface_token else "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"
 user_header = f"\"Authorization: Bearer {hf_token}\""
 
@@ -342,9 +341,6 @@ def get_data_from_api(model_id, api_key): # get model data
         response = requests.get(base_url, headers=headers)
         if response.status_code == 200:
             return response.json()
-        else:
-            print(f"Failed to retrieve data. Status code: {response.status_code}")
-            return None
     except requests.exceptions.RequestException as e:
         print(f"An error occurred: {e}")
         return None
@@ -354,9 +350,11 @@ def extract_file_and_image_info(data):
     image_url = data['images'][0]['url'] # get preview: first image
     return model_name, image_url
 
-def generate_preview_filename(model_name):
-    file_parts = model_name.split('.')
-    return f"{file_parts[0]}.preview.jpeg" # assigning the original image format
+def generate_preview_filename(model_name, image_url):
+    name = model_name.split('.')
+    img_ext = image_url.split('.')
+    print(f"\n\033[31m[Preview DL]\033[0m: {name[0]}.preview.{img_ext[-1]}")
+    return f"{name[0]}.preview.{img_ext[-1]}" # assigning the original image format
 
 ''' main download code '''
 
@@ -380,27 +378,26 @@ def manual_download(url, dst_dir, file_name):
     basename = url.split("/")[-1] if file_name is None else file_name
     aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'
 
-    print("\033[32m---"*45 + f"\n\033[33mURL: \033[34m{url}\n\033[33mSAVE DIR: \033[34m{dst_dir}\n\033[33mFILE NAME: \033[34m{file_name}\033[32m\n~~~\033[0m")
+    print(f"\n\033[32m{'---'*45}\n\033[33mURL: \033[34m{url}\n\033[33mSAVE DIR: \033[34m{dst_dir}\n\033[33mFILE NAME: \033[34m{file_name}033[0m")
     # print(url, dst_dir, file_name)
 
     # === CivitAi API ===
     civitai_token = "62c0c5956b2f9defbd844d754000180b"
-    if 'civitai' in url and civitai_token:
+    if 'civitai' in url:
         model_id = url.split('/')[-1]
-        data = get_data_from_api(model_id, civitai_token)
-        url = f"{url}?token={civitai_token}"
-
-        if data:
-            model_name, image_url = extract_file_and_image_info(data)
-            if model_name and image_url:
-                image_file_name = generate_preview_filename(model_name if not file_name else file_name)
-                with capture.capture_output() as cap: # clear shit
-                    get_ipython().system('aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}')
-                del cap
+        url = f"{url}&token={civitai_token}" if '?' in url else f"{url}?token={civitai_token}"
+
+        if dst_dir not in [vaes_dir, control_dir, adetailer_dir]: # filter
+            data = get_data_from_api(model_id, civitai_token)
+            if data:
+                model_name, image_url = extract_file_and_image_info(data)
+                if model_name and image_url:
+                    image_file_name = generate_preview_filename(model_name if not file_name else file_name, image_url)
+                    with capture.capture_output() as cap: # clear shit
+                        get_ipython().system('aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}')
+                    del cap
             else:
-                print("File name or image URL missing.")
-        else:
-            print("Failed to retrieve data from the API.")
+                print("\033[31m[Preview DL]\033[0m: Failed to retrieve data from the API.")
 
     # -- GDrive --
     if 'drive.google' in url:
@@ -444,7 +441,7 @@ def download(url):
 
 ## unpucking zip files
 def unpucking_zip_files():
-    # directories = [models_dir, vaes_dir, embeddings_dir, loras_dir , extensions_dir, control_dir , adetailer_dir]
+    # directories - above
     for directory in directories:
         for root, dirs, files in os.walk(directory):
             for file in files:

files_cells/python/ru/downloading_ru.py

@@ -314,7 +314,7 @@ controlnet_list = {
     ]
 }
 
-extension_repo = []
+url = ""
 prefixes = {
     "model": models_dir,
     "vae": vaes_dir,
@@ -325,11 +325,10 @@ prefixes = {
     "adetailer": adetailer_dir
 }
 
-# !mkdir -p {models_dir} {vaes_dir} {loras_dir} {embeddings_dir} {extensions_dir} {control_dir} {adetailer_dir}
+extension_repo = []
 directories = [value for key, value in prefixes.items()] # for unpucking zip files
 get_ipython().system('mkdir -p {" ".join(directories)}')
 
-url = ""
 hf_token = optional_huggingface_token if optional_huggingface_token else "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"
 user_header = f"\"Authorization: Bearer {hf_token}\""
 
@@ -342,9 +341,6 @@ def get_data_from_api(model_id, api_key): # get model data
         response = requests.get(base_url, headers=headers)
         if response.status_code == 200:
             return response.json()
-        else:
-            print(f"Failed to retrieve data. Status code: {response.status_code}")
-            return None
     except requests.exceptions.RequestException as e:
         print(f"An error occurred: {e}")
         return None
@@ -354,9 +350,11 @@ def extract_file_and_image_info(data):
     image_url = data['images'][0]['url'] # get preview: first image
     return model_name, image_url
 
-def generate_preview_filename(model_name):
-    file_parts = model_name.split('.')
-    return f"{file_parts[0]}.preview.jpeg" # assigning the original image format
+def generate_preview_filename(model_name, image_url):
+    name = model_name.split('.')
+    img_ext = image_url.split('.')
+    print(f"\n\033[31m[Preview DL]\033[0m: {name[0]}.preview.{img_ext[-1]}")
+    return f"{name[0]}.preview.{img_ext[-1]}" # assigning the original image format
 
 ''' main download code '''
 
@@ -380,27 +378,26 @@ def manual_download(url, dst_dir, file_name):
     basename = url.split("/")[-1] if file_name is None else file_name
     aria2_args = '--optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -j5 -x16 -s16 -k1M -c'
 
-    print("\033[32m---"*45 + f"\n\033[33mURL: \033[34m{url}\n\033[33mSAVE DIR: \033[34m{dst_dir}\n\033[33mFILE NAME: \033[34m{file_name}\033[32m\n~~~\033[0m")
+    print(f"\n\033[32m{'---'*45}\n\033[33mURL: \033[34m{url}\n\033[33mSAVE DIR: \033[34m{dst_dir}\n\033[33mFILE NAME: \033[34m{file_name}033[0m")
     # print(url, dst_dir, file_name)
 
     # === CivitAi API ===
     civitai_token = "62c0c5956b2f9defbd844d754000180b"
-    if 'civitai' in url and civitai_token:
+    if 'civitai' in url:
         model_id = url.split('/')[-1]
-        data = get_data_from_api(model_id, civitai_token)
-        url = f"{url}?token={civitai_token}"
-
-        if data:
-            model_name, image_url = extract_file_and_image_info(data)
-            if model_name and image_url:
-                image_file_name = generate_preview_filename(model_name if not file_name else file_name)
-                with capture.capture_output() as cap: # clear shit
-                    get_ipython().system('aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}')
-                del cap
+        url = f"{url}&token={civitai_token}" if '?' in url else f"{url}?token={civitai_token}"
+
+        if dst_dir not in [vaes_dir, control_dir, adetailer_dir]: # filter
+            data = get_data_from_api(model_id, civitai_token)
+            if data:
+                model_name, image_url = extract_file_and_image_info(data)
+                if model_name and image_url:
+                    image_file_name = generate_preview_filename(model_name if not file_name else file_name, image_url)
+                    with capture.capture_output() as cap: # clear shit
+                        get_ipython().system('aria2c {aria2_args} -d {dst_dir} -o {image_file_name} {image_url}')
+                    del cap
             else:
-                print("File name or image URL missing.")
-        else:
-            print("Failed to retrieve data from the API.")
+                print("\033[31m[Preview DL]\033[0m: Failed to retrieve data from the API.")
 
     # -- GDrive --
     if 'drive.google' in url:
@@ -444,7 +441,7 @@ def download(url):
 
 ## unpucking zip files
 def unpucking_zip_files():
-    # directories = [models_dir, vaes_dir, embeddings_dir, loras_dir , extensions_dir, control_dir , adetailer_dir]
+    # directories - above
     for directory in directories:
         for root, dirs, files in os.walk(directory):
             for file in files: