import os import subprocess import sys import requests def download_model(url, save_path): # Send a GET request to the URL response = requests.get(url, stream=True) # Check if the request was successful (status code 200) if response.status_code == 200: # Open a file in binary write mode to save the downloaded content with open(save_path, 'wb') as f: # Iterate over the response content in chunks and write to the file for chunk in response.iter_content(chunk_size=1024): f.write(chunk) print("Model downloaded successfully!") else: # Print an error message if the request was not successful print(f"Failed to download model. Status code: {response.status_code}") def set_tokenizers_parallelism(value): """Set the TOKENIZERS_PARALLELISM environment variable.""" os.environ['TOKENIZERS_PARALLELISM'] = 'true' if value else 'false' print(f"TOKENIZERS_PARALLELISM set to {os.environ['TOKENIZERS_PARALLELISM']}") def install_requirements(): """Install packages listed in requirements.txt""" try: subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"]) print("All packages from requirements.txt installed successfully.") except subprocess.CalledProcessError as e: print(f"Failed to install packages from requirements.txt: {e}") sys.exit(1) def install_spacy_model(model_name): """Install a specific spaCy model""" try: subprocess.check_call([sys.executable, "-m", "spacy", "download", model_name]) print(f"spaCy model '{model_name}' installed successfully.") except subprocess.CalledProcessError as e: print(f"Failed to install spaCy model '{model_name}': {e}") sys.exit(1) if __name__ == "__main__": install_requirements() install_spacy_model("de_core_news_lg") install_spacy_model("fr_core_news_lg") install_spacy_model("it_core_news_lg") set_tokenizers_parallelism(True) download_model('https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin', 'lid.176.bin')