install_packages.py

import os
import subprocess
import sys

import requests


def download_model(url, save_path):
 # Send a GET request to the URL
 response = requests.get(url, stream=True)

 # Check if the request was successful (status code 200)
 if response.status_code == 200:
 # Open a file in binary write mode to save the downloaded content
 with open(save_path, 'wb') as f:
 # Iterate over the response content in chunks and write to the file
 for chunk in response.iter_content(chunk_size=1024):
 f.write(chunk)
 print("Model downloaded successfully!")
 else:
 # Print an error message if the request was not successful
 print(f"Failed to download model. Status code: {response.status_code}")


def set_tokenizers_parallelism(value):
 """Set the TOKENIZERS_PARALLELISM environment variable."""
 os.environ['TOKENIZERS_PARALLELISM'] = 'true' if value else 'false'
 print(f"TOKENIZERS_PARALLELISM set to {os.environ['TOKENIZERS_PARALLELISM']}")


def install_requirements():
 """Install packages listed in requirements.txt"""
 try:
 subprocess.check_call([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"])
 print("All packages from requirements.txt installed successfully.")
 except subprocess.CalledProcessError as e:
 print(f"Failed to install packages from requirements.txt: {e}")
 sys.exit(1)


def install_spacy_model(model_name):
 """Install a specific spaCy model"""
 try:
 subprocess.check_call([sys.executable, "-m", "spacy", "download", model_name])
 print(f"spaCy model '{model_name}' installed successfully.")
 except subprocess.CalledProcessError as e:
 print(f"Failed to install spaCy model '{model_name}': {e}")
 sys.exit(1)


if __name__ == "__main__":
 install_requirements()
 install_spacy_model("de_core_news_lg")
 install_spacy_model("fr_core_news_lg")
 install_spacy_model("it_core_news_lg")
 set_tokenizers_parallelism(True)
 download_model('https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.bin', 'lid.176.bin')