initializer.py

# initializer.py
import json
import os
import requests
from huggingface_hub import snapshot_download

def download_json(url):
    """Download JSON content from the given URL."""
    response = requests.get(url)
    response.raise_for_status()  # Check if request was successful
    return response.json()

def download_and_modify_json(url, local_filename, modifications):
    """Download JSON from URL and modify its contents based on modifications."""
    if os.path.exists(local_filename):
        with open(local_filename, 'r', encoding='utf-8') as f:
            data = json.load(f)
        config_version = data.get('config_version', '0.0.0')
        # If the version is older than desired, re-download
        if config_version < '1.1.1':
            data = download_json(url)
    else:
        data = download_json(url)

    # Apply modifications
    data.update(modifications)

    # Save the modified JSON locally
    with open(local_filename, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=4)

def initialize_app(force=False):
    """Initialize models and configuration.
    
    This function downloads the required models and modifies the configuration file.
    It will only perform these actions if they have not been done before.
    """
    # Define the path to the configuration file in the user's home directory.
    home_dir = os.path.expanduser('~')
    config_file_name = 'magic-pdf.json'
    config_file = os.path.join(home_dir, config_file_name)
    
    # If the config file exists, assume initialization is complete.
    if os.path.exists(config_file):
        print(f"Initialization already completed. Using existing configuration at {config_file}")
        return
    elif force:
        print(f"Forced initialization.")

    # Define patterns for model download
    mineru_patterns = [
        "models/Layout/LayoutLMv3/*",
        "models/Layout/YOLO/*",
        "models/MFD/YOLO/*",
        "models/MFR/unimernet_small_2501/*",
        "models/TabRec/TableMaster/*",
        "models/TabRec/StructEqTable/*",
    ]
    model_dir = snapshot_download('opendatalab/PDF-Extract-Kit-1.0', allow_patterns=mineru_patterns)

    layoutreader_pattern = [
        "*.json",
        "*.safetensors",
    ]
    layoutreader_model_dir = snapshot_download('hantian/layoutreader', allow_patterns=layoutreader_pattern)

    model_dir = os.path.join(model_dir, 'models')
    print(f"Downloaded model_dir is: {model_dir}")
    print(f"Downloaded layoutreader_model_dir is: {layoutreader_model_dir}")

    # Download and modify JSON configuration
    json_url = 'https://github.com/opendatalab/MinerU/raw/master/magic-pdf.template.json'
    json_mods = {
        'models-dir': model_dir,
        'layoutreader-model-dir': layoutreader_model_dir,
    }
    download_and_modify_json(json_url, config_file, json_mods)
    print(f"The configuration file has been configured successfully, the path is: {config_file}")

# This block will run if the module is executed directly.
if __name__ == '__main__':
    initialize_app()