import os
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset
from torch.optim import AdamW
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import time
from tqdm import tqdm
from transformers import AutoTokenizer, AutoModelForCausalLM, TrainingArguments
from diffusers import DiffusionPipeline
from huggingface_hub import login, HfApi, Repository
from dotenv import load_dotenv
import gradio as gr
# Cargar variables de entorno
load_dotenv()
class UnifiedModel(nn.Module):
def __init__(self, models):
super(UnifiedModel, self).__init__()
self.models = nn.ModuleList(models)
self.classifier = nn.Linear(sum([model.config.hidden_size for model in models if hasattr(model, 'config')]), 2)
def forward(self, inputs):
hidden_states = []
for model in self.models:
if isinstance(model, nn.Module):
outputs = model(**inputs)
hidden_states.append(outputs.last_hidden_state[:, 0, :])
elif isinstance(model, DiffusionPipeline):
outputs = model(**inputs)
hidden_states.append(torch.tensor(outputs).float())
concatenated_hidden_states = torch.cat(hidden_states, dim=-1)
logits = self.classifier(concatenated_hidden_states)
return logits
class SyntheticDataset(Dataset):
def __init__(self, tokenizers, size=100):
self.tokenizers = tokenizers
self.size = size
self.data = self._generate_data()
def _generate_data(self):
data = []
for _ in range(self.size):
text = "This is a sample sentence for testing purposes."
label = torch.tensor(0) # Sample label
item = {"text": text, "label": label}
for name, tokenizer in self.tokenizers.items():
tokenized = tokenizer(text, padding="max_length", truncation=True, max_length=128)
item[f"input_ids_{name}"] = torch.tensor(tokenized["input_ids"])
item[f"attention_mask_{name}"] = torch.tensor(tokenized["attention_mask"])
data.append(item)
return data
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
return self.data[idx]
def push_to_hub(local_dir, repo_name):
try:
repo_url = HfApi().create_repo(repo_name, exist_ok=True)
repo = Repository(local_dir, clone_from=repo_url)
if not os.path.exists(os.path.join(local_dir, ".git")):
os.system(f"cd {local_dir} && git init && git remote add origin {repo_url} && git pull origin main")
repo.git_add(auto_lfs_track=True)
repo.git_commit("Add model and tokenizer files")
json_files = ["config.json", "generation_config.json", "special_tokens_map.json", "tokenizer.json", "tokenizer.model", "tokenizer_config.json"]
for json_file in json_files:
json_file_path = os.path.join(local_dir, json_file)
if os.path.exists(json_file_path):
repo.git_add(json_file_path)
repo.git_push()
print(f"Pushed model and tokenizer to {repo_url}")
except Exception as e:
print(f"Error pushing to Hugging Face Hub: {e}")
def load_model(model_name):
tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)
model = AutoModelForCausalLM.from_pretrained(model_name)
return tokenizer, model
def train(model, train_loader, eval_loader, args):
model.train()
epoch = 0
total_steps = len(train_loader)
for step, batch in enumerate(train_loader):
start_time = time.time()
input_ids = [batch[f"input_ids_{name}"].to("cpu") for name in tokenizers.keys()]
attention_mask = [batch[f"attention_mask_{name}"].to("cpu") for name in tokenizers.keys()]
labels = batch["label"].to("cpu")
optimizer.zero_grad()
outputs = model(input_ids)
loss = nn.CrossEntropyLoss()(outputs, labels)
loss.backward()
optimizer.step()
elapsed_time = time.time() - start_time
estimated_total_time = total_steps * (elapsed_time / (step + 1))
estimated_remaining_time = estimated_total_time - elapsed_time
if step % args.logging_steps == 0:
train_losses.append(loss.item())
print(f"Step {step}/{total_steps}, Loss: {loss.item()}, Estimated remaining time: {estimated_remaining_time:.2f} seconds")
epoch += 1
model.eval()
eval_loss = 0
with torch.no_grad():
for batch in eval_loader:
input_ids = [batch[f"input_ids_{name}"].to("cpu") for name in tokenizers.keys()]
attention_mask = [batch[f"attention_mask_{name}"].to("cpu") for name in tokenizers.keys()]
labels = batch["label"].to("cpu")
outputs = model(input_ids)
loss = nn.CrossEntropyLoss()(outputs, labels)
eval_loss += loss.item()
eval_loss /= len(eval_loader)
eval_losses.append(eval_loss)
print(f"Epoch {epoch}/{args.num_train_epochs}, Evaluation Loss: {eval_loss}")
def gradio_interface(input_text):
# Define the Gradio interface function
tokenized_inputs = {name: tokenizer.encode(input_text, return_tensors="pt") for name, tokenizer in tokenizers.items()}
model_output = unified_model(tokenized_inputs)
return model_output
def main():
while True:
try:
os.system("git config --global credential.helper store")
login(token=os.getenv("HUGGINGFACE_TOKEN"), add_to_git_credential=True)
# Definir los modelos que se van a utilizar
models_to_train = [
"openai-community/gpt2-xl",
"google/gemma-2-9b-it",
"google/gemma-2-9b",
"meta-llama/Meta-Llama-3.1-8B-Instruct",
"meta-llama/Meta-Llama-3.1-8B",
"openbmb/MiniCPM-V-2_6",
"bigcode/starcoder",
"WizardLMTeam/WizardCoder-Python-34B-V1.0",
"Qwen/Qwen2-72B-Instruct",
"google/gemma-2-2b-it",
"facebook/bart-large-cnn",
"Falconsai/text_summarization",
"microsoft/speecht5_tts",
"Groq/Llama-3-Groq-70B-Tool-Use",
"Groq/Llama-3-Groq-8B-Tool-Use",
"facebook/musicgen-large",
"facebook/musicgen-melody",
"black-forest-labs/FLUX.1-schnell",
"facebook/musicgen-small",
"stabilityai/stable-video-diffusion-img2vid-xt-1-1",
"openai/whisper-small",
"black-forest-labs/FLUX.1-dev",
"stabilityai/stable-diffusion-2-1"
]
# Inicializar los modelos y tokenizadores
tokenizers = {}
models = []
for model_name in models_to_train:
tokenizer, model = load_model(model_name)
tokenizers[model_name] = tokenizer
models.append(model)
# Crear un dataset sintético para entrenamiento y evaluación
synthetic_dataset = SyntheticDataset(tokenizers, size=100)
# Dividir el dataset en entrenamiento y evaluación
train_size = int(0.8 * len(synthetic_dataset))
val_size = len(synthetic_dataset) - train_size
train_dataset, val_dataset = torch.utils.data.random_split(synthetic_dataset, [train_size, val_size])
# Crear DataLoaders para entrenamiento y evaluación
train_loader = DataLoader(train_dataset, batch_size=2, shuffle=True)
eval_loader = DataLoader(val_dataset, batch_size=16)
# Unificar los modelos en uno solo
unified_model = UnifiedModel(models)
unified_model.to(torch.device("cpu"))
# Mostrar la cantidad de parámetros totales a entrenar
total_params = sum(p.numel() for p in unified_model.parameters())
print(f"Total parameters to train: {total_params}")
# Definir los argumentos de entrenamiento
training_args = TrainingArguments(
per_device_train_batch_size=2,
per_device_eval_batch_size=16,
num_train_epochs=1,
logging_steps=10,
save_steps=10,
evaluation_strategy="steps"
)
# Definir el optimizador
optimizer = AdamW(unified_model.parameters(), lr=5e-5)
# Listas para almacenar las pérdidas
train_losses = []
eval_losses = []
# Entrenar el modelo
train(unified_model, train_loader, eval_loader, training_args)
# Visualizar pérdidas
fig, ax = plt.subplots()
ax.set_xlabel("Epochs")
ax.set_ylabel("Loss")
ax.plot(train_losses, label="Training Loss")
ax.plot(eval_losses, label="Evaluation Loss")
ax.legend()
def animate(i):
ax.clear()
ax.plot(train_losses, label="Training Loss")
ax.plot(eval_losses, label="Evaluation Loss")
ax.set_xlabel("Epochs")
ax.set_ylabel("Loss")
ax.legend()
ani = animation.FuncAnimation(fig, animate, interval=1000)
plt.show()
# Guardar el modelo y el tokenizador unificados
if not os.path.exists("./outputs/unified_model"):
os.makedirs("./outputs/unified_model")
# Guardar el modelo unificado en un directorio local
local_dir = "./outputs/unified_model"
torch.save(unified_model.state_dict(), os.path.join(local_dir, "pytorch_model.bin"))
# Guardar el tokenizador en un directorio local
for name, tokenizer in tokenizers.items():
tokenizer.save_pretrained(local_dir)
# Subir el modelo y el tokenizador a Hugging Face
push_to_hub(local_dir, repo_name="Ffftdtd5dtft/my_model")
# Configurar y lanzar la interfaz Gradio
interface = gr.Interface(fn=gradio_interface, inputs="text", outputs="text")
interface.launch()
break
except Exception as e:
print(f"Error: {e}")
time.sleep(2)
if __name__ == "__main__":
main()