app.py

import gradio as gr
from transformers import AutoProcessor, AutoModelForCausalLM
from PIL import Image
import torch
from gtts import gTTS
import spacy
import requests
import nltk.tree
import re
import os

# Carregar o modelo de português do spaCy
nlp = spacy.load("pt_core_news_sm")

# Chave para o LX-Parser
key = "eb159d39469d84f0ff47167a4d89cada"

# Funções de manipulação gramatical
def invert_adj_n(doc, tags):
    frase = []
    already = False
    for i in range(len(doc)):
        if already:
            already = False
            continue
        if doc[i].tag_ != "PUNCT":
            if tags[i] == "A":
                if i + 1 < len(tags) and tags[i + 1] == "N":
                    frase.append(doc[i + 1].text)
                    frase.append(doc[i].text)
                    already = True
                else:
                    frase.append(doc[i].text)
            else:
                frase.append(doc[i].text)
        else:
            frase.append(doc[i].text)
    return frase

def adjust_adj(doc, tags):
    frase = []
    for i in range(len(doc)):
        frase.append(doc[i].text)
        if tags[i] == "A":
            if i + 1 < len(tags) and tags[i + 1] == "A":
                frase.append("e")
    return frase

def adjust_art(doc, tags):
    frase = []
    already = False
    for i in range(len(doc)):
        if already:
            already = False
            continue
        text = doc[i].text
        if tags[i] == "ART" and text.lower() == "a":
            if i + 1 < len(doc):
                gender = doc[i + 1].morph.get("Gender")
                number = doc[i + 1].morph.get("Number")
                if gender and number:
                    if gender[0] == "Masc" and number[0] == "Sing":
                        frase.append("um")
                    elif gender[0] == "Fem" and number[0] == "Sing":
                        frase.append("uma")
                    elif gender[0] == "Masc" and number[0] != "Sing":
                        frase.append("os")
                    else:
                        frase.append("as")
                else:
                    frase.append(text)
            else:
                frase.append(text)
        else:
            frase.append(text)
    return frase

def create_sentence(doc, tags, frase):
    tmp = frase
    for i in range(len(doc)):
        text = doc[i].text
        if doc[i].is_sent_start:
            tmp[i] = tmp[i].capitalize()
        if doc[i].tag_ == "PUNCT":
            tmp[i - 1] += text
    return tmp

def get_productions(texto):
    format = 'parentheses'
    url = "https://portulanclarin.net/workbench/lx-parser/api/"
    request_data = {
        'method': 'parse',
        'jsonrpc': '2.0',
        'id': 0,
        'params': {
            'text': texto,
            'format': format,
            'key': key,
        },
    }
    request = requests.post(url, json=request_data)
    response_data = request.json()
    if "error" in response_data:
        print("Error:", response_data["error"])
        return []
    else:
        result = response_data["result"]
        productions = []
        tree = nltk.tree.Tree.fromstring(result)
        for tag in tree.productions():
            if len(re.findall(r"'.*'", str(tag))) > 0:
                productions.append(str(tag))
        return productions

def get_tags(productions):
    tags = []
    for item in productions:
        if isinstance(item, str):
            tags.append(item[:item.find(' ->')])
        else:
            tags.append(item)
    for item in tags:
        if "'" in item:
            tags.remove(item)
    return tags

def reordenar_sentenca(sentenca):
    if not sentenca.strip():
        return sentenca
    sentenca = sentenca.lower()
    sentence = get_productions(sentenca)
    tags = get_tags(sentence)
    doc = nlp(sentenca)
    if tags[0] != "ART":
        sentenca = "A " + sentenca.strip()
    sentence = get_productions(sentenca)
    tags = get_tags(sentence)
    doc = nlp(sentenca)
    if not sentence:
        return sentenca.strip()
    aux = []
    if len(tags) > 2 and tags[1] == "N" and tags[2] == "N":
        aux = sentenca.split()
        tmp = aux[1]
        aux[1] = aux[2]
        aux.insert(2, "de")
        aux[3] = tmp
        sentenca = " ".join(aux)
        sentence = get_productions(sentenca)
        tags = get_tags(sentence)
        doc = nlp(sentenca)
    frase = []
    already = False
    person = 3
    tmp_doc = []
    for token in doc:
        tmp_doc.append(token)
    frase = invert_adj_n(tmp_doc, tags)
    nova_sentenca = ' '.join(frase)
    productions = get_productions(nova_sentenca)
    tags = get_tags(productions)
    doc = nlp(nova_sentenca)
    while nova_sentenca != sentenca:
        frase = invert_adj_n(doc, tags)
        sentenca = nova_sentenca
        nova_sentenca = ' '.join(frase)
        productions = get_productions(nova_sentenca)
        tags = get_tags(productions)
        doc = nlp(nova_sentenca)
    frase = adjust_adj(doc, tags)
    nova_sentenca = ' '.join(frase)
    productions = get_productions(nova_sentenca)
    tags = get_tags(productions)
    doc = nlp(nova_sentenca)
    while nova_sentenca != sentenca:
        frase = adjust_adj(doc, tags)
        sentenca = nova_sentenca
        nova_sentenca = ' '.join(frase)
        productions = get_productions(nova_sentenca)
        tags = get_tags(productions)
        doc = nlp(nova_sentenca)
    frase = adjust_art(doc, tags)
    sentenca = ' '.join(frase)
    productions = get_productions(sentenca)
    tags = get_tags(productions)
    doc = nlp(sentenca)
    frase = create_sentence(doc, tags, frase)
    sentenca_normalizada = ""
    for i in range(len(frase)):
        sentenca_normalizada += frase[i] + " "
    return sentenca_normalizada.strip()

def prepare_image(image_path):
    image = Image.open(image_path).convert("RGB")
    inputs = processor(images=image, return_tensors="pt").to(device)
    return image, inputs.pixel_values

def generate_caption(pixel_values):
    model.eval()
    with torch.no_grad():
        generated_ids = model.generate(
            pixel_values=pixel_values,
            max_length=50,
            num_beams=4,
            early_stopping=True,
            no_repeat_ngram_size=2
        )
    return processor.batch_decode(generated_ids, skip_special_tokens=True)[0]

def text_to_speech_gtts(text, lang='pt'):
    tts = gTTS(text=text, lang=lang)
    tts.save("output.mp3")
    return "output.mp3"

# Carregar os modelos
processor = AutoProcessor.from_pretrained("histlearn/microsoft-git-portuguese-neuro-simbolic")
model = AutoModelForCausalLM.from_pretrained("histlearn/microsoft-git-portuguese-neuro-simbolic")

# Configurar o dispositivo (GPU ou CPU)
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)

# Função principal para processar a imagem e gerar a voz
def process_image(image):
    _, pixel_values = prepare_image(image)
    caption_pt = generate_caption(pixel_values)
    sentenca_normalizada = reordenar_sentenca(caption_pt)
    audio_file = text_to_speech_gtts(sentenca_normalizada)
    productions = get_productions(sentenca_normalizada)
    return sentenca_normalizada, productions, audio_file

# Caminhos para as imagens de exemplo
example_image_paths = [
    "example1.jpeg",
    "example2.jpeg",
    "example3.jpeg"
]

# Interface Gradio
iface = gr.Interface(
    fn=process_image,
    inputs=gr.Image(type="filepath"),
    outputs=[gr.Textbox(label="Sentença Normalizada"), gr.Textbox(label="Classes Gramaticais"), gr.Audio(type="filepath", label="Áudio")],
    examples=example_image_paths,
    title="Image to Voice",
    description="Gera uma descrição em português e a converte em voz a partir de uma imagem."
)

if __name__ == "__main__":
    iface.launch()