app.py

import os.path

from openai import OpenAI
import os
from groq import Groq
import requests
import time
from html.parser import HTMLParser
from bs4 import BeautifulSoup
import json
from datetime import datetime
import pandas as pd
from serpapi import GoogleSearch
import gradio as gr

GROQ_API_KEY=os.environ.get("GROQ_API_KEY")
client_groq = Groq(api_key=GROQ_API_KEY,)
openai_key=os.environ.get("OPENAI_API_KEY")
os.environ["OPENAI_API_KEY"] = openai_key
client = OpenAI()

SERPAPI_KEY=os.environ.get("SERPAPI_KEY")



def scrape_website(url):
    headers = {'User-Agent': 'Mozilla/5.0'}
    try:
        response = requests.get(url, headers=headers, timeout=20)
        response.encoding = response.apparent_encoding  # Set encoding to match the content

        if response.status_code == 200:
            page_content = response.content
            soup = BeautifulSoup(page_content, 'html.parser')
            paragraphs = soup.find_all('p')
            scraped_data = [p.get_text() for p in paragraphs]
            formatted_data = u"\n".join(scraped_data)

            return formatted_data  # Return only content
        else:
            return "Failed to retrieve the webpage (Status Code: {})".format(response.status_code)
   
    except requests.exceptions.ReadTimeout:
        # Handle the timeout exception
        return "Request timed out after 20 seconds."
    except requests.exceptions.SSLError as e:
        return "Request Error: {}".format(e)
    except requests.exceptions.RequestException as e:
        # Handle other requests-related exceptions
        return "An error occurred: {}".format(e)


def update_dataframe_with_results(organic_results):
    # Prepare data for DataFrame
    max_chars = 100000  # Maximum characters allowed in a cell
    data = []
    for result in organic_results:
        # Scrape the website content
        scraped_content = scrape_website(result.get('link'))
        # Truncate the content if it exceeds the limit
        if len(scraped_content) > max_chars:
          scraped_content = scraped_content[:max_chars]
        data.append({
            "Title": result.get('title'),
            "Link": result.get('link'),
            "Snippet": result.get('snippet'),
            "Displayed Link": result.get('displayed_link'),
            "Date": result.get('date'),  # Might not always be present
            "Rich Snippet": result.get('rich_snippet'),  # Might not always be present
            "Scraped Content": scraped_content  # Add scraped content
        })

    df = pd.DataFrame(data)
    return df

def opencall(text,user_query):
    print("Calling opencall function with", len(text), "characters")
    #completion = client_groq.chat.completions.create(   
    completion = client.chat.completions.create(
            model="gpt-4-0125-preview",
            #model="mixtral-8x7b-32768",
            temperature=0.1,
            messages=[
               {"role": "system", "content": "You are a helpful assistant, specialised in preparing contents for preparing a presentation."},
                {"role": "system", "content": "Your task is to prepare a base report on the topics, themes and trends addressed in the latest conferences, seminars and symposiums." },
                {"role": "system", "content": "For this matter I will be providing you in the Information Pool a compilation of several scraped google search results from the latest conferences, seminars and symposiums on the topic: "+user_query},
                {"role": "system", "content": "Each piece of Scraped Content start with the tag '### Title:' indicating the title, followed by the URL reference '### Link:' , followed by the contents '### Content:'"},
                {"role": "system", "content": "Process all the information in the Information Pool to provide:"},
               {"role": "system", "content": "1) Perspective of Relevant Information: Assess and extract the most relevant information from the point of view of this aspect: "+user_query+"."},
                 {"role": "system", "content": "2) Perspective of Key Topics: Highlight the key topics and themes.Cite the URLs that source those topics and themes"},
                  {"role": "system", "content": "3) Perspective of Emergent Trends: Highlight the emergent trends.Cite the URLs that source those trends."},
                  {"role": "system", "content": "In the response, use the indicated structure of 1)Perspective of Relevant Information 2)Perspective of Key Topics 3)Perspective of Emergent Trends"},
                {"role": "user", "content":"Information Pool:"+text}
            ]
        )
    response = completion.choices[0].message.content
    response = response + "\n" + "XXXXX" + "\n"
    return response


def split_large_content(content, max_length=30000):
    # Extract the title and source URL, assuming they end with the second newline
    title_and_source_end = content.find('\n\n') + 2
    title_and_source = content[:title_and_source_end]
    title_and_source_length = len(title_and_source)

    # Ensure each segment has space for the title and source by reducing max_length
    max_segment_length = max_length - title_and_source_length

    segments = []
    content_body = content[title_and_source_end:]

    # Start splitting the content_body into segments
    while len(content_body) > 0:
        # Take a slice of content up to max_segment_length
        segment = content_body[:max_segment_length]

        # If we're not at the end of content_body, back-track to the last complete word
        if len(content_body) > max_segment_length:
            last_space = segment.rfind(' ')
            segment = segment[:last_space]

        # Add the title and source URL to the start of this segment
        full_segment = title_and_source + segment
        segments.append(full_segment)

        # Move forward in content_body by the length of the segment minus the title/source
        content_body = content_body[len(segment):]

    return segments



def main(df,google_search_query):
    # Initialize a string to accumulate the information
    information_pool = ""
    archivo1=""
    # Open or create a plain text file in append mode
    with open('respuestas.txt', mode='a', encoding='utf-8') as file:
        
        # Iterate over the rows of the DataFrame
        for index, row in df.iterrows():
            # Combine title, link, and content into a single string
            document_name = row['Title']  # Using title as document_name
            raw_content = str(row['Scraped Content'])  # Convert to string to ensure compatibility
            link = row['Link']  # Retrieve link for additional usage or logging

            # Assuming process_document_content is a function you've defined to process the content
            processed_content = "### Title: " + row['Title'] + "\n" + "### Link: " + row['Link'] + "\n" + "### Content: " + str(row['Scraped Content']) + "\n" + "\n"

            print(document_name, ":", len(processed_content))
            #print("Contenido:", processed_content)
            print("acumulado:", len(information_pool + processed_content))

            # Handle long content by splitting and processing in segments
            if len(processed_content) > 30000:
                content_segments = split_large_content(processed_content)
                for segment in content_segments:
                    print("EN C, Nuevo valor de Text:", len(segment))
                    #print("segmen:",segment)
                    response = opencall(segment,google_search_query)  # Replace 'opencall' with your actual function call
                    archivo1=archivo1+response+'\n'
                    file.write(response + '\n')

            else:
                # Check if adding processed content exceeds the size limit
                if len(information_pool + processed_content) <= 30000:
                    information_pool += processed_content
                    print("EN A, Nuevo valor de Text:", len(information_pool))
                else:
                    # Process current accumulated content and start new accumulation
                    print("EN B1, llamando con valor de Text:", len(information_pool))
                    #print("Information pool", information_pool)
                    response = opencall(information_pool,google_search_query)
                    file.write(response + '\n')
                    archivo1=archivo1+response+'\n'
                    information_pool = processed_content
                    print("EN B2, nuevo valor de Text:", len(information_pool), " Con documento:", document_name)

        # Handle any remaining content after loop
        if information_pool:
            print("Final call")
            response = opencall(information_pool,google_search_query)
            file.write(response + '\n')
            archivo1=archivo1+response+'\n'
    return archivo1         

def rearrange_text(text):
    # Split the text into batches using 'XXXXX'
    batches = text.split('XXXXX')

    # Initialize variables to store concatenated texts
    all_texta = ""
    all_textb = ""
    all_textc = ""

    # Define markers for different sections
    markers = {
        'texta_marker': "Perspective of Relevant Information",
        'textb_marker': "Perspective of Key Emerging Aspects",
        'textc_marker': "Perspective of Key Entities"
    }

    # Process each batch
    for batch in batches:
        # Initialize indices for each section
        texta_start = batch.find(markers['texta_marker'])
        textb_start = batch.find(markers['textb_marker'])
        textc_start = batch.find(markers['textc_marker'])

        # Extract TEXTA, TEXTB, and TEXTC using the found indices
        # Check if the markers are found; if not, skip to the next marker
        texta = batch[texta_start:textb_start] if textb_start != -1 else batch[texta_start:]
        textb = batch[textb_start:textc_start] if textc_start != -1 else batch[textb_start:]
        textc = batch[textc_start:]

        # Remove the markers from the beginning of each text
        texta = texta.replace(markers['texta_marker'], '').strip()
        textb = textb.replace(markers['textb_marker'], '').strip()
        textc = textc.replace(markers['textc_marker'], '').strip()

        # Concatenate texts from all batches
        all_texta += "\n" + texta if all_texta else texta
        all_textb += "\n" + textb if all_textb else textb
        all_textc += "\n" + textc if all_textc else textc

    # You can now use all_texta, all_textb, and all_textc for further summarization or processing
    return all_texta, all_textb, all_textc

def resumen(text):
    texta, textb, textc = rearrange_text(text)
    
    completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[
         {"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."},
         {"role": "system", "content": "Your task is to provide an integrated comprehensive 2000 words narrative of the different points indicated in the Information Pool text for a internal report on recent news." },
         {"role": "system", "content": "Instructions. Elaborate the text following these rules:" },         
         {"role": "system", "content": "Be exhaustive, comprehensive and detailed in addressing the relation of different points indicated in the Information Pool text." },       
         {"role": "system", "content": "Arrange paragraphs and information around each entity or related entities and concepts, integrating them with a fluent narrative." },         
         {"role": "system", "content": "Start directly with the narrative, do not introduce the text, as it is part of a broader report." },   
         {"role": "system", "content": "Use a formal writing style, yet plain and easy to read. Avoid pomposity and making up artificial descriptions. The audience is well acquainted with technical and defence/military vocabulary, information and entities. " },         
         
         
         {"role": "user", "content":"Information Pool:"+texta}  ] )


    
    response1 = completion.choices[0].message.content if completion.choices[0].message else ""
    response_1="1) Perspective of Relevant Information:"+"\n"+response1+"\n"
   
    completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[
         {"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."},
         {"role": "system", "content": "Your task is to provide a comprehensive and integrated relation of about 2000 words in length of the different emerging aspects indicated in the Information Pool text for a internal report on recent news." },
         {"role": "system", "content": "Instructions. Elaborate the text following these rules:" },     
         {"role": "system", "content": "Be exhaustive, comprehensive and detailed in the relation." },         
         {"role": "system", "content": "Arrange paragraphs and information around each entity or related entities and concepts." },         
         {"role": "system", "content": "Start directly with the relation, do not introduce the text, as it is part of a broader report." },         
         {"role": "system", "content": "Use a formal writing style, yet plain and easy to read. The audience is well acquainted with technical and defence/military vocabulary, information and entities. " },         
         

         {"role": "user", "content":"Information Pool:"+textb}  ] )
    response2 = completion.choices[0].message.content if completion.choices[0].message else ""
    response_2=" 2)Perspective of Key emerging aspects:"+"\n"+response2+"\n"
    
    completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[
         {"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."},
         {"role": "system", "content": "Your task is to consolidate and sore the relation of the different entities indicated in the Information Pool text for a internal report on recent news." },
         {"role": "system", "content": "Instructions. Elaborate the text following these rules:" },     
         {"role": "system", "content": "Be exhaustive in the sorting. Sort around similar entry types: Organization, Program, Technology, Entity, ... You can merge similar entry types (i.e. Technologies and Technology Terms and Concepts, People and Officials,...)" },         
         {"role": "system", "content": "Arrange and integrate entries around similar or related concepts. Discard duplicated concepts or elements." },         
         {"role": "system", "content": "Start directly with the relation, do not introduce the text, as it is part of a broader report." },         
         {"role": "system", "content": "The audience is well acquainted with technical and defence/military vocabulary, information and entities. " },         
         

         {"role": "user", "content":"Information Pool:"+textc}  ] )
    
    response3 = completion.choices[0].message.content if completion.choices[0].message else ""
    response_3=" 3)Perspective of of Key Entities"+"\n"+response3+"\n"
    compilacion=response_1+"\n"+response_2+"\n"+response_3
    print(compilacion)
    print("\n\n")
    print("\n\n")
    return compilacion

# Define the function to get news results
def get_organic_results(query, periodo_tbs, num_results):
    params = {
        "q": query,
        "num": str(num_results),
        "tbs": periodo_tbs,  # quiero los resultados del último año
        "api_key": SERPAPI_KEY
    }
    search = GoogleSearch(params)
    results = search.get_dict()
    organic_results = results.get("organic_results", [])  # Change from "news_results" to "organic_results"
            
    for result in organic_results:
        title = result.get('title')
        print("Title:", title)
        print()  # Print a newline for better readability between results

    return organic_results


def process_inputs(task_type, topic, integration_period, num_results):
    # Construct the query based on user input
    google_search_query = f'"{topic}" Conferences OR seminars OR SYMPOSIUMS'
    periodo_tbs = integration_period
    num_resultados = int(num_results)

    # Fetch results based on the user's query
    results = get_organic_results(google_search_query, periodo_tbs, num_resultados)
    df = update_dataframe_with_results(results)
    archivo1 = main(df, google_search_query)
    resumen_text = resumen(archivo1)
    # Save outputs to temporary files
    path_intermediate = "intermediate_results.txt"
    path_final = "final_results.txt"
    with open(path_intermediate, "w") as f:
        f.write(archivo1)
    with open(path_final, "w") as f:
        f.write(resumen_text)

    # Return both the text for display and the file paths for downloading
    return archivo1, resumen_text, path_intermediate, path_final

# Create the Gradio blocks interface
with gr.Blocks() as app:
    with gr.Row():
        with gr.Column():
            task_type = gr.Dropdown(choices=["Conferencias", "Seminarios", "Simposios"], label="Selecciona el tipo de tarea:")
            topic = gr.Textbox(label="Aspecto o Tema sobre el que trabajar", placeholder="Ingrese el tema aquí...")
            integration_period = gr.Dropdown(choices=["1M", "3M", "6M", "1Y"], label="Periodo de integración de información")
            num_results = gr.Number(label="Número de resultados sobre los que trabajar", value=10)
            submit_button = gr.Button("Submit")
        with gr.Column():
            output_text_intermedio = gr.Textbox(label="Resultados Intermedios", interactive=True, lines=10)
            output_text_final = gr.Textbox(label="Resultados Compilados", interactive=True, lines=10)
            download_intermediate = gr.File(label="Download Intermediate Results")
            download_final = gr.File(label="Download Final Results")


    # Define what happens when you click the Submit button
    submit_button.click(
        fn=process_inputs,
        inputs=[task_type, topic, integration_period, num_results],
        outputs=[output_text_intermedio, output_text_final, download_intermediate, download_final]
    )

if __name__ == "__main__":
    app.launch()