Create app.py

app.py

@@ -0,0 +1,340 @@
+import os.path
+import openai
+from openai import OpenAI
+import os
+from groq import Groq
+import requests
+import time
+from html.parser import HTMLParser
+from bs4 import BeautifulSoup
+import json
+from datetime import datetime
+import pandas as pd
+from serpapi import GoogleSearch
+import gradio as gr
+
+GROQ_API_KEY=getenv("GROQ_API_KEY")
+client_groq = Groq(api_key=GROQ_API_KEY,)
+openai_key=getenv("OPENAI_API_KEY")
+os.environ["OPENAI_API_KEY"] = openai_key
+client = OpenAI()
+
+SERPAPI_KEY=getenv("SERPAPI_KEY")
+
+api_key = os.getenv("API_KEY")
+
+def scrape_website(url):
+ headers = {'User-Agent': 'Mozilla/5.0'}
+ try:
+ response = requests.get(url, headers=headers, timeout=20)
+ response.encoding = response.apparent_encoding # Set encoding to match the content
+
+ if response.status_code == 200:
+ page_content = response.content
+ soup = BeautifulSoup(page_content, 'html.parser')
+ paragraphs = soup.find_all('p')
+ scraped_data = [p.get_text() for p in paragraphs]
+ formatted_data = u"\n".join(scraped_data)
+
+ return formatted_data # Return only content
+ else:
+ return "Failed to retrieve the webpage (Status Code: {})".format(response.status_code)
+ 
+ except requests.exceptions.ReadTimeout:
+ # Handle the timeout exception
+ return "Request timed out after 20 seconds."
+ except requests.exceptions.SSLError as e:
+ return "Request Error: {}".format(e)
+ except requests.exceptions.RequestException as e:
+ # Handle other requests-related exceptions
+ return "An error occurred: {}".format(e)
+
+
+def update_dataframe_with_results(organic_results):
+ # Prepare data for DataFrame
+ max_chars = 100000 # Maximum characters allowed in a cell
+ data = []
+ for result in organic_results:
+ # Scrape the website content
+ scraped_content = scrape_website(result.get('link'))
+ # Truncate the content if it exceeds the limit
+ if len(scraped_content) > max_chars:
+ scraped_content = scraped_content[:max_chars]
+ data.append({
+ "Title": result.get('title'),
+ "Link": result.get('link'),
+ "Snippet": result.get('snippet'),
+ "Displayed Link": result.get('displayed_link'),
+ "Date": result.get('date'), # Might not always be present
+ "Rich Snippet": result.get('rich_snippet'), # Might not always be present
+ "Scraped Content": scraped_content # Add scraped content
+ })
+
+ df = pd.DataFrame(data)
+ return df
+
+def opencall(text,user_query):
+ print("Calling opencall function with", len(text), "characters")
+ #completion = client_groq.chat.completions.create( 
+ completion = client.chat.completions.create(
+ model="gpt-4-0125-preview",
+ #model="mixtral-8x7b-32768",
+ temperature=0.1,
+ messages=[
+ {"role": "system", "content": "You are a helpful assistant, specialised in preparing contents for preparing a presentation."},
+ {"role": "system", "content": "Your task is to prepare a base report on the topics, themes and trends addressed in the latest conferences, seminars and symposiums." },
+ {"role": "system", "content": "For this matter I will be providing you in the Information Pool a compilation of several scraped google search results from the latest conferences, seminars and symposiums on the topic: "+user_query},
+ {"role": "system", "content": "Each piece of Scraped Content start with the tag '### Title:' indicating the title, followed by the URL reference '### Link:' , followed by the contents '### Content:'"},
+ {"role": "system", "content": "Process all the information in the Information Pool to provide:"},
+ {"role": "system", "content": "1) Perspective of Relevant Information: Assess and extract the most relevant information from the point of view of this aspect: "+user_query+"."},
+ {"role": "system", "content": "2) Perspective of Key Topics: Highlight the key topics and themes.Cite the URLs that source those topics and themes"},
+ {"role": "system", "content": "3) Perspective of Emergent Trends: Highlight the emergent trends.Cite the URLs that source those trends."},
+ {"role": "system", "content": "In the response, use the indicated structure of 1)Perspective of Relevant Information 2)Perspective of Key Topics 3)Perspective of Emergent Trends"},
+ {"role": "user", "content":"Information Pool:"+text}
+ ]
+ )
+ response = completion.choices[0].message.content
+ response = response + "\n" + "XXXXX" + "\n"
+ return response
+
+
+def split_large_content(content, max_length=30000):
+ # Extract the title and source URL, assuming they end with the second newline
+ title_and_source_end = content.find('\n\n') + 2
+ title_and_source = content[:title_and_source_end]
+ title_and_source_length = len(title_and_source)
+
+ # Ensure each segment has space for the title and source by reducing max_length
+ max_segment_length = max_length - title_and_source_length
+
+ segments = []
+ content_body = content[title_and_source_end:]
+
+ # Start splitting the content_body into segments
+ while len(content_body) > 0:
+ # Take a slice of content up to max_segment_length
+ segment = content_body[:max_segment_length]
+
+ # If we're not at the end of content_body, back-track to the last complete word
+ if len(content_body) > max_segment_length:
+ last_space = segment.rfind(' ')
+ segment = segment[:last_space]
+
+ # Add the title and source URL to the start of this segment
+ full_segment = title_and_source + segment
+ segments.append(full_segment)
+
+ # Move forward in content_body by the length of the segment minus the title/source
+ content_body = content_body[len(segment):]
+
+ return segments
+
+
+
+def main(df,google_search_query):
+ # Initialize a string to accumulate the information
+ information_pool = ""
+ archivo1=""
+ # Open or create a plain text file in append mode
+ with open('respuestas.txt', mode='a', encoding='utf-8') as file:
+ 
+ # Iterate over the rows of the DataFrame
+ for index, row in df.iterrows():
+ # Combine title, link, and content into a single string
+ document_name = row['Title'] # Using title as document_name
+ raw_content = str(row['Scraped Content']) # Convert to string to ensure compatibility
+ link = row['Link'] # Retrieve link for additional usage or logging
+
+ # Assuming process_document_content is a function you've defined to process the content
+ processed_content = "### Title: " + row['Title'] + "\n" + "### Link: " + row['Link'] + "\n" + "### Content: " + str(row['Scraped Content']) + "\n" + "\n"
+
+ print(document_name, ":", len(processed_content))
+ #print("Contenido:", processed_content)
+ print("acumulado:", len(information_pool + processed_content))
+
+ # Handle long content by splitting and processing in segments
+ if len(processed_content) > 30000:
+ content_segments = split_large_content(processed_content)
+ for segment in content_segments:
+ print("EN C, Nuevo valor de Text:", len(segment))
+ #print("segmen:",segment)
+ response = opencall(segment,google_search_query) # Replace 'opencall' with your actual function call
+ archivo1=archivo1+response+'\n'
+ file.write(response + '\n')
+
+ else:
+ # Check if adding processed content exceeds the size limit
+ if len(information_pool + processed_content) <= 30000:
+ information_pool += processed_content
+ print("EN A, Nuevo valor de Text:", len(information_pool))
+ else:
+ # Process current accumulated content and start new accumulation
+ print("EN B1, llamando con valor de Text:", len(information_pool))
+ #print("Information pool", information_pool)
+ response = opencall(information_pool,google_search_query)
+ file.write(response + '\n')
+ archivo1=archivo1+response+'\n'
+ information_pool = processed_content
+ print("EN B2, nuevo valor de Text:", len(information_pool), " Con documento:", document_name)
+
+ # Handle any remaining content after loop
+ if information_pool:
+ print("Final call")
+ response = opencall(information_pool,google_search_query)
+ file.write(response + '\n')
+ archivo1=archivo1+response+'\n'
+ return archivo1 
+
+def rearrange_text(text):
+ # Split the text into batches using 'XXXXX'
+ batches = text.split('XXXXX')
+
+ # Initialize variables to store concatenated texts
+ all_texta = ""
+ all_textb = ""
+ all_textc = ""
+
+ # Define markers for different sections
+ markers = {
+ 'texta_marker': "Perspective of Relevant Information",
+ 'textb_marker': "Perspective of Key Emerging Aspects",
+ 'textc_marker': "Perspective of Key Entities"
+ }
+
+ # Process each batch
+ for batch in batches:
+ # Initialize indices for each section
+ texta_start = batch.find(markers['texta_marker'])
+ textb_start = batch.find(markers['textb_marker'])
+ textc_start = batch.find(markers['textc_marker'])
+
+ # Extract TEXTA, TEXTB, and TEXTC using the found indices
+ # Check if the markers are found; if not, skip to the next marker
+ texta = batch[texta_start:textb_start] if textb_start != -1 else batch[texta_start:]
+ textb = batch[textb_start:textc_start] if textc_start != -1 else batch[textb_start:]
+ textc = batch[textc_start:]
+
+ # Remove the markers from the beginning of each text
+ texta = texta.replace(markers['texta_marker'], '').strip()
+ textb = textb.replace(markers['textb_marker'], '').strip()
+ textc = textc.replace(markers['textc_marker'], '').strip()
+
+ # Concatenate texts from all batches
+ all_texta += "\n" + texta if all_texta else texta
+ all_textb += "\n" + textb if all_textb else textb
+ all_textc += "\n" + textc if all_textc else textc
+
+ # You can now use all_texta, all_textb, and all_textc for further summarization or processing
+ return all_texta, all_textb, all_textc
+
+def resumen(text):
+ texta, textb, textc = rearrange_text(text)
+ 
+ completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[
+ {"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."},
+ {"role": "system", "content": "Your task is to provide an integrated comprehensive 2000 words narrative of the different points indicated in the Information Pool text for a internal report on recent news." },
+ {"role": "system", "content": "Instructions. Elaborate the text following these rules:" }, 
+ {"role": "system", "content": "Be exhaustive, comprehensive and detailed in addressing the relation of different points indicated in the Information Pool text." }, 
+ {"role": "system", "content": "Arrange paragraphs and information around each entity or related entities and concepts, integrating them with a fluent narrative." }, 
+ {"role": "system", "content": "Start directly with the narrative, do not introduce the text, as it is part of a broader report." }, 
+ {"role": "system", "content": "Use a formal writing style, yet plain and easy to read. Avoid pomposity and making up artificial descriptions. The audience is well acquainted with technical and defence/military vocabulary, information and entities. " }, 
+ 
+ 
+ {"role": "user", "content":"Information Pool:"+texta} ] )
+
+
+ 
+ response1 = completion.choices[0].message.content if completion.choices[0].message else ""
+ response_1="1) Perspective of Relevant Information:"+"\n"+response1+"\n"
+ 
+ completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[
+ {"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."},
+ {"role": "system", "content": "Your task is to provide a comprehensive and integrated relation of about 2000 words in length of the different emerging aspects indicated in the Information Pool text for a internal report on recent news." },
+ {"role": "system", "content": "Instructions. Elaborate the text following these rules:" }, 
+ {"role": "system", "content": "Be exhaustive, comprehensive and detailed in the relation." }, 
+ {"role": "system", "content": "Arrange paragraphs and information around each entity or related entities and concepts." }, 
+ {"role": "system", "content": "Start directly with the relation, do not introduce the text, as it is part of a broader report." }, 
+ {"role": "system", "content": "Use a formal writing style, yet plain and easy to read. The audience is well acquainted with technical and defence/military vocabulary, information and entities. " }, 
+ 
+
+ {"role": "user", "content":"Information Pool:"+textb} ] )
+ response2 = completion.choices[0].message.content if completion.choices[0].message else ""
+ response_2=" 2)Perspective of Key emerging aspects:"+"\n"+response2+"\n"
+ 
+ completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[
+ {"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."},
+ {"role": "system", "content": "Your task is to consolidate and sore the relation of the different entities indicated in the Information Pool text for a internal report on recent news." },
+ {"role": "system", "content": "Instructions. Elaborate the text following these rules:" }, 
+ {"role": "system", "content": "Be exhaustive in the sorting. Sort around similar entry types: Organization, Program, Technology, Entity, ... You can merge similar entry types (i.e. Technologies and Technology Terms and Concepts, People and Officials,...)" }, 
+ {"role": "system", "content": "Arrange and integrate entries around similar or related concepts. Discard duplicated concepts or elements." }, 
+ {"role": "system", "content": "Start directly with the relation, do not introduce the text, as it is part of a broader report." }, 
+ {"role": "system", "content": "The audience is well acquainted with technical and defence/military vocabulary, information and entities. " }, 
+ 
+
+ {"role": "user", "content":"Information Pool:"+textc} ] )
+ 
+ response3 = completion.choices[0].message.content if completion.choices[0].message else ""
+ response_3=" 3)Perspective of of Key Entities"+"\n"+response3+"\n"
+ compilacion=response_1+"\n"+response_2+"\n"+response_3
+ print(compilacion)
+ print("\n\n")
+ print("\n\n")
+ return compilacion
+
+# Define the function to get news results
+def get_organic_results(query, periodo_tbs, num_results):
+ params = {
+ "q": query,
+ "num": str(num_results),
+ "tbs": periodo_tbs, # quiero los resultados del último año
+ "api_key": SERPAPI_KEY
+ }
+ search = GoogleSearch(params)
+ results = search.get_dict()
+ organic_results = results.get("organic_results", []) # Change from "news_results" to "organic_results"
+ 
+ for result in organic_results:
+ title = result.get('title')
+ print("Title:", title)
+ print() # Print a newline for better readability between results
+
+ return organic_results
+
+
+def process_inputs(task_type, topic, integration_period, num_results):
+ # Construct the query based on user input
+ google_search_query = f'"{topic}" Conferences OR seminars OR SYMPOSIUMS'
+ periodo_tbs = integration_period
+ num_resultados = int(num_results)
+
+ # Fetch results based on the user's query
+ results = get_organic_results(google_search_query, periodo_tbs, num_resultados)
+ df = update_dataframe_with_results(results)
+ archivo1 = main(df, google_search_query)
+ resumen_text = resumen(archivo1)
+
+ return archivo1,resumen_text
+
+# Create the Gradio blocks interface
+with gr.Blocks() as app:
+ with gr.Row():
+ with gr.Column():
+ task_type = gr.Dropdown(choices=["Conferencias", "Seminarios", "Simposios"], label="Selecciona el tipo de tarea:")
+ topic = gr.Textbox(label="Aspecto o Tema sobre el que trabajar", placeholder="Ingrese el tema aquí...")
+ integration_period = gr.Dropdown(choices=["1M", "3M", "6M", "1Y"], label="Periodo de integración de información")
+ num_results = gr.Number(label="Número de resultados sobre los que trabajar", value=10)
+ submit_button = gr.Button("Submit")
+ with gr.Column():
+ output_text_intermedio = gr.Textbox(label="Resultados Intermedios", interactive=True, lines=10)
+ output_text_final = gr.Textbox(label="Resultados Compilados", interactive=True, lines=10)
+ 
+
+ # Define what happens when you click the Submit button
+ submit_button.click(
+ fn=process_inputs,
+ inputs=[task_type, topic, integration_period, num_results],
+ outputs=[output_text_intermedio,output_text_final]
+ )
+
+if __name__ == "__main__":
+ app.launch()