Mayordomo / app.py
LAPINILLA's picture
Update app.py
69430c1 verified
raw
history blame
17.8 kB
import os.path
from openai import OpenAI
import os
from groq import Groq
import requests
import time
from html.parser import HTMLParser
from bs4 import BeautifulSoup
import json
from datetime import datetime
import pandas as pd
from serpapi import GoogleSearch
import gradio as gr
GROQ_API_KEY=getenv("GROQ_API_KEY")
client_groq = Groq(api_key=GROQ_API_KEY,)
openai_key=getenv("OPENAI_API_KEY")
os.environ["OPENAI_API_KEY"] = openai_key
client = OpenAI()
SERPAPI_KEY=getenv("SERPAPI_KEY")
def scrape_website(url):
headers = {'User-Agent': 'Mozilla/5.0'}
try:
response = requests.get(url, headers=headers, timeout=20)
response.encoding = response.apparent_encoding # Set encoding to match the content
if response.status_code == 200:
page_content = response.content
soup = BeautifulSoup(page_content, 'html.parser')
paragraphs = soup.find_all('p')
scraped_data = [p.get_text() for p in paragraphs]
formatted_data = u"\n".join(scraped_data)
return formatted_data # Return only content
else:
return "Failed to retrieve the webpage (Status Code: {})".format(response.status_code)
except requests.exceptions.ReadTimeout:
# Handle the timeout exception
return "Request timed out after 20 seconds."
except requests.exceptions.SSLError as e:
return "Request Error: {}".format(e)
except requests.exceptions.RequestException as e:
# Handle other requests-related exceptions
return "An error occurred: {}".format(e)
def update_dataframe_with_results(organic_results):
# Prepare data for DataFrame
max_chars = 100000 # Maximum characters allowed in a cell
data = []
for result in organic_results:
# Scrape the website content
scraped_content = scrape_website(result.get('link'))
# Truncate the content if it exceeds the limit
if len(scraped_content) > max_chars:
scraped_content = scraped_content[:max_chars]
data.append({
"Title": result.get('title'),
"Link": result.get('link'),
"Snippet": result.get('snippet'),
"Displayed Link": result.get('displayed_link'),
"Date": result.get('date'), # Might not always be present
"Rich Snippet": result.get('rich_snippet'), # Might not always be present
"Scraped Content": scraped_content # Add scraped content
})
df = pd.DataFrame(data)
return df
def opencall(text,user_query):
print("Calling opencall function with", len(text), "characters")
#completion = client_groq.chat.completions.create(
completion = client.chat.completions.create(
model="gpt-4-0125-preview",
#model="mixtral-8x7b-32768",
temperature=0.1,
messages=[
{"role": "system", "content": "You are a helpful assistant, specialised in preparing contents for preparing a presentation."},
{"role": "system", "content": "Your task is to prepare a base report on the topics, themes and trends addressed in the latest conferences, seminars and symposiums." },
{"role": "system", "content": "For this matter I will be providing you in the Information Pool a compilation of several scraped google search results from the latest conferences, seminars and symposiums on the topic: "+user_query},
{"role": "system", "content": "Each piece of Scraped Content start with the tag '### Title:' indicating the title, followed by the URL reference '### Link:' , followed by the contents '### Content:'"},
{"role": "system", "content": "Process all the information in the Information Pool to provide:"},
{"role": "system", "content": "1) Perspective of Relevant Information: Assess and extract the most relevant information from the point of view of this aspect: "+user_query+"."},
{"role": "system", "content": "2) Perspective of Key Topics: Highlight the key topics and themes.Cite the URLs that source those topics and themes"},
{"role": "system", "content": "3) Perspective of Emergent Trends: Highlight the emergent trends.Cite the URLs that source those trends."},
{"role": "system", "content": "In the response, use the indicated structure of 1)Perspective of Relevant Information 2)Perspective of Key Topics 3)Perspective of Emergent Trends"},
{"role": "user", "content":"Information Pool:"+text}
]
)
response = completion.choices[0].message.content
response = response + "\n" + "XXXXX" + "\n"
return response
def split_large_content(content, max_length=30000):
# Extract the title and source URL, assuming they end with the second newline
title_and_source_end = content.find('\n\n') + 2
title_and_source = content[:title_and_source_end]
title_and_source_length = len(title_and_source)
# Ensure each segment has space for the title and source by reducing max_length
max_segment_length = max_length - title_and_source_length
segments = []
content_body = content[title_and_source_end:]
# Start splitting the content_body into segments
while len(content_body) > 0:
# Take a slice of content up to max_segment_length
segment = content_body[:max_segment_length]
# If we're not at the end of content_body, back-track to the last complete word
if len(content_body) > max_segment_length:
last_space = segment.rfind(' ')
segment = segment[:last_space]
# Add the title and source URL to the start of this segment
full_segment = title_and_source + segment
segments.append(full_segment)
# Move forward in content_body by the length of the segment minus the title/source
content_body = content_body[len(segment):]
return segments
def main(df,google_search_query):
# Initialize a string to accumulate the information
information_pool = ""
archivo1=""
# Open or create a plain text file in append mode
with open('respuestas.txt', mode='a', encoding='utf-8') as file:
# Iterate over the rows of the DataFrame
for index, row in df.iterrows():
# Combine title, link, and content into a single string
document_name = row['Title'] # Using title as document_name
raw_content = str(row['Scraped Content']) # Convert to string to ensure compatibility
link = row['Link'] # Retrieve link for additional usage or logging
# Assuming process_document_content is a function you've defined to process the content
processed_content = "### Title: " + row['Title'] + "\n" + "### Link: " + row['Link'] + "\n" + "### Content: " + str(row['Scraped Content']) + "\n" + "\n"
print(document_name, ":", len(processed_content))
#print("Contenido:", processed_content)
print("acumulado:", len(information_pool + processed_content))
# Handle long content by splitting and processing in segments
if len(processed_content) > 30000:
content_segments = split_large_content(processed_content)
for segment in content_segments:
print("EN C, Nuevo valor de Text:", len(segment))
#print("segmen:",segment)
response = opencall(segment,google_search_query) # Replace 'opencall' with your actual function call
archivo1=archivo1+response+'\n'
file.write(response + '\n')
else:
# Check if adding processed content exceeds the size limit
if len(information_pool + processed_content) <= 30000:
information_pool += processed_content
print("EN A, Nuevo valor de Text:", len(information_pool))
else:
# Process current accumulated content and start new accumulation
print("EN B1, llamando con valor de Text:", len(information_pool))
#print("Information pool", information_pool)
response = opencall(information_pool,google_search_query)
file.write(response + '\n')
archivo1=archivo1+response+'\n'
information_pool = processed_content
print("EN B2, nuevo valor de Text:", len(information_pool), " Con documento:", document_name)
# Handle any remaining content after loop
if information_pool:
print("Final call")
response = opencall(information_pool,google_search_query)
file.write(response + '\n')
archivo1=archivo1+response+'\n'
return archivo1
def rearrange_text(text):
# Split the text into batches using 'XXXXX'
batches = text.split('XXXXX')
# Initialize variables to store concatenated texts
all_texta = ""
all_textb = ""
all_textc = ""
# Define markers for different sections
markers = {
'texta_marker': "Perspective of Relevant Information",
'textb_marker': "Perspective of Key Emerging Aspects",
'textc_marker': "Perspective of Key Entities"
}
# Process each batch
for batch in batches:
# Initialize indices for each section
texta_start = batch.find(markers['texta_marker'])
textb_start = batch.find(markers['textb_marker'])
textc_start = batch.find(markers['textc_marker'])
# Extract TEXTA, TEXTB, and TEXTC using the found indices
# Check if the markers are found; if not, skip to the next marker
texta = batch[texta_start:textb_start] if textb_start != -1 else batch[texta_start:]
textb = batch[textb_start:textc_start] if textc_start != -1 else batch[textb_start:]
textc = batch[textc_start:]
# Remove the markers from the beginning of each text
texta = texta.replace(markers['texta_marker'], '').strip()
textb = textb.replace(markers['textb_marker'], '').strip()
textc = textc.replace(markers['textc_marker'], '').strip()
# Concatenate texts from all batches
all_texta += "\n" + texta if all_texta else texta
all_textb += "\n" + textb if all_textb else textb
all_textc += "\n" + textc if all_textc else textc
# You can now use all_texta, all_textb, and all_textc for further summarization or processing
return all_texta, all_textb, all_textc
def resumen(text):
texta, textb, textc = rearrange_text(text)
completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[
{"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."},
{"role": "system", "content": "Your task is to provide an integrated comprehensive 2000 words narrative of the different points indicated in the Information Pool text for a internal report on recent news." },
{"role": "system", "content": "Instructions. Elaborate the text following these rules:" },
{"role": "system", "content": "Be exhaustive, comprehensive and detailed in addressing the relation of different points indicated in the Information Pool text." },
{"role": "system", "content": "Arrange paragraphs and information around each entity or related entities and concepts, integrating them with a fluent narrative." },
{"role": "system", "content": "Start directly with the narrative, do not introduce the text, as it is part of a broader report." },
{"role": "system", "content": "Use a formal writing style, yet plain and easy to read. Avoid pomposity and making up artificial descriptions. The audience is well acquainted with technical and defence/military vocabulary, information and entities. " },
{"role": "user", "content":"Information Pool:"+texta} ] )
response1 = completion.choices[0].message.content if completion.choices[0].message else ""
response_1="1) Perspective of Relevant Information:"+"\n"+response1+"\n"
completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[
{"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."},
{"role": "system", "content": "Your task is to provide a comprehensive and integrated relation of about 2000 words in length of the different emerging aspects indicated in the Information Pool text for a internal report on recent news." },
{"role": "system", "content": "Instructions. Elaborate the text following these rules:" },
{"role": "system", "content": "Be exhaustive, comprehensive and detailed in the relation." },
{"role": "system", "content": "Arrange paragraphs and information around each entity or related entities and concepts." },
{"role": "system", "content": "Start directly with the relation, do not introduce the text, as it is part of a broader report." },
{"role": "system", "content": "Use a formal writing style, yet plain and easy to read. The audience is well acquainted with technical and defence/military vocabulary, information and entities. " },
{"role": "user", "content":"Information Pool:"+textb} ] )
response2 = completion.choices[0].message.content if completion.choices[0].message else ""
response_2=" 2)Perspective of Key emerging aspects:"+"\n"+response2+"\n"
completion = client.chat.completions.create(model="gpt-4-0125-preview",temperature=0.5, messages=[
{"role": "system", "content": "You are a helpful assistant, specialised in composing and integrating information."},
{"role": "system", "content": "Your task is to consolidate and sore the relation of the different entities indicated in the Information Pool text for a internal report on recent news." },
{"role": "system", "content": "Instructions. Elaborate the text following these rules:" },
{"role": "system", "content": "Be exhaustive in the sorting. Sort around similar entry types: Organization, Program, Technology, Entity, ... You can merge similar entry types (i.e. Technologies and Technology Terms and Concepts, People and Officials,...)" },
{"role": "system", "content": "Arrange and integrate entries around similar or related concepts. Discard duplicated concepts or elements." },
{"role": "system", "content": "Start directly with the relation, do not introduce the text, as it is part of a broader report." },
{"role": "system", "content": "The audience is well acquainted with technical and defence/military vocabulary, information and entities. " },
{"role": "user", "content":"Information Pool:"+textc} ] )
response3 = completion.choices[0].message.content if completion.choices[0].message else ""
response_3=" 3)Perspective of of Key Entities"+"\n"+response3+"\n"
compilacion=response_1+"\n"+response_2+"\n"+response_3
print(compilacion)
print("\n\n")
print("\n\n")
return compilacion
# Define the function to get news results
def get_organic_results(query, periodo_tbs, num_results):
params = {
"q": query,
"num": str(num_results),
"tbs": periodo_tbs, # quiero los resultados del último año
"api_key": SERPAPI_KEY
}
search = GoogleSearch(params)
results = search.get_dict()
organic_results = results.get("organic_results", []) # Change from "news_results" to "organic_results"
for result in organic_results:
title = result.get('title')
print("Title:", title)
print() # Print a newline for better readability between results
return organic_results
def process_inputs(task_type, topic, integration_period, num_results):
# Construct the query based on user input
google_search_query = f'"{topic}" Conferences OR seminars OR SYMPOSIUMS'
periodo_tbs = integration_period
num_resultados = int(num_results)
# Fetch results based on the user's query
results = get_organic_results(google_search_query, periodo_tbs, num_resultados)
df = update_dataframe_with_results(results)
archivo1 = main(df, google_search_query)
resumen_text = resumen(archivo1)
return archivo1,resumen_text
# Create the Gradio blocks interface
with gr.Blocks() as app:
with gr.Row():
with gr.Column():
task_type = gr.Dropdown(choices=["Conferencias", "Seminarios", "Simposios"], label="Selecciona el tipo de tarea:")
topic = gr.Textbox(label="Aspecto o Tema sobre el que trabajar", placeholder="Ingrese el tema aquí...")
integration_period = gr.Dropdown(choices=["1M", "3M", "6M", "1Y"], label="Periodo de integración de información")
num_results = gr.Number(label="Número de resultados sobre los que trabajar", value=10)
submit_button = gr.Button("Submit")
with gr.Column():
output_text_intermedio = gr.Textbox(label="Resultados Intermedios", interactive=True, lines=10)
output_text_final = gr.Textbox(label="Resultados Compilados", interactive=True, lines=10)
# Define what happens when you click the Submit button
submit_button.click(
fn=process_inputs,
inputs=[task_type, topic, integration_period, num_results],
outputs=[output_text_intermedio,output_text_final]
)
if __name__ == "__main__":
app.launch()