Update app.py

app.py

@@ -14,17 +14,22 @@ import mimetypes
 import validators
 import requests
 import tempfile
-from bs4 import BeautifulSoup
 from langchain.chains import create_extraction_chain
 from GoogleNews import GoogleNews
 import pandas as pd
+import requests
 import gradio as gr
 import re
 from langchain.document_loaders import WebBaseLoader
-from langchain.chains.llm import LLMChain
 from langchain.chains.combine_documents.stuff import StuffDocumentsChain
 from transformers import pipeline
 import plotly.express as px
+from langchain.document_loaders.csv_loader import CSVLoader
+from langchain.chains.llm import LLMChain
+import yfinance as yf
+import pandas as pd
+import nltk
+from nltk.tokenize import sent_tokenize
 
 class KeyValueExtractor:
 
@@ -38,153 +43,189 @@ class KeyValueExtractor:
  """
  self.model = "facebook/bart-large-mnli"
 
- def get_news(self,keyword):
+ def get_url(self,keyword):
+ return f"https://finance.yahoo.com/quote/{keyword}?p={keyword}"
 
- googlenews = GoogleNews(lang='en', region='US', period='1d', encode='utf-8')
- googlenews.clear()
- googlenews.search(keyword)
- googlenews.get_page(2)
- news_result = googlenews.result(sort=True)
- news_data_df = pd.DataFrame.from_dict(news_result)
+ def get_each_link_summary(self,url):
 
- news_data_df.info()
+ loader = WebBaseLoader(url)
+ docs = loader.load()
+ text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
+ chunk_size=3000, chunk_overlap=200
+ )
 
- # Display header of dataframe.
- news_data_df.head()
+ # Split the documents into chunks
+ split_docs = text_splitter.split_documents(docs)
+
+ # Prepare the prompt template for summarization
+ prompt_template = """The give text is Finance Stock Details for one company i want to get values for
+ Previous Close : [value]
+ Open : [value]
+ Bid : [value]
+ Ask : [value]
+ Day's Range : [value]
+ 52 Week Range : [value]
+ Volume : [value]
+ Avg. Volume : [value]
+ Market Cap : [value]
+ Beta (5Y Monthly) : [value]
+ PE Ratio (TTM) : [value]
+ EPS (TTM) : [value]
+ Earnings Date : [value]
+ Forward Dividend & Yield : [value]
+ Ex-Dividend Date : [value]
+ 1y Target Est : [value]
+ these details form that and Write a abractive summary about those details:
+ Given Text: {text}
+ CONCISE SUMMARY:"""
+ prompt = PromptTemplate.from_template(prompt_template)
+
+ # Prepare the template for refining the summary with additional context
+ refine_template = (
+ "Your job is to produce a final summary\n"
+ "We have provided an existing summary up to a certain point: {existing_answer}\n"
+ "We have the opportunity to refine the existing summary"
+ "(only if needed) with some more context below.\n"
+ "------------\n"
+ "{text}\n"
+ "------------\n"
+ "Given the new context, refine the original summary"
+ "If the context isn't useful, return the original summary."
+ )
+ refine_prompt = PromptTemplate.from_template(refine_template)
+
+ # Load the summarization chain using the ChatOpenAI language model
+ chain = load_summarize_chain(
+ llm = ChatOpenAI(temperature=0),
+ chain_type="refine",
+ question_prompt=prompt,
+ refine_prompt=refine_prompt,
+ return_intermediate_steps=True,
+ input_key="input_documents",
+ output_key="output_text",
+ )
+
+ # Generate the refined summary using the loaded summarization chain
+ result = chain({"input_documents": split_docs}, return_only_outputs=True)
+ print(result["output_text"])
+
+ return result["output_text"]
+
+ def one_day_summary(self,content) -> None:
+
+
+ # Use OpenAI's Completion API to analyze the text and extract key-value pairs
+ response = openai.Completion.create(
+ engine="text-davinci-003", # You can choose a different engine as well
+ temperature = 0,
+ prompt=f"i want detailed Summary from given finance details. i want information like what happen today comparing last day good or bad Bullish or Bearish like these details i want summary. content in backticks.```{content}```.",
+ max_tokens=1000 # You can adjust the length of the response
+ )
+
+ # Extract and return the chatbot's reply
+ result = response['choices'][0]['text'].strip()
+ print(result)
+ return result
 
- tot_news_link = []
- for index, headers in news_data_df.iterrows():
- news_link = str(headers['link'])
- tot_news_link.append(news_link)
+ def extract_key_value_pair(self,content) -> None:
 
- return tot_news_link
+ """
+ Extract key-value pairs from the refined summary.
 
- def url_format(self,urls):
+ Prints the extracted key-value pairs.
+ """
 
- tot_url_links = []
- for url_text in urls:
- # Define a regex pattern to match URLs starting with 'http' or 'https'
- pattern = r'(https?://[^\s]+)'
+ try:
 
- # Search for the URL in the text using the regex pattern
- match = re.search(pattern, url_text)
+ # Use OpenAI's Completion API to analyze the text and extract key-value pairs
+ response = openai.Completion.create(
+ engine="text-davinci-003", # You can choose a different engine as well
+ temperature = 0,
+ prompt=f"Get maximum count meaningfull key value pairs. content in backticks.```{content}```.",
+ max_tokens=1000 # You can adjust the length of the response
+ )
 
- if match:
- extracted_url = match.group(1)
- tot_url_links.append(extracted_url)
+ # Extract and return the chatbot's reply
+ result = response['choices'][0]['text'].strip()
+ return result
+ except Exception as e:
+ # If an error occurs during the key-value extraction process, log the error
+ logging.error(f"Error while extracting key-value pairs: {e}")
+ print("Error:", e)
 
- else:
- print("No URL found in the given text.")
+ def analyze_sentiment_for_graph(self, text):
 
- return tot_url_links
+ pipe = pipeline("zero-shot-classification", model=self.model)
+ label=["Positive", "Negative", "Neutral"]
+ result = pipe(text, label)
+ sentiment_scores = {
+ result['labels'][0]: result['scores'][0],
+ result['labels'][1]: result['scores'][1],
+ result['labels'][2]: result['scores'][2]
+ }
+ return sentiment_scores
 
- def clear_error_ulr(self,urls):
- error_url = []
- for url in urls:
- if validators.url(url):
- headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36',}
- r = requests.get(url,headers=headers)
- if r.status_code != 200:
- # raise ValueError("Check the url of your file; returned status code %s" % r.status_code)
- print(f"Error fetching {url}:")
- error_url.append(url)
- continue
- cleaned_list_url = [item for item in urls if item not in error_url]
- return cleaned_list_url
+ def display_graph(self,text):
 
- def get_each_link_summary(self,urls):
+ sentiment_scores = self.analyze_sentiment_for_graph(text)
+ labels = sentiment_scores.keys()
+ scores = sentiment_scores.values()
+ fig = px.bar(x=scores, y=labels, orientation='h', color=labels, color_discrete_map={"Negative": "red", "Positive": "green", "Neutral": "gray"})
+ fig.update_traces(texttemplate='%{x:.2f}%', textposition='outside')
+ fig.update_layout(title="Sentiment Analysis",width=800)
 
- each_link_summary = ""
+ formatted_pairs = []
+ for key, value in sentiment_scores.items():
+ formatted_value = round(value, 2) # Round the value to two decimal places
+ formatted_pairs.append(f"{key} : {formatted_value}")
 
- for url in urls:
- loader = WebBaseLoader(url)
- docs = loader.load()
- text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
- chunk_size=3000, chunk_overlap=200
- )
+ result_string = '\t'.join(formatted_pairs)
 
- # Split the documents into chunks
- split_docs = text_splitter.split_documents(docs)
+ return fig
 
- # Prepare the prompt template for summarization
- prompt_template = """Write a concise summary of the following:
- {text}
- CONCISE SUMMARY:"""
- prompt = PromptTemplate.from_template(prompt_template)
+ def get_finance_data(self,symbol):
 
- # Prepare the template for refining the summary with additional context
- refine_template = (
- "Your job is to produce a final summary\n"
- "We have provided an existing summary up to a certain point: {existing_answer}\n"
- "We have the opportunity to refine the existing summary"
- "(only if needed) with some more context below.\n"
- "------------\n"
- "{text}\n"
- "------------\n"
- "Given the new context, refine the original summary"
- "If the context isn't useful, return the original summary."
- )
- refine_prompt = PromptTemplate.from_template(refine_template)
+ # Define the stock symbol and date range
+ start_date = '2022-08-19'
+ end_date = '2023-08-19'
 
- # Load the summarization chain using the ChatOpenAI language model
- chain = load_summarize_chain(
- llm = ChatOpenAI(temperature=0),
- chain_type="refine",
- question_prompt=prompt,
- refine_prompt=refine_prompt,
- return_intermediate_steps=True,
- input_key="input_documents",
- output_key="output_text",
- )
+ # Fetch historical OHLC data using yfinance
+ data = yf.download(symbol, start=start_date, end=end_date)
 
- # Generate the refined summary using the loaded summarization chain
- result = chain({"input_documents": split_docs}, return_only_outputs=True)
- print(result["output_text"])
+ # Select only the OHLC columns
+ ohlc_data = data[['Open', 'High', 'Low', 'Close']]
 
- # Return the refined summary
- each_link_summary = each_link_summary + result["output_text"]
+ csv_path = "ohlc_data.csv"
+ # Save the OHLC data to a CSV file
+ ohlc_data.to_csv(csv_path)
+ return csv_path
 
- return each_link_summary
+ def csv_to_dataframe(self,csv_path):
 
- def save_text_to_file(self,each_link_summary) -> str:
+ # Replace 'your_file.csv' with the actual path to your CSV file
+ csv_file_path = csv_path
+ # Read the CSV file into a DataFrame
+ df = pd.read_csv(csv_file_path)
+ # Now you can work with the 'df' DataFrame
+ return df # Display the first few rows of the DataFrame
 
- """
- Load the text from the saved file and split it into documents.
+ def save_dataframe_in_text_file(self,df):
 
- Returns:
- List[str]: List of document texts.
- """
+ output_file_path = 'output.txt'
 
- # Get the path to the text file where the extracted text will be saved
- file_path = "extracted_text.txt"
- try:
- with open(file_path, 'w') as file:
- # Write the extracted text into the text file
- file.write(each_link_summary)
- # Return the file path of the saved text file
- return file_path
- except IOError as e:
- # If an IOError occurs during the file saving process, log the error
- logging.error(f"Error while saving text to file: {e}")
+ # Convert the DataFrame to a text file
+ df.to_csv(output_file_path, sep='\t', index=False)
 
- def document_loader(self,file_path) -> List[str]:
+ return output_file_path
 
- """
- Load the text from the saved file and split it into documents.
+ def csv_loader(self,output_file_path):
 
- Returns:
- List[str]: List of document texts.
- """
-
- # Initialize the UnstructuredFileLoader
- loader = UnstructuredFileLoader(file_path, strategy="fast")
- # Load the documents from the file
+ loader = UnstructuredFileLoader(output_file_path, strategy="fast")
  docs = loader.load()
 
- # Return the list of loaded document texts
  return docs
 
- def document_text_spilliter(self,docs) -> List[str]:
+ def document_text_spilliter(self,docs):
 
  """
  Split documents into chunks for efficient processing.
@@ -195,7 +236,7 @@ class KeyValueExtractor:
 
  # Initialize the text splitter with specified chunk size and overlap
  text_splitter = CharacterTextSplitter.from_tiktoken_encoder(
- chunk_size=3000, chunk_overlap=200
+ chunk_size=1000, chunk_overlap=200
  )
 
  # Split the documents into chunks
@@ -204,45 +245,33 @@ class KeyValueExtractor:
  # Return the list of split document chunks
  return split_docs
 
- def extract_key_value_pair(self,content) -> None:
-
- """
- Extract key-value pairs from the refined summary.
-
- Prints the extracted key-value pairs.
- """
+ def change_bullet_points(self,text):
 
- try:
+ nltk.download('punkt') # Download the sentence tokenizer data (only need to run this once)
 
- # Use OpenAI's Completion API to analyze the text and extract key-value pairs
- response = openai.Completion.create(
- engine="text-davinci-003", # You can choose a different engine as well
- temperature = 0,
- prompt=f"Get maximum count meaningfull key value pairs. content in backticks.```{content}```.",
- max_tokens=1000 # You can adjust the length of the response
- )
+ # Example passage
+ passage = text
 
- # Extract and return the chatbot's reply
- result = response['choices'][0]['text'].strip()
- return result
- except Exception as e:
- # If an error occurs during the key-value extraction process, log the error
- logging.error(f"Error while extracting key-value pairs: {e}")
- print("Error:", e)
+ # Tokenize the passage into sentences
+ sentences = sent_tokenize(passage)
+ bullet_string = ""
+ # Print the extracted sentences
+ for sentence in sentences:
+ bullet_string+="* "+sentence+"\n"
 
- def refine_summary(self,split_docs) -> str:
+ return bullet_string
 
- """
- Refine the summary using the provided context.
+ def one_year_summary(self,keyword):
 
- Returns:
- str: Refined summary.
- """
+ csv_path = self.get_finance_data(keyword)
+ df = self.csv_to_dataframe(csv_path)
+ output_file_path = self.save_dataframe_in_text_file(df)
+ docs = self.csv_loader(output_file_path)
+ split_docs = self.document_text_spilliter(docs)
 
- # Prepare the prompt template for summarization
- prompt_template = """Write a detalied broad abractive summary of the following:
- {text}
- CONCISE SUMMARY:"""
+ prompt_template = """Analyze the Financial Details and Write a abractive quick short summary how the company perform up and down,Bullish/Bearish of the following:
+ {text}
+ CONCISE SUMMARY:"""
  prompt = PromptTemplate.from_template(prompt_template)
 
  # Prepare the template for refining the summary with additional context
@@ -256,6 +285,7 @@ class KeyValueExtractor:
  "------------\n"
  "Given the new context, refine the original summary"
  "If the context isn't useful, return the original summary."
+ "10 line summary is enough"
  )
  refine_prompt = PromptTemplate.from_template(refine_template)
 
@@ -272,54 +302,19 @@ class KeyValueExtractor:
 
  # Generate the refined summary using the loaded summarization chain
  result = chain({"input_documents": split_docs}, return_only_outputs=True)
-
- key_value_pair = self.extract_key_value_pair(result["output_text"])
-
+ one_year_perfomance_summary = self.change_bullet_points(result["output_text"])
  # Return the refined summary
- return result["output_text"],key_value_pair
-
- def analyze_sentiment_for_graph(self, text):
- pipe = pipeline("zero-shot-classification", model=self.model)
- label=["Positive", "Negative", "Neutral"]
- result = pipe(text, label)
- sentiment_scores = {
- result['labels'][0]: result['scores'][0],
- result['labels'][1]: result['scores'][1],
- result['labels'][2]: result['scores'][2]
- }
- return sentiment_scores
-
- def display_graph(self,text):
-
- sentiment_scores = self.analyze_sentiment_for_graph(text)
- labels = sentiment_scores.keys()
- scores = sentiment_scores.values()
- fig = px.bar(x=scores, y=labels, orientation='h', color=labels, color_discrete_map={"Negative": "red", "Positive": "green", "Neutral": "gray"})
- fig.update_traces(texttemplate='%{x:.2f}%', textposition='outside')
- fig.update_layout(title="Sentiment Analysis",width=800)
-
- formatted_pairs = []
- for key, value in sentiment_scores.items():
- formatted_value = round(value, 2) # Round the value to two decimal places
- formatted_pairs.append(f"{key} : {formatted_value}")
-
- result_string = '\t'.join(formatted_pairs)
-
- return fig
+ return one_year_perfomance_summary
 
  def main(self,keyword):
 
- urls = self.get_news(keyword)
- tot_urls = self.url_format(urls)
- clean_url = self.clear_error_ulr(tot_urls)
- each_link_summary = self.get_each_link_summary(clean_url)
- file_path = self.save_text_to_file(each_link_summary)
- docs = self.document_loader(file_path)
- split_docs = self.document_text_spilliter(docs)
- result = self.refine_summary(split_docs)
 
+ clean_url = self.get_url(keyword)
+ link_summary = self.get_each_link_summary(clean_url)
+ clean_summary = self.one_day_summary(link_summary)
+ key_value = self.extract_key_value_pair(clean_summary)
 
- return result
+ return clean_summary, key_value
 
  def gradio_interface(self):
 
@@ -329,24 +324,38 @@ class KeyValueExtractor:
  <br><h1 style="color:#fff">summarizer</h1></center>""")
  with gr.Row(elem_id="col-container"):
  with gr.Column(scale=1.0, min_width=150, ):
- input_news = gr.Textbox(label="NEWS")
+ input_news = gr.Textbox(label="Company Name")
  with gr.Row(elem_id="col-container"):
  with gr.Column(scale=1.0, min_width=150):
  analyse = gr.Button("Analyse")
  with gr.Row(elem_id="col-container"):
  with gr.Column(scale=0.50, min_width=150):
- result_summary = gr.Textbox(label="Summary")
+ result_summary = gr.Textbox(label="Summary", lines = 20)
  with gr.Column(scale=0.50, min_width=150):
- key_value_pair_result = gr.Textbox(label="Key Value Pair")
+ key_value_pair_result = gr.Textbox(label="Key Value Pair", lines = 20)
+ with gr.Row(elem_id="col-container"):
+ with gr.Column(scale=1.0, min_width=0):
+ plot_for_day =gr.Plot(label="Sentiment", size=(500, 600))
+ with gr.Row(elem_id="col-container"):
+ with gr.Column(scale=1.0, min_width=150):
+ analyse_sentiment = gr.Button("Analyse Sentiment")
+ with gr.Row(elem_id="col-container"):
+ with gr.Column(scale=1.0, min_width=150, ):
+ one_year_summary = gr.Textbox(label="Summary Of One Year Perfomance",lines = 20)
+ with gr.Row(elem_id="col-container"):
+ with gr.Column(scale=1.0, min_width=150):
+ one_year = gr.Button("Analyse One Year Summary")
  with gr.Row(elem_id="col-container"):
- with gr.Column(scale=0.70, min_width=0):
- plot =gr.Plot(label="Customer", size=(500, 600))
+ with gr.Column(scale=1.0, min_width=0):
+ plot_for_year =gr.Plot(label="Sentiment", size=(500, 600))
  with gr.Row(elem_id="col-container"):
  with gr.Column(scale=1.0, min_width=150):
- analyse_sentiment = gr.Button("Analyse")
+ analyse_sentiment_for_year = gr.Button("Analyse Sentiment")
 
  analyse.click(self.main, input_news, [result_summary,key_value_pair_result])
- analyse_sentiment.click(self.display_graph,result_summary,[plot])
+ analyse_sentiment.click(self.display_graph,result_summary,[plot_for_day])
+ one_year.click(self.one_year_summary,input_news,one_year_summary)
+ analyse_sentiment_for_year.click(self.display_graph,one_year_summary,[plot_for_year])
 
  app.launch(debug=True)