"Multi Threaded Approach"

Dockerfile

@@ -5,8 +5,8 @@ FROM python:3.9-slim
 WORKDIR /code
 
 # Copy the required files to the working directory
-COPY app.py .
-COPY ./templates/index.html /code/templates/index.html 
+COPY app_socket_multi_threaded.py .
+COPY ./templates/index_socket.html /code/templates/index_socket.html 
 COPY ./requirements.txt /code/requirements.txt
 
 # Install the required packages
@@ -27,4 +27,4 @@ COPY --chown=user . $HOME/app
 EXPOSE 7860
 
 # Start the Flask app
-CMD ["python", "app.py"]
+CMD ["python", "app_socket_multi_threaded.py"]

app_graph_test.py

@@ -0,0 +1,46 @@
+from flask import Flask, render_template, request, jsonify
+from flask_socketio import SocketIO
+import yfinance as yf
+from datetime import datetime, timedelta
+import plotly.graph_objs as go
+
+app = Flask(__name__)
+socketio = SocketIO(app)
+
+@app.route("/")
+def index():
+ return render_template("test.html")
+
+@socketio.on('get_graph')
+def get_graph(ticker):
+ # Define the stock ticker symbol and the date range
+ ticker_symbol = ticker
+ end_date = datetime.today()
+ start_date = end_date - timedelta(days=90)
+
+ # Fetch historical data using yfinance
+ data = yf.download(ticker_symbol, start=start_date, end=end_date)
+
+ # Create a candlestick graph using Plotly
+ fig = go.Figure(data=[go.Candlestick(x=data.index.strftime('%Y-%m-%d').tolist(),
+ open=data['Open'].tolist(),
+ high=data['High'].tolist(),
+ low=data['Low'].tolist(),
+ close=data['Close'].tolist())])
+
+ # Customize the layout
+ fig.update_layout(title=f'Candlestick Chart for {ticker_symbol} in the Last 90 Days',
+ xaxis_title='Date',
+ yaxis_title='Price',
+ xaxis_rangeslider_visible=False)
+
+ # # convert the fig to HTML DIV element
+ # graph_html = fig.to_html(full_html=False,include_plotlyjs = False)
+ # print(len(graph_html))
+ # # socketio.emit('graph', fig.to_html(full_html=False,include_plotlyjs = False))
+ graph_data = fig.to_plotly_json()
+ socketio.emit('graph',graph_data)
+ print("Graph is finished")
+
+if __name__ == "__main__":
+ socketio.run(app, host='0.0.0.0', port=7860)

app_socket_multi_threaded.py

@@ -0,0 +1,176 @@
+from flask import Flask, render_template
+from flask_socketio import SocketIO
+from transformers import pipeline
+import yfinance as yf
+import requests
+from newspaper import Article
+from newspaper import Config
+from bs4 import BeautifulSoup
+from datetime import datetime, timedelta
+import plotly.graph_objs as go
+from transformers import BertTokenizer, BertForSequenceClassification
+from transformers import BartForConditionalGeneration, BartTokenizer
+import threading
+
+app = Flask(__name__)
+socketio = SocketIO(app)
+
+news_data = []
+
+# Set up sentiment analysis and summarization pipelines
+finbert = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-tone', num_labels=3)
+sentiment_tokenizer = BertTokenizer.from_pretrained('yiyanghkust/finbert-tone')
+summarization_model = BartForConditionalGeneration.from_pretrained("facebook/bart-large", forced_bos_token_id=0)
+summarization_tokenizer = BartTokenizer.from_pretrained("facebook/bart-large")
+sentiment_analysis_pipeline = pipeline("sentiment-analysis", model=finbert, tokenizer=sentiment_tokenizer, truncation=True, max_length=512)
+summarization_pipeline = pipeline("summarization", model=summarization_model, tokenizer=summarization_tokenizer, max_length=512, truncation=True)
+
+# News API setup
+def get_news_articles_info(ticker_name):
+ # URL of the search results page
+ url = "https://www.marketwatch.com/search?q=" + ticker_name + "&ts=0&tab=All%20News"
+
+ # Send an HTTP GET request to the URL
+ response = requests.get(url)
+
+ # Parse the HTML content using BeautifulSoup
+ soup = BeautifulSoup(response.content, "html.parser")
+
+ article_links = []
+ 
+ for content in soup.find_all("h3",class_="article__headline"):
+ for link in content.find_all("a"):
+ if link['href'] != "#":
+ article_links.append(link['href'])
+
+ article_links = article_links[18:36]
+ ticker_news_extracted = []
+ count = 0
+ for link in article_links:
+ article_info = {}
+ article_info['text'] = ''
+ article_info['url'] = link
+ try:
+ url = article_info['url']
+ url = requests.head(url, allow_redirects=True).url
+ 
+ user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36'
+ config = Config()
+ config.browser_user_agent = user_agent
+
+ article = Article(url, config=config)
+ article.download()
+ article.parse()
+ article_info['title'] = article.title
+ article_info['text'] = article.text
+ article_info['url'] = url
+ count = count + 1
+ print(count,url)
+ socketio.emit('log',str(count) + " " + url)
+ except Exception as error:
+ print("Error",url,error)
+ continue
+ if article_info['text'] == '':
+ print('No text',url)
+ continue
+ if count > 5:
+ break
+ ticker_news_extracted.append(article_info)
+ return ticker_news_extracted
+
+@app.route("/")
+def index():
+ global news_data 
+ news_data = []
+ socketio.emit('news_data',news_data)
+ return render_template("index_socket.html")
+
+@socketio.on('get_graph')
+def get_graph(ticker):
+ # Define the stock ticker symbol and the date range
+
+ ticker_symbol = ticker # Example: Apple Inc.
+ end_date = datetime.today()
+ start_date = end_date - timedelta(days=90)
+
+ # Fetch historical data using yfinance
+ data = yf.download(ticker_symbol, start=start_date, end=end_date)
+
+ # Create a candlestick graph using Plotly
+ fig = go.Figure(data=[go.Candlestick(x=data.index.strftime('%Y-%m-%d').tolist(),
+ open=data['Open'].tolist(),
+ high=data['High'].tolist(),
+ low=data['Low'].tolist(),
+ close=data['Close'].tolist())])
+
+ # Customize the layout
+ fig.update_layout(title=f'Candlestick Chart for {ticker_symbol} in the Last 90 Days',
+ xaxis_title='Date',
+ yaxis_title='Price',
+ xaxis_rangeslider_visible=False)
+
+ # convert the fig to HTML DIV element
+ graph_data = fig.to_plotly_json()
+ socketio.emit('graph',graph_data)
+ print("Graph is finished")
+
+def get_sentiment(_index,_content):
+ _sentiment = sentiment_analysis_pipeline(_content)[0]
+ news_data[_index]['sentiment'] = _sentiment["label"]
+ news_data[_index]['sentiment_score'] = round(_sentiment["score"], 3)
+ socketio.emit('news_data',news_data)
+
+def get_summary(_index,_content):
+ _summary = summarization_pipeline(_content, max_length=100, min_length=30, do_sample=False)[0]["summary_text"]
+ news_data[_index]['summary'] = _summary
+ socketio.emit('news_data', news_data)
+
+
+@socketio.on('get_news_data')
+def get_news_data(ticker):
+
+ global news_data 
+ news_data = []
+ socketio.emit('news_data',news_data)
+
+ # Get top 5 news articles for the given ticker
+ search_results = get_news_articles_info(ticker_name=ticker)
+ articles = search_results[:5]
+
+ for article in articles:
+ news_data.append({
+ "title": article['title'],
+ "link": article['url'],
+ "sentiment": 'Loading...',
+ "sentiment_score": 'Loading...',
+ "summary": 'Loading...',
+ "Ticker": ticker
+ })
+
+ socketio.emit('news_data',news_data)
+
+ threads_list = []
+
+ for i in range(0,len(articles)):
+ article = articles[i]
+ link = article['url']
+ content = article['text']
+
+ t1 = threading.Thread(target=get_sentiment, args=(i,content))
+ t2 = threading.Thread(target=get_summary, args=(i,content))
+
+ t1.start()
+ t2.start()
+
+ threads_list.append(t1)
+ threads_list.append(t2)
+
+ socketio.emit('log',link + " Analysis started")
+
+ for thread in threads_list:
+ thread.join()
+ 
+ socketio.emit('log','Analysis is finished')
+
+if __name__ == "__main__":
+ socketio.run(app, host='0.0.0.0', port=7860)

static/css/styles.css

@@ -0,0 +1,110 @@
+/* Body styles */
+body {
+ font-family: Arial, sans-serif;
+ background-color: #f3f3f3;
+ margin: 0;
+ padding: 0;
+}
+
+/* Page container styles */
+.container {
+ max-width: 800px;
+ margin: 0 auto;
+ padding: 20px;
+ background-color: #fff;
+ border: 1px solid #ccc;
+ box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
+}
+
+/* Header styles */
+h1 {
+ text-align: center;
+ color: #333;
+}
+
+/* Header styles */
+h2 {
+ text-align: center;
+ color: #333;
+}
+
+/* Form styles */
+form {
+ text-align: center;
+ margin-top: 20px;
+}
+
+label {
+ font-weight: bold;
+}
+
+input[type="text"] {
+ width: 15%; /* Set the width to 25% of the parent container */
+ padding: 10px;
+ margin: 5px auto; /* Center the input horizontally */
+ display: block; /* Make it a block-level element for centering */
+ border: 1px solid #ccc;
+ border-radius: 4px;
+}
+
+button {
+ background-color: #007BFF;
+ color: #fff;
+ border: none;
+ padding: 10px 20px;
+ cursor: pointer;
+ border-radius: 4px;
+}
+
+button:hover {
+ background-color: #0056b3;
+}
+
+/* Log and News Data styles */
+#log, #newsData {
+ padding: 20px;
+ margin-top: 20px;
+ /* background-color: #fff; */
+ /* border: 1px solid #ccc; */
+ /* box-shadow: 0 0 5px rgba(0, 0, 0, 0.1); */
+}
+
+/* Stock Graph styles */
+#stock_graph {
+ margin-top: 20px;
+ /* background-color: #fff; */
+ /* border: 1px solid #ccc; */
+ /* box-shadow: 0 0 5px rgba(0, 0, 0, 0.1); */
+ padding: 20px;
+}
+
+/* Table styles for News Data */
+#newsTable {
+ width: 100%;
+ border-collapse: collapse;
+}
+
+#newsTable th, #newsTable td {
+ border: 1px solid #ccc;
+ padding: 10px;
+}
+
+#newsTable th {
+ background-color: #f0f0f0;
+}
+
+/* Links in News Data */
+#newsTable a {
+ text-decoration: none;
+ color: #007BFF;
+}
+
+#newsTable a:hover {
+ text-decoration: underline;
+}
+
+/* No news data message */
+#newsData p {
+ text-align: center;
+ color: #888;
+}

templates/index_socket.html

@@ -0,0 +1,105 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <title>Stock Information App</title>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.0.1/socket.io.js"></script>
+ <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+ <link rel= "stylesheet" type= "text/css" href= "{{ url_for('static',filename='css/styles.css') }}">
+</head>
+<body>
+ <h1>Stock Information App</h1>
+ 
+ <form onsubmit="getInformation(); return false;">
+ <label for="ticker">Enter Ticker:</label>
+ <input type="text" id="ticker" required>
+ <button type="submit">Get Information</button>
+ </form>
+
+ <div id = "log">
+ <!-- Here the log will be displayed -->
+ </div>
+
+ <div id = "newsData">
+ <!-- News data will be displayed here -->
+ </div>
+
+ <div>
+ <h2 id = "graph_title"></h2>
+ <div id = "stock_graph">
+ <!-- The candlestick graph will be displayed here -->
+ </div>
+ </div>
+
+ <script>
+ var socket = io.connect('http://' + document.domain + ':' + location.port);
+
+ function getInformation() {
+ var ticker = document.getElementById('ticker').value;
+ socket.emit('get_graph',ticker);
+ socket.emit('get_news_data', ticker);
+ }
+
+ socket.on('log', function(data) {
+ var graphDiv = document.getElementById('log');
+ graphDiv.innerHTML = "Log: " + data;
+ console.log(data)
+ });
+
+ socket.on('graph', function(graphData) {
+ document.getElementById("graph_title").innerHTML = '';
+ var graphDiv = document.getElementById('stock_graph');
+ graphDiv.innerHTML = '';
+ console.log(graphData);
+ if (graphData != '')
+ {
+ Plotly.newPlot(graphDiv, graphData.data, graphData.layout);
+ document.getElementById("graph_title").innerHTML = "Stock Opening and Closing Data";
+ }
+ else
+ {
+ document.getElementById("graph_title").innerHTML = "Stock Opening and Closing Data";
+ graphDiv.innerHTML = '<center><p>Not a valid Ticker.</p></center>';
+ }
+ });
+
+ socket.on('news_data', function(data) {
+ // Handle the received news data and update the DOM
+ var newsDataDiv = document.getElementById('newsData');
+ newsDataDiv.innerHTML = ''; // Clear previous data
+
+ if (data.length > 0) {
+ var table = document.createElement('table');
+ table.id = 'newsTable';
+ table.innerHTML = `
+ <thead>
+ <tr>
+ <th>Title</th>
+ <th>Sentiment</th>
+ <th>Summary</th>
+ <th>Full Article</th>
+ </tr>
+ </thead>
+ <tbody>
+ </tbody>
+ `;
+
+ var tbody = table.querySelector('tbody');
+ data.forEach(function(article) {
+ var row = document.createElement('tr');
+ row.innerHTML = `
+ <td>${article.title}</td>
+ <td>${article.sentiment} (Score: ${article.sentiment_score})</td>
+ <td>${article.summary}</td>
+ <td><a href="${article.link}" target="_blank">Read full article</a></td>
+ `;
+ tbody.appendChild(row);
+ });
+
+ newsDataDiv.appendChild(table);
+ } else {
+ newsDataDiv.innerHTML = '<p>No news data available. or still searching for News</p>';
+ }
+ });
+ </script>
+</body>
+</html>

templates/test.html

@@ -0,0 +1,40 @@
+<!DOCTYPE html>
+<html>
+<head>
+ <title>Stock Information App Testing</title>
+ <script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/4.0.1/socket.io.js"></script>
+ <script src="https://cdn.plot.ly/plotly-latest.min.js"></script>
+ <link rel= "stylesheet" type= "text/css" href= "{{ url_for('static',filename='css/styles.css') }}">
+</head>
+<body>
+ <h1>Stock Information App</h1>
+ 
+ <form onsubmit="getInformation(); return false;">
+ <label for="ticker">Enter Ticker:</label>
+ <input type="text" id="ticker" required>
+ <button type="submit">Get Information</button>
+ </form>
+
+ <div>
+ <h2>Stock Opening and Closing Data</h2>
+ <div id = "stock_graph">
+ <!-- The candlestick graph will be displayed here -->
+ </div>
+ </div>
+
+ <script>
+ var socket = io.connect('http://' + document.domain + ':' + location.port);
+
+ function getInformation() {
+ var ticker = document.getElementById('ticker').value;
+ socket.emit('get_graph',ticker);
+ }
+
+ socket.on('graph', function(graphData) {
+ var graphDiv = document.getElementById('stock_graph');
+ graphDiv.innerHTML = '';
+ Plotly.newPlot(graphDiv, graphData.data, graphData.layout);
+ });
+ </script>
+</body>
+</html>