File size: 6,055 Bytes
d64b41c df68e4f cecc930 d64b41c 2986de4 d64b41c 1834f20 d64b41c 1834f20 d64b41c 1834f20 d64b41c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 |
import streamlit as st
import torch
import torch.nn as nn
import pandas as pd
import numpy as np
import pickle
from nltk.tokenize import RegexpTokenizer
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
import re
import string
from nltk.stem import WordNetLemmatizer
import time
import transformers
import json
import nltk
nltk.download('stopwords')
nltk.download('wordnet')
nltk.download('punkt')
nltk.download('averaged_perceptron_tagger')
from biLSTM1 import biLSTM
from lstm_preprocessing import (
data_preprocessing,
get_words_by_freq,
padding,
preprocess_single_string
)
# 1-Lesha, 2-Lena, 3-Gal
# +++++++++++
# 1 -Lesha
# Load the saved model
with open('logistic_regression_model.pkl', 'rb') as file:
loaded_model_1 = pickle.load(file)
with open('tfidf_vectorizer.pkl', 'rb') as file:
vectorizer_1 = pickle.load(file)
# Load the stop words
stop_words = stopwords.words('english')
# Create a tokenizer
tokenizer = RegexpTokenizer(r'\w+')
def data_preprocessing(text: str) -> str:
"""preprocessing string: lowercase, removing html-tags, punctuation and stopwords
Args:
text (str): input string for preprocessing
Returns:
str: preprocessed string
"""
text = text.lower()
text = re.sub('<.*?>', '', text) # html tags
text = ''.join([c for c in text if c not in string.punctuation])# Remove punctuation
lemmatizer = WordNetLemmatizer()
tokens = tokenizer.tokenize(text)
tokens = [lemmatizer.lemmatize(word) for word in tokens if not word.isdigit() and word not in stop_words]
return ' '.join(tokens)
# ++++
# Lena
def load_model_l():
model_finetuned = transformers.AutoModel.from_pretrained(
"nghuyong/ernie-2.0-base-en",
output_attentions = False,
output_hidden_states = False
)
model_finetuned.load_state_dict(torch.load('ErnieModel_imdb.pt', map_location=torch.device('cpu')))
tokenizer = transformers.AutoTokenizer.from_pretrained("nghuyong/ernie-2.0-base-en")
return model_finetuned, tokenizer
def preprocess_text(text_input, max_len, tokenizer):
input_tokens = tokenizer(
text_input,
return_tensors='pt',
padding=True,
max_length=max_len,
truncation = True
)
return input_tokens
def predict_sentiment(model, input_tokens):
id2label = {0: "negative", 1: "positive"}
output = model(**input_tokens).pooler_output.detach().numpy()
with open('LogReg_imdb_Ernie.pkl', 'rb') as file:
cls = pickle.load(file)
result = id2label[int(cls.predict(output))]
return result
# ++++
# Gala
with open('vocab_to_int.json', 'r') as fp:
vocab_to_int = json.load(fp)
VOCAB_SIZE = len(vocab_to_int)+1
EMBEDDING_DIM = 32
HIDDEN_DIM = 64
N_LAYERS = 3
SEQ_LEN = 128
def load_model_g():
model = biLSTM(
vocab_size=VOCAB_SIZE,
embedding_dim=EMBEDDING_DIM,
hidden_dim=HIDDEN_DIM,
n_layers=N_LAYERS
)
model.load_state_dict(torch.load('biLSTM_model_do_05_lr001_best.pt', map_location=torch.device('cpu')))
return model
device = 'cuda' if torch.cuda.is_available() else 'cpu'
def predict_sentence(text: str, model: nn.Module) -> str:
id2label = {0: "negative", 1: "positive"}
output = model.to(device)(preprocess_single_string(text, SEQ_LEN, vocab_to_int).unsqueeze(0).to(device))
pred = int(output.round().item())
result = id2label[pred]
return result
# ++++++
# Lesha
# Create the Streamlit app
def main():
st.title('Sentiment Analysis App')
st.header('Classic ML, ErnieModel, bidirectional LSTM')
user_input = st.text_area('Please enter your review:')
st.write(user_input)
submit = st.button("Predict!")
col1, col2,col3 = st.columns(3)
if user_input is not None and submit:
with col1:
# Preprocess the user input
preprocessed_input_1 = data_preprocessing(user_input)
# Vectorize the preprocessed input
input_vector = vectorizer_1.transform([preprocessed_input_1])
start_time = time.time()
proba_1 = loaded_model_1.predict_proba(input_vector)[:, 1]
# Predict the sentiment using the loaded model
#prediction = loaded_model.predict(input_vector)[0]
prediction_1 = round(proba_1[0])
end_time = time.time()
st.header('Classic ML (LogReg on TF-IDF)')
# Display the predicted sentiment
if prediction_1 == 0:
st.write('The sentiment of your review is negative.')
st.write('Predicted probability:', (1 - round(proba_1[0], 2))*100, '%')
else:
st.write('The sentiment of your review is positive.')
st.write('Processing time:', round(end_time - start_time, 4), 'seconds')
# Lena
if user_input is not None and submit:
with col2:
model2, tokenizer = load_model_l()
start_time = time.time()
input_tokens = preprocess_text(user_input, 500, tokenizer)
output = predict_sentiment(model2, input_tokens)
end_time = time.time()
st.header('ErnieModel')
st.write('The sentiment of your review is', output)
st.write('Processing time:', round(end_time - start_time, 4), 'seconds')
# Gala
if user_input is not None and submit:
with col3:
model3 = load_model_g()
start_time = time.time()
output = predict_sentence(user_input,model3)
end_time = time.time()
st.header('bidirectional LSTM')
st.write('The sentiment of your review is', output)
st.write('Processing time:', round(end_time - start_time, 4), 'seconds')
if __name__ == '__main__':
main()
|