Update app.py

app.py

@@ -4,42 +4,20 @@ import json
 import gradio as gr
 import spacy
 from spacy import displacy
-from transformers import AutoTokenizer, AutoModelForTokenClassification,RobertaTokenizer,pipeline
+from transformers import RobertaTokenizer,pipeline
 import torch
 import nltk
 from nltk.tokenize import sent_tokenize
 from fin_readability_sustainability import BERTClass, do_predict
 import pandas as pd
 import en_core_web_sm
-#from fincat_utils import extract_context_words
-#from fincat_utils import bert_embedding_extract
 from score_fincat import score_fincat
-import pickle
-#lr_clf = pickle.load(open("lr_clf_FiNCAT.pickle",'rb'))
-
+from sus_fls import get_sustainability,fls
 nlp = en_core_web_sm.load()
 nltk.download('punkt')
 device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-#SUSTAINABILITY STARTS
-tokenizer_sus = RobertaTokenizer.from_pretrained('roberta-base')
-model_sustain = BERTClass(2, "sustanability")
-model_sustain.to(device)
-model_sustain.load_state_dict(torch.load('sustainability_model.bin', map_location=device)['model_state_dict'])
 
-def get_sustainability(text):
-  df = pd.DataFrame({'sentence':sent_tokenize(text)})
-  actual_predictions_sustainability = do_predict(model_sustain, tokenizer_sus, df)
-  highlight = []
-  for sent, prob in zip(df['sentence'].values, actual_predictions_sustainability[1]):
-    if prob>=4.384316:
-      highlight.append((sent, 'non-sustainable'))
-    elif prob<=1.423736:
-      highlight.append((sent, 'sustainable'))
-    else:
-      highlight.append((sent, '-'))
-  return highlight
-#SUSTAINABILITY ENDS
 
 
 ##Summarization
@@ -49,22 +27,6 @@ def summarize_text(text):
     stext = resp[0]['summary_text']
     return stext
 
-##Forward Looking Statement
-def split_in_sentences(text):
-    doc = nlp(text)
-    return [str(sent).strip() for sent in doc.sents]
-def make_spans(text,results):
-    results_list = []
-    for i in range(len(results)):
-        results_list.append(results[i]['label'])
-    facts_spans = []
-    facts_spans = list(zip(split_in_sentences(text),results_list))
-    return facts_spans    
-
-fls_model = pipeline("text-classification", model="yiyanghkust/finbert-fls", tokenizer="yiyanghkust/finbert-fls")
-def fls(text):
-    results = fls_model(split_in_sentences(text))
-    return make_spans(text,results) 
     
 ##Company Extraction
 ner=pipeline('ner',model='Jean-Baptiste/camembert-ner-with-dates',tokenizer='Jean-Baptiste/camembert-ner-with-dates', aggregation_strategy="simple")