shainaraza
commited on
Commit
•
7eca2e2
1
Parent(s):
36d3b4c
Update app.py
Browse files
app.py
CHANGED
@@ -1,47 +1,57 @@
|
|
|
|
1 |
import streamlit as st
|
2 |
-
from transformers import AutoTokenizer, pipeline
|
3 |
-
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
#
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
12 |
-
|
13 |
-
|
14 |
-
|
15 |
-
)
|
16 |
-
|
17 |
-
|
18 |
-
|
19 |
-
|
20 |
-
|
21 |
-
|
22 |
-
|
23 |
-
|
24 |
-
|
25 |
-
|
26 |
-
|
27 |
-
|
28 |
-
|
29 |
-
|
30 |
-
|
31 |
-
|
32 |
-
)
|
33 |
-
|
34 |
-
|
35 |
-
|
36 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
37 |
|
38 |
# Streamlit interface
|
39 |
st.title('UnBIAS App')
|
40 |
-
input_text = st.text_area("Enter text
|
41 |
-
if st.button("
|
42 |
if input_text:
|
43 |
-
|
44 |
-
|
|
|
|
|
45 |
else:
|
46 |
-
st.write("Please enter some text to
|
47 |
-
|
|
|
1 |
+
%%writefile debias_app.py
|
2 |
import streamlit as st
|
3 |
+
from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification, pipeline
|
4 |
+
|
5 |
+
# Define the BiasPipeline class with text processing methods
|
6 |
+
class BiasPipeline:
|
7 |
+
def __init__(self):
|
8 |
+
# Load models and tokenizers
|
9 |
+
self.load_resources()
|
10 |
+
|
11 |
+
def load_resources(self):
|
12 |
+
"""Load models and tokenizers."""
|
13 |
+
self.classifier_tokenizer = AutoTokenizer.from_pretrained("newsmediabias/UnBIAS-classification-bert")
|
14 |
+
self.classifier_model = AutoModelForSequenceClassification.from_pretrained("newsmediabias/UnBIAS-classification-bert")
|
15 |
+
|
16 |
+
self.ner_tokenizer = AutoTokenizer.from_pretrained("newsmediabias/UnBIAS-Named-Entity-Recognition")
|
17 |
+
self.ner_model = AutoModelForTokenClassification.from_pretrained("newsmediabias/UnBIAS-Named-Entity-Recognition")
|
18 |
+
|
19 |
+
self.classifier = pipeline("text-classification", model=self.classifier_model, tokenizer=self.classifier_tokenizer)
|
20 |
+
self.ner = pipeline("ner", model=self.ner_model, tokenizer=self.ner_tokenizer)
|
21 |
+
|
22 |
+
def clean_text(self, text):
|
23 |
+
"""Clean up the text by removing any redundant spaces."""
|
24 |
+
return ' '.join(text.split())
|
25 |
+
|
26 |
+
def complete_sentence(self, text):
|
27 |
+
"""If the text ends mid-sentence, remove all words after the last full stop."""
|
28 |
+
sentences = text.split(". ")
|
29 |
+
if len(sentences) > 1 and not sentences[-1].endswith("."):
|
30 |
+
return ". ".join(sentences[:-1]) + "."
|
31 |
+
return text
|
32 |
+
|
33 |
+
def create_token_limit(self, text):
|
34 |
+
words = text.split()
|
35 |
+
max_length = round(len(words) + 1.5 * len(words))
|
36 |
+
return max_length
|
37 |
+
|
38 |
+
def process(self, texts):
|
39 |
+
"""Process texts to classify and find named entities."""
|
40 |
+
classification_results = self.classifier(texts)
|
41 |
+
ner_results = self.ner(texts)
|
42 |
+
return classification_results, ner_results
|
43 |
+
|
44 |
+
# Initialize the BiasPipeline
|
45 |
+
pipeline = BiasPipeline()
|
46 |
|
47 |
# Streamlit interface
|
48 |
st.title('UnBIAS App')
|
49 |
+
input_text = st.text_area("Enter text:", height=150)
|
50 |
+
if st.button("Process Text"):
|
51 |
if input_text:
|
52 |
+
cleaned_text = pipeline.clean_text(input_text)
|
53 |
+
classification_results, ner_results = pipeline.process(cleaned_text)
|
54 |
+
st.write("Classification Results:", classification_results)
|
55 |
+
st.write("Named Entity Recognition Results:", ner_results)
|
56 |
else:
|
57 |
+
st.write("Please enter some text to process.")
|
|