shainaraza
commited on
Commit
•
2b3dec6
1
Parent(s):
be61171
Update app.py
Browse files
app.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
#%%writefile debias_app.py
|
2 |
-
|
3 |
import streamlit as st
|
4 |
from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification, pipeline
|
|
|
5 |
|
6 |
# Define the BiasPipeline class with text processing methods
|
7 |
class BiasPipeline:
|
@@ -24,18 +24,6 @@ class BiasPipeline:
|
|
24 |
"""Clean up the text by removing any redundant spaces."""
|
25 |
return ' '.join(text.split())
|
26 |
|
27 |
-
def complete_sentence(self, text):
|
28 |
-
"""If the text ends mid-sentence, remove all words after the last full stop."""
|
29 |
-
sentences = text.split(". ")
|
30 |
-
if len(sentences) > 1 and not sentences[-1].endswith("."):
|
31 |
-
return ". ".join(sentences[:-1]) + "."
|
32 |
-
return text
|
33 |
-
|
34 |
-
def create_token_limit(self, text):
|
35 |
-
words = text.split()
|
36 |
-
max_length = round(len(words) + 1.5 * len(words))
|
37 |
-
return max_length
|
38 |
-
|
39 |
def process(self, texts):
|
40 |
"""Process texts to classify and find named entities."""
|
41 |
classification_results = self.classifier(texts)
|
@@ -50,23 +38,7 @@ st.title('UnBIAS App')
|
|
50 |
# List of preloaded example sentences
|
51 |
example_sentences = [
|
52 |
"Women are just too emotional to be leaders.",
|
53 |
-
|
54 |
-
"People from that country are always dishonest and corrupt.",
|
55 |
-
"Men are inherently better at science and math than women.",
|
56 |
-
"The elderly are just burdens to society; they contribute nothing.",
|
57 |
-
"All poor people are lazy and don't want to work.",
|
58 |
-
"People who follow that religion are all terrorists.",
|
59 |
-
"Immigrants are taking all our jobs and ruining the country.",
|
60 |
-
"All wealthy people achieved success only through manipulation and deceit.",
|
61 |
-
"People with tattoos are not professional.",
|
62 |
-
"All politicians are liars and cannot be trusted.",
|
63 |
-
"Blue-collar workers don't have any real skills.",
|
64 |
-
"Anyone who doesn't attend college is unintelligent.",
|
65 |
-
"Only people from the city are sophisticated.",
|
66 |
-
"Rural folks have backward mindsets and don't understand progress.",
|
67 |
-
"All artists are always broke and unreliable.",
|
68 |
-
"Anyone from that region is uneducated and narrow-minded.",
|
69 |
-
"People without jobs are simply not trying hard enough."
|
70 |
]
|
71 |
|
72 |
# Dropdown for selecting an example sentence or entering your own
|
@@ -77,7 +49,13 @@ if st.button("Process Text"):
|
|
77 |
if input_text:
|
78 |
cleaned_text = pipeline.clean_text(input_text)
|
79 |
classification_results, ner_results = pipeline.process(cleaned_text)
|
80 |
-
|
81 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
82 |
else:
|
83 |
st.write("Please enter some text to process.")
|
|
|
1 |
#%%writefile debias_app.py
|
|
|
2 |
import streamlit as st
|
3 |
from transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoModelForTokenClassification, pipeline
|
4 |
+
import pandas as pd
|
5 |
|
6 |
# Define the BiasPipeline class with text processing methods
|
7 |
class BiasPipeline:
|
|
|
24 |
"""Clean up the text by removing any redundant spaces."""
|
25 |
return ' '.join(text.split())
|
26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
27 |
def process(self, texts):
|
28 |
"""Process texts to classify and find named entities."""
|
29 |
classification_results = self.classifier(texts)
|
|
|
38 |
# List of preloaded example sentences
|
39 |
example_sentences = [
|
40 |
"Women are just too emotional to be leaders.",
|
41 |
+
# More examples...
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
42 |
]
|
43 |
|
44 |
# Dropdown for selecting an example sentence or entering your own
|
|
|
49 |
if input_text:
|
50 |
cleaned_text = pipeline.clean_text(input_text)
|
51 |
classification_results, ner_results = pipeline.process(cleaned_text)
|
52 |
+
label = classification_results[0]['label']
|
53 |
+
score = classification_results[0]['score']
|
54 |
+
st.write(f"**Classification:** {label} (Confidence: {score:.2f})")
|
55 |
+
|
56 |
+
# Extract biased words from NER results
|
57 |
+
biased_words = [result['word'] for result in ner_results if result['entity'].startswith('B-BIAS')]
|
58 |
+
st.write("**Biased Words Identified:**")
|
59 |
+
st.write(", ".join(biased_words))
|
60 |
else:
|
61 |
st.write("Please enter some text to process.")
|