File size: 8,410 Bytes
fd2d5c8 99e8b58 fd2d5c8 4ec4c3d fd2d5c8 99e8b58 4a454ea 99e8b58 4a454ea 1188a0f 99e8b58 fd2d5c8 2d3c2b4 fd2d5c8 2d3c2b4 fd2d5c8 1188a0f fd2d5c8 5d0f703 2d3c2b4 fd2d5c8 5d0f703 fd2d5c8 2d3c2b4 fd2d5c8 1188a0f 4a454ea fd2d5c8 99e8b58 fd2d5c8 99e8b58 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 |
import warnings
import matplotlib.colors as mcolors
import matplotlib.pyplot as plt
import streamlit as st
from transformers import (
AutoModelForTokenClassification,
AutoTokenizer,
logging,
pipeline,
)
warnings.simplefilter(action="ignore", category=Warning)
logging.set_verbosity(logging.ERROR)
st.set_page_config(page_title="CAROLL Language Models", page_icon="🐠", layout="wide")
st.markdown(
"""
<style>
body {
font-family: 'Poppins', sans-serif;
background-color: #f4f4f8;
}
.header {
background-color: rgba(220, 219, 219, 0.25);
color: #000;
padding: 5px 0;
text-align: center;
border-radius: 7px;
margin-bottom: 13px;
border-bottom: 2px solid #333;
}
.container {
background-color: #fff;
padding: 30px;
border-radius: 10px;
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
width: 100%;
max-width: 1000px;
margin: 0 auto;
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
}
.btn-primary {
background-color: #5477d1;
border: none;
transition: background-color 0.3s, transform 0.2s;
border-radius: 25px;
box-shadow: 0 1px 3px rgba(0, 0, 0, 0.08);
}
.btn-primary:hover {
background-color: #4c6cbe;
transform: translateY(-1px);
}
h2 {
font-weight: 600;
font-size: 24px;
margin-bottom: 20px;
}
label {
font-weight: 500;
}
.tip {
background-color: rgba(180, 47, 109, 0.25);
padding: 7px;
border-radius: 7px;
display: inline-block;
margin-top: 15px;
margin-bottom: 15px;
}
.sec {
background-color: rgba(220, 219, 219, 0.10);
padding: 7px;
border-radius: 5px;
display: inline-block;
margin-top: 15px;
margin-bottom: 15px;
}
.tooltip {
position: relative;
display: inline-block;
cursor: pointer;
}
.tooltip .tooltiptext {
visibility: hidden;
width: 120px;
background-color: #6c757d;
color: #fff;
text-align: center;
border-radius: 3px;
padding: 3px;
position: absolute;
z-index: 1;
bottom: 125%;
left: 50%;
margin-left: -60px;
opacity: 0;
transition: opacity 0.3s;
}
.tooltip:hover .tooltiptext {
visibility: visible;
opacity: 1;
}
</style>
""",
unsafe_allow_html=True,
)
# Initialization for German Legal NER
tokenizer_legal = AutoTokenizer.from_pretrained("PaDaS-Lab/gbert-legal-ner")
model_legal = AutoModelForTokenClassification.from_pretrained(
"PaDaS-Lab/gbert-legal-ner"
)
ner_legal = pipeline("ner", model=model_legal, tokenizer=tokenizer_legal)
# Initialization for GDPR Privacy Policy NER
tokenizer_gdpr = AutoTokenizer.from_pretrained("PaDaS-Lab/gdpr-privacy-policy-ner")
model_gdpr = AutoModelForTokenClassification.from_pretrained(
"PaDaS-Lab/gdpr-privacy-policy-ner"
)
ner_gdpr = pipeline("ner", model=model_gdpr, tokenizer=tokenizer_gdpr)
# Define class labels for German Legal and GDPR NER models
classes_legal = {
"AN": "Lawyer",
"EUN": "European legal norm",
"GRT": "Court",
"GS": "Law",
"INN": "Institution",
"LD": "Country",
"LDS": "Landscape",
"LIT": "Legal literature",
"MRK": "Brand",
"ORG": "Organization",
"PER": "Person",
"RR": "Judge",
"RS": "Court decision",
"ST": "City",
"STR": "Street",
"UN": "Company",
"VO": "Ordinance",
"VS": "Regulation",
"VT": "Contract",
}
classes_gdpr = {
"DC": "Data Controller",
"DP": "Data Processor",
"DPO": "Data Protection Officer",
"R": "Recipient",
"TP": "Third Party",
"A": "Authority",
"DS": "Data Subject",
"DSO": "Data Source",
"RP": "Required Purpose",
"NRP": "Not-Required Purpose",
"P": "Processing",
"NPD": "Non-Personal Data",
"PD": "Personal Data",
"OM": "Organisational Measure",
"TM": "Technical Measure",
"LB": "Legal Basis",
"CONS": "Consent",
"CONT": "Contract",
"LI": "Legitimate Interest",
"ADM": "Automated Decision Making",
"RET": "Retention",
"SEU": "Scale EU",
"SNEU": "Scale Non-EU",
"RI": "Right",
"DSR15": "Art. 15 Right of access by the data subject",
"DSR16": "Art. 16 Right to rectification",
"DSR17": "Art. 17 Right to erasure (‘right to be forgotten’)",
"DSR18": "Art. 18 Right to restriction of processing",
"DSR19": "Art. 19 Notification obligation regarding rectification or erasure of personal data or restriction of processing",
"DSR20": "Art. 20 Right to data portability",
"DSR21": "Art. 21 Right to object",
"DSR22": "Art. 22 Automated individual decision-making, including profiling",
"LC": "Lodge Complaint",
}
# Extract the keys (labels) from the class dictionaries
ner_labels_legal = list(classes_legal.keys())
ner_labels_gdpr = list(classes_gdpr.keys())
# Function to generate a list of colors for visualization
def generate_colors(num_colors):
cm = plt.get_cmap("tab20")
colors = [mcolors.rgb2hex(cm(1.0 * i / num_colors)) for i in range(num_colors)]
return colors
# Function to color substrings based on NER results
def color_substrings(input_string, model_output, ner_labels, current_classes):
colors = generate_colors(len(ner_labels))
label_to_color = {
label: colors[i % len(colors)] for i, label in enumerate(ner_labels)
}
last_end = 0
html_output = ""
for entity in sorted(model_output, key=lambda x: x["start"]):
start, end, label = entity["start"], entity["end"], entity["label"]
html_output += input_string[last_end:start]
tooltip = current_classes.get(label, "")
html_output += f'<span class="tooltip" style="color: {label_to_color.get(label)}; font-weight: bold;">{input_string[start:end]}<span class="tooltiptext">{tooltip}</span></span>'
last_end = end
html_output += input_string[last_end:]
return html_output
st.title("CAROLL Language Models - Demo")
st.markdown("<hr>", unsafe_allow_html=True)
# Example texts
example_text_legal = "1. Das Bundesarbeitsgericht ist gemäß § 9 Abs. 2 Satz 2 ArbGG iVm. § 201 Abs. 1 Satz 2 GVG für die beabsichtigte Klage gegen den Bund zuständig ."
example_text_gdpr = "We do not knowingly collect personal information from anyone under 16. We may limit how we collect, use and store some of the information of EU or EEA users between ages 13 and 16."
model_choice = st.radio(
"Choose a model:", ["German Legal NER", "GDPR Privacy Policy NER"], index=0
)
# Set example text based on model choice
if model_choice == "German Legal NER":
test_sentence = st.text_area("Enter Text:", value=example_text_legal, height=150)
else:
test_sentence = st.text_area("Enter Text:", value=example_text_gdpr, height=150)
if st.button("Analyze"):
if model_choice == "German Legal NER":
ner_model = ner_legal
current_classes = classes_legal
current_ner_labels = ner_labels_legal
else:
ner_model = ner_gdpr
current_classes = classes_gdpr
current_ner_labels = ner_labels_gdpr
results = ner_model(test_sentence)
processed_results = [
{
"start": result["start"],
"end": result["end"],
"label": result["entity"].split("-")[-1],
}
for result in results
]
colored_html = color_substrings(
test_sentence, processed_results, current_ner_labels, current_classes
)
st.markdown(
'<div class="sec"><strong>Analyzed text</strong></div><br>{}<br><br>'.format(
colored_html
),
unsafe_allow_html=True,
)
st.markdown(
'<div class="tip"><strong>Tip:</strong> Hover over the colored words to see its class.</div>',
unsafe_allow_html=True,
)
|