Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
@@ -5,8 +5,6 @@ import datetime
|
|
5 |
import dotenv
|
6 |
import pandas as pd
|
7 |
import streamlit as st
|
8 |
-
import streamlit.components.v1 as components
|
9 |
-
from annotated_text import annotated_text
|
10 |
from streamlit_tags import st_tags
|
11 |
from PyPDF2 import PdfReader, PdfWriter
|
12 |
from presidio_helpers import (
|
@@ -53,7 +51,7 @@ st_model_package = st_model.split("/")[0]
|
|
53 |
st_model = st_model if st_model_package.lower() not in ("spacy", "huggingface") else "/".join(st_model.split("/")[1:])
|
54 |
|
55 |
analyzer_params = (st_model_package, st_model)
|
56 |
-
st.sidebar.warning("Note: Models might take some time to download.")
|
57 |
|
58 |
st_operator = st.sidebar.selectbox(
|
59 |
"De-identification approach",
|
@@ -87,80 +85,101 @@ with col1:
|
|
87 |
uploaded_file = st.file_uploader("Upload PDF", type=["pdf"])
|
88 |
|
89 |
if uploaded_file:
|
90 |
-
|
91 |
-
|
92 |
-
|
93 |
-
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
-
|
104 |
-
|
105 |
-
|
106 |
-
|
107 |
-
|
108 |
-
|
109 |
-
|
110 |
-
|
111 |
-
|
112 |
-
|
113 |
-
|
114 |
-
|
115 |
-
|
116 |
-
|
117 |
-
|
118 |
-
|
119 |
-
|
120 |
-
|
121 |
-
)
|
122 |
-
|
123 |
-
# Create new PDF
|
124 |
-
pdf_writer = PdfWriter()
|
125 |
-
for page in pdf_reader.pages:
|
126 |
-
pdf_writer.add_page(page)
|
127 |
-
|
128 |
-
# Generate output filename with timestamp
|
129 |
-
timestamp = datetime.datetime.now().strftime("%I%M%p_%d-%m-%y")
|
130 |
-
output_filename = f"{timestamp}_{uploaded_file.name}"
|
131 |
-
|
132 |
-
# Save modified PDF
|
133 |
-
with open(output_filename, "wb") as f:
|
134 |
-
pdf_writer.write(f)
|
135 |
-
|
136 |
-
# Generate base64 download link
|
137 |
-
with open(output_filename, "rb") as f:
|
138 |
-
pdf_bytes = f.read()
|
139 |
-
b64 = base64.b64encode(pdf_bytes).decode()
|
140 |
-
href = f'<a href="data:application/pdf;base64,{b64}" download="{output_filename}">Download de-identified PDF</a>'
|
141 |
-
st.markdown(href, unsafe_allow_html=True)
|
142 |
-
|
143 |
-
# Display findings
|
144 |
-
with col2:
|
145 |
-
st.subheader("Findings")
|
146 |
-
if st_analyze_results:
|
147 |
-
df = pd.DataFrame.from_records([r.to_dict() for r in st_analyze_results])
|
148 |
-
df["text"] = [text[res.start:res.end] for res in st_analyze_results]
|
149 |
-
df_subset = df[["entity_type", "text", "start", "end", "score"]].rename(
|
150 |
-
{
|
151 |
-
"entity_type": "Entity type",
|
152 |
-
"text": "Text",
|
153 |
-
"start": "Start",
|
154 |
-
"end": "End",
|
155 |
-
"score": "Confidence",
|
156 |
-
},
|
157 |
-
axis=1,
|
158 |
-
)
|
159 |
-
if st_return_decision_process:
|
160 |
-
analysis_explanation_df = pd.DataFrame.from_records(
|
161 |
-
[r.analysis_explanation.to_dict() for r in st_analyze_results]
|
162 |
-
)
|
163 |
-
df_subset = pd.concat([df_subset, analysis_explanation_df], axis=1)
|
164 |
-
st.dataframe(df_subset.reset_index(drop=True), use_container_width=True)
|
165 |
else:
|
166 |
-
st.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
5 |
import dotenv
|
6 |
import pandas as pd
|
7 |
import streamlit as st
|
|
|
|
|
8 |
from streamlit_tags import st_tags
|
9 |
from PyPDF2 import PdfReader, PdfWriter
|
10 |
from presidio_helpers import (
|
|
|
51 |
st_model = st_model if st_model_package.lower() not in ("spacy", "huggingface") else "/".join(st_model.split("/")[1:])
|
52 |
|
53 |
analyzer_params = (st_model_package, st_model)
|
54 |
+
st.sidebar.warning("Note: Models might take some time to download on first run.")
|
55 |
|
56 |
st_operator = st.sidebar.selectbox(
|
57 |
"De-identification approach",
|
|
|
85 |
uploaded_file = st.file_uploader("Upload PDF", type=["pdf"])
|
86 |
|
87 |
if uploaded_file:
|
88 |
+
try:
|
89 |
+
# Read PDF
|
90 |
+
pdf_reader = PdfReader(uploaded_file)
|
91 |
+
text = ""
|
92 |
+
for page in pdf_reader.pages:
|
93 |
+
text += page.extract_text() + "\n"
|
94 |
+
|
95 |
+
# Initialize analyzer
|
96 |
+
try:
|
97 |
+
analyzer = analyzer_engine(*analyzer_params)
|
98 |
+
except Exception as e:
|
99 |
+
st.error(f"Failed to load model: {str(e)}")
|
100 |
+
st.info("Ensure models are downloaded (e.g., 'python -m spacy download en_core_web_lg') and check network/permissions.")
|
101 |
+
raise
|
102 |
+
|
103 |
+
# Analyze
|
104 |
+
st_analyze_results = analyze(
|
105 |
+
analyzer=analyzer,
|
106 |
+
text=text,
|
107 |
+
entities=get_supported_entities(*analyzer_params),
|
108 |
+
language="en",
|
109 |
+
score_threshold=st_threshold,
|
110 |
+
return_decision_process=st_return_decision_process,
|
111 |
+
allow_list=st_allow_list,
|
112 |
+
deny_list=st_deny_list,
|
113 |
+
)
|
114 |
+
|
115 |
+
# Process results
|
116 |
+
phi_types = set(res.entity_type for res in st_analyze_results)
|
117 |
+
if phi_types:
|
118 |
+
st.success(f"Removed PHI types: {', '.join(phi_types)}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
119 |
else:
|
120 |
+
st.info("No PHI detected")
|
121 |
+
|
122 |
+
# Anonymize
|
123 |
+
anonymized_result = anonymize(
|
124 |
+
text=text,
|
125 |
+
operator=st_operator,
|
126 |
+
analyze_results=st_analyze_results,
|
127 |
+
)
|
128 |
+
|
129 |
+
# Create new PDF
|
130 |
+
pdf_writer = PdfWriter()
|
131 |
+
for page in pdf_reader.pages:
|
132 |
+
pdf_writer.add_page(page)
|
133 |
+
|
134 |
+
# Generate output filename with timestamp
|
135 |
+
timestamp = datetime.datetime.now().strftime("%I%M%p_%d-%m-%y")
|
136 |
+
output_filename = f"{timestamp}_{uploaded_file.name}"
|
137 |
+
|
138 |
+
# Save modified PDF
|
139 |
+
try:
|
140 |
+
with open(output_filename, "wb") as f:
|
141 |
+
pdf_writer.write(f)
|
142 |
+
except PermissionError as e:
|
143 |
+
st.error(f"Permission denied when saving PDF: {str(e)}")
|
144 |
+
st.info("Check write permissions in the current directory.")
|
145 |
+
raise
|
146 |
+
|
147 |
+
# Generate base64 download link
|
148 |
+
try:
|
149 |
+
with open(output_filename, "rb") as f:
|
150 |
+
pdf_bytes = f.read()
|
151 |
+
b64 = base64.b64encode(pdf_bytes).decode()
|
152 |
+
href = f'<a href="data:application/pdf;base64,{b64}" download="{output_filename}">Download de-identified PDF</a>'
|
153 |
+
st.markdown(href, unsafe_allow_html=True)
|
154 |
+
except Exception as e:
|
155 |
+
st.error(f"Error generating download link: {str(e)}")
|
156 |
+
raise
|
157 |
+
|
158 |
+
# Display findings
|
159 |
+
with col2:
|
160 |
+
st.subheader("Findings")
|
161 |
+
if st_analyze_results:
|
162 |
+
df = pd.DataFrame.from_records([r.to_dict() for r in st_analyze_results])
|
163 |
+
df["text"] = [text[res.start:res.end] for res in st_analyze_results]
|
164 |
+
df_subset = df[["entity_type", "text", "start", "end", "score"]].rename(
|
165 |
+
{
|
166 |
+
"entity_type": "Entity type",
|
167 |
+
"text": "Text",
|
168 |
+
"start": "Start",
|
169 |
+
"end": "End",
|
170 |
+
"score": "Confidence",
|
171 |
+
},
|
172 |
+
axis=1,
|
173 |
+
)
|
174 |
+
if st_return_decision_process:
|
175 |
+
analysis_explanation_df = pd.DataFrame.from_records(
|
176 |
+
[r.analysis_explanation.to_dict() for r in st_analyze_results]
|
177 |
+
)
|
178 |
+
df_subset = pd.concat([df_subset, analysis_explanation_df], axis=1)
|
179 |
+
st.dataframe(df_subset.reset_index(drop=True), use_container_width=True)
|
180 |
+
else:
|
181 |
+
st.text("No findings")
|
182 |
+
|
183 |
+
except Exception as e:
|
184 |
+
st.error(f"An error occurred: {str(e)}")
|
185 |
+
logger.error(f"Processing error: {str(e)}")
|