Update Model/MultimodelNER/VLSP2021/MNER_2021.py
Browse files
Model/MultimodelNER/VLSP2021/MNER_2021.py
CHANGED
@@ -1,151 +1,151 @@
|
|
1 |
-
import streamlit as st
|
2 |
-
from spacy import displacy
|
3 |
-
from Model.NER.VLSP2021.Predict_Ner import ViTagger
|
4 |
-
import re
|
5 |
-
from thunghiemxuly import save_uploaded_image,convert_text_to_txt,add_string_to_txt
|
6 |
-
|
7 |
-
import os
|
8 |
-
from transformers import AutoTokenizer, BertConfig
|
9 |
-
from Model.MultimodelNER.VLSP2021.train_umt_2021 import load_model,predict
|
10 |
-
from Model.MultimodelNER.Ner_processing import format_predictions,process_predictions,combine_entities,remove_B_prefix,combine_i_tags
|
11 |
-
|
12 |
-
from Model.MultimodelNER.predict import get_test_examples_predict
|
13 |
-
from Model.MultimodelNER import resnet as resnet
|
14 |
-
from Model.MultimodelNER.resnet_utils import myResnet
|
15 |
-
import torch
|
16 |
-
import numpy as np
|
17 |
-
from Model.MultimodelNER.VLSP2021.dataset_roberta import MNERProcessor_2021
|
18 |
-
|
19 |
-
|
20 |
-
CONFIG_NAME = 'bert_config.json'
|
21 |
-
WEIGHTS_NAME = 'pytorch_model.bin'
|
22 |
-
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
23 |
-
|
24 |
-
|
25 |
-
net = getattr(resnet, 'resnet152')()
|
26 |
-
net.load_state_dict(torch.load(os.path.join('
|
27 |
-
encoder = myResnet(net, True, device)
|
28 |
-
def process_text(text):
|
29 |
-
# Loại bỏ dấu cách thừa và dấu cách ở đầu và cuối văn bản
|
30 |
-
processed_text = re.sub(r'\s+', ' ', text.strip())
|
31 |
-
return processed_text
|
32 |
-
|
33 |
-
|
34 |
-
|
35 |
-
def show_mner_2021():
|
36 |
-
multimodal_text = st.text_area("Enter your text for MNER:", height=300)
|
37 |
-
multimodal_text = process_text(multimodal_text) # Xử lý văn bản
|
38 |
-
image = st.file_uploader("Upload an image (only jpg):", type=["jpg"])
|
39 |
-
if st.button("Process Multimodal NER"):
|
40 |
-
save_image = '
|
41 |
-
save_txt = '
|
42 |
-
image_name = image.name
|
43 |
-
save_uploaded_image(image, save_image)
|
44 |
-
convert_text_to_txt(multimodal_text, save_txt)
|
45 |
-
add_string_to_txt(image_name, save_txt)
|
46 |
-
st.image(image, caption="Uploaded Image", use_column_width=True)
|
47 |
-
|
48 |
-
bert_model = 'vinai/phobert-base-v2'
|
49 |
-
output_dir = '
|
50 |
-
output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
|
51 |
-
output_encoder_file = os.path.join(output_dir, "pytorch_encoder.bin")
|
52 |
-
processor = MNERProcessor_2021()
|
53 |
-
label_list = processor.get_labels()
|
54 |
-
auxlabel_list = processor.get_auxlabels()
|
55 |
-
num_labels = len(label_list) + 1
|
56 |
-
auxnum_labels = len(auxlabel_list) + 1
|
57 |
-
trans_matrix = np.zeros((auxnum_labels, num_labels), dtype=float)
|
58 |
-
trans_matrix[0, 0] = 1 # pad to pad
|
59 |
-
trans_matrix[1, 1] = 1 # O to O
|
60 |
-
trans_matrix[2, 2] = 0.25 # B to B-MISC
|
61 |
-
trans_matrix[2, 4] = 0.25 # B to B-PER
|
62 |
-
trans_matrix[2, 6] = 0.25 # B to B-ORG
|
63 |
-
trans_matrix[2, 8] = 0.25 # B to B-LOC
|
64 |
-
trans_matrix[3, 3] = 0.25 # I to I-MISC
|
65 |
-
trans_matrix[3, 5] = 0.25 # I to I-PER
|
66 |
-
trans_matrix[3, 7] = 0.25 # I to I-ORG
|
67 |
-
trans_matrix[3, 9] = 0.25 # I to I-LOC
|
68 |
-
trans_matrix[4, 10] = 1 # X to X
|
69 |
-
trans_matrix[5, 11] = 1 # [CLS] to [CLS]
|
70 |
-
trans_matrix[6, 12] = 1
|
71 |
-
tokenizer = AutoTokenizer.from_pretrained(bert_model, do_lower_case=False)
|
72 |
-
model_umt, encoder_umt = load_model(output_model_file, output_encoder_file, encoder, num_labels,
|
73 |
-
auxnum_labels)
|
74 |
-
eval_examples = get_test_examples_predict(
|
75 |
-
'
|
76 |
-
|
77 |
-
y_pred, a = predict(model_umt, encoder_umt, eval_examples, tokenizer, device, save_image, trans_matrix)
|
78 |
-
formatted_output = format_predictions(a, y_pred[0])
|
79 |
-
final = process_predictions(formatted_output)
|
80 |
-
final2 = combine_entities(final)
|
81 |
-
final3 = remove_B_prefix(final2)
|
82 |
-
final4 = combine_i_tags(final3)
|
83 |
-
|
84 |
-
words_and_labels = final4
|
85 |
-
# Tạo danh sách từ
|
86 |
-
words = [word for word, _ in words_and_labels]
|
87 |
-
# Tạo danh sách thực thể và nhãn cho mỗi từ, loại bỏ nhãn 'O'
|
88 |
-
entities = [{'start': sum(len(word) + 1 for word, _ in words_and_labels[:i]),
|
89 |
-
'end': sum(len(word) + 1 for word, _ in words_and_labels[:i + 1]), 'label': label} for
|
90 |
-
i, (word, label)
|
91 |
-
in enumerate(words_and_labels) if label != 'O']
|
92 |
-
# print(entities)
|
93 |
-
|
94 |
-
# Render the visualization without color for 'O' labels
|
95 |
-
html = displacy.render(
|
96 |
-
{"text": " ".join(words), "ents": entities, "title": None},
|
97 |
-
style="ent",
|
98 |
-
manual=True,
|
99 |
-
options={"colors": {"DATETIME-DATERANGE": "#66c2ff",
|
100 |
-
"LOCATION-GPE": "#ffcc99",
|
101 |
-
"O": None, # Màu cho nhãn 'O'
|
102 |
-
"QUANTITY-NUM": "#ffdf80",
|
103 |
-
"EVENT-CUL": "#bfbfbf",
|
104 |
-
"DATETIME": "#80ff80",
|
105 |
-
"PERSONTYPE": "#ff80ff",
|
106 |
-
"PERSON": "#bf80ff",
|
107 |
-
"QUANTITY-PER": "#80cccc",
|
108 |
-
"ORGANIZATION": "#ff6666",
|
109 |
-
"LOCATION-GEO": "#66cc66",
|
110 |
-
"LOCATION-STRUC": "#cccc66",
|
111 |
-
"PRODUCT-COM": "#ffff66",
|
112 |
-
"DATETIME-DATE": "#66cccc",
|
113 |
-
"QUANTITY-DIM": "#6666ff",
|
114 |
-
"PRODUCT": "#cc6666",
|
115 |
-
"QUANTITY": "#6666cc",
|
116 |
-
"DATETIME-DURATION": "#9966ff",
|
117 |
-
"QUANTITY-CUR": "#ff9966",
|
118 |
-
"DATETIME-TIME": "#cdbf93",
|
119 |
-
"QUANTITY-TEM": "#cc9966",
|
120 |
-
"DATETIME-TIMERANGE": "#cc8566",
|
121 |
-
"EVENT-GAMESHOW": "#8c8c5a",
|
122 |
-
"QUANTITY-AGE": "#70db70",
|
123 |
-
"QUANTITY-ORD": "#e699ff",
|
124 |
-
"PRODUCT-LEGAL": "#806699",
|
125 |
-
"LOCATION": "#993366",
|
126 |
-
"ORGANIZATION-MED": "#339933",
|
127 |
-
"URL": "#ff4d4d",
|
128 |
-
"PHONENUMBER": "#99cc99",
|
129 |
-
"ORGANIZATION-SPORTS": "#6666ff",
|
130 |
-
"EVENT-SPORT": "#ffff80",
|
131 |
-
"SKILL": "#b38f66",
|
132 |
-
"EVENT-NATURAL": "#ff9966",
|
133 |
-
"ADDRESS": "#cc9966",
|
134 |
-
"IP": "#b38f66",
|
135 |
-
"EMAIL": "#cc8566",
|
136 |
-
"ORGANIZATION-STOCK": "#666633",
|
137 |
-
"DATETIME-SET": "#70db70",
|
138 |
-
"PRODUCT-AWARD": "#e699ff",
|
139 |
-
"MISCELLANEOUS": "#806699",
|
140 |
-
"LOCATION-GPE-GEO": "#99ffff"}}
|
141 |
-
)
|
142 |
-
# print(html)
|
143 |
-
st.markdown(html, unsafe_allow_html=True)
|
144 |
-
|
145 |
-
# Sử dụng widget st.html để hiển thị HTML
|
146 |
-
|
147 |
-
# Hiển thị văn bản đã nhập
|
148 |
-
# st.write("Văn bản đã nhập:", text)
|
149 |
-
|
150 |
-
|
151 |
###Ví dụ 1 : Một trận hỗn chiến đã xảy ra tại trận đấu khúc côn cầu giữa Penguins và Islanders ở Mỹ (image:penguin)
|
|
|
1 |
+
import streamlit as st
|
2 |
+
from spacy import displacy
|
3 |
+
from Model.NER.VLSP2021.Predict_Ner import ViTagger
|
4 |
+
import re
|
5 |
+
from thunghiemxuly import save_uploaded_image,convert_text_to_txt,add_string_to_txt
|
6 |
+
|
7 |
+
import os
|
8 |
+
from transformers import AutoTokenizer, BertConfig
|
9 |
+
from Model.MultimodelNER.VLSP2021.train_umt_2021 import load_model,predict
|
10 |
+
from Model.MultimodelNER.Ner_processing import format_predictions,process_predictions,combine_entities,remove_B_prefix,combine_i_tags
|
11 |
+
|
12 |
+
from Model.MultimodelNER.predict import get_test_examples_predict
|
13 |
+
from Model.MultimodelNER import resnet as resnet
|
14 |
+
from Model.MultimodelNER.resnet_utils import myResnet
|
15 |
+
import torch
|
16 |
+
import numpy as np
|
17 |
+
from Model.MultimodelNER.VLSP2021.dataset_roberta import MNERProcessor_2021
|
18 |
+
|
19 |
+
|
20 |
+
CONFIG_NAME = 'bert_config.json'
|
21 |
+
WEIGHTS_NAME = 'pytorch_model.bin'
|
22 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
23 |
+
|
24 |
+
|
25 |
+
net = getattr(resnet, 'resnet152')()
|
26 |
+
net.load_state_dict(torch.load(os.path.join('/Model/Resnet/', 'resnet152.pth')))
|
27 |
+
encoder = myResnet(net, True, device)
|
28 |
+
def process_text(text):
|
29 |
+
# Loại bỏ dấu cách thừa và dấu cách ở đầu và cuối văn bản
|
30 |
+
processed_text = re.sub(r'\s+', ' ', text.strip())
|
31 |
+
return processed_text
|
32 |
+
|
33 |
+
|
34 |
+
|
35 |
+
def show_mner_2021():
|
36 |
+
multimodal_text = st.text_area("Enter your text for MNER:", height=300)
|
37 |
+
multimodal_text = process_text(multimodal_text) # Xử lý văn bản
|
38 |
+
image = st.file_uploader("Upload an image (only jpg):", type=["jpg"])
|
39 |
+
if st.button("Process Multimodal NER"):
|
40 |
+
save_image = '/Model/MultimodelNER/VLSP2021/Image'
|
41 |
+
save_txt = '/Model/MultimodelNER/VLSP2021/Filetxt/test.txt'
|
42 |
+
image_name = image.name
|
43 |
+
save_uploaded_image(image, save_image)
|
44 |
+
convert_text_to_txt(multimodal_text, save_txt)
|
45 |
+
add_string_to_txt(image_name, save_txt)
|
46 |
+
st.image(image, caption="Uploaded Image", use_column_width=True)
|
47 |
+
|
48 |
+
bert_model = 'vinai/phobert-base-v2'
|
49 |
+
output_dir = '/Model/MultimodelNER/VLSP2021/best_model'
|
50 |
+
output_model_file = os.path.join(output_dir, WEIGHTS_NAME)
|
51 |
+
output_encoder_file = os.path.join(output_dir, "pytorch_encoder.bin")
|
52 |
+
processor = MNERProcessor_2021()
|
53 |
+
label_list = processor.get_labels()
|
54 |
+
auxlabel_list = processor.get_auxlabels()
|
55 |
+
num_labels = len(label_list) + 1
|
56 |
+
auxnum_labels = len(auxlabel_list) + 1
|
57 |
+
trans_matrix = np.zeros((auxnum_labels, num_labels), dtype=float)
|
58 |
+
trans_matrix[0, 0] = 1 # pad to pad
|
59 |
+
trans_matrix[1, 1] = 1 # O to O
|
60 |
+
trans_matrix[2, 2] = 0.25 # B to B-MISC
|
61 |
+
trans_matrix[2, 4] = 0.25 # B to B-PER
|
62 |
+
trans_matrix[2, 6] = 0.25 # B to B-ORG
|
63 |
+
trans_matrix[2, 8] = 0.25 # B to B-LOC
|
64 |
+
trans_matrix[3, 3] = 0.25 # I to I-MISC
|
65 |
+
trans_matrix[3, 5] = 0.25 # I to I-PER
|
66 |
+
trans_matrix[3, 7] = 0.25 # I to I-ORG
|
67 |
+
trans_matrix[3, 9] = 0.25 # I to I-LOC
|
68 |
+
trans_matrix[4, 10] = 1 # X to X
|
69 |
+
trans_matrix[5, 11] = 1 # [CLS] to [CLS]
|
70 |
+
trans_matrix[6, 12] = 1
|
71 |
+
tokenizer = AutoTokenizer.from_pretrained(bert_model, do_lower_case=False)
|
72 |
+
model_umt, encoder_umt = load_model(output_model_file, output_encoder_file, encoder, num_labels,
|
73 |
+
auxnum_labels)
|
74 |
+
eval_examples = get_test_examples_predict(
|
75 |
+
'/Model/MultimodelNER/VLSP2021/Filetxt/')
|
76 |
+
|
77 |
+
y_pred, a = predict(model_umt, encoder_umt, eval_examples, tokenizer, device, save_image, trans_matrix)
|
78 |
+
formatted_output = format_predictions(a, y_pred[0])
|
79 |
+
final = process_predictions(formatted_output)
|
80 |
+
final2 = combine_entities(final)
|
81 |
+
final3 = remove_B_prefix(final2)
|
82 |
+
final4 = combine_i_tags(final3)
|
83 |
+
|
84 |
+
words_and_labels = final4
|
85 |
+
# Tạo danh sách từ
|
86 |
+
words = [word for word, _ in words_and_labels]
|
87 |
+
# Tạo danh sách thực thể và nhãn cho mỗi từ, loại bỏ nhãn 'O'
|
88 |
+
entities = [{'start': sum(len(word) + 1 for word, _ in words_and_labels[:i]),
|
89 |
+
'end': sum(len(word) + 1 for word, _ in words_and_labels[:i + 1]), 'label': label} for
|
90 |
+
i, (word, label)
|
91 |
+
in enumerate(words_and_labels) if label != 'O']
|
92 |
+
# print(entities)
|
93 |
+
|
94 |
+
# Render the visualization without color for 'O' labels
|
95 |
+
html = displacy.render(
|
96 |
+
{"text": " ".join(words), "ents": entities, "title": None},
|
97 |
+
style="ent",
|
98 |
+
manual=True,
|
99 |
+
options={"colors": {"DATETIME-DATERANGE": "#66c2ff",
|
100 |
+
"LOCATION-GPE": "#ffcc99",
|
101 |
+
"O": None, # Màu cho nhãn 'O'
|
102 |
+
"QUANTITY-NUM": "#ffdf80",
|
103 |
+
"EVENT-CUL": "#bfbfbf",
|
104 |
+
"DATETIME": "#80ff80",
|
105 |
+
"PERSONTYPE": "#ff80ff",
|
106 |
+
"PERSON": "#bf80ff",
|
107 |
+
"QUANTITY-PER": "#80cccc",
|
108 |
+
"ORGANIZATION": "#ff6666",
|
109 |
+
"LOCATION-GEO": "#66cc66",
|
110 |
+
"LOCATION-STRUC": "#cccc66",
|
111 |
+
"PRODUCT-COM": "#ffff66",
|
112 |
+
"DATETIME-DATE": "#66cccc",
|
113 |
+
"QUANTITY-DIM": "#6666ff",
|
114 |
+
"PRODUCT": "#cc6666",
|
115 |
+
"QUANTITY": "#6666cc",
|
116 |
+
"DATETIME-DURATION": "#9966ff",
|
117 |
+
"QUANTITY-CUR": "#ff9966",
|
118 |
+
"DATETIME-TIME": "#cdbf93",
|
119 |
+
"QUANTITY-TEM": "#cc9966",
|
120 |
+
"DATETIME-TIMERANGE": "#cc8566",
|
121 |
+
"EVENT-GAMESHOW": "#8c8c5a",
|
122 |
+
"QUANTITY-AGE": "#70db70",
|
123 |
+
"QUANTITY-ORD": "#e699ff",
|
124 |
+
"PRODUCT-LEGAL": "#806699",
|
125 |
+
"LOCATION": "#993366",
|
126 |
+
"ORGANIZATION-MED": "#339933",
|
127 |
+
"URL": "#ff4d4d",
|
128 |
+
"PHONENUMBER": "#99cc99",
|
129 |
+
"ORGANIZATION-SPORTS": "#6666ff",
|
130 |
+
"EVENT-SPORT": "#ffff80",
|
131 |
+
"SKILL": "#b38f66",
|
132 |
+
"EVENT-NATURAL": "#ff9966",
|
133 |
+
"ADDRESS": "#cc9966",
|
134 |
+
"IP": "#b38f66",
|
135 |
+
"EMAIL": "#cc8566",
|
136 |
+
"ORGANIZATION-STOCK": "#666633",
|
137 |
+
"DATETIME-SET": "#70db70",
|
138 |
+
"PRODUCT-AWARD": "#e699ff",
|
139 |
+
"MISCELLANEOUS": "#806699",
|
140 |
+
"LOCATION-GPE-GEO": "#99ffff"}}
|
141 |
+
)
|
142 |
+
# print(html)
|
143 |
+
st.markdown(html, unsafe_allow_html=True)
|
144 |
+
|
145 |
+
# Sử dụng widget st.html để hiển thị HTML
|
146 |
+
|
147 |
+
# Hiển thị văn bản đã nhập
|
148 |
+
# st.write("Văn bản đã nhập:", text)
|
149 |
+
|
150 |
+
|
151 |
###Ví dụ 1 : Một trận hỗn chiến đã xảy ra tại trận đấu khúc côn cầu giữa Penguins và Islanders ở Mỹ (image:penguin)
|