Spaces:

osiria
/

bert-cased-ner-english

Sleeping

File size: 8,965 Bytes

import os
import gradio as gr
import subprocess
import sys

def install(package):
    subprocess.check_call([sys.executable, "-m", "pip", "install", package])

install("numpy")
install("torch")
install("transformers")
install("unidecode")

import numpy as np
import torch
from transformers import AutoTokenizer
from transformers import BertForTokenClassification
from collections import Counter
from unidecode import unidecode
import string
import re

auth_token = os.environ.get("AUTH_TOKEN")

tokenizer = AutoTokenizer.from_pretrained("osiria/bert-base-cased-ner-en", token=auth_token)
model = BertForTokenClassification.from_pretrained("osiria/bert-base-cased-ner-en", num_labels = 5, token=auth_token)
device = torch.device("cpu")
model = model.to(device)
model.eval()

from transformers import pipeline
ner = pipeline('ner', model=model, tokenizer=tokenizer, device=-1)


header = '''--------------------------------------------------------------------------------------------------
<style>
.vertical-text {
    writing-mode: vertical-lr;
    text-orientation: upright;
    background-color:red;
}
</style>
<center>
<body>
<span class="vertical-text" style="background-color:lightgreen;border-radius: 3px;padding: 3px;"> </span>
<span class="vertical-text" style="background-color:orange;border-radius: 3px;padding: 3px;"> D</span>
<span class="vertical-text" style="background-color:lightblue;border-radius: 3px;padding: 3px;">    E</span>
<span class="vertical-text" style="background-color:tomato;border-radius: 3px;padding: 3px;">    M</span>
<span class="vertical-text" style="background-color:lightgrey;border-radius: 3px;padding: 3px;"> O</span>
<span class="vertical-text" style="background-color:#CF9FFF;border-radius: 3px;padding: 3px;"> </span>
</body>
</center>
<br>
'''

maps = {"O": "NONE", "PER": "PER", "LOC": "LOC", "ORG": "ORG", "MISC": "MISC", "DATE": "DATE"}
reg_month = "(?:gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|novembre|dicembre|january|february|march|april|may|june|july|august|september|october|november|december)"
reg_date = "(?:\d{1,2}\°{0,1}|primo|\d{1,2}\º{0,1})" + " " + reg_month + " " + "\d{4}|"
reg_date = reg_date + reg_month + " " + "\d{4}|"
reg_date = reg_date + "\d{1,2}" + " " + reg_month
reg_date = reg_date + "\d{1,2}" + "(?:\/|\.)\d{1,2}(?:\/|\.)" + "\d{4}|"
reg_date = reg_date + "(?<=dal )\d{4}|(?<=al )\d{4}|(?<=nel )\d{4}|(?<=anno )\d{4}|(?<=del )\d{4}|"
reg_date = reg_date + "\d{1,5} a\.c\.|\d{1,5} d\.c\."
map_punct = {"’": "'", "«": '"', "»": '"', "”": '"', "“": '"', "–": "-", "$": ""}
unk_tok = 9005
merge_th_1 = 0.8
merge_th_2 = 0.4
min_th = 0.5

def extract(text):

    text = text.strip()
    for mp in map_punct:
        text = text.replace(mp, map_punct[mp])
    text = re.sub("\[\d+\]", "", text)

    warn_flag = False
    
    res_total = []
    out_text = ""
    
    for p_text in text.split("\n"):
    
        if p_text:

            toks = tokenizer.encode(p_text)
            if unk_tok in toks:
                warn_flag = True
    
            res_orig = ner(p_text, aggregation_strategy = "first")
            res_orig = [el for r, el in enumerate(res_orig) if len(el["word"].strip()) > 1]
            res = []
            
            for r, ent in enumerate(res_orig):
                if r > 0 and ent["score"] < merge_th_1 and ent["start"] <= res[-1]["end"] + 1 and ent["score"] <= res[-1]["score"]:
                    res[-1]["word"] = res[-1]["word"] + " " + ent["word"]
                    res[-1]["score"] = merge_th_1*(res[-1]["score"] > merge_th_2)
                    res[-1]["end"] = ent["end"]
                elif r < len(res_orig) - 1 and ent["score"] < merge_th_1 and res_orig[r+1]["start"] <= ent["end"] + 1 and res_orig[r+1]["score"] > ent["score"]:
                    res_orig[r+1]["word"] = ent["word"] + " " + res_orig[r+1]["word"]
                    res_orig[r+1]["score"] = merge_th_1*(res_orig[r+1]["score"] > merge_th_2)
                    res_orig[r+1]["start"] = ent["start"]
                else:
                    res.append(ent)
                    
            res = [el for r, el in enumerate(res) if el["score"] >= min_th]
        
            dates = [{"entity_group": "DATE", "score": 1.0, "word": p_text[el.span()[0]:el.span()[1]], "start": el.span()[0], "end": el.span()[1]} for el in re.finditer(reg_date, p_text, flags = re.IGNORECASE)]
            res.extend(dates)
            res = sorted(res, key = lambda t: t["start"])
            res_total.extend(res)
    
            chunks = [("", "", 0, "NONE")]
    
            for el in res:
                if maps[el["entity_group"]] != "NONE":
                    tag = maps[el["entity_group"]]
                    chunks.append((p_text[el["start"]: el["end"]], p_text[chunks[-1][2]:el["end"]], el["end"], tag))

            if chunks[-1][2] < len(p_text):
                chunks.append(("END", p_text[chunks[-1][2]:], -1, "NONE"))
            chunks = chunks[1:]
            
            n_text = []
    
            for i, chunk in enumerate(chunks):

                rep = chunk[0]
    
                if chunk[3] == "PER":
                    rep = '<span style="background-color:lightgreen;border-radius: 3px;padding: 3px;"><b>ᴘᴇʀ</b> ' + chunk[0] + '</span>'
                elif chunk[3] == "LOC":
                    rep = '<span style="background-color:orange;border-radius: 3px;padding: 3px;"><b>ʟᴏᴄ</b> ' + chunk[0] + '</span>'
                elif chunk[3] == "ORG":
                    rep = '<span style="background-color:lightblue;border-radius: 3px;padding: 3px;"><b>ᴏʀɢ</b> ' + chunk[0] + '</span>'
                elif chunk[3] == "MISC":
                    rep = '<span style="background-color:tomato;border-radius: 3px;padding: 3px;"><b>ᴍɪsᴄ</b> ' + chunk[0] + '</span>'
                elif chunk[3] == "DATE":
                    rep = '<span style="background-color:lightgrey;border-radius: 3px;padding: 3px;"><b>ᴅᴀᴛᴇ</b> ' + chunk[0] + '</span>'
    
                n_text.append(chunk[1].replace(chunk[0], rep))
    
            n_text = "".join(n_text)
            if out_text:
                out_text = out_text + "<br>" + n_text
            else:
                out_text = n_text
    

    tags = [el["word"] for el in res_total if el["entity_group"] not in ['DATE', None]]
    cnt = Counter(tags)
    tags = sorted(list(set([el for el in tags if cnt[el] > 1])), key = lambda t: cnt[t]*np.exp(-tags.index(t)))[::-1]
    tags = [" ".join(re.sub("[^A-Za-z0-9\s]", "", unidecode(tag)).split()) for tag in tags]
    tags = ['<span style="background-color:#CF9FFF;border-radius: 3px;padding: 3px;"><b>ᴛᴀɢ </b> ' + el + '</span>' for el in tags]
    tags = "    ".join(tags)

    if tags:
        out_text = out_text + "<br><br><b>Tags:</b> " + tags

    if warn_flag:
        out_text = out_text + "<br><br><b>Warning ⚠️:</b> Unknown tokens detected in text.  The model might behave erratically"
    
    return out_text



init_text = '''The American Academy of Arts and Sciences (AAA&S) is one of the oldest learned societies in the United States. It was founded in 1780 during the American Revolution by John Adams, John Hancock, James Bowdoin, Andrew Oliver, and other Founding Fathers of the United States. It is headquartered in Cambridge, Massachusetts.
Membership in the academy is achieved through a thorough petition, review, and election process. The academy's quarterly journal, Dædalus, is published by the MIT Press on behalf of the academy. The academy also conducts multidisciplinary public policy research.
The Academy was established by the Massachusetts legislature on May 4, 1780, charted in order "to cultivate every art and science which may tend to advance the interest, honor, dignity, and happiness of a free, independent, and virtuous people." The sixty-two incorporating fellows represented varying interests and high standing in the political, professional, and commercial sectors of the state. The first class of new members, chosen by the Academy in 1781, included Benjamin Franklin and George Washington as well as several international honorary members.
'''

init_output = extract(init_text)




with gr.Blocks(css="footer {visibility: hidden}", theme=gr.themes.Default(text_size="lg", spacing_size="lg")) as interface:
    
    with gr.Row():
        gr.Markdown(header)
    with gr.Row():
        text = gr.Text(label="Extract entities", lines = 10, value = init_text)
    with gr.Row():
        with gr.Column():
            button = gr.Button("Extract").style(full_width=False)
    with gr.Row():
        with gr.Column():
            entities = gr.Markdown(init_output)

    with gr.Row():
        with gr.Column():
            gr.Markdown("<center>The input examples in this demo are extracted from https://it.wikipedia.org</center>") 

    button.click(extract, inputs=[text], outputs = [entities])


interface.launch()