SCBconsulting commited on
Commit
d90e47c
·
verified ·
1 Parent(s): 2c2937b

Create metadata.py

Browse files
Files changed (1) hide show
  1. utils/metadata.py +25 -0
utils/metadata.py ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import pipeline
2
+
3
+ model = "dslim/bert-base-NER"
4
+ ner = pipeline("ner", model=model, tokenizer=model, grouped_entities=True)
5
+
6
+ def extract_metadata(text):
7
+ if not text.strip():
8
+ return {"error": "No input provided."}
9
+
10
+ text = text[:1000]
11
+ entities = ner(text)
12
+
13
+ result = {
14
+ "DATE": [],
15
+ "PERSON": [],
16
+ "ORGANIZATION": [],
17
+ "LOCATION": []
18
+ }
19
+
20
+ for ent in entities:
21
+ label = ent["entity_group"]
22
+ if label in result and ent["word"] not in result[label]:
23
+ result[label].append(ent["word"])
24
+
25
+ return result