import os os.environ["CUDA_VISIBLE_DEVICES"] = "1,6" #GPUs to use os.environ["HF_HUB_CACHE"] = "/eos/jeodpp/home/users/consose/cache/huggingface/hub" os.environ["HUGGINGFACE_HUB_CACHE"] = "/eos/jeodpp/home/users/consose/cache/huggingface/hub" os.environ["HF_HOME"] = "/eos/jeodpp/home/users/consose/cache/huggingface/hub" from transformers import file_utils print(file_utils.default_cache_path) import sys, re import time import pandas as pd from tqdm import tqdm from concurrent.futures import ThreadPoolExecutor, as_completed from collections import Counter from gliner import GLiNER, GLiNERConfig, data_processing #os.environ["PYTORCH_CUDA_ALLOC_CONF"] = "max_split_size_mb:512" os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True' import torch torch.cuda.empty_cache() # Clear cache ot torch import logging import tiktoken from langchain.text_splitter import TokenTextSplitter import requests import re from common import strtobool, split_camel_case, chunk_tokens, update_nested_dict, cleanInputText, token_counter, encoding_getter, extract_words, all_words_in_list, row_to_dict_string, strip_quotes, rescale_exponential_to_linear, rescale_exponential_to_logarithmic from accelerate import Accelerator # device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # print(f"Device: {device}...") # if device.type == "cuda": # print("GPU number:", torch.cuda.current_device()) accelerator = Accelerator() #accelerator = Accelerator(mixed_precision='fp16', device_placement=False) #fp16 bf16 device = accelerator.device print("Using accelerator device = "+ str(device)) from transformers import AutoTokenizer, AutoModelForCausalLM from transformers import pipeline from transformers.pipelines.pt_utils import KeyDataset from virtuosoQueryRest import sparqlQuery from llmqueryNer import call_model, call_model_with_caching, process_list, setup_gptjrc, api_call_gptjrc, model_list_gptjrc #api_call_HFonPremises import string import datasets import argparse import json import random import numpy as np #from retrieverRAG_testing import RAG_retrieval_Base, RAG_retrieval_Z_scores, RAG_retrieval_Percentile, RAG_retrieval_TopK, retrievePassageSimilarities from retrieverRAG_SF import RAG_retrieval_Base from joblib import Memory cachedir = 'cached' mem = Memory(cachedir, verbose=False) # this is to completely delete the cache: # mem.clear(warn=False) # aa = data_processing.tokenizer.WordsSplitter('whitespace' ) # bb=aa("this is what i wanted to say") # count = 0 # for item in bb: # count = count+1 # print(item) # print(count) POSSIBLE_KGchoices_List = ["AI", "AIO", "AEO", "BFO", "BIM", "BCGO", "CL", "CHIRO", "CHEBI", "DCM", "FMA", "GO", "GENO", "GeoSPARQL", "HL7", "DOID", "HP", "HP_O", "IDO", "IAO", "ICD10", "LOINC", "MESH", "MONDO", "NCIT", "NCBITAXON", "NCBITaxon_", "NIFCELL", "NIFSTD", "GML", "OBCS", "OCHV", "OHPI", "OPB", "TRANS", "PLOSTHES", "RADLEX", "RO", "STY", "SO", "SNOMED", "STATO", "SYMP", "FoodOn", "UBERON", "ORDO", "HOOM", "VO", "OGMS", "EuroSciVoc"] ONLY_Ontologies_OnBIOPORTAL = ["AI", "AIO", "AEO", "BCGO", "BFO", "BIM", "CHEBI", "CHIRO", "CL", "DCM", "DOID", "FMA", "FOODON", "GENO", "GML", "GO", "GEOSPARQL", "HL7", "HP", "HP_O", "IAO", "ICD10", "IDO", "LOINC", "MESH", "MONDO", "NCBITAXON", "NCIT", "NIFCELL", "NIFSTD", "OBCS", "OCHV", "OHPI", "OPB", "PLOSTHES", "RADLEX", "OBOREL", "SNOMEDCT", "SO", "STATO", "STY", "SYMP", "PTRANS", "UBERON", "ORDO", "HOOM", "VO", "OGMS"] # query = 'A man is eating a piece of bread' # # # Define the passages list # passages = [ # "A man is eating food.", # "A man is eating pasta.", # "The girl is carrying a baby.", # "A man is riding a horse.", # ] # # df_retr = RAG_retrieval_Base(questionText, passages, min_threshold=0.5, max_num_passages=3) # print(df_retr) # Define a handler for non-serializable objects def default_serializer(obj): if isinstance(obj, np.floating): return float(obj) raise TypeError(f"Object of type {obj.__class__.__name__} is not JSON serializable") def is_json(myjson): try: # json.loads(myjson, default=default_serializer) json.dumps(myjson, default=default_serializer) except ValueError as e: return False return True def get_filtered_entities(txt_ents): filtered_ent_list = [] for ent_dct in txt_ents: if ent_dct["score"] > 0.0: # 0.5 if ent_dct["entity_group"]: # in ["LOC", "MISC"]: filtered_ent_list.append(ent_dct) return filtered_ent_list def process_row_Gliner(args, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels, row): context_to_annotate = row[args.source_column] tokens = tokenizerGliner.tokenize(context_to_annotate) entities = [] offset = 0 if "gliner_large_bio" in args.model_id: max_chunk_length = modelGlinerBio.config.max_len else: max_chunk_length = modelGliner.config.max_len for chunk in chunk_tokens(tokens, (max_chunk_length - 1)): chunk_text = tokenizerGliner.convert_tokens_to_string(chunk) if "gliner_large_bio" in args.model_id: chunk_entities = modelGlinerBio.predict_entities(chunk_text, glinerlabels, threshold=args.entities_filter_threshold) else: chunk_entities = modelGliner.predict_entities(chunk_text, glinerlabels, threshold=args.entities_filter_threshold) adjusted_entities = [] for entity in chunk_entities: adjusted_entity = { 'text': entity['text'], 'score': entity['score'], 'start': entity['start'] + offset, 'end': entity['end'] + offset, 'label': entity['label'] } adjusted_entities.append(adjusted_entity) entities.extend(adjusted_entities) offset += len(chunk_text) if entities and isinstance(entities, list): for d in entities: d['entity_group'] = d.pop('label') d['word'] = d.pop('text') d['entity_group'] = d['entity_group'].upper() return row.name, entities def process_row_BioPortal_api(args, key_bioportal, row): #context_to_annotate = row[args.source_column] if isinstance(row, list) or isinstance(row, pd.Series): context_to_annotate = row[args.source_column] elif isinstance(row, str): context_to_annotate = row else: raise ValueError("Unsupported type for row. Expected list or string.") url="" if getattr(args, 'KG_restriction', None): # api call if strtobool(args.debug): print("--- BIOPORTAL: " + context_to_annotate) # args.KG_restriction exists and is not empty if strtobool(args.debug): print("KG_restriction is provided and not empty:", args.KG_restriction) onto_clauses = "" for choice in args.KG_restriction: if choice == "SNOMED": choice="SNOMEDCT" elif choice == "RO": choice = "OBOREL" elif choice == "TRANS": choice = "PTRANS" elif choice == "FoodOn": choice = "FOODON" elif choice == "GeoSPARQL": choice = "GEOSPARQL" # elif choice == "NCBITAXON": # choice = "NCBITAXON,NCBITaxon_" elif choice == "NCBITaxon_": choice = "NCBITAXON" if choice in ONLY_Ontologies_OnBIOPORTAL: onto_clauses=onto_clauses+choice+"," if onto_clauses and onto_clauses[-1] == ",": onto_clauses=onto_clauses[:-1] url = f"https://services.data.bioontology.org/annotatorplus/?text={context_to_annotate}&ontologies={onto_clauses}&longest_only=true&exclude_numbers=true&whole_word_only=true&exclude_synonyms=false&negation=false&experiencer=false&temporality=false&score_threshold=0&confidence_threshold=0&display_links=false&display_context=false&score=cvalue&apikey={key_bioportal}" else: kg_restriction = getattr(args, 'KG_restriction', None) if kg_restriction is not None and len(kg_restriction) == 0: print("KG_restriction is provided but empty") return pd.DataFrame() # args.KG_restriction does not exist or is empty if strtobool(args.debug): print("--- BIOPORTAL: " + context_to_annotate) print("KG_restriction is not provided or empty - Consider all the KGs") url = f"https://services.data.bioontology.org/annotatorplus/?text={context_to_annotate}&ontologies=AEO,BFO,BIM,BCGO,CL,CHIRO,CHEBI,DCM,FMA,GO,GENO,GEOSPARQL,HL7,DOID,HP,HP_O,IDO,IAO,ICD10,LOINC,MESH,MONDO,NCIT,NCBITAXON,NIFCELL,NIFSTD,GML,OBCS,OCHV,OHPI,OPB,PTRANS,PLOSTHES,RADLEX,OBOREL,STY,SO,SNOMEDCT,STATO,SYMP,FOODON,UBERON,VO&longest_only=true&exclude_numbers=true&whole_word_only=true&exclude_synonyms=false&negation=false&experiencer=false&temporality=false&score_threshold=0&confidence_threshold=0&display_links=false&display_context=false&score=cvalue&apikey={key_bioportal}" response = requests.get(url) try: data = response.json() if not data: # nothing found from Bioportal return pd.DataFrame() dff = pd.DataFrame(data) dff = dff.drop(columns=['hierarchy', 'mappings']) # If the columns are dictionary-like, use pd.json_normalize: expanded_annotated_class = pd.json_normalize(dff['annotatedClass']) expanded_annotations = pd.DataFrame(dff['annotations'].tolist(), index=dff.index) expanded_annotations = pd.json_normalize(expanded_annotations[0]) # Join the expanded columns back to the original DataFrame df_expanded = dff.drop(columns=['annotatedClass', 'annotations']).join(expanded_annotated_class).join( expanded_annotations) # Snomed id replacement because in our internal knolwedgebase we have this base uri df_expanded['@id'] = df_expanded['@id'].str.replace( "http://purl.bioontology.org/ontology/SNOMEDCT/", "http://snomed.info/id/" ) return df_expanded except Exception as err: logging.error( f'ERROR ON BioPortal Annotator API Call\n\tError: {err}\n TextToAnnotate: {context_to_annotate}\n Have a check...') return pd.DataFrame() # empty dataframe def parallel_process_df_Gliner(args, df, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels): results = [] if args.num_cores_Gliner > 0: with ThreadPoolExecutor(max_workers=args.num_cores_Gliner) as executor: futures = [ executor.submit( process_row_Gliner, args, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels, row ) for _, row in df.iterrows() ] for future in tqdm(futures): drm_idx, entities = future.result() df.at[drm_idx, 'annotation'] = entities else: # Apply the function to each row and extract only the entities part df['annotation'] = df.apply( lambda row: process_row_Gliner(args, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels, row)[1], axis=1 ) return df def annotate(df, args, pipeInner, tokenizerGliner, modelGliner, modelGlinerBio, device="cpu"): if strtobool(args.debug): print("\nAnnotate using " + args.model_id) print("device=" + str(device)) startAnnotate = time.time() if "gliner" in args.model_id: df['model'] = args.model_id df['annotation'] = None glinerlabels = ["location", "disease", "date", "numerical value", "number"] # Parallel CPU computation for Gliner: df = parallel_process_df_Gliner(args, df, tokenizerGliner, modelGlinerBio, modelGliner, glinerlabels) # for drm_idx, row in tqdm(df.iterrows()): # context_to_annotate = row[args.source_column] # # # Tokenize the text # tokens = tokenizerGliner.tokenize(context_to_annotate) # # # Process each chunk and predict entities # entities = [] # offset = 0 # Initialize the offset # # if "gliner_large_bio" in args.model_id: # maxchunckslen=modelGlinerBio.config.max_len # else: # maxchunckslen = modelGliner.config.max_len # for chunk in chunk_tokens(tokens, (maxchunckslen - 1)): # # Convert tokens back to text for the chunk # chunk_text = tokenizerGliner.convert_tokens_to_string(chunk) # # Predict entities for the chunk # if "gliner_large_bio" in args.model_id: # chunk_entities = modelGlinerBio.predict_entities(chunk_text, glinerlabels, threshold=args.entities_filter_threshold) # else: # chunk_entities = modelGliner.predict_entities(chunk_text, glinerlabels, threshold=args.entities_filter_threshold) # # # Adjust the start and end positions of entities to reflect their positions in the original text # adjusted_entities = [] # for entity in chunk_entities: # adjusted_entity = { # 'text': entity['text'], # 'score': entity['score'], # 'start': entity['start'] + offset, # 'end': entity['end'] + offset, # 'label': entity['label'] # } # adjusted_entities.append(adjusted_entity) # # # Append adjusted entities to all_entities # entities.extend(adjusted_entities) # # # Update the offset for the next chunk by adding the length of the current chunk # offset += len(chunk_text) # # # Now `all entities` contains all entities with adjusted positions # if entities and isinstance(entities, list): # # # if strtobool(args.debug): # # for entity in entities: # # print(entity["text"], "=>", entity["label"]) # # for d in entities: # d['entity_group'] = d.pop('label') # Change 'label' to 'entity_group' # d['word'] = d.pop('text') # Change 'text' to 'word' # d['entity_group'] = d['entity_group'].upper() # # df.at[drm_idx, 'annotation'] = entities df_annot = df.explode('annotation').dropna(subset=['annotation']).reset_index(drop=True) elif "NCBO" in args.model_id: #NCBO/BioPortal" annotator #https://data.bioontology.org/documentation#nav_annotator #https://bioportal.bioontology.org/annotatorplus key_bioportal = "" if args.bioportalkey_filename and os.path.exists(args.bioportalkey_filename): fkeyname = args.bioportalkey_filename with open(fkeyname) as f: key_bioportal = f.read() else: key_bioportal = os.environ['key_bioportal'] df_annot = pd.DataFrame() for drm_idx, row in tqdm(df.iterrows()): df_BioPortalAnnotation=process_row_BioPortal_api(args, key_bioportal, row) if not df_BioPortalAnnotation.empty: df_BioPortalAnnotation = df_BioPortalAnnotation.sort_values( by=['from', 'text', 'score', 'matchType'], ascending=[True, True, False, False]) df_biop_minimised = df_BioPortalAnnotation.copy() # Group the dataframe by "from" and "to" columns grouped_biop = df_biop_minimised.groupby(['from', 'to']) # Get the index of the row with the maximum score in each group idx_biop = grouped_biop['score'].idxmax() # Filter the dataframe using these indices df_max_score_biop = df_biop_minimised.loc[idx_biop] # Optional: Reset index if you want a clean index df_max_score_biop = df_max_score_biop.reset_index(drop=True) # Create the "pippo" column by grouping and collecting "@id" values pippo_lists = grouped_biop['@id'].apply(list).reset_index() # Merge the lists back into the df_max_score_biop by 'from' and 'to' df_max_score_biop = df_max_score_biop.merge(pippo_lists, on=['from', 'to']) # Rename the merged column to "pippo" df_max_score_biop = df_max_score_biop.rename(columns={'@id_x': '@id'}) df_max_score_biop = df_max_score_biop.rename(columns={'@id_y': 'ALLURIScontextFromNCBO'}) # The df_max_score_biop now contains the "pippo" column with lists of "@id" values. # Filter the dataframe to keep only rows where the score is greater than 3.0 df_max_score_biop = df_max_score_biop[df_max_score_biop['score'] > 3.0] if "semantic_groups" not in df_max_score_biop.columns: # Drop the '@id' column df_max_score_biop["semantic_groups"] = None # Specify the columns you want to keep columns_to_keep = ["score", "from", "to", "prefLabel", "text", "semantic_groups", "@id", "ALLURIScontextFromNCBO"] # Subset the dataframe to keep only the specified columns df_max_score_biop = df_max_score_biop[columns_to_keep] # Rename the specified columns df_max_score_biop = df_max_score_biop.rename(columns={"from": "start", "to": "end", "text": "word", "semantic_groups": "entity_group"}) # Optional: Reset index if you want a clean index after filtering df_max_score_biop = df_max_score_biop.reset_index(drop=True) df_max_score_biop['score'] = df_max_score_biop['score'].round(2) # Each row of "semantic_groups" --> 'entity_group' is a list of string ... keep only the first part if not NaN # df_max_score_biop['entity_group'] = df_max_score_biop['entity_group'].apply(lambda x: x[0] if pd.notna(x).all() else x) df_max_score_biop['entity_group'] = df_max_score_biop['entity_group'].apply(lambda x: x[0] if isinstance(x, list) and len(x) > 0 else (np.nan if x is None or (isinstance(x, float) and pd.isna(x)) else x)) #df_max_score_biop = df_max_score_biop.dropna(subset=['entity_group']) all_empty_or_nan_or_empty_string = df_max_score_biop['entity_group'].replace('', pd.NA).isna().all() if not all_empty_or_nan_or_empty_string: #print("Is the 'entity_group' column completely NaN, None, or empty strings?", all_empty_or_nan_or_empty_string) # Identify the minimum score min_score_biop = df_max_score_biop['score'].min() # Apply filter to find rows where entity_group is None and score is the minimum one conditionBiop = (df_max_score_biop['entity_group'].isna()) & (df_max_score_biop['score'] == min_score_biop) df_max_score_biop = df_max_score_biop[~conditionBiop] # Replace all NaN values in 'entity_group' with 'BIOP' df_max_score_biop['entity_group'] = df_max_score_biop['entity_group'].fillna('BIOP') # To delete the columns "start" and "end" #df_max_score_biop = df_max_score_biop.drop(columns=['start', 'end', 'word', 'entity_group', 'score', 'prefLabel']) if not df_max_score_biop.empty: row_df = pd.DataFrame([row] * len(df_max_score_biop), columns=row.index) row_df['model'] = args.model_id #'NCBO/BioPortal' df_max_score_biop = pd.concat([row_df.reset_index(drop=True), df_max_score_biop.reset_index(drop=True)], axis=1) df_annot = pd.concat([df_annot, df_max_score_biop], ignore_index=True) #here I finish the for cycle if not df_annot.empty: mmax_score = df_annot['score'].max() mmin_score = df_annot['score'].min() if mmax_score == mmin_score: df_annot['score'] = 0.3 df_annot.loc[df_annot['score'].notnull(), 'score'] = 0.7 else: # Apply the transformation function #df_annot = rescale_exponential_to_linear(df_annot, 'score', new_min=0.5, new_max=1.0) df_annot = rescale_exponential_to_logarithmic(df_annot, 'score', new_min=0.7, new_max=1.0) columnsDict = ['start', 'end', 'word', 'entity_group', 'score', 'prefLabel'] df_annot['annotation'] = df_annot.apply(row_to_dict_string, axis=1, columnsDict=columnsDict) # Convert JSON string to a dictionary, if needed df_annot['annotation'] = df_annot['annotation'].apply( lambda x: json.loads(x) if isinstance(x, str) else x ) df_annot = df_annot.drop(columns=columnsDict) else: HF_dataset = datasets.Dataset.from_pandas(pd.DataFrame(data=df)) # tsk = "token-classification" # # pipe = pipeline( # tsk, # model=args.model_id, # aggregation_strategy="simple", # device=device, # ) if strtobool(args.debug): print('Annotating texts...') annotated_texts = [ # out for out in tqdm(pipe(KeyDataset(HF_dataset, args.source_column), batch_size=args.batch_size)) # out for out in tqdm(ner_pipeline(KeyDataset(HF_dataset, args.source_column), batch_size=args.batch_size)) out for out in tqdm(pipeInner(KeyDataset(HF_dataset, args.source_column), batch_size=args.batch_size)) ] if strtobool(args.debug): print('looping annotations...') df['model'] = args.model_id df['annotation'] = annotated_texts df_annot = df.explode('annotation').dropna(subset=['annotation']).reset_index(drop=True) # # Initialize an empty dataframe with the same columns as df # df_annot = pd.DataFrame(columns=df.columns.tolist()) #+ ['model', 'annotation']) # # for drm_idx, txt_ents in enumerate(tqdm(annotated_texts)): # # filtered_entities = get_filtered_entities(txt_ents) # # for x_ent in filtered_entities: # # Create a new row with the annotation # new_row = df.iloc[drm_idx].copy() # new_row['model'] = args.model_id # new_row['annotation'] = x_ent # df_annot = pd.concat([df_annot, new_row.to_frame().T], ignore_index=True) # Append the new row to the dataframe if strtobool(args.debug): endAnnotate = time.time() hours, rem = divmod(endAnnotate - startAnnotate, 3600) minutes, seconds = divmod(rem, 60) print("...end annotation - Time... {:0>2}:{:0>2}:{:05.2f}\n".format(int(hours), int(minutes), seconds)) print('\n') return df_annot ### def is_cross_inside(df_sorted, args, valuecutCross=0.75): df_sorted['IsCrossInside'] = 0 df_sorted = df_sorted.reset_index(drop=True) # I reset the index so that I can loop on inner loop inside # this works but it is absolutely too heavy!!! Got: mask1 = (SentenceRef_array[:, None] == SentenceRef_array) & \ # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ # numpy.core._exceptions._ArrayMemoryError: Unable to allocate 73.4 TiB for an array with shape (8984940, 8984940) and data type bool # # # Assuming df_sorted is your DataFrame # df_sorted['IsCrossInside'] = 0 # # # Convert necessary columns to numpy arrays # SentenceRef_array = df_sorted['SentenceRef'].to_numpy() # start_array = df_sorted['start'].to_numpy() # end_array = df_sorted['end'].to_numpy() # score_array = df_sorted['score'].to_numpy() # ToLink_array = df_sorted['ToLink'].to_numpy() # # # Identify NaN or empty strings in ToLink array # toLink_nan_or_empty = pd.isna(ToLink_array) | (ToLink_array == '') # # # Create the mask for the first set of conditions # mask1 = (SentenceRef_array[:, None] == SentenceRef_array) & \ # (start_array[:, None] <= start_array) & \ # (end_array[:, None] > start_array) & \ # (end_array[:, None] <= end_array) & \ # (score_array[:, None] < score_array) & \ # toLink_nan_or_empty[:, None] # # # Create the mask for the second set of conditions # mask2 = (SentenceRef_array[:, None] == SentenceRef_array) & \ # (end_array[:, None] >= end_array) & \ # (start_array[:, None] < end_array) & \ # (start_array[:, None] >= start_array) & \ # (score_array[:, None] < score_array) & \ # toLink_nan_or_empty[:, None] # # # Combine the masks # mask = mask1 | mask2 # # # Aggregate the masks and set 'IsCrossInside' where the condition is True # df_sorted.loc[mask.any(axis=1), 'IsCrossInside'] = 1 # df_sorted['IsCrossInside'] = 0 # # #questo for e' altamente inefficiente...guardare per alternative piu' ottimizzate!!! # for index, row in df_sorted.iterrows(): # # print(row) # # mask = (df_sorted['SentenceRef'] == row['SentenceRef']) & (df_sorted['start'] <= row['start']) & (df_sorted['end'] > row['start']) & ( # df_sorted['end'] <= row['end']) & (df_sorted['score'] < row['score']) & (df_sorted['ToLink'].isnull() | (df_sorted['ToLink'] == '')) # mask |= (df_sorted['SentenceRef'] == row['SentenceRef']) & (df_sorted['end'] >= row['end']) & (df_sorted['start'] < row['end']) & ( # df_sorted['start'] >= row['start']) & (df_sorted['score'] < row['score']) & (df_sorted['ToLink'].isnull() | (df_sorted['ToLink'] == '')) # # df_sorted.loc[mask, 'IsCrossInside'] = 1 # Iterate over the DataFrame with an outer loop - I know that df_sorted is ordered by 'SentenceRef' and 'start' indexes_list = [] scores_list = [] IsToLinkContained = [] for i, row_outer in tqdm(df_sorted.iterrows()): #print(row_outer) # if (i>0) and (df_sorted.iloc[i-1]['SentenceRef'] == row_outer['SentenceRef']): # if the previous row is of the same SenteceRef, it has been alredy computed and I skip # continue if len(indexes_list)==0: scores_list.append(row_outer['score']) indexes_list.append(i) if (pd.isnull(row_outer['ToLink']==False) and len(row_outer['ToLink'])>0): IsToLinkContained.append(True) else: IsToLinkContained.append(False) # if (pd.isnull(row_outer['ToLink']) or row_outer['ToLink'] == ''): # IsToLinkContained.append(False) # elif len(row_outer['ToLink'])>0: # IsToLinkContained.append(True) else: if i in indexes_list: if (i == indexes_list[-1]): # indexes_list = [] # scores_list = [] # IsToLinkContained = [] indexes_list = [indexes_list[-1]] scores_list = [scores_list[-1]] IsToLinkContained = [IsToLinkContained[-1]] else: continue # Inner loop only needs to consider rows starting from the current position for j in range(i + 1, len(df_sorted)): #for j in range(0, len(df_sorted)): # if i==j: continue #print(j) row_inner = df_sorted.iloc[j] # Break the inner loop if SentenceRef changes (due to sorting) if row_inner['SentenceRef'] != row_outer['SentenceRef']: break elif row_inner['start'] >= row_outer['end']: break else: scores_list.append(row_inner['score']) indexes_list.append(j) if (pd.isnull(row_inner['ToLink'] == False) and len(row_inner['ToLink']) > 0): IsToLinkContained.append(True) else: IsToLinkContained.append(False) if len(indexes_list)>1: first_true_index = -1 try: first_true_index = IsToLinkContained.index(True) #print(f"The first index with value True is: {first_true_index}") except ValueError: first_true_index = -1 #print("No True value found in the list") topinlist=-1 if first_true_index >=0: topinlist = first_true_index else: topinlist = scores_list.index(max(scores_list)) #print(f"The position with the maximum score is: {topinlist}") if topinlist >= 0: for xx in range(0, len(indexes_list)): if xx == topinlist: continue df_sorted.at[indexes_list[xx], 'IsCrossInside'] = 1 else: indexes_list = [] scores_list = [] IsToLinkContained = [] #Now I want to delete all the words that are contained within another . For example I want to drop "osis" from Legionellosis if not df_sorted.empty: # punctuation_and_space = set(string.punctuation + ' ') # condition = df_sorted.apply(lambda row: # (row['IsCrossInside'] == 0) or # ((row['IsCrossInside'] == 1) and # ((row[args.source_column][row['start'] - 1] in punctuation_and_space) if row[ # 'start'] - 1 >= 0 else True) and # ((row[args.source_column][row['end']] in punctuation_and_space) if row['end'] <= len( # row[args.source_column]) else True)), # axis=1) condition = df_sorted.apply(lambda row: (row['IsCrossInside'] == 0) or ( (row['IsCrossInside'] == 1) and (row['score'] >=valuecutCross )), axis=1) # Filter the DataFrame to keep only rows where the condition is False df_sorted = df_sorted[condition] return df_sorted def entitiesFusion(df_annotated, args): if strtobool(args.debug): print("\nStart entities fusion and filtering ...") areJson = df_annotated["annotation"].apply(is_json) if False in areJson.unique(): for idxr, rr in df_annotated.iterrows(): # for idxr in range(df_annotated["annotation"].shape[0]): if areJson[idxr] == False: print("PROBLEM WITH JSON AT INDEX " + str(idxr) + ":\n" + df_annotated["annotation"][idxr]) replacement_empty_myjson = '{\"entity_group\": \"\", \"score\": \"\", "word": \"\", \"start\": \"\", \"end\": \"\"}' # {'entity_group': 'ORG', 'score': 0.9999951, 'word': 'Health Ministry', 'start': 4, 'end': 19} df_annotated.at[idxr, "annotation"] = replacement_empty_myjson print(" ...... Then replacing it with empty JSON --> " + df_annotated["annotation"][idxr]) try: # df_extract = df_annotated.apply(lambda x: pd.Series( # json.loads(x['annotation'], default=default_serializer).values(), # index=json.loads(x['annotation'], default=default_serializer).keys()), axis=1) df_extract = df_annotated.apply(lambda x: pd.Series(x['annotation'].values(), index=x['annotation'].keys()), axis=1) # Check if '@id' column exists in df_Extract if '@id' in df_extract.columns: # Drop the '@id' column df_extract = df_extract.drop(columns='@id') df_annotated = pd.merge(df_annotated, df_extract, left_index=True, right_index=True) except Exception as err: logging.error( f'FAILED to extract json results\n\tError: {err}\nLeaving it as a single column then and not decompressing! Have a check...') # # Delete all the rows where EXACT MATCHING NOT MET: # Apply the conditions condition_to_delete = ( df_annotated[args.source_column].str.startswith('"') & df_annotated[args.source_column].str.endswith('"') & (df_annotated[args.source_column].apply(strip_quotes).str.lower() != df_annotated['word'].str.lower()) ) # Now Filter out the rows where condition_to_delete is True df_annotated = df_annotated[~condition_to_delete].copy() # #delete all the rows with score smaller than entities_filter_threshold: if args.entities_filter_threshold > 0: #df_annotated = df_annotated[df_annotated['score'] >= args.entities_filter_threshold] df_annotated = df_annotated[df_annotated['score'] > args.entities_filter_threshold] if df_annotated.empty: return df_annotated # #delete all the rows where the concept that was asked to link, in the column "ToLink", is present and it is different from the column "word", that was recognised # df_annotated = df_annotated[(df_annotated['ToLink'] == df_annotated['word']) | df_annotated['ToLink'].isna()] # in all the rows having a value not null for the column "ToLink", compare this value to that of the column "word". If they are different, set the value in "ToLink" to None # df_annotated.loc[ # (~df_annotated['ToLink'].isnull()) & (df_annotated['ToLink'] != df_annotated['word']), 'ToLink'] = None df_annotated.loc[ (~df_annotated['ToLink'].isnull()) & ( df_annotated['ToLink'].str.casefold() != df_annotated['word'].str.casefold()), 'ToLink'] = None # now fill all the values of the column "toLink" that are empty with the values of the row "word": # df_annotated['ToLink'] = df_annotated['ToLink'].fillna(df_annotated['word']) if "IsGeo" not in df_annotated.columns: #df_annotated["IsGeo"] = None # 0 df_annotated.loc[:, "IsGeo"] = None if "IsBio" not in df_annotated.columns: #df_annotated["IsBio"] = None # 0 df_annotated.loc[:, "IsBio"] = None df_annotated.loc[df_annotated['entity_group'] == 'LOCATION', 'entity_group'] = "LOC" df_annotated.loc[df_annotated['entity_group'] == 'LOC', 'IsGeo'] = 1 #df_annotated.loc[df_annotated['entity_group'] == 'DISEASE', 'IsBio'] = 1 df_annotated.loc[df_annotated['entity_group'].str.lower().str.contains('disease'), 'IsBio'] = 1 df_annotated.loc[(df_annotated['model'].str.contains('Medical-NER')) & ( df_annotated['entity_group'].isin(['LOC', 'DATE', 'PER', 'ORG', 'DOSAGE', 'LAB_VALUE', 'DURATION']) == False), 'IsBio'] = 1 df_annotated.loc[(df_annotated['model'].str.contains('NCBO')) & ( df_annotated['entity_group'].isin(['CONC']) == False), 'IsBio'] = 1 # !!! THIS CHECK HAS BEEN PLACED HERE BECAUSE NCBO Bioportal put Start+1 !!! SO I NEED TO DROP A ONE TO DROP DUPLICATES AND MAX SCORES df_annotated.loc[df_annotated['model'].str.lower().str.contains('ncbo'), 'start'] -= 1 # !!! THIS CHECK HAS BEEN PLACED HERE BECAUSE MEDICAL-NER PUT A VALUE OF START WHICH IS THE REAL ONE MINUS ONE, IN THE CASE THAT THE IDENTIFIED ENTITY STARTS WITH A SPACE!!! SO I NEED TO ADD A ONE TO DROP DUPLICATES AND MAX SCORES df_annotated.loc[(df_annotated['model'] == 'blaze999/Medical-NER') & df_annotated.apply(lambda row: row[args.source_column][row['start']] == ' ', axis=1), 'start'] += 1 # !!! THIS CHECK HAS BEEN PLACED HERE BECAUSE GLINER MODELS PUT A VALUE OF START and of END WHICH IS THE REAL ONE MINUS ONE, IN THE CASE THAT THE IDENTIFIED ENTITY STARTS WITH A SPACE!!! SO I NEED TO ADD A ONE to START and END TO DROP DUPLICATES AND MAX SCORES df_annotated.loc[df_annotated['model'].str.lower().str.contains('gliner') & df_annotated.apply(lambda row: row[args.source_column][row['start']] == ' ', axis=1), 'end'] += 1 df_annotated.loc[df_annotated['model'].str.lower().str.contains('gliner') & df_annotated.apply(lambda row: row[args.source_column][row['start']] == ' ', axis=1), 'start'] += 1 #### Here I drop all the identified NER words which are strictly contained in other words # Apply first the function to the "args.source_column" to create lists of words df_annotated['extracted_words'] = df_annotated[args.source_column].apply(extract_words,putInLower=True) # Filter the DataFrame df_annotated = df_annotated[df_annotated.apply(lambda row: all_words_in_list(row['word'], row['extracted_words'], putInLower=True), axis=1)] # drop the 'extracted_words' column afterwards: df_annotated = df_annotated.drop(columns=['extracted_words']) ##### # This operation sorts the DataFrame by the "ToLink" column in descending order, with null values at the end, and then uses the drop_duplicates method to drop all duplicate rows, # except the first one, based on all columns except "ToLink". # This way, it will keep the rows with the non-null value in "ToLink" if there are multiple rows with the same values in all columns except "ToLink". df_annotated = df_annotated.sort_values(by='ToLink', ascending=False, na_position='last') for col in df_annotated.columns: if df_annotated[col].apply(lambda x: isinstance(x, dict)).any(): if strtobool(args.debug): print( f"Column '{col}' contains dictionaries...converting it to strings otherwise it will not work the concat etc..") df_annotated[col] = df_annotated[col].apply(lambda x: str(x)) df_annotated = df_annotated.drop_duplicates(subset=[col for col in df_annotated.columns if col != 'ToLink' and col != 'ALLURIScontextFromNCBO' and not df_annotated[col].apply( lambda x: isinstance(x, dict)).any()], keep='first') # df_annotated = df_annotated.loc[df_annotated.groupby(['SentenceRef', 'ToLink', args.source_column, 'end', 'start', df_annotated['word'].str.lower()])['score'].idxmax()] # df_annotated = df_annotated.loc[df_annotated.groupby(['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower()])['score'].idxmax()] df_annotated_Geo = df_annotated.loc[df_annotated.groupby( ['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower(), 'IsGeo'])[ 'score'].idxmax()] df_annotated_Bio = df_annotated.loc[df_annotated.groupby( ['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower(), 'IsBio'])[ 'score'].idxmax()] df_annotated_all = df_annotated.loc[ df_annotated.groupby(['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower()])[ 'score'].idxmax()] # now you can concat df_annotated_combined = pd.concat([df_annotated_Geo, df_annotated_Bio, df_annotated_all]) df_annotated_combined = df_annotated_combined.drop_duplicates(subset=[col for col in df_annotated_combined.columns if col != 'ToLink' and col != 'ALLURIScontextFromNCBO' and not df_annotated_combined[col].apply( lambda x: isinstance(x, dict)).any()], keep='first') # df_annotated_combined['IsBioGeo'] = df_annotated_combined['IsGeo'].fillna(0) + df_annotated_combined['IsBio'].fillna(0) # df_annotated_combined['IsBioGeo'] = pd.to_numeric(df_annotated_combined['IsGeo'], errors='coerce').fillna(0) + pd.to_numeric(df_annotated_combined['IsBio'], errors='coerce').fillna(0) # df_annotated_combined['IsBioGeo'] = df_annotated_combined['IsGeo'].infer_objects(copy=False).fillna(0) + df_annotated_combined['IsBio'].infer_objects(copy=False).fillna(0) df_annotated_combined.loc[:, "IsBioGeo"] = df_annotated_combined.loc[:, 'IsGeo'].infer_objects(copy=False).fillna(0) + df_annotated_combined.loc[:, 'IsBio'].infer_objects(copy=False).fillna(0) df_annotated_combined = df_annotated_combined.loc[df_annotated_combined.groupby( ['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower()])['IsBioGeo'].idxmax()] df_annotated_combined = df_annotated_combined.loc[ df_annotated_combined.groupby( ['SentenceRef', args.source_column, 'end', 'start', df_annotated['word'].str.lower(), 'IsBioGeo'])[ 'score'].idxmax()] df_annotated_combined = df_annotated_combined.drop('IsBioGeo', axis=1) df_annotated_combined.loc[df_annotated_combined['IsBio'] == 0, 'IsBio'] = None df_annotated_combined.loc[df_annotated_combined['IsGeo'] == 0, 'IsGeo'] = None df_annotated_combined = df_annotated_combined.sort_values(by=['SentenceRef', 'start', 'ToLink', 'word', 'score'], ascending=[True, True, True, True, False]) #df_annotated_combined = df_annotated_combined.reindex(range(len(df_annotated_combined))) return df_annotated_combined def geonames_api_call(word,args, key_geonames,cache_map_geonames): context = "" singleContext = None globalContext = None singleTriples = None globalTriples = None if cache_map_geonames is not None: if word in cache_map_geonames: if context in cache_map_geonames[word]: url_text = cache_map_geonames[word][context] if strtobool(args.debug): print("RETRIEVED CACHED RESULT FOR:\n", word, " => ", url_text, "\n") return url_text, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames #url = f"http://api.geonames.org/search?name_equals={word}&maxRows=1&featureClass=A&featureCode=ADM1&featureCode=ADM2&featureCode=ADM3&featureCode=ADM4&featureCode=ADM5&type=json&username={key_geonames}" url = f"http://api.geonames.org/search?name_equals={word}&maxRows=1&type=json&username={key_geonames}" response = requests.get(url) try: data = response.json() if data['geonames']: # geoname = data['geonames'][0]['name'] geonameId = data['geonames'][0]['geonameId'] geonameUrl = "https://sws.geonames.org/" + str(geonameId) + "/" if cache_map_geonames is not None: if not word in cache_map_geonames: cache_map_geonames[word] = {} cache_map_geonames[word][context] = geonameUrl return geonameUrl, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames else: if cache_map_geonames is not None: if not word in cache_map_geonames: cache_map_geonames[word] = {} cache_map_geonames[word][context] = None return None, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames except Exception as err: # if cache_map_geonames is not None: # if not word in cache_map_geonames: # cache_map_geonames[word] = {} # cache_map_geonames[word][context] = None return None, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames def getLinearTextualContextFromTriples(word,labelTriplesLIST, text_splitter, args, map_query_input_output, cleanInput=True, questionText=""): # trial #return None, map_query_input_output word = word.lower() word = word.capitalize() labelTriples="" if labelTriplesLIST and getattr(args, 'maxTriplesContextComputation', None): # it means it exists if args.maxTriplesContextComputation > 0: if len(labelTriplesLIST) > args.maxTriplesContextComputation: labelTriplesLIST = labelTriplesLIST[:args.maxTriplesContextComputation] if (strtobool(args.UseRetrieverForContextCreation) == True): # if strtobool(args.debug): # print("Start reranking - num passages : ", len(labelTriplesLIST), "\n") # startRerank = time.time() # # labelTriples = "" # passages = [] # nn = 200 # # OverallListRAGtriples = [] # labelTriplesLIST_RAGGED = [] # # if len(labelTriplesLIST) <= nn: # passages = [] # for i, triple in enumerate(labelTriplesLIST, start=1): # # for triple in labelTriplesLIST: # TriplesString = (" ".join(str(element).capitalize() for element in triple)) # passages.append(TriplesString) # # df_retrieved = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20, # min_threshold=0.7) # # if not df_retrieved.empty: # # labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist() # labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist() # labelTriplesAPP = ". ".join( # " ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED) # # if not labelTriples: # labelTriples = labelTriplesAPP # else: # labelTriples = labelTriples + ". " + labelTriplesAPP # # else: # # OverallListRAGtriples = labelTriplesLIST.copy() # # while len(OverallListRAGtriples) > nn: # Oinnerlistiterative = [] # for i, triple in enumerate(OverallListRAGtriples, start=1): # # for triple in labelTriplesLIST: # TriplesString = (" ".join(str(element).capitalize() for element in triple)) # passages.append(TriplesString) # # Check if the current index is a multiple of nn # if i % nn == 0: # # print("elaborate RAG triples") # # # df_retrieved_Base = RAG_retrieval_Base(questionText, passages, min_threshold=0.7, max_num_passages=20) # # df_retrievedZscore = RAG_retrieval_Z_scores(questionText, passages, z_threshold=1.0, max_num_passages=20, min_threshold=0.7) # # df_retrievedPercentile = RAG_retrieval_Percentile(questionText, passages, percentile=90, max_num_passages=20, min_threshold=0.7) # df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, # max_num_passages=20, # min_threshold=0.7) # # passages = [] # # df_retrieved = df_retrievedtopk.copy() # if not df_retrieved.empty: # # labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist() # labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist() # if not Oinnerlistiterative: # Oinnerlistiterative = labelTriplesLIST_RAGGED # else: # Oinnerlistiterative.extend(labelTriplesLIST_RAGGED) # # if passages: # df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20, # min_threshold=0.7) # # df_retrieved = df_retrievedtopk.copy() # if not df_retrieved.empty: # # labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist() # labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist() # if not Oinnerlistiterative: # Oinnerlistiterative = labelTriplesLIST_RAGGED # else: # Oinnerlistiterative.extend(labelTriplesLIST_RAGGED) # # OverallListRAGtriples = Oinnerlistiterative.copy() # # if OverallListRAGtriples: # labelTriplesAPP = ". ".join( # " ".join(str(element).capitalize() for element in triple) for triple in OverallListRAGtriples) # # if not labelTriples: # labelTriples = labelTriplesAPP # else: # labelTriples = labelTriples + ". " + labelTriplesAPP # # labelTriples = labelTriples.strip().replace("..", ".").strip() # # # labelTriples = "" # # passages = [] # # nn=200 # # for i, triple in enumerate(labelTriplesLIST, start=1): # # #for triple in labelTriplesLIST: # # TriplesString = (" ".join(str(element).capitalize() for element in triple)) # # passages.append(TriplesString) # # # Check if the current index is a multiple of nn # # if i % nn == 0: # # #print("elaborate RAG triples") # # # # #df_retrieved_Base = RAG_retrieval_Base(questionText, passages, min_threshold=0.7, max_num_passages=20) # # #df_retrievedZscore = RAG_retrieval_Z_scores(questionText, passages, z_threshold=1.0, max_num_passages=20, min_threshold=0.7) # # #df_retrievedPercentile = RAG_retrieval_Percentile(questionText, passages, percentile=90, max_num_passages=20, min_threshold=0.7) # # df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20, min_threshold=0.7) # # # # passages = [] # # # # df_retrieved = df_retrievedtopk.copy() # # if not df_retrieved.empty: # # #labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist() # # labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist() # # labelTriplesAPP = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED) # # # # if not labelTriples: # # labelTriples =labelTriplesAPP # # else: # # labelTriples = labelTriples + ". " + labelTriplesAPP # # # # if passages: # # df_retrievedtopk = RAG_retrieval_TopK(questionText, passages, top_fraction=0.1, max_num_passages=20, min_threshold=0.7) # # # # df_retrieved = df_retrievedtopk.copy() # # if not df_retrieved.empty: # # #labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist() # # labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist() # # labelTriplesAPP = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED) # # if not labelTriples: # # labelTriples = labelTriplesAPP # # else: # # labelTriples = labelTriples + ". " + labelTriplesAPP # # # # if labelTriples: # # labelTriples.strip().replace("..",".").strip() # # if strtobool(args.debug): # numfinal = 0 # if OverallListRAGtriples: # numfinal = len(OverallListRAGtriples) # elif labelTriplesLIST_RAGGED: # numfinal = len(labelTriplesLIST_RAGGED) # print("End reranking - found final passages : ", numfinal, "\n") # # # endRerank = time.time() # hours, rem = divmod(endRerank - startRerank, 3600) # minutes, seconds = divmod(rem, 60) # print("Rerank Time... {:0>2}:{:0>2}:{:05.2f}\n".format(int(hours), int(minutes), seconds)) # # # if len(labelTriplesLIST) > 10000: # print("debug") if strtobool(args.debug): print("Start reranking2 - num passages : ", len(labelTriplesLIST), "\n") startRerank2 = time.time() labelTriples = "" try: passages = [] for i, triple in enumerate(labelTriplesLIST, start=1): # for triple in labelTriplesLIST: TriplesString = (" ".join(str(element).capitalize() for element in triple)) passages.append(TriplesString) nback = 1 if len(passages) <= 10: nback = len(passages) elif len(passages) <= 1000: nback = 10+int(0.1 * len(passages)) # 10% of the number of passages elif len(passages) <= 5000: nback = 200 elif len(passages) <= 10000: nback = 300 else: nback = 400 df_retrieved = RAG_retrieval_Base(questionText, passages, min_threshold=0, max_num_passages=nback) if not df_retrieved.empty: countRetr = 0 min_threshold = 0.80 countRetr = (df_retrieved['score'] > min_threshold).sum() countRetrThreshold = int(nback / 2) if nback > 10: countRetrThreshold = 10 else: countRetrThreshold = int(nback/2) if countRetrThreshold <=0: countRetrThreshold = 1 while countRetr <= countRetrThreshold: min_threshold = min_threshold - 0.05 countRetr = (df_retrieved['score'] >= min_threshold).sum() if min_threshold < 0.2: break # countRetrThreshold = int(0.1 + nback) # if countRetrThreshold > 5: # countRetrThreshold = 5 # # countRetr=0 # min_threshold = 0.90 # countRetr = (df_retrieved['score'] > min_threshold).sum() # while countRetr<=countRetrThreshold: # min_threshold = min_threshold - 0.05 # if min_threshold<0.7: # countRetrThreshold=0 # if min_threshold == 0: # min_threshold = 0.01 # countRetr = (df_retrieved['score'] > min_threshold).sum() # if min_threshold <= 0.01: # break if countRetr > 0: df_retrieved = df_retrieved[df_retrieved['score'] > min_threshold] # labelTriplesLIST_RAGGED = df_retrieved.to_records(index=False).tolist() labelTriplesLIST_RAGGED = df_retrieved['Passage'].apply(lambda x: (x,)).tolist() labelTriplesAPP = ". ".join( " ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST_RAGGED) if not labelTriples: labelTriples = labelTriplesAPP else: labelTriples = labelTriples + ". " + labelTriplesAPP else: labelTriplesLIST_RAGGED = [] labelTriples = "" if strtobool(args.debug): numfinal = 0 if labelTriplesLIST_RAGGED: numfinal = len(labelTriplesLIST_RAGGED) print("End reranking2 - found final passages : ", numfinal, "\n") endRerank2 = time.time() hours, rem = divmod(endRerank2 - startRerank2, 3600) minutes, seconds = divmod(rem, 60) print("Rerank2 Time... {:0>2}:{:0>2}:{:05.2f}\n".format(int(hours), int(minutes), seconds)) # except Exception as err: print("SOMETHING HAPPENED on PASSAGE RERANKING for Question :"+questionText+"\n") print(err) #status_code: 422, body: type='validation_error' url='https://www.mixedbread.ai/api-reference' message='Your request is invalid. Please check your input and try again.' details=[[{'type': 'too_long', 'loc': ['body', 'input', 'list[str]'], 'msg': 'List should have at most 1000 items after validation, not 4249', else: # NO RAG on triples labelTriples = ". ".join(" ".join(str(element).capitalize() for element in triple) for triple in labelTriplesLIST) if not(labelTriples) or labelTriples.strip=="": logging.warning("getLinearTextualContextFromTriples - No text or prompt supplied! No relevant contextual triples retrieved...Skypping it! Word: "+str(word)) return "", map_query_input_output if token_counter(labelTriples, args.model_name) > args.tokens_max: # THE CONTEXT IS TOO BIG, BIGGER THAN tokens_max, I need to split texts = text_splitter.create_documents([labelTriples]) labelTriples = texts[0].page_content if not (labelTriples) or labelTriples.strip == "": logging.warning("after splitting ...No text or prompt supplied! Skypping it! Word: "+str(word)) return "", map_query_input_output contextText = "" if (strtobool(args.UseRetrieverForContextCreation) == True): contextText = labelTriples else: #USE the LLM for summarise the triples # Can you elaborate and express better the following notes, delimited by triple backticks, about "{word}"? # Don't add explanations for your answer. Do not invent. Don't use a structure or indenting. Be concise. Don't discard relevant information. # made of RDF-like statements, # myPromt = f""" # Can you elaborate and express better the given notes below, delimited by triple backticks, about "{word}"? # Don't add explanations for your answer. # Do not invent. # Don't use a structure or indenting. # Be concise but exhaustive. Don't discard information reported in the notes. # """ myPromt = f""" Can you reformulate the following notes, provided between triple backticks, into clear and complete sentences about "{word}"? Ensure the rewriting is human-readable and easily interpretable. Maintain conciseness and exhaustiveness, including all information from the notes. Avoid using note formats or lists, and refrain from inventing additional information. """ myDelimiter = "```" if cleanInput==True: labelTriples = cleanInputText(labelTriples) # try to read cache if map_query_input_output is not None: key = args.model_name + "__" + str(args.temperature) + "__" + myPromt if key in map_query_input_output: if labelTriples in map_query_input_output[key]: output = map_query_input_output[key][labelTriples] # if input_text.strip() == "": # print("here") # if handler == api_call_dglc: # output = clean_gpt_out(output) #clean output if strtobool(args.debug): print("RETRIEVED CACHED RESULT FOR:\n", myPromt, "\n", myDelimiter, word, myDelimiter, "\n=>\n", output, "\n") return output, map_query_input_output # call try: contextText = "" if args.service_provider == "gptjrc": contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name, temperature=args.temperature, delimiter=myDelimiter, InContextExamples=[], handler=api_call_gptjrc, verbose=True, args=args) # elif args.service_provider == "HFonPremises": # contextText = call_model(input_text=labelTriples, prompt=myPromt, model=args.model_name, # temperature=args.temperature, delimiter=myDelimiter, # InContextExamples=[], # handler=api_call_HFonPremises, # verbose=True, args=args) if contextText: if not isinstance(contextText, str): contextText = contextText['choices'][0]['message']['content'] if map_query_input_output is not None: if not key in map_query_input_output: map_query_input_output[key] = {} if contextText: if contextText != "": map_query_input_output[key][labelTriples] = contextText except Exception as err: return None, map_query_input_output return contextText, map_query_input_output def getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso, endpoint, VirtuosoUsername, contextWordVirtuoso, UseBioportalForLinking=False, questionText="" ): #UseBioportalForLinking = False #trial to del if strtobool(args.debug): print("--- start getUrlBioAndAllOtherBioConcepts for " + word.lower()) entityBioeUrl = None ALLURIScontext = [] key_bioportal = "" if args.bioportalkey_filename and os.path.exists(args.bioportalkey_filename): fkeyname = args.bioportalkey_filename with open(fkeyname) as f: key_bioportal = f.read() else: key_bioportal = os.environ['key_bioportal'] # Check if args.KG_restriction exists and is not empty if getattr(args, 'KG_restriction', None): # api call if strtobool(args.debug): print("--- " + word.lower()) # args.KG_restriction exists and is not empty if strtobool(args.debug): print("KG_restriction is provided and not empty:", args.KG_restriction) from_clauses = ' '.join([f"FROM <{choice}>" for choice in args.KG_restriction]) # Construct the full SPARQL query query = f""" prefix skosxl: SELECT ?concept ?label (COUNT(?edge) AS ?score) {from_clauses} WHERE {{ ?concept skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?label . FILTER (LCASE(STR(?label)) = "{word.lower()}") ?concept ?edge ?o . }} GROUP BY ?concept ?label ORDER BY DESC(?score) """ ### this is for Bioportal url api: onto_clauses = "" # for choice in args.KG_restriction: # if choice == "SNOMEDCT": # choice = "SNOMED" # elif choice == "OBOREL": # choice = "RO" # elif choice == "PTRANS": # choice = "TRANS" # elif choice == "FOODON": # choice = "FoodOn" # elif choice == "GEOSPARQL": # choice = "GeoSPARQL" # elif choice == "NCBITAXON": # choice = "NCBITAXON,NCBITaxon_" # onto_clauses = onto_clauses + choice + "," for choice in args.KG_restriction: if choice == "SNOMED": choice="SNOMEDCT" elif choice == "RO": choice = "OBOREL" elif choice == "TRANS": choice = "PTRANS" elif choice == "FoodOn": choice = "FOODON" elif choice == "GeoSPARQL": choice = "GEOSPARQL" # elif choice == "NCBITAXON": # choice = "NCBITAXON,NCBITaxon_" elif choice == "NCBITaxon_": choice = "NCBITAXON" if choice in ONLY_Ontologies_OnBIOPORTAL: onto_clauses=onto_clauses+choice+"," if onto_clauses and onto_clauses[-1] == ",": onto_clauses = onto_clauses[:-1] url = f"https://services.data.bioontology.org/annotatorplus/?text={word.lower()}&ontologies={onto_clauses}&longest_only=true&exclude_numbers=true&whole_word_only=true&exclude_synonyms=false&negation=false&experiencer=false&temporality=false&score_threshold=0&confidence_threshold=0&display_links=false&display_context=false&score=cvalue&apikey={key_bioportal}" else: # args.KG_restriction does not exist or is empty kg_restriction = getattr(args, 'KG_restriction', None) if kg_restriction is not None and len(kg_restriction) == 0: print("KG_restriction is provided but empty") return None, None, cache_map_virtuoso if strtobool(args.debug): print("--- " + word.lower()) print("KG_restriction is not provided or empty - Consider all the KGs in the virtuoso endpoint") query = f""" prefix skosxl: SELECT ?concept ?label (COUNT(?edge) AS ?score) WHERE {{ ?concept skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?label . FILTER (LCASE(STR(?label)) = "{word.lower()}") ?concept ?edge ?o . }} GROUP BY ?concept ?label ORDER BY DESC(?score) """ ### this is for Bioportal url api: url = f"https://services.data.bioontology.org/annotatorplus/?text={word.lower()}&ontologies=AEO,BFO,BIM,BCGO,CL,CHIRO,CHEBI,DCM,FMA,GO,GENO,GEOSPARQL,HL7,DOID,HP,HP_O,IDO,IAO,ICD10,LOINC,MESH,MONDO,NCIT,NCBITAXON,NIFCELL,NIFSTD,GML,OBCS,OCHV,OHPI,OPB,PTRANS,PLOSTHES,RADLEX,OBOREL,STY,SO,SNOMEDCT,STATO,SYMP,FOODON,UBERON,VO&longest_only=true&exclude_numbers=true&whole_word_only=true&exclude_synonyms=false&negation=false&experiencer=false&temporality=false&score_threshold=0&confidence_threshold=0&display_links=false&display_context=false&score=cvalue&apikey={key_bioportal}" try: if UseBioportalForLinking == False: if strtobool(args.debug): print("Use Virtuoso Sparql endpoint for linking ... " + word.lower()) responseText = sparqlQuery(endpoint, questionText, VirtuosoUsername, key_virtuoso, strtobool(args.USE_CACHE)) # Parse the response as JSON results = json.loads(responseText) if len(results) > 0 and results['results']['bindings']: entityBioeUrl = str(results['results']['bindings'][0]['concept']['value']) if cache_map_virtuoso is not None: if not word in cache_map_virtuoso: cache_map_virtuoso[word] = {} cache_map_virtuoso[word][contextWordVirtuoso] = entityBioeUrl # # loop the results for result in results['results']['bindings']: # print(result) contextConcept = result['concept']['value'] if contextConcept not in ALLURIScontext: ALLURIScontext.append(contextConcept) if cache_map_virtuoso is not None: if not word in cache_map_virtuoso: cache_map_virtuoso[word] = {} cache_map_virtuoso[word][contextConcept] = None if ALLURIScontext and isinstance(ALLURIScontext, list): ALLURIScontext = list(set(ALLURIScontext)) if cache_map_virtuoso is not None: if not word in cache_map_virtuoso: cache_map_virtuoso[word] = {} cache_map_virtuoso[word]['ALLURIScontext'] = ALLURIScontext else: if cache_map_virtuoso is not None: if not word in cache_map_virtuoso: cache_map_virtuoso[word] = {} cache_map_virtuoso[word][contextWordVirtuoso] = None cache_map_virtuoso[word]['ALLURIScontext'] = [] else: #this is instead using Bioportal API for linking if strtobool(args.debug): print("Use Bioportal for linking ... " + word.lower()) response = requests.get(url) try: data = response.json() if not data: # nothing found from Bioportal #return None, None, None, None, None, cache_map_virtuoso return None, None, cache_map_virtuoso dff = pd.DataFrame(data) dff = dff.drop(columns=['hierarchy', 'mappings']) # If the columns are dictionary-like, use pd.json_normalize: expanded_annotated_class = pd.json_normalize(dff['annotatedClass']) expanded_annotations = pd.DataFrame(dff['annotations'].tolist(), index=dff.index) expanded_annotations = pd.json_normalize(expanded_annotations[0]) # Join the expanded columns back to the original DataFrame df_expanded = dff.drop(columns=['annotatedClass', 'annotations']).join(expanded_annotated_class).join( expanded_annotations) # Snomed id replacement because in our internal knolwedgebase we have this base uri df_expanded['@id'] = df_expanded['@id'].str.replace( "http://purl.bioontology.org/ontology/SNOMEDCT/", "http://snomed.info/id/" ) if not df_expanded.empty: df_expanded = df_expanded.sort_values( by=['from', 'text', 'score', 'matchType'], ascending=[True, True, False, False]) df_expanded = df_expanded.drop_duplicates(subset=['@id']) # Filter rows where 'prefLabel' is exactly equal to 'word.lower()' or 'word.lower()' is in 'synonym' # filtered_df = df_expanded[ # df_expanded['prefLabel'].str.lower() == word.lower() | # df_expanded['synonym'].apply( # lambda x: True if isinstance(x, list) and len(x) > 0 and word.lower() in [item.lower() for # item in x] else ( # np.nan if x is None or (isinstance(x, float) and pd.isna(x)) else x)) # ] if 'prefLabel' in df_expanded.columns: if 'synonym' in df_expanded.columns: df_expanded = df_expanded[ df_expanded['prefLabel'].apply( lambda x: isinstance(x, str) and x.lower() == word.lower() ) | df_expanded['synonym'].apply( lambda x: isinstance(x, list) and any(item.lower() == word.lower() for item in x) ) ] else: df_expanded = df_expanded[ df_expanded['prefLabel'].apply( lambda x: isinstance(x, str) and x.lower() == word.lower() ) ] if df_expanded.empty: # nothing found from Bioportal #return None, None, None, None, None, cache_map_virtuoso return None, None, cache_map_virtuoso # Specify the columns you want to keep columns_to_keep = ["score", "from", "to", "prefLabel", "text", "@id"] # Subset the dataframe to keep only the specified columns df_expanded = df_expanded[columns_to_keep] # Rename the specified columns df_expanded = df_expanded.rename( columns={"from": "start", "to": "end", "text": "word"}) # Optional: Reset index if you want a clean index after filtering df_expanded = df_expanded.reset_index(drop=True) df_expanded['score'] = df_expanded['score'].round(2) # Find the index of the row with the maximum 'score' max_score_index = df_expanded['score'].idxmax() max_score_row = df_expanded.loc[df_expanded['score'].idxmax()] entityBioeUrl = str(max_score_row['@id']) if cache_map_virtuoso is not None: if not word in cache_map_virtuoso: cache_map_virtuoso[word] = {} cache_map_virtuoso[word][contextWordVirtuoso] = entityBioeUrl # Drop the row with the maximum 'score' #df_expanded = df_expanded.drop(max_score_index) # Reset the index if desired (optional) df_expanded.reset_index(drop=True, inplace=True) # Iterate over each row in the DataFrame for index, row in df_expanded.iterrows(): # Append the '@id' value to the list if row['@id'] is not None and pd.notna(row['@id']): contextConcept=row['@id'] ALLURIScontext.append(contextConcept) if cache_map_virtuoso is not None: if not word in cache_map_virtuoso: cache_map_virtuoso[word] = {} cache_map_virtuoso[word][contextConcept] = None if ALLURIScontext and isinstance(ALLURIScontext, list): ALLURIScontext = list(set(ALLURIScontext)) if cache_map_virtuoso is not None: if not word in cache_map_virtuoso: cache_map_virtuoso[word] = {} cache_map_virtuoso[word]['ALLURIScontext'] = ALLURIScontext return entityBioeUrl, ALLURIScontext, cache_map_virtuoso else: #nothing found from Bioportal #return None, None, None, None, None, cache_map_virtuoso return None, None, cache_map_virtuoso except Exception as err: logging.error( f'ERROR ON BioPortal Annotator API Call\n\tError: {err}\n TextToAnnotate: {word.lower()}\n Have a check...') #return None, None, None, None, None, cache_map_virtuoso return None, None, cache_map_virtuoso except Exception as err: # if cache_map_virtuoso is not None: # if not word in cache_map_virtuoso: # cache_map_virtuoso[word] = {} # cache_map_virtuoso[word][contextWordVirtuoso] = None #return None, None, None, None, None, cache_map_virtuoso return None, None, cache_map_virtuoso return entityBioeUrl, ALLURIScontext, cache_map_virtuoso #@mem.cache def virtuoso_api_call(word, text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=None, iALLURIScontextFromNCBO=None,UseBioportalForLinking=True,questionText=""): if strtobool(args.debug): print(f"\n----- Starting virtuoso_api_call for {word}") word = word.lower() word = strip_quotes(word) endpoint = 'https://api-vast.jrc.service.ec.europa.eu/sparql' VirtuosoUsername = 'dba' if getattr(args, 'KG_restriction', None): contextWordVirtuoso = ', '.join(sorted(args.KG_restriction)) else: contextWordVirtuoso = "" singleContext = None globalContext = None sssingleTriples = None ggglobalTriples = None unique_listLabelTriples = [] unique_listGlobalTriples = [] ALLURIScontext = [] url_text = None if id: url_text = id if iALLURIScontextFromNCBO and isinstance(iALLURIScontextFromNCBO, list): ALLURIScontext=iALLURIScontextFromNCBO ALLURIScontext = list(set(ALLURIScontext)) if (cache_map_virtuoso is not None) and (not url_text): if word in cache_map_virtuoso: if contextWordVirtuoso in cache_map_virtuoso[word]: url_text = cache_map_virtuoso[word][contextWordVirtuoso] if strtobool(args.debug): print("RETRIEVED CACHED RESULT FOR:\n", word, " => ", url_text, "\n") if not url_text: return None, None, None, None, None, None, cache_map_virtuoso, load_map_query_input_output if url_text and not ALLURIScontext: if cache_map_virtuoso is not None: if word in cache_map_virtuoso: if 'ALLURIScontext' in cache_map_virtuoso[word]: ALLURIScontext = cache_map_virtuoso[word]['ALLURIScontext'] entityBioeUrl = None if url_text and ALLURIScontext: entityBioeUrl = url_text else: try: entityBioeUrl, ALLURIScontext, cache_map_virtuoso = getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso, endpoint, VirtuosoUsername, contextWordVirtuoso, UseBioportalForLinking=UseBioportalForLinking, questionText=questionText ) if ALLURIScontext and isinstance(ALLURIScontext, list): ALLURIScontext = list(set(ALLURIScontext)) except Exception as err: # if cache_map_virtuoso is not None: # if not word in cache_map_virtuoso: # cache_map_virtuoso[word] = {} # cache_map_virtuoso[word][contextWordVirtuoso] = None return None, None, None, None, None, None, cache_map_virtuoso, load_map_query_input_output if entityBioeUrl: if strtobool(args.computeEntityContext) and (strtobool(args.computeEntityGlobalContext)==False): if strtobool(args.debug): print("START computeEntityContext") unique_listLabelTriples = [] singleContext = None if cache_map_virtuoso is not None: if entityBioeUrl in cache_map_virtuoso: if "LabelTriples" in cache_map_virtuoso[entityBioeUrl]: unique_listLabelTriples = cache_map_virtuoso[entityBioeUrl]["LabelTriples"] if strtobool(args.debug): print("RETRIEVED CACHED RESULT FOR:\n", entityBioeUrl, " => ", "LabelTriples", "\n") if ("SingleContext" in cache_map_virtuoso[entityBioeUrl]) and (strtobool(args.UseRetrieverForContextCreation)==False): singleContext = cache_map_virtuoso[entityBioeUrl]["SingleContext"] if strtobool(args.debug): print("RETRIEVED CACHED RESULT FOR:\n", entityBioeUrl, " => ", "SingleContext", "\n") if not singleContext: if unique_listLabelTriples: singleContext, load_map_query_input_output = getLinearTextualContextFromTriples(word, unique_listLabelTriples, text_splitter, args, load_map_query_input_output,cleanInput=True,questionText=questionText) else: query = f""" prefix skosxl: SELECT DISTINCT ?labelS ?labelP ?labelO WHERE {{ {{ <{entityBioeUrl}> ?p ?o. <{entityBioeUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS . ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP . ?o skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelO . }} UNION {{ SELECT ?labelS ?labelP ?labelO WHERE {{ <{entityBioeUrl}> ?p ?labelO . <{entityBioeUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS . ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP . FILTER (isLiteral(?labelO)) }} }} UNION {{ SELECT DISTINCT ?labelS ?labelP ?labelO WHERE {{ <{entityBioeUrl}> ?ppp ?ooo . ?ooo rdf:type owl:Restriction . ?ooo owl:onProperty ?p . ?ooo owl:someValuesFrom ?o . <{entityBioeUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS . ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP . ?o skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelO . }} }} }} """ try: responseText = sparqlQuery(endpoint, query, VirtuosoUsername, key_virtuoso, strtobool(args.USE_CACHE)) # Parse the response as JSON results = json.loads(responseText) if len(results) > 0 and results['results']['bindings']: # word = "subClassOf" # split_word = split_camel_case(word) # # loop the results labelTriples="" listLabelTriples = [] pattern = r'\^\^' for result in results['results']['bindings']: #print(result) ss = str(result['labelS']['value']).strip().replace("..",".").replace("@en","") ss = re.sub(pattern, '', ss) pp = split_camel_case(str(result['labelP']['value'])).replace("_"," ").strip().replace("..",".").replace("@en","") pp = re.sub(pattern, '', pp) oo = str(result['labelO']['value']).replace("_"," ").strip().replace("..",".").replace("@en","") oo = re.sub(pattern, '', oo) listLabelTriples.append([ss, pp, oo]) # Remove duplicates while preserving order unique_listLabelTriples = list(dict.fromkeys(tuple(triple) for triple in listLabelTriples)) # # If you need the result back in list of lists format # unique_listLabelTriples = [list(triple) for triple in unique_listLabelTriples] if unique_listLabelTriples: if cache_map_virtuoso is not None: if not entityBioeUrl in cache_map_virtuoso: cache_map_virtuoso[entityBioeUrl] = {} cache_map_virtuoso[entityBioeUrl]["LabelTriples"] = unique_listLabelTriples singleContext, load_map_query_input_output = getLinearTextualContextFromTriples(word, unique_listLabelTriples, text_splitter, args, load_map_query_input_output,cleanInput=True,questionText=questionText) except Exception as err: singleContext = None if singleContext and (strtobool(args.UseRetrieverForContextCreation)==False): if cache_map_virtuoso is not None: if not entityBioeUrl in cache_map_virtuoso: cache_map_virtuoso[entityBioeUrl] = {} cache_map_virtuoso[entityBioeUrl]["SingleContext"] = singleContext if strtobool(args.computeEntityGlobalContext): if strtobool(args.debug): print("START computeEntityGlobalContext") unique_listGlobalTriples = [] globalContext = None if cache_map_virtuoso is not None: if word in cache_map_virtuoso: if ("GlobalTriples"+" "+contextWordVirtuoso).strip() in cache_map_virtuoso[word]: unique_listGlobalTriples = cache_map_virtuoso[word][("GlobalTriples"+" "+contextWordVirtuoso).strip()] if strtobool(args.debug): print("RETRIEVED CACHED RESULT FOR:\n", word, " => ", ("GlobalTriples"+" "+contextWordVirtuoso).strip(), "\n") if (("GlobalContext"+" "+contextWordVirtuoso).strip() in cache_map_virtuoso[word]) and (strtobool(args.UseRetrieverForContextCreation)==False): globalContext = cache_map_virtuoso[word][("GlobalContext"+" "+contextWordVirtuoso).strip()] if strtobool(args.debug): print("RETRIEVED CACHED RESULT FOR:\n", word, " => ", ("GlobalContext"+" "+contextWordVirtuoso).strip(), "\n") if not globalContext: BreakenBeforeAll = False if unique_listGlobalTriples: globalContext, load_map_query_input_output = getLinearTextualContextFromTriples(word, unique_listGlobalTriples, text_splitter, args, load_map_query_input_output,cleanInput=True,questionText=questionText) else: if not ALLURIScontext: if cache_map_virtuoso is not None: if word in cache_map_virtuoso: ALLURIScontext = list(cache_map_virtuoso[word].keys()) ALLURIScontext = [element for element in ALLURIScontext if element and ("GlobalTriples" in element == False) and ("GlobalContext" in element == False) and "http" in element ] if not ALLURIScontext: # THIS CASE SHOULD BE VERY DIFFICULT TO HAPPEN...IT WILL HAPPEN IN CASE IT COMES FROM AN INITIAL BIOPORTAL ANNOTATION; WHICH ALREADY RECOGNISED THE FIRST URL # # Print the error message to stderr # print("THIS CASE SHOULD NEVER HAPPEN!!!! Check what's happening...exiting now...") # # Exit the program with a non-zero status code (commonly used to indicate an error) # sys.exit(1) try: entityBioeUrl, ALLURIScontext, cache_map_virtuoso = getUrlBioAndAllOtherBioConcepts(word, args, key_virtuoso, cache_map_virtuoso, endpoint, VirtuosoUsername, contextWordVirtuoso, UseBioportalForLinking=UseBioportalForLinking, questionText=questionText) if ALLURIScontext and isinstance(ALLURIScontext, list): ALLURIScontext = list(set(ALLURIScontext)) except Exception as err: # if cache_map_virtuoso is not None: # if not word in cache_map_virtuoso: # cache_map_virtuoso[word] = {} # cache_map_virtuoso[word][contextWordVirtuoso] = None return None, None, None, None, None, None, cache_map_virtuoso, load_map_query_input_output if not ALLURIScontext: # Print the error message to stderr print("THIS CASE SHOULD NEVER HAPPEN NOW!!!! Check what's happening...exiting now...") # Exit the program with a non-zero status code (commonly used to indicate an error) sys.exit(1) else: for xxUrl in ALLURIScontext: unique_listLabelTriples = [] # singleContext = None if cache_map_virtuoso is not None: if xxUrl in cache_map_virtuoso: if "LabelTriples" in cache_map_virtuoso[xxUrl]: unique_listLabelTriples = cache_map_virtuoso[xxUrl]["LabelTriples"] if strtobool(args.debug): print("RETRIEVED CACHED RESULT FOR:\n", xxUrl, " => ", "LabelTriples", "\n") # if "SingleContext" in cache_map_virtuoso[xxUrl] and (strtobool(args.UseRetrieverForContextCreation)==False): # singleContext = cache_map_virtuoso[xxUrl]["SingleContext"] # if strtobool(args.debug): # print("RETRIEVED CACHED RESULT FOR:\n", xxUrl, " => ", # "SingleContext", "\n") # if not singleContext: # if unique_listLabelTriples: # singleContext, load_map_query_input_output = getLinearTextualContextFromTriples( # word, unique_listLabelTriples, # text_splitter, args, # load_map_query_input_output, cleanInput=True, questionText=questionText) # else: if not unique_listLabelTriples: query = f""" prefix skosxl: SELECT DISTINCT ?labelS ?labelP ?labelO WHERE {{ {{ <{xxUrl}> ?p ?o. <{xxUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS . ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP . ?o skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelO . }} UNION {{ SELECT ?labelS ?labelP ?labelO WHERE {{ <{xxUrl}> ?p ?labelO . <{xxUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS . ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP . FILTER (isLiteral(?labelO)) }} }} UNION {{ SELECT DISTINCT ?labelS ?labelP ?labelO WHERE {{ <{xxUrl}> ?ppp ?ooo . ?ooo rdf:type owl:Restriction . ?ooo owl:onProperty ?p . ?ooo owl:someValuesFrom ?o . <{xxUrl}> skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelS . ?p skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelP . ?o skos:prefLabel|rdfs:label|skos:altLabel|skosxl:literalForm|obo:hasRelatedSynonym ?labelO . }} }} }} """ try: responseText = sparqlQuery(endpoint, query, VirtuosoUsername, key_virtuoso, strtobool(args.USE_CACHE)) # Parse the response as JSON results = json.loads(responseText) if len(results) > 0 and results['results']['bindings']: # word = "subClassOf" # split_word = split_camel_case(word) # # loop the results labelTriples = "" listLabelTriples = [] pattern = r'\^\^' for result in results['results']['bindings']: # print(result) ss = str(result['labelS']['value']).strip().replace("..", ".").replace("@en","") ss = re.sub(pattern, '', ss) pp = split_camel_case(str(result['labelP']['value'])).replace("_"," ").strip().replace("..", ".").replace("@en","") pp = re.sub(pattern, '', pp) oo = str(result['labelO']['value']).replace("_"," ").strip().replace("..", ".").replace("@en","") oo = re.sub(pattern, '', oo) listLabelTriples.append([ss, pp, oo]) # Remove duplicates while preserving order unique_listLabelTriples = list( dict.fromkeys(tuple(triple) for triple in listLabelTriples)) # # If you need the result back in list of lists format # unique_listLabelTriples = [list(triple) for triple in unique_listLabelTriples] if unique_listLabelTriples: if cache_map_virtuoso is not None: if not xxUrl in cache_map_virtuoso: cache_map_virtuoso[xxUrl] = {} cache_map_virtuoso[xxUrl][ "LabelTriples"] = unique_listLabelTriples # singleContext, load_map_query_input_output = getLinearTextualContextFromTriples( # word, unique_listLabelTriples, text_splitter, args, load_map_query_input_output, cleanInput=True, questionText=questionText) # # if singleContext and (strtobool(args.UseRetrieverForContextCreation)==False): # if cache_map_virtuoso is not None: # if not xxUrl in cache_map_virtuoso: # cache_map_virtuoso[xxUrl] = {} # cache_map_virtuoso[xxUrl][ # "SingleContext"] = singleContext except Exception as err: unique_listLabelTriples = [] if unique_listLabelTriples: unique_listGlobalTriples.extend(unique_listLabelTriples) # This is to speed up, so I break here the global, but in this case I will not store the triples for the other uris in the cache, which maybe useful in the future # #if token_counter(str(unique_listGlobalTriples),args.model_name) > args.tokens_max: if getattr(args, 'maxTriplesContextComputation', None): #it means it exists if args.maxTriplesContextComputation > 0: if len(unique_listGlobalTriples) > args.maxTriplesContextComputation: unique_listGlobalTriples = unique_listGlobalTriples[:args.maxTriplesContextComputation] BreakenBeforeAll = True break # BREAK THE FOR LOOP IF THE GLOBAL CONTEXT IS ALREADY TOO BIG, BIGGER THAN tokens_max if unique_listGlobalTriples: # Remove duplicates while preserving order unique_listGlobalTriples = list( dict.fromkeys(tuple(triple) for triple in unique_listGlobalTriples)) if cache_map_virtuoso is not None: if not word in cache_map_virtuoso: cache_map_virtuoso[word] = {} if BreakenBeforeAll == False: cache_map_virtuoso[word][("GlobalTriples"+" "+contextWordVirtuoso).strip()] = unique_listGlobalTriples globalContext, load_map_query_input_output = getLinearTextualContextFromTriples(word, unique_listGlobalTriples, text_splitter, args, load_map_query_input_output, cleanInput=True, questionText=questionText) if globalContext and (strtobool(args.UseRetrieverForContextCreation)==False): if cache_map_virtuoso is not None: if not word in cache_map_virtuoso: cache_map_virtuoso[word] = {} if BreakenBeforeAll == False: cache_map_virtuoso[word][("GlobalContext"+" "+contextWordVirtuoso).strip()] = globalContext if unique_listLabelTriples: sssingleTriples = " ,., ".join( " ,,, ".join(str(element).capitalize() for element in triple) for triple in unique_listLabelTriples) while "\\n" in sssingleTriples: sssingleTriples = sssingleTriples.replace("\\n", " ") sssingleTriples = sssingleTriples.strip() while "\t" in sssingleTriples: sssingleTriples = sssingleTriples.replace("\t", " ") sssingleTriples = sssingleTriples.strip() if unique_listGlobalTriples: ggglobalTriples = " ,., ".join( " ,,, ".join(str(element).capitalize() for element in triple) for triple in unique_listGlobalTriples) while "\\n" in ggglobalTriples: ggglobalTriples = ggglobalTriples.replace("\\n", " ") ggglobalTriples = ggglobalTriples.strip() while "\t" in ggglobalTriples: ggglobalTriples = ggglobalTriples.replace("\t", " ") ggglobalTriples = ggglobalTriples.strip() return entityBioeUrl, ALLURIScontext, singleContext, globalContext, sssingleTriples, ggglobalTriples, cache_map_virtuoso, load_map_query_input_output # def process_geo_row(row, key_geonames, cache_map_geonames): # if row['IsGeo'] == 1: # result, cache_map_geonames = geonames_api_call(row['word'], key_geonames, cache_map_geonames) # return result, cache_map_geonames # else: # return None, cache_map_geonames def process_row4Linking(row, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output): result = "" #None singleContext = "" #None globalContext = "" #None singleTriples = "" #None globalTriples = "" #None ALLURIScontext = [] try: if row.empty: return result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames, cache_map_virtuoso, load_map_query_input_output, row.name InRagMode=False if hasattr(args, 'useBioKgRAG') and (strtobool(args.useBioKgRAG)==True): InRagMode = True if (InRagMode==False): if row['IsGeo'] == 1: if strtobool(args.debug): print(f"\n----- IsGeo ... COMPUTING {row['word']} IN THE TEXT:") print(row[args.source_column]) result, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames = geonames_api_call(row['word'], args, key_geonames, cache_map_geonames) elif row['IsBio'] == 1: # Check if '@id' column exists in df_Extract iiid = None # Check if the '@id' exists in the Series if '@id' in row: # Check if the value is not None or NaN if row['@id'] is not None and not pd.isna(row['@id']): # Assign the value to the variable iiid iiid = row['@id'] iiiALLURIScontextFromNCBO = None if 'ALLURIScontextFromNCBO' in row: if row['ALLURIScontextFromNCBO'] is not None and isinstance(row['ALLURIScontextFromNCBO'], list): #and not pd.isna(row['ALLURIScontextFromNCBO']): iiiALLURIScontextFromNCBO=row['ALLURIScontextFromNCBO'] iiiALLURIScontextFromNCBO = list(set(iiiALLURIScontextFromNCBO)) if strtobool(args.debug): print(f"\n----- isBio COMPUTING ... {row['word']} IN THE TEXT:") print(row[args.source_column]) result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call(row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO, UseBioportalForLinking=True, questionText=row[args.source_column]) else: if row['model'] == "Forced": # Check if '@id' column exists in df_Extract iiid = None # Check if the '@id' exists in the Series if '@id' in row: # Check if the value is not None or NaN if row['@id'] is not None and not pd.isna(row['@id']): # Assign the value to the variable iiid iiid = row['@id'] iiiALLURIScontextFromNCBO = None if 'ALLURIScontextFromNCBO' in row: if row['ALLURIScontextFromNCBO'] is not None and isinstance(row['ALLURIScontextFromNCBO'], list): # and not pd.isna(row['ALLURIScontextFromNCBO']): iiiALLURIScontextFromNCBO = row['ALLURIScontextFromNCBO'] iiiALLURIScontextFromNCBO = list(set(iiiALLURIScontextFromNCBO)) if strtobool(args.debug): print(f"\n----- isForced COMPUTING ... {row['word']} IN THE TEXT:") print(row[args.source_column]) result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call( row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO,UseBioportalForLinking=True,questionText=row[args.source_column]) if not result: #try annotation without bioportal result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call( row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO, UseBioportalForLinking=False,questionText=row[args.source_column]) else: # this case is only considered if it is not IsBio or IsGeo or RagMode, etcc..but there is already an annoptation for the NEL coming from NCBO if '@id' in row: # Check if the value is not None or NaN if row['@id'] is not None and not pd.isna(row['@id']): # Assign the value to the variable iiid iiid = row['@id'] iiiALLURIScontextFromNCBO = None if 'ALLURIScontextFromNCBO' in row: if row['ALLURIScontextFromNCBO'] is not None and isinstance( row['ALLURIScontextFromNCBO'], list): # and not pd.isna(row['ALLURIScontextFromNCBO']): iiiALLURIScontextFromNCBO = row['ALLURIScontextFromNCBO'] iiiALLURIScontextFromNCBO = list(set(iiiALLURIScontextFromNCBO)) if strtobool(args.debug): print(f"\n----- It is not IsBio or IsGeo, but it has id from NCBO ...forcing COMPUTING ... {row['word']} IN THE TEXT:") print(row[args.source_column]) result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call( row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO, UseBioportalForLinking=True, questionText=row[args.source_column]) else: if (row['IsBio'] == 1) or ( (pd.isnull(row["IsBio"]) or row["IsBio"] == '' or row['IsBio'] == 0 or row["IsBio"] is None) and (row['entity_group'] == "MISC") ): if strtobool(args.debug): print(f"\n----- InRagMode ...COMPUTING ... {row['word']} IN THE TEXT:") print(row[args.source_column]) # Check if '@id' column exists in df_Extract iiid = None # Check if the '@id' exists in the Series if '@id' in row: # Check if the value is not None or NaN if row['@id'] is not None and not pd.isna(row['@id']): # Assign the value to the variable iiid iiid = row['@id'] iiiALLURIScontextFromNCBO = None if 'ALLURIScontextFromNCBO' in row: if row['ALLURIScontextFromNCBO'] is not None and isinstance(row['ALLURIScontextFromNCBO'], list): iiiALLURIScontextFromNCBO = row['ALLURIScontextFromNCBO'] iiiALLURIScontextFromNCBO = list(set(iiiALLURIScontextFromNCBO)) result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_virtuoso, load_map_query_input_output = virtuoso_api_call( row['word'], text_splitter, args, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, id=iiid, iALLURIScontextFromNCBO=iiiALLURIScontextFromNCBO,UseBioportalForLinking=True,questionText=row[args.source_column]) return result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames, cache_map_virtuoso, load_map_query_input_output, row.name except Exception as e: #print(f"Error occurred: {e}") return result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames, cache_map_virtuoso, load_map_query_input_output, row.name def parallel_process_Row4Linking(df, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output): results = [] with ThreadPoolExecutor(max_workers=args.num_cores_eLinking) as executor: # Submit tasks to ThreadPoolExecutor futures = [executor.submit(process_row4Linking, row, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output) for _, row in df.iterrows()] # Collect results for future in as_completed(futures): try: result, ALLURIScontext, singleContext, globalContext, singleTriples, globalTriples, cache_map_geonames_Inner, cache_map_virtuoso_Inner, load_map_query_input_output_Inner, drm_idx = future.result() df.at[drm_idx,'namedEntity'] = result df.at[drm_idx, 'ALLURIScontext'] = ALLURIScontext df.at[drm_idx,'Context'] = singleContext df.at[drm_idx,'ContextGlobal'] = globalContext df.at[drm_idx, 'Triples'] = singleTriples df.at[drm_idx, 'TriplesGlobal'] = globalTriples # Recursively update cache_map files with values from _Inner --> actually, this will never happen, because when you pass a mutable object like a dictionary to a function, # changes made to that object within the function are reflected outside the function as well. This is because the function receives a reference to the original object, # rather than a copy of it. # cache_map_geonames, cache_map_virtuoso, load_map_query_input_output are like global variables, which is very good for the parallelization! if (cache_map_geonames == cache_map_geonames_Inner)==False: update_nested_dict(cache_map_geonames, cache_map_geonames_Inner) if (cache_map_virtuoso == cache_map_virtuoso_Inner) == False: update_nested_dict(cache_map_virtuoso, cache_map_virtuoso_Inner) if (load_map_query_input_output == load_map_query_input_output_Inner) == False: update_nested_dict(load_map_query_input_output, load_map_query_input_output_Inner) except Exception as e: print(f"Error occurred: {e}") return df, cache_map_geonames, cache_map_virtuoso, load_map_query_input_output def elinking(df_annotated_combined, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, device): if "ALLURIScontext" not in df_annotated_combined.columns: df_annotated_combined["ALLURIScontext"] = None if args.num_cores_eLinking>1: # parallel elinking process #result df_annotated_combined, cache_map_geonames_AFTER, cache_map_virtuoso_AFTER, load_map_query_input_output_AFTER = parallel_process_Row4Linking(df_annotated_combined, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output) #if isinstance(result, list): # result=pd.Series(result) # I need this after the parallel processing else: # single processing result = df_annotated_combined.apply(lambda row: process_row4Linking(row, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output), axis=1) # try: df_annotated_combined['namedEntity'] = result.str[0] df_annotated_combined['ALLURIScontext'] = result.str[1] df_annotated_combined['Context'] = result.str[2] df_annotated_combined['ContextGlobal'] = result.str[3] df_annotated_combined['Triples'] = result.str[4] df_annotated_combined['TriplesGlobal'] = result.str[5] cache_map_geonames_AFTER = result.str[6].iloc[-1] cache_map_virtuoso_AFTER = result.str[7].iloc[-1] load_map_query_input_output_AFTER = result.str[8].iloc[-1] # except Exception as e: # print(f"Error occurred: {e}") df_annotated_combined['namedEntity'] = "" df_annotated_combined['ALLURIScontext'] = "" df_annotated_combined['Context'] = "" df_annotated_combined['ContextGlobal'] = "" df_annotated_combined['Triples'] = "" df_annotated_combined['TriplesGlobal'] = "" cache_map_geonames_AFTER = cache_map_geonames cache_map_virtuoso_AFTER = cache_map_virtuoso load_map_query_input_output_AFTER = load_map_query_input_output def fill_alluriscontext(row): if not row['ALLURIScontext'] and pd.notnull(row['namedEntity']): return [row['namedEntity']] return row['ALLURIScontext'] # Apply the function to the DataFrame df_annotated_combined['ALLURIScontext'] = df_annotated_combined.apply(fill_alluriscontext, axis=1) if args.num_cores_eLinking>1: # if parallel elinking process I need to sort again, because rows in late will arrive later df_annotated_combined = df_annotated_combined.sort_values(by=['SentenceRef', 'start', 'ToLink', 'word', 'score'], ascending=[True, True, True, True, False]) return df_annotated_combined, cache_map_geonames_AFTER, cache_map_virtuoso_AFTER, load_map_query_input_output_AFTER if __name__ == '__main__': # df_rrr = pd.read_csv("ToAnnotateTrial2_OutputAnnotated-withTriples.csv", sep=',', header=0, dtype=str, encoding='utf-8') # print("try") # accelerator = Accelerator() # # accelerator = Accelerator(mixed_precision='fp16', device_placement=False) #fp16 bf16 # device = accelerator.device # print("Using accelerator device = " + str(device)) start = time.time() encod = encoding_getter('microsoft/deberta-v3-large') text_splitter = TokenTextSplitter( # separators=separators, encoding_name=encod.name, chunk_size=80000, chunk_overlap=50, length_function=len, add_start_index=True, ) #models_List = ["FacebookAI/xlm-roberta-large-finetuned-conll03-english", "Babelscape/wikineural-multilingual-ner", "blaze999/Medical-NER", "urchade/gliner_large-v2.1", "urchade/gliner_large_bio-v0.1", "NCBO/BioPortal" ] #"urchade/gliner_large-v2.1", "knowledgator/gliner-multitask-large-v0.5" #models_List = ["urchade/gliner_large_bio-v0.1"] #models_List = [ "Babelscape/wikineural-multilingual-ner", "urchade/gliner_large-v2.1", "NCBO/BioPortal"] models_List = ["NCBO/BioPortal"] # Babelscape/wikineural-multilingual-ner # https://huggingface.co/Babelscape/wikineural-multilingual-ner? # https://aclanthology.org/2021.findings-emnlp.215.pdf # PERSON → PER, ORG → ORG, GPE → LOC, # LOC → LOC, FAC → LOC, PRODUCT → MISC, # EVENT → MISC, WORK_OF_ART → MISC, # LAW → O, LANGUAGE → MISC, NORP → MISC, # DATA → O, TIME → O, PERCENT → O, # MONEY → O, QUANTITY → O, ORDINAL → O, # CARDINAL → O # --> "LOC" "MISC" # blaze999/Medical-NER # https://huggingface.co/blaze999/Medical-NER # 41 recognised entities: https://huggingface.co/blaze999/Medical-NER/blob/main/config.json # # FacebookAI/xlm-roberta-large-finetuned-conll03-english # https://huggingface.co/FacebookAI/xlm-roberta-large-finetuned-conll03-english # urchade/gliner_large-v2.1 # https://huggingface.co/urchade/gliner_large-v2.1 # # urchade/gliner_large_bio-v0.1 # https://huggingface.co/urchade/gliner_large_bio-v0.1 # model = GLiNER.from_pretrained("urchade/gliner_large-v2.1") # model = GLiNER.from_pretrained("urchade/gliner_large_bio-v0.1") # # text = """ # The doctor diagnosed the patient with basal cell carcinoma, a common type of skin cancer. The Health Ministry has detected about 4000 suspected chikungunya cases nationwide this year [2008], Minister Datuk Liow Tiong Lai said Wednesday [17 Dec 2008]. # He said the disease which was 1st detected in Johor had spread to Negeri Sembilan, Melaka, Perak, Selangor and the latest Kedah. He said the disease was 1st detected to have spread to Malaysia from Africa in 1997. # """ # # glinerlabels = ["location", "disease"] # # entities = model.predict_entities(text, glinerlabels) # # for entity in entities: # print(entity["text"], "=>", entity["label"]) df_annotated = pd.DataFrame() for model_id in models_List: parser = argparse.ArgumentParser() parser.add_argument("--model_id", type=str, default=model_id, help="model to use") parser.add_argument("--debug", type=str, default="True", help="set debug mode") # parser.add_argument("--input_dir", type=str, default="/eos/jeodpp/data/products/JRC-OpenData/DIGLIFE/ner2/") #/eos/jeodpp/home/users/consose/PycharmProjects/ner-virtuoso/ # parser.add_argument("--filename", type=str, default="OutputAnnotatedTexts-LLMs-ENSEMBLE.csv") #ToAnnotateTrial.csv # parser.add_argument("--source_column", type=str, default="texts") # # parser.add_argument("--input_dir", type=str, default="/eos/jeodpp/home/users/consose/PycharmProjects/ner-virtuoso/") # parser.add_argument("--filename", type=str, default="ToAnnotateTrial2.csv") #ToAnnotateTrial.csv #ToAnnotateTrial2.csv #ToAnnotateTrialGliner.csv #ToAnnotateTrial3.csv parser.add_argument("--source_column", type=str, default="ContextToAnnotate") # "" parser.add_argument("--entities_filter_threshold", type=int, default=0.7) parser.add_argument("--SEED", type=int, default=41) parser.add_argument("--batch_size", type=int, default=4) # 2 - 4 - 8 - 16 parser.add_argument("--num_cores_Gliner", type=int, default=0, help="parallel processing for Gliner annotation") # 0 means use the GPU for Gliner ! parser.add_argument("--entity_linking", type=str, default="True", help="whether to make entities linking or not") parser.add_argument("--geonameskey_filename", type=str, default="GEONAMES-API.key", help="file location where it is stored the geonames api key") parser.add_argument("--virtuosokey_filename", type=str, default="VIRTUOSO-dba.key", help="file location where it is stored the virtuoso endpoint dba pwd") parser.add_argument("--bioportalkey_filename", type=str, default="NCBO-BioPortal.key", help="file location where it is stored the NCBO BioPortal api key") # Add the --KG_restriction argument # Since nargs='+' is used, this will accept one or more values from the command line KGchoices = POSSIBLE_KGchoices_List # KGchoices = ['SNOMED', 'LOINC', 'ICD10', 'NCIT'] # KGchoices = ['SNOMED', 'LOINC', 'ICD10', 'MESH', 'NCIT'] # restricts the input to these values only if KGchoices: KGchoices.sort() parser.add_argument("--KG_restriction", nargs='+', choices=KGchoices, default=KGchoices, help="List of ontologies to which restrict the entity linking task.") # consose 20250502: if Counter(KGchoices) == Counter(POSSIBLE_KGchoices_List): parser.add_argument("--USE_CACHE", type=str, default="True", #True help="whether to use cache for the NER and NEL tasks or not") else: # print("Lists do not have the same elements") parser.add_argument("--USE_CACHE", type=str, default="False", help="whether to use cache for the NER and NEL tasks or not") parser.add_argument("--num_cores_eLinking", type=int, default=10, help="parallel processing for the entity linking process") parser.add_argument("--computeEntityContext", type=str, default="False", help="whether to extract a readable context from the extracted triples for the concept") parser.add_argument("--computeEntityGlobalContext", type=str, default="False", help="whether to extract a readable context from the extracted triples of all the entities extracted from the endpoint for the concept") parser.add_argument("--maxTriplesContextComputation", type=int, default=20000, help="maximum number of triples to consider for global context computation") # if 0 or None it is not considered parser.add_argument("--UseRetrieverForContextCreation", type=str, default="True", help="whether to use a retriever for the creation of the context of the entities from the triples coming from the KGs") parser.add_argument("--service_provider", type=str, default="gptjrc", help="llm service provider") # "gptjrc", "HFonPremises" parser.add_argument("--model_name", type=str, default="llama-3.1-70b-instruct-fp8", help="llm to use") # llama-3.1-70b-instruct "llama-3.1-70b-instruct" "llama-3.1-70b-instruct-fp8" parser.add_argument("--tokens_max", type=int, default=80000, help="max number of tokens to supply to the llm") parser.add_argument("--max_new_tokens", type=int, default=4096, help="max number of tokens for the output of the LLM in premises") parser.add_argument("--temperature", type=int, default=0.01) args = parser.parse_args() print("ARGS:") print(args) # %% n machine learning tasks, particularly when dealing with models that have stochasticity involved (like text generation), it's important to set seeds for random number generators to ensure reproducibility of results. In the case of using models from the transformers library, you need to set seeds for both Python's random module, NumPy, and PyTorch to ensure that the results are the same every time you run the code. # Before you create the pipeline and run the text generation, set the seeds like this: random.seed(args.SEED) np.random.seed(args.SEED) torch.manual_seed(args.SEED) torch.cuda.manual_seed_all(args.SEED) ### # cache_map_bioportal = None # if strtobool(args.USE_CACHE): # cachebioportal_filename = "CACHE_bioportal.json" # if os.path.exists(cachebioportal_filename): # with open(cachebioportal_filename) as f: # cache_map_bioportal = json.load(f) # else: # cache_map_bioportal = {} # # key_bioportal = "" # if args.bioportalkey_filename and os.path.exists(args.bioportalkey_filename): # fkeyname = args.bioportalkey_filename # with open(fkeyname) as f: # key_bioportal = f.read() in_filename = args.input_dir + args.filename df_ToAnnotate = pd.read_csv(in_filename, sep=',', header=0, dtype=str, encoding='utf-8') #df_ToAnnotate = df_ToAnnotate.head() #for debugging #df_ToAnnotate = df_ToAnnotate.head(1) # for debugging if "ToLink" not in df_ToAnnotate.columns: df_ToAnnotate["ToLink"] = None if "SentenceRef" not in df_ToAnnotate.columns: df_ToAnnotate["SentenceRef"] = None df_ToAnnotate = df_ToAnnotate[['SentenceRef'] + [col for col in df_ToAnnotate.columns if col != 'SentenceRef']] # this moves it to the first position #df_ToAnnotate['SentenceRef'] = df_ToAnnotate[args.source_column].rank(method='dense', ascending=True).astype(int) # df_ToAnnotate['rank'] = df_ToAnnotate.index + 1 # df_ToAnnotate['rank2'] = df_ToAnnotate['rank'].argsort().groupby(df_ToAnnotate[args.source_column]).transform('min').astype(int) # df_ToAnnotate['rank2'] = df_ToAnnotate['rank2'].rank(method='dense').astype(int) df_ToAnnotate['SentenceRef'] = df_ToAnnotate.index + 1 df_ToAnnotate['SentenceRef'] = df_ToAnnotate['SentenceRef'].argsort().groupby(df_ToAnnotate[args.source_column]).transform('min').astype(int) df_ToAnnotate['SentenceRef'] = df_ToAnnotate['SentenceRef'].rank(method='dense').astype(int) # device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') # if strtobool(args.debug): # print(f"Device: {device}...") # if device.type == "cuda": # print("GPU number:", torch.cuda.current_device()) tsk = "token-classification" pipe = None tokenizerGliner = None modelGliner = None modelGlinerBio = None if (("/gliner" in args.model_id) == False) and (("NCBO" in args.model_id) == False): pipe = pipeline( tsk, model=args.model_id, aggregation_strategy="simple", device=device, ) elif ("/gliner" in args.model_id): if not tokenizerGliner: tokenizerGliner = AutoTokenizer.from_pretrained('microsoft/deberta-v3-large') if "_bio-" in args.model_id: if args.num_cores_Gliner > 0: modelGlinerBio = GLiNER.from_pretrained(args.model_id) #"urchade/gliner_large_bio-v0.1") else: modelGlinerBio = GLiNER.from_pretrained(args.model_id, map_location=device) else: if args.num_cores_Gliner > 0: modelGliner = GLiNER.from_pretrained(args.model_id) #"knowledgator/gliner-multitask-large-v0.5" - "urchade/gliner_large-v2.1" else: modelGliner = GLiNER.from_pretrained(args.model_id, map_location=device) new_annotations = annotate(df_ToAnnotate, args, pipe, tokenizerGliner, modelGliner, modelGlinerBio, device) if not new_annotations.empty: if df_annotated.empty: # If df_annotated is empty, just assign new_annotations to it df_annotated = new_annotations else: # If df_annotated is not empty, concatenate new_annotations to it df_annotated = pd.concat([df_annotated, new_annotations], ignore_index=True) if not df_annotated.empty: df_annotated_combined = entitiesFusion(df_annotated,args) if strtobool(args.debug): print("\nStart is_cross_inside function ...") df_annotated_combined = is_cross_inside(df_annotated_combined, args, 0.75) else: df_annotated_combined = df_annotated #### GPT@JRC API if args.service_provider == "gptjrc": key_gptjrc = "" fkeyname = "GPTJRC-APItoken.key" if os.path.exists(fkeyname): with open(fkeyname) as f: key_gptjrc = f.read() else: key_gptjrc = os.environ['key_gptjrc'] if key_gptjrc and key_gptjrc != "": setup_gptjrc(key_gptjrc) ##### cache_prefix_fp = "LLMQUERYNER" cache_nameLLMs = cache_prefix_fp + "___" + "__".join( [args.service_provider, args.model_name, str(args.temperature)]).replace( " ", "_") + ".json" load_map_query_input_output = None if strtobool(args.USE_CACHE): if os.path.exists(cache_nameLLMs): with open(cache_nameLLMs) as f: load_map_query_input_output = json.load(f) else: load_map_query_input_output = {} ### entity linking part: if strtobool(args.entity_linking): cache_map_geonames = None if strtobool(args.USE_CACHE): cacheGeonames_filename = "CACHE_geonames.json" if os.path.exists(cacheGeonames_filename): with open(cacheGeonames_filename) as f: cache_map_geonames = json.load(f) else: cache_map_geonames = {} key_geonames = "" if args.geonameskey_filename and os.path.exists(args.geonameskey_filename): fkeyname = args.geonameskey_filename with open(fkeyname) as f: key_geonames = f.read() else: key_geonames = os.environ['key_geonames'] cache_map_virtuoso = None if strtobool(args.USE_CACHE): cacheVirtuoso_filename = "CACHE_virtuoso.json" if os.path.exists(cacheVirtuoso_filename): with open(cacheVirtuoso_filename) as f: cache_map_virtuoso = json.load(f) else: cache_map_virtuoso = {} key_virtuoso = "" if args.virtuosokey_filename and os.path.exists(args.virtuosokey_filename): fkeyname = args.virtuosokey_filename with open(fkeyname) as f: key_virtuoso = f.read() else: key_virtuoso = os.environ['key_virtuoso'] # Here for the EXACT MATCHING "" - if the desired term has not been identified in the NER, add to the dataframe: # Define the condition to find missing SentenceRefs missing_sentence_refs = ~df_ToAnnotate['SentenceRef'].isin(df_annotated_combined['SentenceRef']) # Define the condition to check if ContextToAnnotate starts and ends with quotes quoted_context = df_ToAnnotate[args.source_column].str.startswith('"') & df_ToAnnotate[ args.source_column].str.endswith('"') # Combine both conditions condition = missing_sentence_refs & quoted_context # Select rows from df_ToAnnotate that meet the condition rows_to_add = df_ToAnnotate[condition].copy() rows_to_add['model'] = "Forced" rows_to_add['entity_group'] = "MISC" rows_to_add['word'] = rows_to_add[args.source_column] rows_to_add['word'] = rows_to_add[args.source_column].apply(strip_quotes) rows_to_add['score'] = 1.0 rows_to_add['start'] = int(1) rows_to_add['end'] = rows_to_add['word'].apply(len)+int(1) rows_to_add['IsGeo'] = None rows_to_add['IsBio'] = None rows_to_add['IsCrossInside'] = 0.0 # Append these rows to df_annotated_combined df_annotated_combined = pd.concat([df_annotated_combined, rows_to_add], ignore_index=True) df_annotated_combined = df_annotated_combined.sort_values( by=['SentenceRef', 'start', 'ToLink', 'word', 'score'], ascending=[True, True, True, True, False]) # Now df_annotated_combined contains the additional rows df_annotated_combined, cache_map_geonames_AFTER, cache_map_virtuoso_AFTER, load_map_query_input_output_AFTER = elinking(df_annotated_combined, text_splitter, args, key_geonames, cache_map_geonames, key_virtuoso, cache_map_virtuoso, load_map_query_input_output, device) if strtobool(args.USE_CACHE): if cache_map_geonames_AFTER is not None: with open(cacheGeonames_filename, "w") as f: json.dump(cache_map_geonames_AFTER, f) if cache_map_virtuoso_AFTER is not None: with open(cacheVirtuoso_filename, "w") as f: json.dump(cache_map_virtuoso_AFTER, f) if load_map_query_input_output_AFTER is not None: with open(cache_nameLLMs, "w") as f: json.dump(load_map_query_input_output_AFTER, f) ### end entity linking part # # delete Triples and TriplesGlobal if not needed, too space all_emptyTriples = None all_emptyGlobalTriples = None if 'Triples' in df_annotated_combined.columns: all_emptyTriples = df_annotated_combined['Triples'].apply(lambda x: pd.isnull(x) or x == '' or x is None).all() #print("All values in 'Triples' are empty (NaN, None, or ''):", all_emptyTriples) if 'TriplesGlobal' in df_annotated_combined.columns: all_emptyGlobalTriples = df_annotated_combined['TriplesGlobal'].apply(lambda x: pd.isnull(x) or x == '' or x is None).all() #print("All values in 'TriplesGlobal' are empty (NaN, None, or ''):", all_emptyGlobalTriples) if '@id' in df_annotated_combined.columns: df_annotated_combined = df_annotated_combined.rename(columns={'@id': '@idfromNCBO'}) # df_annotated_combined.drop('@id', axis=1, inplace=True) # if 'ALLURIScontextFromNCBO' in df_annotated_combined.columns: df_annotated_combined.drop('ALLURIScontextFromNCBO', axis=1, inplace=True) if (all_emptyTriples == False) or (all_emptyGlobalTriples==False): output_texts_filename = args.input_dir + args.filename.replace(".csv", "_OutputAnnotated-withTriples.csv") df_annotated_combined.to_csv(output_texts_filename, sep=',', header=True, index=False, encoding='utf-8') df_annotated_combined.drop('Triples', axis=1, inplace=True) df_annotated_combined.drop('TriplesGlobal', axis=1, inplace=True) output_texts_filename = args.input_dir + args.filename.replace(".csv", "_OutputAnnotated.csv") df_annotated_combined.to_csv(output_texts_filename, sep=',', header=True, index=False, encoding='utf-8') print("\nEnd script") # end = time.time() hours, rem = divmod(end - start, 3600) minutes, seconds = divmod(rem, 60) print("Overall Computational Time... {:0>2}:{:0>2}:{:05.2f}\n".format(int(hours), int(minutes), seconds)) #