# coding=utf-8 # Copyright 2024 The Google Research Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Language processing utilities.""" import spacy def load_spacy_model(model='en_core_web_trf'): nlp = spacy.load(model) return nlp def process_sentence(sentence, nlp): """Process a sentence.""" doc = nlp(sentence) sentence_for_spacy = [] for _, token in enumerate(doc): if token.text == ' ': continue sentence_for_spacy.append(token.text) sentence_for_spacy = ' '.join(sentence_for_spacy) noun_phrase, _, _ = extract_noun_phrase( sentence_for_spacy, nlp, need_index=True ) return noun_phrase def extract_noun_phrase(text, nlp, need_index=False): """Extract noun phrase from text. nlp is a spacy model. Args: text: str, text to be processed. nlp: spacy model. need_index: bool, whether to return the index of the noun phrase. Returns: noun_phrase: str, noun phrase of the text. """ # text = text.lower() doc = nlp(text) chunks = {} chunks_index = {} for chunk in doc.noun_chunks: for i in range(chunk.start, chunk.end): chunks[i] = chunk chunks_index[i] = (chunk.start, chunk.end) for token in doc: if token.head.i == token.i: head = token.head if head.i not in chunks: children = list(head.children) if children and children[0].i in chunks: head = children[0] else: if need_index: return text, [], text else: return text head_noun = head.text head_index = chunks_index[head.i] head_index = [i for i in range(head_index[0], head_index[1])] sentence_index = [i for i in range(len(doc))] not_phrase_index = [] for i in sentence_index: # not_phrase_index.append(i) if i not in head_index else None if i not in head_index: not_phrase_index.append(i) head = chunks[head.i] if need_index: return head.text, not_phrase_index, head_noun else: return head.text