Update functions.py

functions.py

@@ -167,21 +167,61 @@ def extract_terms(cell):
         return []
  
     
-def remove_excluded_from_list(keywords_list, excluded_keywords):
+# def remove_excluded_from_list(keywords_list, excluded_keywords):
+#     """
+#     Remove items from the keywords_list if they contain any of the excluded keywords.
+#     This function checks for partial matches in a case-insensitive manner.
+#     """
+#     if not isinstance(keywords_list, list):
+#         return keywords_list  # If it's not a list, return as is
+
+#     filtered_list = []
+#     for item in keywords_list:
+#         # Check if item contains any excluded keyword (case-insensitive)
+#         if any(kw.lower() in item.lower() for kw in excluded_keywords):
+#             # Skip this item if it matches an excluded keyword
+#             continue
+#         else:
+#             filtered_list.append(item)
+
+#     return filtered_list
+
+
+
+def remove_excluded_from_text(text, excluded_keywords):
+    """
+    Remove occurrences of any excluded keyword from the text.
+    Matching is case-insensitive. Extra whitespace is cleaned.
+    """
+    if not isinstance(text, str):
+        return text
+    filtered_text = text
+    for kw in excluded_keywords:
+        # Create a regex pattern for the keyword (case-insensitive)
+        pattern = re.compile(re.escape(kw), re.IGNORECASE)
+        # Replace any occurrence of the keyword with an empty string
+        filtered_text = pattern.sub("", filtered_text)
+    # Remove extra spaces and strip the result
+    filtered_text = re.sub(r'\s+', ' ', filtered_text).strip()
+    return filtered_text
+
+
+
+# Example of processing an extracted result (which may be a list of phrases)
+def process_extracted_result(result, excluded_keywords):
     """
-    Remove items from the keywords_list if they contain any of the excluded keywords.
-    This function checks for partial matches in a case-insensitive manner.
+    Process an extracted result by removing excluded keywords from each string.
+    If result is a list, process each element; if it's a string, process it directly.
+    Return a list of non-empty cleaned strings.
     """
-    if not isinstance(keywords_list, list):
-        return keywords_list  # If it's not a list, return as is
-
-    filtered_list = []
-    for item in keywords_list:
-        # Check if item contains any excluded keyword (case-insensitive)
-        if any(kw.lower() in item.lower() for kw in excluded_keywords):
-            # Skip this item if it matches an excluded keyword
-            continue
-        else:
-            filtered_list.append(item)
-
-    return filtered_list
+    cleaned_items = []
+    if isinstance(result, list):
+        for item in result:
+            cleaned_item = remove_excluded_from_text(item, excluded_keywords)
+            if cleaned_item:  # Only add non-empty strings
+                cleaned_items.append(cleaned_item)
+    elif isinstance(result, str):
+        cleaned_item = remove_excluded_from_text(result, excluded_keywords)
+        if cleaned_item:
+            cleaned_items.append(cleaned_item)
+    return cleaned_items