Embedding class fix: neighbors bug, added max_n neighbors, typing, etc.

app.py

@@ -4,26 +4,34 @@ import pandas as pd
 
 
 # --- Imports modules ---
-from modules.model_embbeding import Embedding
+from modules.model_embbeding import Embedding # Fix and Updated
+
 
 # --- Imports interfaces ---
-from interfaces.interface_WordExplorer import interface as wordExplorer_interface
+from interfaces.interface_WordExplorer import interface as wordExplorer_interface # Updated
 from interfaces.interface_BiasWordExplorer import interface as biasWordExplorer_interface
 
+
 # --- Tool config ---
 AVAILABLE_LOGS      = True                          # [True     | False]
 LANGUAGE            = "spanish"                     # [spanish  | english]
 EMBEDDINGS_PATH     = "data/fasttext-sbwc.100k.vec"
+MAX_NEIGHBORS       = 20 # Updated
+
 
 # --- Init classes ---
 embedding = Embedding(
     path=EMBEDDINGS_PATH,
     binary=EMBEDDINGS_PATH.endswith('.bin'),
     limit=None,
-    randomizedPCA=False
+    randomizedPCA=False,
+    max_neighbors=MAX_NEIGHBORS # Updated
 )
+
+# --- Init Vars ---
 labels = pd.read_json(f"language/{LANGUAGE}.json")["app"]
 
+
 # --- Main App ---
 INTERFACE_LIST = [
     biasWordExplorer_interface(
@@ -33,6 +41,7 @@ INTERFACE_LIST = [
     wordExplorer_interface(
         embedding=embedding,
         available_logs=AVAILABLE_LOGS,
+        max_neighbors=MAX_NEIGHBORS, # Updated
         lang=LANGUAGE),
 ]
 

data/.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+data_loader.py

interfaces/.gitignore

@@ -0,0 +1 @@
+__pycache__/

interfaces/interface_WordExplorer.py

@@ -3,13 +3,19 @@ import pandas as pd
 import matplotlib.pyplot as plt
 
 from tool_info import TOOL_INFO
-from modules.module_connection import WordExplorerConnector
+from modules.module_connection import WordExplorerConnector # Updated
 from modules.module_logsManager import HuggingFaceDatasetSaver
 from examples.examples import examples_explorar_relaciones_entre_palabras
 
 plt.rcParams.update({'font.size': 14})
 
-def interface(embedding, available_logs, lang="spanish"):
+def interface(
+    embedding, 
+    available_logs: bool, 
+    max_neighbors: int, # Updated
+    lang: str="spanish",
+) -> gr.Blocks:
+
     # --- Init logs ---
     log_callback = HuggingFaceDatasetSaver(
         available_logs=available_logs
@@ -53,10 +59,10 @@ def interface(embedding, available_logs, lang="spanish"):
                 with gr.Row():
                     with gr.Row():
                         gr.Markdown(labels["plotNeighbours"]["title"])
-                        n_neighbors = gr.Slider(minimum=0,maximum=100,step=1,label=labels["plotNeighbours"]["quantity"])
+                        n_neighbors = gr.Slider(minimum=0,maximum=max_neighbors,step=1,label=labels["plotNeighbours"]["quantity"])
                     with gr.Row():
                         alpha = gr.Slider(minimum=0.1,maximum=0.9, value=0.3, step=0.1,label=labels["options"]["transparency"])
-                        fontsize=gr.Number(value=18, label=labels["options"]["font-size"])
+                        fontsize=gr.Number(value=25, label=labels["options"]["font-size"])
                     with gr.Row():
                         btn_plot = gr.Button(labels["plot_button"])
                 with gr.Row(): 

modules/.gitignore

@@ -0,0 +1 @@
+__pycache__/

modules/model_embbeding.py

@@ -1,58 +1,127 @@
+from modules.module_ann import Ann
+from memory_profiler import profile
+from sklearn.neighbors import NearestNeighbors
+from sklearn.decomposition import PCA
+from gensim.models import KeyedVectors
+from typing import List
 import os
 import operator
-import numpy as np
 import pandas as pd
+
+import numpy as np
 from numpy import dot
 from gensim import matutils
-from modules.module_ann import Ann
-from memory_profiler import profile
-from sklearn.neighbors import NearestNeighbors
-from data.data_loader import load_embeddings
 
 
 class Embedding:
     @profile
-    def __init__(self, path, binary, limit = None, randomizedPCA = False):
-        # Dataset info
+    def __init__(self, 
+        path: str, 
+        binary: bool, 
+        limit: int=None, 
+        randomizedPCA: bool=False,
+        max_neighbors: int=20
+    ) -> None:
+
+        # Embedding vars
         self.path = path
+        self.limit = limit
+        self.randomizedPCA = randomizedPCA
+        self.binary = binary
+        self.max_neighbors = max_neighbors
         
-        # Pandas dataset
+        # Full embedding dataset 
         self.ds = None
 
-        # All Words embedding List[List[float]]
-        self.embedding = None
-
-        # Estimate AproximateNearestNeighbors
-        self.ann = None
+        # Estimate NearestNeighbors
+        self.ann = None     # Aproximate with Annoy method
+        self.neigh = None   # Exact with Sklearn method
 
         # Load embedding and pca dataset
-        self.__load(binary, limit, randomizedPCA)
+        self.__load()
 
-    def __contains__(self, word):
-        return word in self.ds['word'].to_list()
+    def __load(
+        self, 
+    ) -> None:
 
-    def __load(self, binary, limit, randomizedPCA):
         print(f"Preparing {os.path.basename(self.path)} embeddings...")
 
         # --- Prepare dataset ---
-        self.ds = load_embeddings(self.path, binary, randomizedPCA, limit)
-
-        # --- Get embedding from string
-        self.embedding = self.ds['embedding'].to_list()
+        self.ds = self.__preparate(
+            self.path, self.binary, self.limit, self.randomizedPCA
+        )
 
-        # --- Get forest tree to estimate Nearest Neighbors ---
+        # --- Estimate Nearest Neighbors
+        # Method A: Througth annoy using forest tree
         self.ann = Ann(
             words=self.ds['word'], 
             vectors=self.ds['embedding'], 
             coord=self.ds['pca']
         )
-        self.ann.init(n_trees=20, metric='dot', n_jobs=-1)
+        self.ann.init(
+            n_trees=20, metric='dot', n_jobs=-1
+        )
+
+        # Method B: Througth Sklearn method
+        self.neigh = NearestNeighbors(
+            n_neighbors=self.max_neighbors
+        )
+        self.neigh.fit(
+            X=self.ds['embedding'].to_list()
+        )
+    
+    def __preparate(
+        self, 
+        path: str, 
+        binary: bool, 
+        limit: int,
+        randomizedPCA: bool
+    ) -> pd.DataFrame:
+
+        if randomizedPCA:
+            pca = PCA(
+                n_components=2, 
+                copy=False, 
+                whiten=False, 
+                svd_solver='randomized', 
+                iterated_power='auto'
+            )
+
+        else:
+            pca = PCA(
+                n_components=2
+            )
+        
+        print("--------> PATH:", path)
+        model = KeyedVectors.load_word2vec_format(
+            fname=path, 
+            binary=binary, 
+            limit=limit
+        )
+
+        # Cased Vocab
+        cased_words = model.index_to_key
+        cased_emb = model.get_normed_vectors()
+        cased_pca = pca.fit_transform(cased_emb)
+
+        df_cased = pd.DataFrame(
+            zip(
+                cased_words,
+                cased_emb,
+                cased_pca
+            ),
+            columns=['word', 'embedding', 'pca']
+        )
 
-        # --- Fit Sklearn NN method ---
-        self.neigh = NearestNeighbors(n_neighbors=20)
-        self.neigh.fit(self.embedding)
+        df_cased['word'] = df_cased.word.apply(lambda w: w.lower())
+        df_uncased = df_cased.drop_duplicates(subset='word')
+        return df_uncased
 
-    def __getValue(self, word, feature):
+    def __getValue(
+        self, 
+        word: str, 
+        feature: str
+    ):
         word_id, value = None, None
 
         if word in self:
@@ -63,30 +132,56 @@ class Embedding:
 
         return value
     
-    def getEmbedding(self, word):
+    def getEmbedding(
+        self, 
+        word: str
+    ):
+
         return self.__getValue(word, 'embedding')
 
-    def getPCA(self, word):
+    def getPCA(
+        self, 
+        word: str
+    ):
+
         return self.__getValue(word, 'pca')
     
-    def cosineSimilarities(self, vector_1, vectors_all):
-        norm = np.linalg.norm(vector_1)
-        all_norms = np.linalg.norm(vectors_all, axis=1)
-        dot_products = dot(vectors_all, vector_1)
-        similarities = dot_products / (norm * all_norms)
-        return similarities 
-    
-    def getNearestNeighbors(self, word, n_neighbors=10, nn_method='sklearn'):
+    def getNearestNeighbors(
+        self, 
+        word: str, 
+        n_neighbors: int=10, 
+        nn_method: str='sklearn'
+    ) -> List[str]:
+
+        assert(n_neighbors <= self.max_neighbors), f"Error: The value of the parameter 'n_neighbors:{n_neighbors}' must less than or equal to {self.max_neighbors}!."
+
         if nn_method == 'ann':
             words = self.ann.get(word, n_neighbors)
+            
         elif nn_method == 'sklearn':
-            word_emb = self.getEmbedding(word)
-            neighbors = self.neigh.kneighbors([word_emb], n_neighbors)[1][0]
-            words = operator.itemgetter(*neighbors)(self.ds['word'])
+            word_emb = self.getEmbedding(word).reshape(1,-1)
+            _, nn_ids = self.neigh.kneighbors(word_emb, n_neighbors)
+            words = operator.itemgetter(*nn_ids[0])(self.ds['word'].to_list())
         else: 
             words = []
         return words
 
+    def __contains__(
+        self, 
+        word: str
+    ) -> bool:
+
+        return word in self.ds['word'].to_list()
+
+    # ToDo: Revisar estos dos métodos usados en la pestaña sesgoEnPalabras
+    # ya que ahora los embedding vienen normalizados
+    def cosineSimilarities(self, vector_1, vectors_all):
+        norm = np.linalg.norm(vector_1)
+        all_norms = np.linalg.norm(vectors_all, axis=1)
+        dot_products = dot(vectors_all, vector_1)
+        similarities = dot_products / (norm * all_norms)
+        return similarities 
+
     def getCosineSimilarities(self, w1, w2):
         return dot(
             matutils.unitvec(self.getEmbedding(w1)), 

modules/module_WordExplorer.py

@@ -142,10 +142,13 @@ class WordExplorer:
                 processed_word_list.append(WordToPlot(word, color_dict[color], color, 1))
 
                 if n_neighbors > 0:
+                    # Updated: Con el agregado del parámetro max_neightbors, el (n_neighbors+1) 
+                    # hacia superar ese valor máximo y se producia una aserción
                     neighbors = self.get_neighbors(word, 
-                                                   n_neighbors=n_neighbors+1,
-                                                   nn_method=kwargs.get('nn_method', 'sklearn')
-                                                   )
+                        # n_neighbors=n_neighbors+1, 
+                        n_neighbors=n_neighbors, 
+                        nn_method=kwargs.get('nn_method', 'sklearn')
+                    )
                     for n in neighbors:
                         if n not in [wtp.word for wtp in processed_word_list]:
                             processed_word_list.append(WordToPlot(n, color_dict[color], color, n_alpha))

modules/module_connection.py

@@ -3,7 +3,7 @@ import pandas as pd
 import gradio as gr
 from abc import ABC, abstractmethod
 
-from modules.module_WordExplorer import WordExplorer
+from modules.module_WordExplorer import WordExplorer # Updated
 from modules.module_BiasExplorer import WordBiasExplorer
 
 class Connector(ABC):