Update app.py

app.py

@@ -6,10 +6,9 @@ import json
 from transformers import AutoTokenizer, AutoModel
 from sentence_transformers import SentenceTransformer, models
 from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
 import re
-from collections import Counter
 import spacy
+import joblib
 
 # Load Spacy model for advanced NLP
 try:
@@ -51,9 +50,7 @@ embeddings, patent_numbers, metadata, texts = load_data()
 try:
     tokenizer = AutoTokenizer.from_pretrained('anferico/bert-for-patents')
     bert_model = AutoModel.from_pretrained('anferico/bert-for-patents')
-    word_embedding_model = models.Transformer(model_name='anferico/bert-for-patents', tokenizer=tokenizer, model=bert_model)
-    pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
-    model = SentenceTransformer(modules=[word_embedding_model, pooling_model])
+    model = SentenceTransformer(modules=[models.Transformer(model_name='anferico/bert-for-patents'), models.Pooling(bert_model.config.hidden_size)])
 except Exception as e:
     print(f"Error loading anferico/bert-for-patents: {e}")
     print("Falling back to a general-purpose model.")
@@ -73,9 +70,10 @@ embeddings = embeddings / np.linalg.norm(embeddings, axis=1, keepdims=True)
 index = faiss.IndexFlatIP(embeddings.shape[1])
 index.add(embeddings)
 
-# Create TF-IDF vectorizer
+# Create and save TF-IDF vectorizer
 tfidf_vectorizer = TfidfVectorizer(stop_words='english')
 tfidf_matrix = tfidf_vectorizer.fit_transform(texts)
+joblib.dump(tfidf_vectorizer, 'tfidf_vectorizer.joblib')
 
 def extract_key_features(text):
     # Use Spacy to extract noun phrases and key phrases
@@ -101,59 +99,28 @@ def hybrid_search(query, top_k=5):
     query_embedding = query_embedding / np.linalg.norm(query_embedding)
     
     # Perform semantic similarity search
-    semantic_distances, semantic_indices = index.search(np.array([query_embedding]).astype('float32'), top_k * 2)
-    
-    # Perform TF-IDF based search
-    query_tfidf = tfidf_vectorizer.transform([query])
-    tfidf_similarities = cosine_similarity(query_tfidf, tfidf_matrix).flatten()
-    tfidf_indices = tfidf_similarities.argsort()[-top_k * 2:][::-1]
-    
-    # Combine and rank results
-    combined_results = {}
-    for i, idx in enumerate(semantic_indices[0]):
-        patent_number = patent_numbers[idx].decode('utf-8')
-        text = metadata[patent_number]['text']
-        patent_features = extract_key_features(text)
-        common_features, feature_similarity = compare_features(query_features, patent_features)
-        combined_results[patent_number] = {
-            'score': semantic_distances[0][i] * 1.5 + feature_similarity,
-            'common_features': common_features,
-            'text': text
-        }
-    
-    for idx in tfidf_indices:
-        patent_number = patent_numbers[idx].decode('utf-8')
-        if patent_number not in combined_results:
-            text = metadata[patent_number]['text']
-            patent_features = extract_key_features(text)
-            common_features, feature_similarity = compare_features(query_features, patent_features)
-            combined_results[patent_number] = {
-                'score': tfidf_similarities[idx] + feature_similarity,
-                'common_features': common_features,
-                'text': text
-            }
-    
-    # Sort and get top results
-    top_results = sorted(combined_results.items(), key=lambda x: x[1]['score'], reverse=True)[:top_k]
+    semantic_distances, semantic_indices = index.search(np.array([query_embedding]), top_k)
     
     results = []
-    for patent_number, data in top_results:
-        result = f"Patent Number: {patent_number}\n"
-        result += f"Text: {data['text'][:200]}...\n"
-        result += f"Combined Score: {data['score']:.4f}\n"
-        result += f"Common Key Features: {', '.join(data['common_features'])}\n\n"
-        results.append(result)
-    
-    return "\n".join(results)
+    for i in range(top_k):
+        patent_number = patent_numbers[semantic_indices[0][i]]
+        patent_data = metadata[patent_number]
+        patent_features = extract_key_features(patent_data['text'])
+        common_features, similarity_score = compare_features(query_features, patent_features)
+        
+        results.append({
+            'patent_number': patent_number,
+            'common_features': common_features,
+            'similarity_score': similarity_score,
+            'semantic_score': semantic_distances[0][i]
+        })
+        
+    return results
 
-# Create Gradio interface
 iface = gr.Interface(
     fn=hybrid_search,
-    inputs=gr.Textbox(lines=2, placeholder="Enter your patent query here..."),
-    outputs=gr.Textbox(lines=10, label="Search Results"),
-    title="Patent Similarity Search",
-    description="Enter a patent description to find similar patents based on key features."
+    inputs=gr.inputs.Textbox(label="Enter your search query"),
+    outputs=gr.outputs.JSON(label="Search Results")
 )
 
-if __name__ == "__main__":
-    iface.launch()
+iface.launch()