New Changes

embedding.py

@@ -1,263 +1,263 @@
-from PyPDF2 import PdfReader
-import requests
-import json
-import os
-import concurrent.futures
-import random
-from langchain_google_genai import ChatGoogleGenerativeAI
-from langchain_community.document_loaders import WebBaseLoader
-from langchain_community.document_loaders import PyPDFLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-import google.generativeai as genai
-from io import BytesIO
-
-
-
-gemini = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBmZtXjJgp7yIAo9joNCZGSxK9PbGMcVaA',temperature = 0.1)
-gemini1 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyABsaDjPujPCBlz4LLxcXDX_bDA9uEL7Xc',temperature = 0.1)
-gemini2 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBCIQgt1uK7-sJH5Afg5vUZ99EWkx5gSU0',temperature = 0.1)
-gemini3 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBot9W5Q-BKQ66NAYRUmVeloXWEbXOXTmM',temperature = 0.1)
-
-genai.configure(api_key="AIzaSyBmZtXjJgp7yIAo9joNCZGSxK9PbGMcVaA")
-
-
-def pdf_extractor(link):
-    text = ''
-
-    try:
-        # Fetch the PDF file from the URL
-        response = requests.get(link)
-        response.raise_for_status()  # Raise an error for bad status codes
-
-        # Use BytesIO to handle the PDF content in memory
-        pdf_file = BytesIO(response.content)
-
-        # Load the PDF file
-        reader = PdfReader(pdf_file)
-        for page in reader.pages:
-            text += page.extract_text()  # Extract text from each page
-
-    except requests.exceptions.HTTPError as e:
-        print(f'HTTP error occurred: {e}')
-    except Exception as e:
-        print(f'An error occurred: {e}')
-    
-    return [text]
-
-def web_extractor(link):
-    text = ''
-
-    try:
-        loader = WebBaseLoader(link)
-        pages = loader.load_and_split()
-
-        for page in pages:
-            text+=page.page_content
-    except:
-        pass
-    
-    return [text]
-
-
-def feature_extraction(tag, history , context):
-
-    prompt = f'''
-    You are an intelligent assistant tasked with updating product information. You have two data sources:
-    1. Tag_History: Previously gathered information about the product.
-    2. Tag_Context: New data that might contain additional details.
-    Your job is to read the Tag_Context and update the relevant field in the Tag_History with any new details found. The field to be updated is the {tag} FIELD.
-    Guidelines:
-    - Only add new details that are relevant to the {tag} FIELD.
-    - Do not add or modify any other fields in the Tag_History.
-    - Ensure your response is in coherent sentences, integrating the new details seamlessly into the existing information.
-    Here is the data:
-    Tag_Context: {str(context)}
-    Tag_History: {history}
-    Respond with the updated Tag_History.
-    '''
-
-    # model = random.choice([gemini,gemini1,gemini2,gemini3])
-    result = gemini1.invoke(prompt)
-
-    return result.content
-
-def detailed_feature_extraction(find, context):
-
-    prompt = f'''
-    You are an intelligent assistant tasked with finding product information. You have one data source and one output format:
-    1. Context: The gathered information about the product.
-    2. Format: Details which need to be filled based on Context.
-    Your job is to read the Context and update the relevant field in Format using Context.
-    Guidelines:
-    - Only add details that are relevant to the individual FIELD.
-    - Do not add or modify any other fields in the Format.
-    - If nothing found return None.
-    Here is the data:
-    The Context is {str(context)}
-    The Format is {str(find)}
-    '''
-
-    model = random.choice([gemini,gemini1,gemini2,gemini3])
-    result = model.invoke(prompt)
-
-    return result.content
-
-def detailed_history(history):
-
-    details = {
-    "Introduction": {
-        "Product Name": None,
-        "Overview of the product": None,
-        "Purpose of the manual": None,
-        "Audience": None,
-        "Additional Details": None
-    },
-    "Specifications": {
-        "Technical specifications": None,
-        "Performance metrics": None,
-        "Additional Details": None
-    },
-    "Product Overview": {
-        "Product features": None,
-        "Key components and parts": None,
-        "Additional Details": None
-    },
-    "Safety Information": {
-        "Safety warnings and precautions": None,
-        "Compliance and certification information": None,
-        "Additional Details": None
-    },
-    "Installation Instructions": {
-        "Unboxing and inventory checklist": None,
-        "Step-by-step installation guide": None,
-        "Required tools and materials": None,
-        "Additional Details": None
-    },
-    "Setup and Configuration": {
-        "Initial setup procedures": None,
-        "Configuration settings": None,
-        "Troubleshooting setup issues": None,
-        "Additional Details": None
-    },
-    "Operation Instructions": {
-        "How to use the product": None,
-        "Detailed instructions for different functionalities": None,
-        "User interface guide": None,
-        "Additional Details": None
-    },
-    "Maintenance and Care": {
-        "Cleaning instructions": None,
-        "Maintenance schedule": None,
-        "Replacement parts and accessories": None,
-        "Additional Details": None
-    },
-    "Troubleshooting": {
-        "Common issues and solutions": None,
-        "Error messages and their meanings": None,
-        "Support Information": None,
-        "Additional Details": None
-    },
-    "Warranty Information": {
-        "Terms and Conditions": None,
-        "Service and repair information": None,
-        "Additional Details": None
-    },
-    "Legal Information": {
-        "Copyright information": None,
-        "Trademarks and patents": None,
-        "Disclaimers": None,
-        "Additional Details": None
-
-    }
-}
-
-    for key,val in history.items():
-
-        find = details[key]
-
-        details[key] = str(detailed_feature_extraction(find,val))
-
-    return details
-
-
-def get_embeddings(link,tag_option): 
-
-        print(f"\nCreating Embeddings ----- {link}")
-
-        if tag_option=='Single':
-            history = { "Details": "" }
-
-        else:
-            history = {
-                    "Introduction": "",
-                    "Specifications": "",
-                    "Product Overview": "",
-                    "Safety Information": "",
-                    "Installation Instructions": "",
-                    "Setup and Configuration": "",
-                    "Operation Instructions": "",
-                    "Maintenance and Care": "",
-                    "Troubleshooting": "",
-                    "Warranty Information": "",
-                    "Legal Information": ""
-                }
-
-        # Extract Text -----------------------------
-        print("Extracting Text")
-        if link[-3:] == '.md' or link[8:11] == 'en.':
-            text = web_extractor(link)
-        else:
-            text = pdf_extractor(link)
-
-        # Create Chunks ----------------------------
-        print("Writing Tag Data")
-
-        if tag_option=="Single":
-            history["Details"] = feature_extraction("Details", history["Details"], text[0][:50000])
-            
-        else:
-            chunks = text_splitter.create_documents(text)
-
-            for chunk in chunks:
-
-                with concurrent.futures.ThreadPoolExecutor() as executor:
-                        future_to_key = {
-                            executor.submit(
-                                feature_extraction, f"Product {key}", history[key], chunk.page_content
-                            ): key for key in history
-                        }
-                        for future in concurrent.futures.as_completed(future_to_key):
-                            key = future_to_key[future]
-                            try:
-                                response = future.result()
-                                history[key] = response
-                            except Exception as e:
-                                print(f"Error processing {key}: {e}")
-            
-        print("Creating Vectors")
-        genai_embeddings=[]
-            
-        for tag in history:
-            result = genai.embed_content(
-                    model="models/embedding-001",
-                    content=history[tag],
-                    task_type="retrieval_document")
-            genai_embeddings.append(result['embedding'])
-
-
-        return history,genai_embeddings
-            
-global text_splitter
-global data
-global history
-
-
-text_splitter = RecursiveCharacterTextSplitter(
-    chunk_size = 10000,
-    chunk_overlap  = 100,
-    separators = ["",''," "]
-)
-
-if __name__ == '__main__':
-    # print(get_embeddings('https://www.galaxys24manual.com/wp-content/uploads/pdf/galaxy-s24-manual-SAM-S921-S926-S928-OS14-011824-FINAL-US-English.pdf',"Single"))
+from PyPDF2 import PdfReader
+import requests
+import json
+import os
+import concurrent.futures
+import random
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import google.generativeai as genai
+from io import BytesIO
+
+
+
+gemini = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBmZtXjJgp7yIAo9joNCZGSxK9PbGMcVaA',temperature = 0.1)
+gemini1 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyABsaDjPujPCBlz4LLxcXDX_bDA9uEL7Xc',temperature = 0.1)
+gemini2 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBCIQgt1uK7-sJH5Afg5vUZ99EWkx5gSU0',temperature = 0.1)
+gemini3 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBot9W5Q-BKQ66NAYRUmVeloXWEbXOXTmM',temperature = 0.1)
+
+genai.configure(api_key="AIzaSyBmZtXjJgp7yIAo9joNCZGSxK9PbGMcVaA")
+
+
+def pdf_extractor(link):
+    text = ''
+
+    try:
+        # Fetch the PDF file from the URL
+        response = requests.get(link)
+        response.raise_for_status()  # Raise an error for bad status codes
+
+        # Use BytesIO to handle the PDF content in memory
+        pdf_file = BytesIO(response.content)
+
+        # Load the PDF file
+        reader = PdfReader(pdf_file)
+        for page in reader.pages:
+            text += page.extract_text()  # Extract text from each page
+
+    except requests.exceptions.HTTPError as e:
+        print(f'HTTP error occurred: {e}')
+    except Exception as e:
+        print(f'An error occurred: {e}')
+    
+    return [text]
+
+def web_extractor(link):
+    text = ''
+
+    try:
+        loader = WebBaseLoader(link)
+        pages = loader.load_and_split()
+
+        for page in pages:
+            text+=page.page_content
+    except:
+        pass
+    
+    return [text]
+
+
+def feature_extraction(tag, history , context):
+
+    prompt = f'''
+    You are an intelligent assistant tasked with updating product information. You have two data sources:
+    1. Tag_History: Previously gathered information about the product.
+    2. Tag_Context: New data that might contain additional details.
+    Your job is to read the Tag_Context and update the relevant field in the Tag_History with any new details found. The field to be updated is the {tag} FIELD.
+    Guidelines:
+    - Only add new details that are relevant to the {tag} FIELD.
+    - Do not add or modify any other fields in the Tag_History.
+    - Ensure your response is in coherent sentences, integrating the new details seamlessly into the existing information.
+    Here is the data:
+    Tag_Context: {str(context)}
+    Tag_History: {history}
+    Respond with the updated Tag_History.
+    '''
+
+    # model = random.choice([gemini,gemini1,gemini2,gemini3])
+    result = gemini1.invoke(prompt)
+
+    return result.content
+
+def detailed_feature_extraction(find, context):
+
+    prompt = f'''
+    You are an intelligent assistant tasked with finding product information. You have one data source and one output format:
+    1. Context: The gathered information about the product.
+    2. Format: Details which need to be filled based on Context.
+    Your job is to read the Context and update the relevant field in Format using Context.
+    Guidelines:
+    - Only add details that are relevant to the individual FIELD.
+    - Do not add or modify any other fields in the Format.
+    - If nothing found return None.
+    Here is the data:
+    The Context is {str(context)}
+    The Format is {str(find)}
+    '''
+
+    model = random.choice([gemini,gemini1,gemini2,gemini3])
+    result = model.invoke(prompt)
+
+    return result.content
+
+def detailed_history(history):
+
+    details = {
+    "Introduction": {
+        "Product Name": None,
+        "Overview of the product": None,
+        "Purpose of the manual": None,
+        "Audience": None,
+        "Additional Details": None
+    },
+    "Specifications": {
+        "Technical specifications": None,
+        "Performance metrics": None,
+        "Additional Details": None
+    },
+    "Product Overview": {
+        "Product features": None,
+        "Key components and parts": None,
+        "Additional Details": None
+    },
+    "Safety Information": {
+        "Safety warnings and precautions": None,
+        "Compliance and certification information": None,
+        "Additional Details": None
+    },
+    "Installation Instructions": {
+        "Unboxing and inventory checklist": None,
+        "Step-by-step installation guide": None,
+        "Required tools and materials": None,
+        "Additional Details": None
+    },
+    "Setup and Configuration": {
+        "Initial setup procedures": None,
+        "Configuration settings": None,
+        "Troubleshooting setup issues": None,
+        "Additional Details": None
+    },
+    "Operation Instructions": {
+        "How to use the product": None,
+        "Detailed instructions for different functionalities": None,
+        "User interface guide": None,
+        "Additional Details": None
+    },
+    "Maintenance and Care": {
+        "Cleaning instructions": None,
+        "Maintenance schedule": None,
+        "Replacement parts and accessories": None,
+        "Additional Details": None
+    },
+    "Troubleshooting": {
+        "Common issues and solutions": None,
+        "Error messages and their meanings": None,
+        "Support Information": None,
+        "Additional Details": None
+    },
+    "Warranty Information": {
+        "Terms and Conditions": None,
+        "Service and repair information": None,
+        "Additional Details": None
+    },
+    "Legal Information": {
+        "Copyright information": None,
+        "Trademarks and patents": None,
+        "Disclaimers": None,
+        "Additional Details": None
+
+    }
+}
+
+    for key,val in history.items():
+
+        find = details[key]
+
+        details[key] = str(detailed_feature_extraction(find,val))
+
+    return details
+
+
+def get_embeddings(link,tag_option): 
+
+        print(f"\nCreating Embeddings ----- {link}")
+
+        if tag_option=='Complete Document Similarity':
+            history = { "Details": "" }
+
+        else:
+            history = {
+                    "Introduction": "",
+                    "Specifications": "",
+                    "Product Overview": "",
+                    "Safety Information": "",
+                    "Installation Instructions": "",
+                    "Setup and Configuration": "",
+                    "Operation Instructions": "",
+                    "Maintenance and Care": "",
+                    "Troubleshooting": "",
+                    "Warranty Information": "",
+                    "Legal Information": ""
+                }
+
+        # Extract Text -----------------------------
+        print("Extracting Text")
+        if link[-3:] == '.md' or link[8:11] == 'en.':
+            text = web_extractor(link)
+        else:
+            text = pdf_extractor(link)
+
+        # Create Chunks ----------------------------
+        print("Writing Tag Data")
+
+        if tag_option=="Complete Document Similarity":
+            history["Details"] = feature_extraction("Details", history["Details"], text[0][:50000])
+            
+        else:
+            chunks = text_splitter.create_documents(text)
+
+            for chunk in chunks:
+
+                with concurrent.futures.ThreadPoolExecutor() as executor:
+                        future_to_key = {
+                            executor.submit(
+                                feature_extraction, f"Product {key}", history[key], chunk.page_content
+                            ): key for key in history
+                        }
+                        for future in concurrent.futures.as_completed(future_to_key):
+                            key = future_to_key[future]
+                            try:
+                                response = future.result()
+                                history[key] = response
+                            except Exception as e:
+                                print(f"Error processing {key}: {e}")
+            
+        print("Creating Vectors")
+        genai_embeddings=[]
+            
+        for tag in history:
+            result = genai.embed_content(
+                    model="models/embedding-001",
+                    content=history[tag],
+                    task_type="retrieval_document")
+            genai_embeddings.append(result['embedding'])
+
+
+        return history,genai_embeddings
+            
+global text_splitter
+global data
+global history
+
+
+text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size = 10000,
+    chunk_overlap  = 100,
+    separators = ["",''," "]
+)
+
+if __name__ == '__main__':
+    # print(get_embeddings('https://www.galaxys24manual.com/wp-content/uploads/pdf/galaxy-s24-manual-SAM-S921-S926-S928-OS14-011824-FINAL-US-English.pdf',"Single"))
     pass

infridgement_score.py

@@ -0,0 +1,174 @@
+import streamlit as st
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor,as_completed
+from functools import partial
+import numpy as np
+from io import StringIO
+import sys
+import time
+
+# File Imports
+from embedding import get_embeddings  # Ensure this file/module is available
+from preprocess import filtering  # Ensure this file/module is available
+from search import *
+
+# Cosine Similarity Function
+def cosine_similarity(vec1, vec2):
+    vec1 = np.array(vec1)
+    vec2 = np.array(vec2)
+    
+    dot_product = np.dot(vec1, vec2)
+    magnitude_vec1 = np.linalg.norm(vec1)
+    magnitude_vec2 = np.linalg.norm(vec2)
+    
+    if magnitude_vec1 == 0 or magnitude_vec2 == 0:
+        return 0.0
+    
+    cosine_sim = dot_product / (magnitude_vec1 * magnitude_vec2)
+    return cosine_sim
+
+# Logger class to capture output
+class StreamCapture:
+    def __init__(self):
+        self.output = StringIO()
+        self._stdout = sys.stdout
+
+    def __enter__(self):
+        sys.stdout = self.output
+        return self.output
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        sys.stdout = self._stdout
+
+# Main Function
+def score(main_product, main_url, product_count, link_count, search, logger, log_area):
+    data = {}
+    similar_products = extract_similar_products(main_product)[:product_count]
+    
+    if search == 'All':
+
+        def process_product(product, search_function, main_product):
+            search_result = search_function(product)
+            return filtering(search_result, main_product, product, link_count)
+        
+        
+        search_functions = {
+            'google': search_google,
+            'duckduckgo': search_duckduckgo,
+            # 'archive': search_archive,
+            'github': search_github,
+            'wikipedia': search_wikipedia
+        }
+
+        with ThreadPoolExecutor() as executor:
+            future_to_product_search = {
+                executor.submit(process_product, product, search_function, main_product): (product, search_name)
+                for product in similar_products
+                for search_name, search_function in search_functions.items()
+            }
+
+            for future in as_completed(future_to_product_search):
+                product, search_name = future_to_product_search[future]
+                try:
+                    if product not in data:
+                        data[product] = {}
+                    data[product] = future.result()
+                except Exception as e:
+                    print(f"Error processing product {product} with {search_name}: {e}")
+
+    else:
+
+        for product in similar_products:
+
+            if search == 'google':
+                data[product] = filtering(search_google(product), main_product, product, link_count)
+            elif search == 'duckduckgo':
+                data[product] = filtering(search_duckduckgo(product), main_product, product, link_count)
+            elif search == 'archive':
+                data[product] = filtering(search_archive(product), main_product, product, link_count)
+            elif search == 'github':
+                data[product] = filtering(search_github(product), main_product, product, link_count)
+            elif search == 'wikipedia':
+                data[product] = filtering(search_wikipedia(product), main_product, product, link_count)
+
+    logger.write("\n\nFiltered Links ------------------>\n")
+    logger.write(str(data) + "\n")
+    log_area.text(logger.getvalue())
+
+    logger.write("\n\nCreating Main product Embeddings ---------->\n")
+    main_result, main_embedding = get_embeddings(main_url,tag_option)
+    log_area.text(logger.getvalue())
+
+    print("main",main_embedding)
+    
+    cosine_sim_scores = []
+
+    logger.write("\n\nCreating Similar product Embeddings ---------->\n")
+    log_area.text(logger.getvalue())
+
+
+    for product in data:
+
+        if len(data[product])==0:
+            logger.write("\n\nNo Product links Found Increase No of Links or Change Search Source\n")
+            log_area.text(logger.getvalue())
+    
+            cosine_sim_scores.append((product,'No Product links Found Increase Number of Links or Change Search Source',None,None))
+        
+        else:
+            for link in data[product][:link_count]:
+
+                similar_result, similar_embedding = get_embeddings(link,tag_option)
+                log_area.text(logger.getvalue())
+
+                print(similar_embedding)
+                for i in range(len(main_embedding)):
+                    score = cosine_similarity(main_embedding[i], similar_embedding[i])
+                    cosine_sim_scores.append((product, link, i, score))
+                    log_area.text(logger.getvalue())
+
+    logger.write("--------------- DONE -----------------\n")
+    log_area.text(logger.getvalue())
+    return cosine_sim_scores, main_result
+
+# Streamlit Interface
+st.title("Check Infringement")
+
+
+# Inputs
+main_product = st.text_input('Enter Main Product Name', 'Philips led 7w bulb')
+main_url = st.text_input('Enter Main Product Manual URL', 'https://www.assets.signify.com/is/content/PhilipsConsumer/PDFDownloads/Colombia/technical-sheets/ODLI20180227_001-UPD-es_CO-Ficha_Tecnica_LED_MR16_Master_7W_Dim_12V_CRI90.pdf')
+search_method = st.selectbox('Choose Search Engine', ['All','duckduckgo', 'google', 'archive', 'github', 'wikipedia'])
+
+col1, col2 = st.columns(2)
+with col1:
+    product_count = st.number_input("Number of Simliar Products",min_value=1, step=1, format="%i")
+with col2:
+    link_count = st.number_input("Number of Links per product",min_value=1, step=1, format="%i")
+
+
+tag_option = st.selectbox('Choose Similarity Method', ["Complete Document Similarity","Feild Wise Document Similarity"])
+
+
+if st.button('Check for Infringement'):
+    log_output = st.empty()  # Placeholder for log output
+
+    with st.spinner('Processing...'):
+        with StreamCapture() as logger:
+            cosine_sim_scores, main_result = score(main_product, main_url,product_count, link_count, search_method, logger, log_output)
+
+    st.success('Processing complete!')
+
+    st.subheader("Cosine Similarity Scores")
+
+    #  = score(main_product, main_url, search, logger, log_output)
+    if tag_option == 'Complete Document Similarity':
+        tags = ['Details']
+    else:    
+        tags = ['Introduction', 'Specifications', 'Product Overview', 'Safety Information', 'Installation Instructions', 'Setup and Configuration', 'Operation Instructions', 'Maintenance and Care', 'Troubleshooting', 'Warranty Information', 'Legal Information']
+
+    for product, link, index, value in cosine_sim_scores:
+        if not index:
+            st.write(f"Product: {product}, Link: {link}")
+        if value!=None:
+            st.write(f"{tags[index]:<20} - Similarity: {value:.2f}")

preprocess.py

@@ -1,168 +1,184 @@
-import requests
-import json
-import random
-import concurrent.futures
-from concurrent.futures import ThreadPoolExecutor
-from langchain_community.document_loaders import PyPDFLoader
-from langdetect import detect_langs
-import requests
-from PyPDF2 import PdfReader
-from io import BytesIO
-from langchain_community.document_loaders import WebBaseLoader
-from langchain_google_genai import ChatGoogleGenerativeAI
-import logging
-
-data = False 
-seen = set()
-
-# API Urls -----
-
-# main_url = "http://127.0.0.1:5000/search/all"
-main_url = "http://127.0.0.1:8000/search/all"
-# main_product = "Samsung Galaxy s23 ultra"
-
-# Revelevance Checking Models -----
-gemini = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBmZtXjJgp7yIAo9joNCZGSxK9PbGMcVaA',temperature = 0.1)
-gemini1 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyABsaDjPujPCBlz4LLxcXDX_bDA9uEL7Xc',temperature = 0.1)
-gemini2 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBCIQgt1uK7-sJH5Afg5vUZ99EWkx5gSU0',temperature = 0.1)
-gemini3 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBot9W5Q-BKQ66NAYRUmVeloXWEbXOXTmM',temperature = 0.1)
-
-
-# API_URL = "https://api-inference.huggingface.co/models/google/flan-t5-xxl"
-# headers = {"Authorization": "Bearer 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"}
-
-# Error Debug
-logging.basicConfig(level=logging.INFO)
-
-
-def get_links(main_product,api_key):
-    params = {
-        "API_KEY": f"{api_key}",
-        "product": f"{main_product}",
-    }
-
-    # Flask
-    response = requests.get(main_url, params=params)
-
-    # FastAPI
-    # response = requests.post(main_url, json=params)
-
-
-    if response.status_code == 200:
-        results = response.json()
-        with open('data.json', 'w') as f:
-            json.dump(results, f)
-    else:
-        print(f"Failed to fetch results: {response.status_code}")
-
-
-
-def language_preprocess(text):
-    try:
-        if detect_langs(text)[0].lang == 'en':
-            return True
-        return False
-    except:
-        return False
-
-
-def relevant(product, similar_product, content):
-
-    try:
-        payload = { "inputs": f'''Do you think that the given content is similar to {similar_product} and {product}, just Respond True or False  \nContent for similar product:  {content}'''}
-        
-        # response = requests.post(API_URL, headers=headers, json=payload)
-        # output = response.json()
-        # return bool(output[0]['generated_text'])
-            
-        model = random.choice([gemini,gemini1,gemini2,gemini3])
-        result = model.invoke(f'''Do you think that the given content is similar to {similar_product} and {product}, just Respond True or False  \nContent for similar product:  {content}''')
-        return bool(result)
-
-    except:
-        return False
-        
-        
-
-def download_pdf(url, timeout=10):
-    try:
-        response = requests.get(url, timeout=timeout)
-        response.raise_for_status()
-        return BytesIO(response.content)
-
-    except requests.RequestException as e:
-        logging.error(f"PDF download error: {e}")
-        return None
-
-def extract_text_from_pdf(pdf_file, pages):
-    reader = PdfReader(pdf_file)
-    extracted_text = ""
-
-    l = len(reader.pages)
-    
-    try:
-        for page_num in pages:
-            if page_num < l:
-                page = reader.pages[page_num]
-                extracted_text += page.extract_text() + "\n"
-            else:
-                print(f"Page {page_num} does not exist in the document.")
-        
-        return extracted_text
-    
-    except:
-        return 'हे चालत नाही'
-    
-def extract_text_online(link):
-
-    loader = WebBaseLoader(link)
-    pages = loader.load_and_split()
-
-    text = ''
-
-    for page in pages[:3]:
-        text+=page.page_content
-    
-    return text
-
-
-def process_link(link, main_product, similar_product):
-    if link in seen:
-        return None
-    seen.add(link)
-    try:
-        if link[-3:]=='.md' or link[8:11] == 'en.':
-            text = extract_text_online(link)
-        else:
-            pdf_file = download_pdf(link)
-            text = extract_text_from_pdf(pdf_file, [0, 2, 4])
-
-        if language_preprocess(text):
-            if relevant(main_product, similar_product, text):
-                print("Accepted",link)
-                return link
-    except:
-        pass
-    print("NOT Accepted",link)
-    return None
-
-def filtering(urls, main_product, similar_product):
-    res = []
-
-    print(f"Filtering Links of ---- {similar_product}")
-    # Main Preprocess ------------------------------
-    with ThreadPoolExecutor() as executor:
-        futures = {executor.submit(process_link, link, main_product, similar_product): link for link in urls}
-        for future in concurrent.futures.as_completed(futures):
-            result = future.result()
-            if result is not None:
-                res.append(result)
-
-    return res
-
-
-# Main Functions -------------------------------------------------->
-
-# get_links()
-# preprocess()
-
+import requests
+import json
+import random
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor
+from langchain_community.document_loaders import PyPDFLoader
+from langdetect import detect_langs
+import requests
+from PyPDF2 import PdfReader
+from io import BytesIO
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_google_genai import ChatGoogleGenerativeAI
+import logging
+
+data = False 
+seen = set()
+
+# API Urls -----
+
+# main_url = "http://127.0.0.1:5000/search/all"
+main_url = "http://127.0.0.1:8000/search/all"
+# main_product = "Samsung Galaxy s23 ultra"
+
+# Revelevance Checking Models -----
+gemini = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBmZtXjJgp7yIAo9joNCZGSxK9PbGMcVaA',temperature = 0.1)
+gemini1 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyABsaDjPujPCBlz4LLxcXDX_bDA9uEL7Xc',temperature = 0.1)
+gemini2 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBCIQgt1uK7-sJH5Afg5vUZ99EWkx5gSU0',temperature = 0.1)
+gemini3 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBot9W5Q-BKQ66NAYRUmVeloXWEbXOXTmM',temperature = 0.1)
+
+
+API_URL = "https://api-inference.huggingface.co/models/google/flan-t5-xxl"
+headers = {"Authorization": "Bearer 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"}
+
+# Error Debug
+logging.basicConfig(level=logging.INFO)
+
+
+def get_links(main_product,api_key):
+    params = {
+        "API_KEY": f"{api_key}",
+        "product": f"{main_product}",
+    }
+
+    # Flask
+    response = requests.get(main_url, params=params)
+
+    # FastAPI
+    # response = requests.post(main_url, json=params)
+
+
+    if response.status_code == 200:
+        results = response.json()
+        with open('data.json', 'w') as f:
+            json.dump(results, f)
+    else:
+        print(f"Failed to fetch results: {response.status_code}")
+
+
+
+def language_preprocess(text):
+    try:
+        if detect_langs(text)[0].lang == 'en':
+            return True
+        return False
+    except:
+        return False
+
+
+def relevant(product, similar_product, content):
+
+    try:
+        payload = { "inputs": f'''Do you think that the given content is similar to {similar_product} and {product}, just Respond True or False  \nContent for similar product:  {content}'''}
+        
+        # response = requests.post(API_URL, headers=headers, json=payload)
+        # output = response.json()
+        # return bool(output[0]['generated_text'])
+            
+        model = random.choice([gemini,gemini1,gemini2,gemini3])
+        result = model.invoke(f'''Do you think that the given content is similar to {similar_product} and {product}, just Respond True or False  \nContent for similar product:  {content}''')
+        return bool(result)
+
+    except:
+        return False
+        
+        
+
+def download_pdf(url, timeout=10):
+    try:
+        response = requests.get(url, timeout=timeout)
+        response.raise_for_status()
+        return BytesIO(response.content)
+
+    except requests.RequestException as e:
+        logging.error(f"PDF download error: {e}")
+        return None
+
+def extract_text_from_pdf(pdf_file, pages):
+    reader = PdfReader(pdf_file)
+    extracted_text = ""
+
+    l = len(reader.pages)
+    
+    try:
+        for page_num in pages:
+            if page_num < l:
+                page = reader.pages[page_num]
+                extracted_text += page.extract_text() + "\n"
+            else:
+                print(f"Page {page_num} does not exist in the document.")
+        
+        return extracted_text
+    
+    except:
+        return 'हे चालत नाही'
+    
+def extract_text_online(link):
+
+    loader = WebBaseLoader(link)
+    pages = loader.load_and_split()
+
+    text = ''
+
+    for page in pages[:3]:
+        text+=page.page_content
+    
+    return text
+
+
+def process_link(link, main_product, similar_product):
+    if link in seen:
+        return None
+    seen.add(link)
+    try:
+        if link[-3:]=='.md' or link[8:11] == 'en.':
+            text = extract_text_online(link)
+        else:
+            pdf_file = download_pdf(link)
+            text = extract_text_from_pdf(pdf_file, [0, 2, 4])
+
+        if language_preprocess(text):
+            if relevant(main_product, similar_product, text):
+                print("Accepted",link)
+                return link
+    except:
+        pass
+    print("NOT Accepted",link)
+    return None
+
+def filtering(urls, main_product, similar_product, link_count):
+    res = []
+
+    # print(f"Filtering Links of ---- {similar_product}")
+    # Main Preprocess ------------------------------
+    # with ThreadPoolExecutor() as executor:
+    #     futures = {executor.submit(process_link, link, main_product, similar_product): link for link in urls}
+    #     for future in concurrent.futures.as_completed(futures):
+    #         result = future.result()
+    #         if result is not None:
+    #             res.append(result)
+
+    # return res
+
+    count = 0
+
+    print(f"Filtering Links of ---- {similar_product}")
+
+    for link in urls:
+        result = process_link(link, main_product, similar_product)
+        
+        if result is not None:
+            res.append(result)
+            count += 1
+        
+        if count == link_count:
+            break
+
+    return res
+
+
+# Main Functions -------------------------------------------------->
+
+# get_links()
+# preprocess()
+