Upload 3 files

embedding.py

@@ -0,0 +1,255 @@
+import requests
+import json
+import os
+import concurrent.futures
+import random
+from langchain_google_genai import ChatGoogleGenerativeAI
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import google.generativeai as genai
+
+
+gemini = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBmZtXjJgp7yIAo9joNCZGSxK9PbGMcVaA',temperature = 0.1)
+gemini1 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyABsaDjPujPCBlz4LLxcXDX_bDA9uEL7Xc',temperature = 0.1)
+gemini2 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBCIQgt1uK7-sJH5Afg5vUZ99EWkx5gSU0',temperature = 0.1)
+gemini3 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBot9W5Q-BKQ66NAYRUmVeloXWEbXOXTmM',temperature = 0.1)
+
+genai.configure(api_key="AIzaSyBmZtXjJgp7yIAo9joNCZGSxK9PbGMcVaA")
+
+
+def pdf_extractor(link):
+    text = ''
+
+    try:
+        loader = PyPDFLoader(link)
+        pages = loader.load_and_split()
+
+        for page in pages:
+            text+=page.page_content
+    except:
+        pass
+    
+    return [text]
+
+def web_extractor(link):
+    text = ''
+
+    try:
+        loader = WebBaseLoader(link)
+        pages = loader.load_and_split()
+
+        for page in pages:
+            text+=page.page_content
+    except:
+        pass
+    
+    return [text]
+
+
+def feature_extraction(tag, history , context):
+
+    prompt = f'''
+    You are an intelligent assistant tasked with updating product information. You have two data sources:
+    1. Tag_History: Previously gathered information about the product.
+    2. Tag_Context: New data that might contain additional details.
+
+    Your job is to read the Tag_Context and update the relevant field in the Tag_History with any new details found. The field to be updated is the {tag} FIELD.
+
+    Guidelines:
+    - Only add new details that are relevant to the {tag} FIELD.
+    - Do not add or modify any other fields in the Tag_History.
+    - Ensure your response is in coherent sentences, integrating the new details seamlessly into the existing information.
+
+    Here is the data:
+
+    Tag_Context: {str(context)}
+    Tag_History: {history}
+
+    Respond with the updated Tag_History.
+    '''
+
+    model = random.choice([gemini,gemini1])
+    result = model.invoke(prompt)
+
+    return result.content
+
+def detailed_feature_extraction(find, context):
+
+    prompt = f'''
+    You are an intelligent assistant tasked with finding product information. You have one data source and one output format:
+    1. Context: The gathered information about the product.
+    2. Format: Details which need to be filled based on Context.
+
+    Your job is to read the Context and update the relevant field in Format using Context.
+
+    Guidelines:
+    - Only add details that are relevant to the individual FIELD.
+    - Do not add or modify any other fields in the Format.
+    - If nothing found return None.
+
+    Here is the data:
+
+    The Context is {str(context)}
+    The Format is {str(find)}
+    '''
+
+    model = random.choice([gemini,gemini1,gemini2,gemini3])
+    result = model.invoke(prompt)
+
+    return result.content
+
+def detailed_history(history):
+
+    details = {
+    "Introduction": {
+        "Product Name": None,
+        "Overview of the product": None,
+        "Purpose of the manual": None,
+        "Audience": None,
+        "Additional Details": None
+    },
+    "Specifications": {
+        "Technical specifications": None,
+        "Performance metrics": None,
+        "Additional Details": None
+    },
+    "Product Overview": {
+        "Product features": None,
+        "Key components and parts": None,
+        "Additional Details": None
+    },
+    "Safety Information": {
+        "Safety warnings and precautions": None,
+        "Compliance and certification information": None,
+        "Additional Details": None
+    },
+    "Installation Instructions": {
+        "Unboxing and inventory checklist": None,
+        "Step-by-step installation guide": None,
+        "Required tools and materials": None,
+        "Additional Details": None
+    },
+    "Setup and Configuration": {
+        "Initial setup procedures": None,
+        "Configuration settings": None,
+        "Troubleshooting setup issues": None,
+        "Additional Details": None
+    },
+    "Operation Instructions": {
+        "How to use the product": None,
+        "Detailed instructions for different functionalities": None,
+        "User interface guide": None,
+        "Additional Details": None
+    },
+    "Maintenance and Care": {
+        "Cleaning instructions": None,
+        "Maintenance schedule": None,
+        "Replacement parts and accessories": None,
+        "Additional Details": None
+    },
+    "Troubleshooting": {
+        "Common issues and solutions": None,
+        "Error messages and their meanings": None,
+        "Support Information": None,
+        "Additional Details": None
+    },
+    "Warranty Information": {
+        "Terms and Conditions": None,
+        "Service and repair information": None,
+        "Additional Details": None
+    },
+    "Legal Information": {
+        "Copyright information": None,
+        "Trademarks and patents": None,
+        "Disclaimers": None,
+        "Additional Details": None
+
+    }
+}
+
+    for key,val in history.items():
+
+        find = details[key]
+
+        details[key] = str(detailed_feature_extraction(find,val))
+
+    return details
+
+
+def get_embeddings(link): 
+
+        print(f"\nCreating Embeddings ----- {link}")
+        history = {
+                "Introduction": "",
+                "Specifications": "",
+                "Product Overview": "",
+                "Safety Information": "",
+                "Installation Instructions": "",
+                "Setup and Configuration": "",
+                "Operation Instructions": "",
+                "Maintenance and Care": "",
+                "Troubleshooting": "",
+                "Warranty Information": "",
+                "Legal Information": ""
+            }
+
+        # Extract Text -----------------------------
+        print("Extracting Text")
+        if link[-3:] == '.md' or link[8:11] == 'en.':
+            text = web_extractor(link)
+        else:
+            text = pdf_extractor(link)
+
+        # Create Chunks ----------------------------
+        print("Writing Tag Data")
+        chunks = text_splitter.create_documents(text)
+
+        for chunk in chunks:
+
+            with concurrent.futures.ThreadPoolExecutor() as executor:
+                    future_to_key = {
+                        executor.submit(
+                            feature_extraction, f"Product {key}", history[key], chunk.page_content
+                        ): key for key in history
+                    }
+                    for future in concurrent.futures.as_completed(future_to_key):
+                        key = future_to_key[future]
+                        try:
+                            response = future.result()
+                            history[key] = response
+                        except Exception as e:
+                            print(f"Error processing {key}: {e}")
+            
+            # history = detailed_history(history)
+        print("Creating Vectors")
+        print(history)
+        genai_embeddings=[]
+
+        for tag in history:
+            try:    
+                result = genai.embed_content(
+                        model="models/embedding-001",
+                        content=history[tag],
+                        task_type="retrieval_document")
+                genai_embeddings.append(result['embedding'])
+            except:
+                genai_embeddings.append([0]*768)
+
+
+        return history,genai_embeddings
+            
+global text_splitter
+global data
+global history
+
+
+text_splitter = RecursiveCharacterTextSplitter(
+    chunk_size = 10000,
+    chunk_overlap  = 100,
+    separators = ["",''," "]
+)
+
+
+if __name__ == '__main__':
+    pass

preprocess.py

@@ -0,0 +1,168 @@
+import requests
+import json
+import random
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor
+from langchain_community.document_loaders import PyPDFLoader
+from langdetect import detect_langs
+import requests
+from PyPDF2 import PdfReader
+from io import BytesIO
+from langchain_community.document_loaders import WebBaseLoader
+from langchain_google_genai import ChatGoogleGenerativeAI
+import logging
+
+data = False 
+seen = set()
+
+# API Urls -----
+
+# main_url = "http://127.0.0.1:5000/search/all"
+main_url = "http://127.0.0.1:8000/search/all"
+# main_product = "Samsung Galaxy s23 ultra"
+
+# Revelevance Checking Models -----
+gemini = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBmZtXjJgp7yIAo9joNCZGSxK9PbGMcVaA',temperature = 0.1)
+gemini1 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyABsaDjPujPCBlz4LLxcXDX_bDA9uEL7Xc',temperature = 0.1)
+gemini2 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBCIQgt1uK7-sJH5Afg5vUZ99EWkx5gSU0',temperature = 0.1)
+gemini3 = ChatGoogleGenerativeAI(model="gemini-1.0-pro-001",google_api_key='AIzaSyBot9W5Q-BKQ66NAYRUmVeloXWEbXOXTmM',temperature = 0.1)
+
+
+API_URL = "https://api-inference.huggingface.co/models/google/flan-t5-xxl"
+headers = {"Authorization": "Bearer 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"}
+
+# Error Debug
+logging.basicConfig(level=logging.INFO)
+
+
+def get_links(main_product,api_key):
+    params = {
+        "API_KEY": f"{api_key}",
+        "product": f"{main_product}",
+    }
+
+    # Flask
+    response = requests.get(main_url, params=params)
+
+    # FastAPI
+    # response = requests.post(main_url, json=params)
+
+
+    if response.status_code == 200:
+        results = response.json()
+        with open('data.json', 'w') as f:
+            json.dump(results, f)
+    else:
+        print(f"Failed to fetch results: {response.status_code}")
+
+
+
+def language_preprocess(text):
+    try:
+        if detect_langs(text)[0].lang == 'en':
+            return True
+        return False
+    except:
+        return False
+
+
+def relevant(product, similar_product, content):
+
+    try:
+        payload = { "inputs": f'''Do you think that the given content is similar to {similar_product} and {product}, just Respond True or False  \nContent for similar product:  {content}'''}
+        
+        # response = requests.post(API_URL, headers=headers, json=payload)
+        # output = response.json()
+        # return bool(output[0]['generated_text'])
+            
+        model = random.choice([gemini,gemini1,gemini2,gemini3])
+        result = model.invoke(f'''Do you think that the given content is similar to {similar_product} and {product}, just Respond True or False  \nContent for similar product:  {content}''')
+        return bool(result)
+
+    except:
+        return False
+        
+        
+
+def download_pdf(url, timeout=10):
+    try:
+        response = requests.get(url, timeout=timeout)
+        response.raise_for_status()
+        return BytesIO(response.content)
+
+    except requests.RequestException as e:
+        logging.error(f"PDF download error: {e}")
+        return None
+
+def extract_text_from_pdf(pdf_file, pages):
+    reader = PdfReader(pdf_file)
+    extracted_text = ""
+
+    l = len(reader.pages)
+    
+    try:
+        for page_num in pages:
+            if page_num < l:
+                page = reader.pages[page_num]
+                extracted_text += page.extract_text() + "\n"
+            else:
+                print(f"Page {page_num} does not exist in the document.")
+        
+        return extracted_text
+    
+    except:
+        return 'हे चालत नाही'
+    
+def extract_text_online(link):
+
+    loader = WebBaseLoader(link)
+    pages = loader.load_and_split()
+
+    text = ''
+
+    for page in pages[:3]:
+        text+=page.page_content
+    
+    return text
+
+
+def process_link(link, main_product, similar_product):
+    if link in seen:
+        return None
+    seen.add(link)
+    try:
+        if link[-3:]=='.md' or link[8:11] == 'en.':
+            text = extract_text_online(link)
+        else:
+            pdf_file = download_pdf(link)
+            text = extract_text_from_pdf(pdf_file, [0, 2, 4])
+
+        if language_preprocess(text):
+            if relevant(main_product, similar_product, text):
+                print("Accepted",link)
+                return link
+    except:
+        pass
+    print("NOT Accepted",link)
+    return None
+
+def filtering(urls, main_product, similar_product):
+    res = []
+
+    print(f"Filtering Links of ---- {similar_product}")
+    # Main Preprocess ------------------------------
+    with ThreadPoolExecutor() as executor:
+        futures = {executor.submit(process_link, link, main_product, similar_product): link for link in urls}
+        for future in concurrent.futures.as_completed(futures):
+            result = future.result()
+            if result is not None:
+                res.append(result)
+
+    return res
+
+
+# Main Functions -------------------------------------------------->
+
+# get_links()
+# preprocess()
+

search.py

@@ -0,0 +1,227 @@
+# Library Imports
+import requests
+from bs4 import BeautifulSoup
+from googlesearch import search
+from duckduckgo_search import DDGS
+import concurrent.futures
+import re
+
+
+
+# Search Functions -------------------------------------------------------------->
+
+# Function to search DuckDuckGo
+def search_duckduckgo(query):
+    print("Fetching Duckduckgo Links -----")
+    try:
+        results = DDGS().text(f"{query} manual filetype:pdf", max_results=5)
+        return [res['href'] for res in results]
+    except:
+        return []
+
+# Function to search Google
+def search_google(query):
+    print("Fetching Google Links -----")
+
+    links = []
+    try:
+        api_key = 'AIzaSyDV_uJwrgNtawqtl6GDfeUj6NqO-H1tA4c'
+        search_engine_id = 'c4ca951b9fc6949cb'
+        
+        url = f"https://www.googleapis.com/customsearch/v1"
+        params = {
+            "key": api_key,
+            "cx": search_engine_id,
+            "q": query + " manual filetype:pdf"
+        }
+
+        response = requests.get(url, params=params)
+        results = response.json()
+
+        for item in results.get('items', []):
+            links.append(item['link'])
+    except:
+        pass
+    
+    try:
+        extension = "ext:pdf"
+        for result in search(query + " manual " + extension, num_results=5):
+            if result.endswith('.pdf'):
+                links.append(result)
+    except:
+        pass
+    
+    return links
+
+# Function to search Internet Archive
+def search_archive(query):
+    print("Fetching Archieve Links -----")
+
+    try:
+        url = "https://archive.org/advancedsearch.php"
+        params = {
+            'q': f'{query} manual',
+            'fl[]': ['identifier', 'title', 'format'],
+            'rows': 50,
+            'page': 1,
+            'output': 'json'
+        }
+
+        # Make the request
+        response = requests.get(url, params=params)
+        data = response.json()
+
+        # Function to extract hyperlinks from a webpage
+        def extract_hyperlinks(url):        
+            # Send a GET request to the URL
+            response = requests.get(url)
+            
+            # Check if the request was successful
+            if response.status_code == 200:
+                # Parse the HTML content of the page
+                soup = BeautifulSoup(response.text, 'html.parser')
+                
+                # Find all <a> tags (hyperlinks)
+                for link in soup.find_all('a', href=True):
+                    href = link['href']
+                    if href.endswith('.pdf'):
+                        pdf_files.append(url+'/'+href)
+                    if href.endswith('.iso'):
+                        # If the link ends with .iso, follow the link and extract .pdf hyperlinks
+                        extract_pdf_from_iso(url+'/'+href+'/')
+
+        # Function to extract .pdf hyperlinks from an .iso file
+        def extract_pdf_from_iso(iso_url):        
+            # Send a GET request to the ISO URL
+            iso_response = requests.get(iso_url)
+            
+            # Check if the request was successful
+            if iso_response.status_code == 200:
+                # Parse the HTML content of the ISO page
+                iso_soup = BeautifulSoup(iso_response.text, 'html.parser')
+                
+                # Find all <a> tags (hyperlinks) in the ISO page
+                for link in iso_soup.find_all('a', href=True):
+                    href = link['href']
+                    if href.endswith('.pdf'):
+                        pdf_files.append('https:'+href)
+
+        pdf_files = []
+
+        def process_doc(doc):
+            identifier = doc.get('identifier', 'N/A')
+            # title = doc.get('title', 'N/A')
+            # format = doc.get('format', 'N/A')
+            pdf_link = f"https://archive.org/download/{identifier}"
+            extract_hyperlinks(pdf_link)
+
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            futures = [executor.submit(process_doc, doc) for doc in data['response']['docs']]
+
+            # Optionally, wait for all futures to complete and handle any exceptions
+            for future in concurrent.futures.as_completed(futures):
+                try:
+                    future.result()  # This will raise an exception if the function call raised
+                except Exception as exc:
+                    print(f'Generated an exception: {exc}')
+
+
+        return pdf_files
+    
+    except:
+        return []
+
+def search_github(query):
+    print("Fetching Github Links -----")
+
+    try:
+        # GitHub Search API endpoint
+        url = f"https://api.github.com/search/code?q={query}+extension:md"
+
+        headers = {
+        'Authorization': 'Token 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'
+        }
+
+        # Make the request
+        response = requests.get(url,headers=headers)
+        data = response.json()
+        links = [item['html_url'] for item in data['items']]
+
+        return links
+    
+    except:
+        return []
+
+def search_wikipedia(product):
+    print("Fetching Duckduckgo Links -----")
+
+    api_url = "https://en.wikipedia.org/w/api.php"
+    params = {
+        "action": "opensearch",
+        "search": product,
+        "limit": 5,
+        "namespace": 0,
+        "format": "json"
+    }
+    
+    try:
+        response = requests.get(api_url, params=params)
+        response.raise_for_status()  # Raise an HTTPError for bad responses (4xx and 5xx)
+        data = response.json()
+        
+        if data and len(data) > 3 and len(data[3]) > 0:
+            return data[3]  # The URL is in the fourth element of the response array
+        else:
+            return []
+    
+    except requests.RequestException as e:
+        print(f"An error occurred: {e}")
+        return []
+
+# def search_all(product,num):
+
+#     similar_products = extract_similar_products(product)[num]
+    
+#     # results = {
+#     #         product : [{'duckduckgo': duckduckgo_search(product)},{'google': google_search(product)},{'github': github_search(product)},{'archive': archive_search(product)}]
+#     #     }
+
+#     results = {}
+        
+#     def search_product(p):
+#             return {
+#                 'product': p,
+#                 'duckduckgo': duckduckgo_search(p),
+#                 'google': google_search(p),
+#                 'github': github_search(p),
+#                 'archive': archive_search(p),
+#                 'wikipedia': wikipedia_search(p)
+#             }
+
+#     with concurrent.futures.ThreadPoolExecutor() as executor:
+#             future_to_product = {executor.submit(search_product, p): p for p in similar_products}
+            
+#             for future in concurrent.futures.as_completed(future_to_product):
+#                 result = future.result()
+#                 product = result['product']
+#                 results[product] = [
+#                     {'duckduckgo': result['duckduckgo']},
+#                     {'google': result['google']},
+#                     {'github': result['github']},
+#                     {'archive': result['archive']},
+#                     {'wikipedia': result['wikipedia']}
+#                 ]
+                
+#     return results
+    
+# Similarity Check  -------------------------------------->
+
+def extract_similar_products(query):
+    print(f"\nFetching similar items of  -----> {query}")
+    results = DDGS().chat(f'{query} Similar Products')
+
+    pattern = r'^\d+\.\s(.+)$'
+    matches = re.findall(pattern, results, re.MULTILINE)
+    matches = [item.split(': ')[0] for item in matches]
+    return matches
+