Update app.py

app.py

@@ -1,327 +1,172 @@
-from flask import Flask, request, jsonify, render_template
-import requests
-from bs4 import BeautifulSoup
-from googlesearch import search
-from duckduckgo_search import DDGS
-import concurrent.futures
-import re
-
-
-app = Flask(__name__)
-
-API_KEY_DEFAULT = '12345'
-
-# Function to search DuckDuckGo
-def duckduckgo_search(query):
-    try:
-        results = DDGS().text(f"{query} manual filetype:pdf", max_results=5)
-        return [res['href'] for res in results]
-    except:
-        return []
-
-# Function to search Google
-def google_search(query):
-
-    links = []
-    try:
-        api_key = 'AIzaSyDV_uJwrgNtawqtl6GDfeUj6NqO-H1tA4c'
-        search_engine_id = 'c4ca951b9fc6949cb'
-        
-        url = f"https://www.googleapis.com/customsearch/v1"
-        params = {
-            "key": api_key,
-            "cx": search_engine_id,
-            "q": query + " manual filetype:pdf"
-        }
-
-        response = requests.get(url, params=params)
-        results = response.json()
-
-        for item in results.get('items', []):
-            links.append(item['link'])
-    except:
-        pass
-    
-    try:
-        extension = "ext:pdf"
-        for result in search(query + " manual " + extension, num_results=5):
-            if result.endswith('.pdf'):
-                links.append(result)
-    except:
-        pass
-    
-
-    return links
-
-# Function to search Internet Archive
-def archive_search(query):
-
-    try:
-        url = "https://archive.org/advancedsearch.php"
-        params = {
-            'q': f'{query} manual',
-            'fl[]': ['identifier', 'title', 'format'],
-            'rows': 50,
-            'page': 1,
-            'output': 'json'
-        }
-
-        # Make the request
-        response = requests.get(url, params=params)
-        data = response.json()
-
-        # Function to extract hyperlinks from a webpage
-        def extract_hyperlinks(url):        
-            # Send a GET request to the URL
-            response = requests.get(url)
-            
-            # Check if the request was successful
-            if response.status_code == 200:
-                # Parse the HTML content of the page
-                soup = BeautifulSoup(response.text, 'html.parser')
-                
-                # Find all <a> tags (hyperlinks)
-                for link in soup.find_all('a', href=True):
-                    href = link['href']
-                    if href.endswith('.pdf'):
-                        pdf_files.append(url+'/'+href)
-                    if href.endswith('.iso'):
-                        # If the link ends with .iso, follow the link and extract .pdf hyperlinks
-                        extract_pdf_from_iso(url+'/'+href+'/')
-
-        # Function to extract .pdf hyperlinks from an .iso file
-        def extract_pdf_from_iso(iso_url):        
-            # Send a GET request to the ISO URL
-            iso_response = requests.get(iso_url)
-            
-            # Check if the request was successful
-            if iso_response.status_code == 200:
-                # Parse the HTML content of the ISO page
-                iso_soup = BeautifulSoup(iso_response.text, 'html.parser')
-                
-                # Find all <a> tags (hyperlinks) in the ISO page
-                for link in iso_soup.find_all('a', href=True):
-                    href = link['href']
-                    if href.endswith('.pdf'):
-                        pdf_files.append('https:'+href)
-
-        pdf_files = []
-
-        def process_doc(doc):
-            identifier = doc.get('identifier', 'N/A')
-            # title = doc.get('title', 'N/A')
-            # format = doc.get('format', 'N/A')
-            pdf_link = f"https://archive.org/download/{identifier}"
-            extract_hyperlinks(pdf_link)
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            futures = [executor.submit(process_doc, doc) for doc in data['response']['docs']]
-
-            # Optionally, wait for all futures to complete and handle any exceptions
-            for future in concurrent.futures.as_completed(futures):
-                try:
-                    future.result()  # This will raise an exception if the function call raised
-                except Exception as exc:
-                    print(f'Generated an exception: {exc}')
-
-
-        return pdf_files
-    
-    except:
-        return []
-
-def github_search(query):
-
-    try:
-        # GitHub Search API endpoint
-        url = f"https://api.github.com/search/code?q={query}+extension:md"
-
-        headers = {
-        'Authorization': 'Token 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'
-        }
-
-        # Make the request
-        response = requests.get(url,headers=headers)
-        data = response.json()
-        links = [item['html_url'].replace('/blob','').replace('//github','//raw.github') for item in data['items']]
-
-        return links
-    
-    except:
-        return []
-    
-
-#Similarity Check 
-
-def extract_similar_products(query):
-    results = DDGS().chat(f'{query} Similar Products')
-
-    pattern = r'^\d+\.\s(.+)$'
-    matches = re.findall(pattern, results, re.MULTILINE)
-    matches = [item.split(': ')[0] for item in matches]
-    print(matches)
-
-    return matches[:5] if matches else []
-
-
-# Define API routes -------------------------------------------------------
-
-@app.route('/')
-def home():
-    return render_template('index.html')
-
-
-@app.route('/search/google', methods=['GET','POST'])
-def search_google():
-    
-    if request.method == 'POST':
-        data = request.get_json()
-        api_key = data.get('API_KEY')
-        product = data.get('product')
-    else:
-        product = request.args.get('product')
-        api_key = request.args.get('API_KEY')
-
-    similar_products = extract_similar_products(product)
-    if api_key == API_KEY_DEFAULT:
-        results = {product: google_search(product)}
-        for p in similar_products:
-            results[p] = google_search(p)
-        return jsonify(results)
-
-    else:
-        return jsonify({'error': 'Invalid API key'}), 401
-
-@app.route('/search/duckduckgo', methods=['GET','POST'])
-def search_duckduckgo():
-
-    if request.method == 'POST':
-        data = request.get_json()
-        api_key = data.get('API_KEY')
-        product = data.get('product')
-    else:
-        product = request.args.get('product')
-        api_key = request.args.get('API_KEY')
-    
-    similar_products = extract_similar_products(product)
-
-
-    if api_key == API_KEY_DEFAULT:
-        results = {product: duckduckgo_search(product)}
-        for p in similar_products:
-            results[p] = duckduckgo_search(p)
-        return jsonify(results)
-    else:
-        return jsonify({'error': 'Invalid API key'}), 401
-
-
-@app.route('/search/archive', methods=['GET','POST'])
-def search_archive():
-
-    if request.method == 'POST':
-        data = request.get_json()
-        api_key = data.get('API_KEY')
-        product = data.get('product')
-    else:
-        product = request.args.get('product')
-        api_key = request.args.get('API_KEY')
-    
-    # Retrieve custom headers if any
-
-    similar_products = extract_similar_products(product)
-
-    if api_key == API_KEY_DEFAULT:
-        results = {product: archive_search(product)}
-        
-        def process_product(product):
-            return product, archive_search(product)
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            # Map the process_product function to similar_products
-            future_to_product = {executor.submit(process_product, p): p for p in similar_products}
-
-            # Collect results as they complete
-            for future in concurrent.futures.as_completed(future_to_product):
-                product, result = future.result()
-                results[product] = result
-
-        return jsonify(results)
-
-    else:
-        return jsonify({'error': 'Invalid API key'}), 401
-
-
-@app.route('/search/github', methods=['GET','POST'])
-def search_github():
-        
-    if request.method == 'POST':
-        data = request.get_json()
-        api_key = data.get('API_KEY')
-        product = data.get('product')
-    else:
-        product = request.args.get('product')
-        api_key = request.args.get('API_KEY')
-
-    similar_products = extract_similar_products(product)
-
-    if api_key == API_KEY_DEFAULT:
-        results = {product: github_search(product)}
-        for p in similar_products:
-            results[p] = github_search(p)
-        return jsonify(results)
-
-    else:
-        return jsonify({'error': 'Invalid API key'}), 401
-
-
-@app.route('/search/all', methods=['GET','POST'])
-def search_all():
-
-    if request.method == 'POST':
-        data = request.get_json()
-        api_key = data.get('API_KEY')
-        product = data.get('product')
-    else:
-        product = request.args.get('product')
-        api_key = request.args.get('API_KEY')
-
-    similar_products = extract_similar_products(product)
-    
-
-    if api_key == API_KEY_DEFAULT:
-
-        results = {
-            product : [{'duckduckgo': duckduckgo_search(product)},{'google': google_search(product)},{'github': github_search(product)},{'archive': archive_search(product)}]
-        }
-        
-        def search_product(p):
-            return {
-                'product': p,
-                'duckduckgo': duckduckgo_search(p),
-                'google': google_search(p),
-                'github': github_search(p),
-                'archive': archive_search(p)
-            }
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
-            future_to_product = {executor.submit(search_product, p): p for p in similar_products}
-            
-            for future in concurrent.futures.as_completed(future_to_product):
-                result = future.result()
-                product = result['product']
-                results[product] = [
-                    {'duckduckgo': result['duckduckgo']},
-                    {'google': result['google']},
-                    {'github': result['github']},
-                    {'archive': result['archive']}
-                ]
-
-        return jsonify(results)
-
-    else:
-        return jsonify({'error': 'Invalid API key'}), 401
-
-# Run the Flask app
-if __name__ == '__main__':
-    app.run(debug=True)
-
+# file: app.py
+
+import gradio as gr
+import requests
+import json
+import concurrent.futures
+from concurrent.futures import ThreadPoolExecutor
+from langchain_community.document_loaders import PyPDFLoader
+from langdetect import detect_langs
+from PyPDF2 import PdfReader
+from io import BytesIO
+import logging
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+data = False 
+seen = set()
+
+main_url = "https://similar-products-api.vercel.app/search/all"
+main_product = "Samsung Galaxy"
+
+API_URL = "https://api-inference.huggingface.co/models/google/flan-t5-xxl"
+headers = {"Authorization": f"Bearer {os.getenv('HUGGINGFACE_API_TOKEN')}"}
+
+logging.basicConfig(level=logging.INFO)
+
+def get_links(product):
+    params = {
+        "API_KEY": "12345",
+        "product": f"{product}",
+    }
+    response = requests.get(main_url, params=params)
+    if response.status_code == 200:
+        results = response.json()
+        return results
+    else:
+        return {}
+
+def language_preprocess(text):
+    try:
+        if detect_langs(text)[0].lang == 'en':
+            return True
+        return False
+    except Exception as e:
+        logging.error(f"Language detection error: {e}")
+        return False
+
+def relevant(product, similar_product, content):
+    try:
+        payload = {"inputs": f'''Do you think that the given content is similar to {similar_product} and {product}, just Respond True or False  \nContent for similar product:  {content[:700]}'''}
+        response = requests.post(API_URL, headers=headers, json=payload)
+        output = response.json()
+        return bool(output[0]['generated_text'])
+    except Exception as e:
+        logging.error(f"Relevance checking error: {e}")
+        return False
+
+def download_pdf(url, timeout=10):
+    try:
+        response = requests.get(url, timeout=timeout)
+        response.raise_for_status()
+        return BytesIO(response.content)
+    except requests.RequestException as e:
+        logging.error(f"PDF download error: {e}")
+        return None
+
+def extract_text_from_pages(pdf_file, pages):
+    reader = PdfReader(pdf_file)
+    extracted_text = ""
+    try:
+        for page_num in pages:
+            if page_num < len(reader.pages):
+                page = reader.pages[page_num]
+                extracted_text += page.extract_text() + "\n"
+            else:
+                logging.warning(f"Page {page_num} does not exist in the document.")
+        return extracted_text
+    except Exception as e:
+        logging.error(f"PDF text extraction error: {e}")
+        return 'हे चालत नाही'
+
+def process_link(link, similar_product):
+    if link in seen:
+        return None
+    seen.add(link)
+    try:
+        pdf_file = download_pdf(link)
+        if pdf_file:
+            text = extract_text_from_pages(pdf_file, [0, 2, 4])
+            if language_preprocess(text):
+                if relevant(main_product, similar_product, text):
+                    return link
+    except Exception as e:
+        logging.error(f"Error processing link: {e}")
+    return None
+
+def filtering(urls, similar_product):
+    res = []
+    with ThreadPoolExecutor() as executor:
+        futures = {executor.submit(process_link, link, similar_product): link for link in urls}
+        for future in concurrent.futures.as_completed(futures):
+            result = future.result()
+            if result is not None:
+                res.append(result)
+    return res
+
+def wikipedia_url(product):
+    api_url = "https://en.wikipedia.org/w/api.php"
+    params = {
+        "action": "opensearch",
+        "search": product,
+        "limit": 5,
+        "namespace": 0,
+        "format": "json"
+    }
+    try:
+        response = requests.get(api_url, params=params)
+        response.raise_for_status()
+        data = response.json()
+        if data and len(data) > 3 and len(data[3]) > 0:
+            return data[3]
+        else:
+            return []
+    except requests.RequestException as e:
+        logging.error(f"Error fetching Wikipedia URLs: {e}")
+        return []
+
+def preprocess_initial(product):
+    return get_links(product)
+
+def preprocess_filter(product, data):
+    for similar_product in data:
+        # if similar_product != product:
+            if list(data[similar_product][0])[0] == 'duckduckgo':
+                s = set(('duckduckgo', 'google', 'archive'))
+                temp = []
+
+                for idx, item in enumerate(data[similar_product]):
+                    if list(item)[0] in s:
+                        urls = data[similar_product][idx][list(item)[0]]
+                        temp += filtering(urls, similar_product)
+                    else:
+                        temp += data[similar_product][idx][list(item)[0]]
+
+                data[similar_product] = temp
+                data[similar_product] += wikipedia_url(similar_product)
+            else:
+                urls = data[similar_product]
+                data[similar_product] = filtering(urls, similar_product)
+                data[similar_product] += wikipedia_url(similar_product)
+    logging.info('Filtering completed')
+    return data
+
+def main(product_name):
+    return preprocess_initial(product_name)
+
+def filter_links(product_name, initial_data):
+    return preprocess_filter(product_name, initial_data)
+
+with gr.Blocks() as demo:
+    product_name = gr.Textbox(label="Product Name")
+    get_links_btn = gr.Button("Get Links")
+    initial_links_output = gr.JSON()
+    filter_btn = gr.Button("Filter Links")
+    filtered_links_output = gr.JSON()
+
+    get_links_btn.click(fn=main, inputs=product_name, outputs=initial_links_output)
+    filter_btn.click(fn=filter_links, inputs=[product_name, initial_links_output], outputs=filtered_links_output)
+
+if __name__ == "__main__":
+    demo.launch()