selvaonline's picture
Upload app.py with huggingface_hub
1ba7ad7 verified
raw
history blame
26.1 kB
import gradio as gr
import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification
import json
import os
import requests
import re
# Function to extract text from HTML (from shopping_assistant.py)
def extract_text_from_html(html):
"""
Extract text from HTML without using BeautifulSoup
"""
# Remove HTML tags
text = re.sub(r'<[^>]+>', ' ', html)
# Remove extra whitespace
text = re.sub(r'\s+', ' ', text)
# Decode HTML entities
text = text.replace('&nbsp;', ' ').replace('&amp;', '&').replace('&lt;', '<').replace('&gt;', '>')
return text.strip()
# Sample deals data to use as fallback
SAMPLE_DEALS = [
{
"id": 1,
"title": {
"rendered": "Apple AirPods Pro (2nd Generation) - 20% Off"
},
"link": "https://www.example.com/deals/airpods-pro",
"date": "2025-02-25T10:00:00",
"content": {
"rendered": "<p>Get the latest Apple AirPods Pro (2nd Generation) for 20% off the regular price. These wireless earbuds feature active noise cancellation, transparency mode, and spatial audio with dynamic head tracking.</p><p>Regular price: $249.99</p><p>Deal price: $199.99</p><p>You save: $50.00</p>"
},
"excerpt": {
"rendered": "<p>Apple AirPods Pro (2nd Generation) with active noise cancellation and transparency mode. Now 20% off - only $199.99!</p>"
}
},
{
"id": 2,
"title": {
"rendered": "Samsung 65\" QLED 4K Smart TV - $300 Off"
},
"link": "https://www.example.com/deals/samsung-qled-tv",
"date": "2025-02-26T09:30:00",
"content": {
"rendered": "<p>Upgrade your home entertainment with this Samsung 65\" QLED 4K Smart TV. Features Quantum HDR, Motion Xcelerator Turbo+, and Object Tracking Sound for an immersive viewing experience.</p><p>Regular price: $1,299.99</p><p>Deal price: $999.99</p><p>You save: $300.00</p>"
},
"excerpt": {
"rendered": "<p>Samsung 65\" QLED 4K Smart TV with Quantum HDR and Object Tracking Sound. Save $300 - now only $999.99!</p>"
}
},
{
"id": 3,
"title": {
"rendered": "Sony WH-1000XM5 Wireless Headphones - 25% Off"
},
"link": "https://www.example.com/deals/sony-wh1000xm5",
"date": "2025-02-26T14:15:00",
"content": {
"rendered": "<p>Experience industry-leading noise cancellation with the Sony WH-1000XM5 wireless headphones. Features 30-hour battery life, quick charging, and exceptional sound quality with the new Integrated Processor V1.</p><p>Regular price: $399.99</p><p>Deal price: $299.99</p><p>You save: $100.00</p>"
},
"excerpt": {
"rendered": "<p>Sony WH-1000XM5 wireless headphones with industry-leading noise cancellation and 30-hour battery life. Now 25% off at $299.99!</p>"
}
},
{
"id": 4,
"title": {
"rendered": "Bose QuietComfort Ultra Headphones - 20% Off"
},
"link": "https://www.example.com/deals/bose-quietcomfort-ultra",
"date": "2025-02-25T15:30:00",
"content": {
"rendered": "<p>Experience the ultimate in noise cancellation with Bose QuietComfort Ultra headphones. Features spatial audio, custom EQ, and up to 24 hours of battery life.</p><p>Regular price: $429.99</p><p>Deal price: $343.99</p><p>You save: $86.00</p>"
},
"excerpt": {
"rendered": "<p>Bose QuietComfort Ultra headphones with advanced noise cancellation and spatial audio. Now 20% off at $343.99!</p>"
}
},
{
"id": 5,
"title": {
"rendered": "Beats Studio Pro Wireless Headphones - 40% Off"
},
"link": "https://www.example.com/deals/beats-studio-pro",
"date": "2025-02-26T16:30:00",
"content": {
"rendered": "<p>The Beats Studio Pro wireless headphones deliver premium sound with active noise cancellation, transparency mode, and up to 40 hours of battery life.</p><p>Regular price: $349.99</p><p>Deal price: $209.99</p><p>You save: $140.00</p>"
},
"excerpt": {
"rendered": "<p>Beats Studio Pro wireless headphones with active noise cancellation and 40-hour battery life. Now 40% off at $209.99!</p>"
}
},
{
"id": 6,
"title": {
"rendered": "Dyson V12 Detect Slim Cordless Vacuum - $150 Off"
},
"link": "https://www.example.com/deals/dyson-v12",
"date": "2025-02-27T08:45:00",
"content": {
"rendered": "<p>The Dyson V12 Detect Slim cordless vacuum features a laser that reveals microscopic dust, an LCD screen that displays particle counts, and powerful suction for deep cleaning.</p><p>Regular price: $649.99</p><p>Deal price: $499.99</p><p>You save: $150.00</p>"
},
"excerpt": {
"rendered": "<p>Dyson V12 Detect Slim cordless vacuum with laser dust detection and powerful suction. Save $150 - now only $499.99!</p>"
}
},
{
"id": 7,
"title": {
"rendered": "Nintendo Switch OLED Model - Bundle Deal"
},
"link": "https://www.example.com/deals/nintendo-switch-oled",
"date": "2025-02-27T11:20:00",
"content": {
"rendered": "<p>Get the Nintendo Switch OLED Model with a vibrant 7-inch OLED screen, plus two games and a carrying case. The perfect gaming package for home or on-the-go play.</p><p>Regular price: $439.99</p><p>Deal price: $379.99</p><p>You save: $60.00</p>"
},
"excerpt": {
"rendered": "<p>Nintendo Switch OLED Model bundle with two games and carrying case. Special bundle price of $379.99!</p>"
}
},
{
"id": 8,
"title": {
"rendered": "MacBook Air M3 - $200 Off"
},
"link": "https://www.example.com/deals/macbook-air-m3",
"date": "2025-02-26T10:45:00",
"content": {
"rendered": "<p>The latest MacBook Air with M3 chip offers incredible performance and battery life in an ultra-thin design. Features a 13.6-inch Liquid Retina display, 8GB RAM, and 256GB SSD storage.</p><p>Regular price: $1,099.99</p><p>Deal price: $899.99</p><p>You save: $200.00</p>"
},
"excerpt": {
"rendered": "<p>MacBook Air with M3 chip, 13.6-inch Liquid Retina display, and all-day battery life. Save $200 - now only $899.99!</p>"
}
},
{
"id": 9,
"title": {
"rendered": "Kindle Paperwhite Signature Edition - 30% Off"
},
"link": "https://www.example.com/deals/kindle-paperwhite",
"date": "2025-02-27T09:15:00",
"content": {
"rendered": "<p>The Kindle Paperwhite Signature Edition features a 6.8-inch display, wireless charging, auto-adjusting front light, and 32GB storage. Perfect for reading anywhere, anytime.</p><p>Regular price: $189.99</p><p>Deal price: $132.99</p><p>You save: $57.00</p>"
},
"excerpt": {
"rendered": "<p>Kindle Paperwhite Signature Edition with 6.8-inch display, wireless charging, and 32GB storage. Now 30% off at $132.99!</p>"
}
},
{
"id": 10,
"title": {
"rendered": "LG C3 65\" OLED 4K Smart TV - $500 Off"
},
"link": "https://www.example.com/deals/lg-c3-oled",
"date": "2025-02-25T13:00:00",
"content": {
"rendered": "<p>Experience stunning picture quality with the LG C3 65\" OLED 4K Smart TV. Features self-lit OLED pixels, Dolby Vision, Dolby Atmos, and NVIDIA G-SYNC for gaming.</p><p>Regular price: $1,799.99</p><p>Deal price: $1,299.99</p><p>You save: $500.00</p>"
},
"excerpt": {
"rendered": "<p>LG C3 65\" OLED 4K Smart TV with self-lit pixels and Dolby Vision. Save $500 - now only $1,299.99!</p>"
}
}
]
# Function to fetch deals from DealsFinders.com (from shopping_assistant.py)
def fetch_deals_data(url="https://www.dealsfinders.com/wp-json/wp/v2/posts", num_pages=2, per_page=100, use_sample_data=False):
"""
Fetch deals data exclusively from the DealsFinders API or use sample data
"""
# If use_sample_data is True, return the sample deals
if use_sample_data:
print("Using sample deals data")
return SAMPLE_DEALS
all_deals = []
# Fetch from the DealsFinders API
for page in range(1, num_pages + 1):
try:
# Add a user agent to avoid being blocked
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
}
response = requests.get(f"{url}?page={page}&per_page={per_page}", headers=headers)
if response.status_code == 200:
deals = response.json()
all_deals.extend(deals)
print(f"Fetched page {page} with {len(deals)} deals from DealsFinders API")
# If we get fewer deals than requested, we've reached the end
if len(deals) < per_page:
print(f"Reached the end of available deals at page {page}")
break
else:
print(f"Failed to fetch page {page} from DealsFinders API: {response.status_code}")
print("Falling back to sample deals data")
return SAMPLE_DEALS
except Exception as e:
print(f"Error fetching page {page} from DealsFinders API: {str(e)}")
print("Falling back to sample deals data")
return SAMPLE_DEALS
# If no deals were fetched, use sample data
if not all_deals:
print("No deals fetched from API. Using sample deals data")
return SAMPLE_DEALS
return all_deals
# Function to process deals data (from shopping_assistant.py)
def process_deals_data(deals_data):
"""
Process the deals data into a structured format
"""
processed_deals = []
for deal in deals_data:
try:
# Extract relevant information using our HTML text extractor
content_html = deal.get('content', {}).get('rendered', '')
excerpt_html = deal.get('excerpt', {}).get('rendered', '')
clean_content = extract_text_from_html(content_html)
clean_excerpt = extract_text_from_html(excerpt_html)
processed_deal = {
'id': deal.get('id'),
'title': deal.get('title', {}).get('rendered', ''),
'link': deal.get('link', ''),
'date': deal.get('date', ''),
'content': clean_content,
'excerpt': clean_excerpt
}
processed_deals.append(processed_deal)
except Exception as e:
print(f"Error processing deal: {str(e)}")
return processed_deals
# Define product categories
category_descriptions = {
"electronics": "Electronic devices like headphones, speakers, TVs, smartphones, and gadgets",
"computers": "Laptops, desktops, computer parts, monitors, and computing accessories",
"mobile": "Mobile phones, smartphones, phone cases, screen protectors, and chargers",
"audio": "Headphones, earbuds, speakers, microphones, and audio equipment",
"clothing": "Clothes, shirts, pants, dresses, and fashion items",
"footwear": "Shoes, boots, sandals, slippers, and all types of footwear",
"home": "Home decor, furniture, bedding, and household items",
"kitchen": "Kitchen appliances, cookware, utensils, and kitchen gadgets",
"toys": "Toys, games, and children's entertainment items",
"sports": "Sports equipment, fitness gear, and outdoor recreation items",
"beauty": "Beauty products, makeup, skincare, and personal care items",
"books": "Books, e-books, audiobooks, and reading materials"
}
# List of categories
categories = list(category_descriptions.keys())
# Try to load the recommended models
try:
# 1. Load BART model for zero-shot classification
from transformers import pipeline
# Initialize the zero-shot classification pipeline
classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli")
print("Using facebook/bart-large-mnli for classification")
# 2. Load MPNet model for semantic search
from sentence_transformers import SentenceTransformer, util
# Load the sentence transformer model
sentence_model = SentenceTransformer('sentence-transformers/all-mpnet-base-v2')
print("Using sentence-transformers/all-mpnet-base-v2 for semantic search")
# Pre-compute embeddings for category descriptions
category_texts = list(category_descriptions.values())
category_embeddings = sentence_model.encode(category_texts, convert_to_tensor=True)
# Using recommended models
using_recommended_models = True
except Exception as e:
# Fall back to local model if recommended models fail to load
print(f"Error loading recommended models: {str(e)}")
print("Falling back to local model")
model_path = os.path.dirname(os.path.abspath(__file__))
tokenizer = AutoTokenizer.from_pretrained(model_path)
model = AutoModelForSequenceClassification.from_pretrained(model_path)
# Load the local categories
try:
with open(os.path.join(model_path, "categories.json"), "r") as f:
categories = json.load(f)
except Exception as e:
print(f"Error loading categories: {str(e)}")
categories = ["electronics", "clothing", "home", "kitchen", "toys", "other"]
# Not using recommended models
using_recommended_models = False
# File path for storing deals data locally
DEALS_DATA_PATH = "deals_data.json"
# Function to fetch and save a large number of deals
def fetch_and_save_deals(max_deals=10000, per_page=100):
"""
Fetch a large number of deals and save them to a local file
"""
print(f"Fetching up to {max_deals} deals...")
all_deals = []
num_pages = min(max_deals // per_page + (1 if max_deals % per_page > 0 else 0), 100) # Limit to 100 pages max
# Fetch from the DealsFinders API
for page in range(1, num_pages + 1):
try:
# Add a user agent to avoid being blocked
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36'
}
response = requests.get(f"https://www.dealsfinders.com/wp-json/wp/v2/posts?page={page}&per_page={per_page}", headers=headers)
if response.status_code == 200:
deals = response.json()
all_deals.extend(deals)
print(f"Fetched page {page} with {len(deals)} deals from DealsFinders API")
# If we get fewer deals than requested, we've reached the end
if len(deals) < per_page:
print(f"Reached the end of available deals at page {page}")
break
# If we've reached the maximum number of deals, stop
if len(all_deals) >= max_deals:
all_deals = all_deals[:max_deals] # Trim to max_deals
print(f"Reached the maximum number of deals ({max_deals})")
break
else:
print(f"Failed to fetch page {page} from DealsFinders API: {response.status_code}")
break
except Exception as e:
print(f"Error fetching page {page} from DealsFinders API: {str(e)}")
break
# Process the deals
processed_deals = process_deals_data(all_deals)
# Save the deals to a local file
try:
with open(DEALS_DATA_PATH, "w") as f:
json.dump(processed_deals, f)
print(f"Saved {len(processed_deals)} deals to {DEALS_DATA_PATH}")
return processed_deals
except Exception as e:
print(f"Error saving deals to file: {str(e)}")
return processed_deals
# Function to load deals from the local file
def load_deals_from_file():
"""
Load deals from the local file
"""
try:
if os.path.exists(DEALS_DATA_PATH):
with open(DEALS_DATA_PATH, "r") as f:
deals = json.load(f)
print(f"Loaded {len(deals)} deals from {DEALS_DATA_PATH}")
return deals
else:
print(f"Deals file {DEALS_DATA_PATH} does not exist")
return None
except Exception as e:
print(f"Error loading deals from file: {str(e)}")
return None
# Global variable to store deals data
deals_cache = None
# Load deals from file on startup
try:
# Try to load from file
deals_cache = load_deals_from_file()
# If file doesn't exist or is empty, use sample data
if deals_cache is None or len(deals_cache) == 0:
print("No deals found in local file. Using sample data...")
deals_cache = process_deals_data(SAMPLE_DEALS)
print(f"Initialized with {len(deals_cache) if deals_cache else 0} deals")
except Exception as e:
print(f"Error initializing deals cache: {str(e)}")
# Fall back to sample data
deals_cache = process_deals_data(SAMPLE_DEALS)
print(f"Initialized with {len(deals_cache)} sample deals")
def classify_text(text, fetch_deals=True):
"""
Classify the text using the model and fetch relevant deals
"""
global deals_cache
# Get the top categories based on the model type
if using_recommended_models:
# Using BART for zero-shot classification
result = classifier(text, categories, multi_label=True)
# Extract categories and scores
top_categories = []
for i, (category, score) in enumerate(zip(result['labels'], result['scores'])):
if score > 0.1: # Lower threshold for zero-shot classification
top_categories.append((category, score))
# Limit to top 3 categories
if i >= 2:
break
else:
# Using the original classification model
inputs = tokenizer(text, return_tensors="pt", truncation=True, padding=True)
# Get the model prediction
with torch.no_grad():
outputs = model(**inputs)
predictions = torch.sigmoid(outputs.logits)
# Get the top categories
top_categories = []
for i, score in enumerate(predictions[0]):
if score > 0.5: # Threshold for multi-label classification
top_categories.append((categories[i], score.item()))
# Sort by score
top_categories.sort(key=lambda x: x[1], reverse=True)
# Format the classification results
if top_categories:
result = f"Top categories for '{text}':\n\n"
for category, score in top_categories:
result += f"- {category}: {score:.4f}\n"
result += f"\nBased on your query, I would recommend looking for deals in the **{top_categories[0][0]}** category.\n\n"
else:
result = f"No categories found for '{text}'. Please try a different query.\n\n"
# Fetch and display deals if requested
if fetch_deals:
result += "## Relevant Deals from DealsFinders.com\n\n"
try:
# Fetch deals data if not already cached
if deals_cache is None:
# Use sample data by default in Hugging Face space environment
deals_data = fetch_deals_data(num_pages=2, use_sample_data=True) # Use sample data for reliability
deals_cache = process_deals_data(deals_data)
# Using MPNet for semantic search if available
if using_recommended_models:
# Create deal texts for semantic search
deal_texts = []
for deal in deals_cache:
# Combine title and excerpt for better matching
deal_text = f"{deal['title']} {deal['excerpt']}"
deal_texts.append(deal_text)
# Encode the query and deals
query_embedding = sentence_model.encode(text, convert_to_tensor=True)
deal_embeddings = sentence_model.encode(deal_texts, convert_to_tensor=True)
# Calculate semantic similarity
similarities = util.cos_sim(query_embedding, deal_embeddings)[0]
# Get top 5 most similar deals
top_indices = torch.topk(similarities, k=min(5, len(deals_cache))).indices
# Extract the relevant deals
relevant_deals = [deals_cache[idx] for idx in top_indices]
else:
# Improved keyword-based search with category awareness
query_terms = text.lower().split()
expanded_terms = list(query_terms)
# Get the top category from the classification results
top_category = top_categories[0][0] if top_categories else None
# Add category-specific terms
if top_category == "electronics":
expanded_terms.extend(['electronic', 'device', 'gadget', 'tech', 'technology'])
if any(term in text.lower() for term in ['headphone', 'headphones']):
expanded_terms.extend(['earbuds', 'earphones', 'earpods', 'airpods', 'audio', 'bluetooth', 'wireless'])
elif any(term in text.lower() for term in ['laptop', 'computer']):
expanded_terms.extend(['notebook', 'macbook', 'chromebook', 'pc'])
elif any(term in text.lower() for term in ['tv', 'television']):
expanded_terms.extend(['smart tv', 'roku', 'streaming'])
elif top_category == "kitchen":
expanded_terms.extend(['appliance', 'cookware', 'utensil', 'blender', 'mixer', 'toaster', 'microwave', 'oven'])
elif top_category == "home":
expanded_terms.extend(['furniture', 'decor', 'decoration', 'bedding', 'household'])
elif top_category == "clothing":
expanded_terms.extend(['clothes', 'shirt', 'pants', 'dress', 'fashion', 'wear', 'apparel'])
elif top_category == "toys":
expanded_terms.extend(['game', 'play', 'children', 'kid', 'kids', 'fun'])
# Score deals based on relevance to the query
scored_deals = []
for deal in deals_cache:
title = deal['title'].lower()
content = deal['content'].lower()
excerpt = deal['excerpt'].lower()
score = 0
# Check original query terms (higher weight)
for term in query_terms:
if term in title:
score += 10
if term in content:
score += 3
if term in excerpt:
score += 3
# Check expanded terms (lower weight)
for term in expanded_terms:
if term not in query_terms: # Skip original terms
if term in title:
score += 5
if term in content:
score += 1
if term in excerpt:
score += 1
# Boost score for deals matching the top category
if top_category:
if top_category.lower() in title.lower():
score += 15
if top_category.lower() in content.lower():
score += 5
if top_category.lower() in excerpt.lower():
score += 5
# Add to scored deals if it has any relevance
if score > 0:
scored_deals.append((deal, score))
# Sort by score (descending)
scored_deals.sort(key=lambda x: x[1], reverse=True)
# Extract the deals from the scored list
relevant_deals = [deal for deal, _ in scored_deals[:5]]
if relevant_deals:
for i, deal in enumerate(relevant_deals, 1):
result += f"{i}. [{deal['title']}]({deal['link']})\n\n"
else:
result += "No specific deals found for your query. Try a different search term or browse the recommended category.\n\n"
except Exception as e:
result += f"Error fetching deals: {str(e)}\n\n"
return result
# Create the Gradio interface
demo = gr.Interface(
fn=classify_text,
inputs=[
gr.Textbox(
lines=2,
placeholder="Enter your shopping query here...",
label="Shopping Query"
),
gr.Checkbox(
label="Fetch Deals",
value=True,
info="Check to fetch and display deals from DealsFinders.com"
)
],
outputs=gr.Markdown(label="Results"),
title="Shopping Assistant",
description="""
This demo shows how to use the Shopping Assistant model to classify shopping queries into categories and find relevant deals.
Enter a shopping query below to see which categories it belongs to and find deals from DealsFinders.com.
Examples:
- "I'm looking for headphones"
- "Do you have any kitchen appliance deals?"
- "Show me the best laptop deals"
- "I need a new smart TV"
""",
examples=[
["I'm looking for headphones", True],
["Do you have any kitchen appliance deals?", True],
["Show me the best laptop deals", True],
["I need a new smart TV", True],
["headphone deals", True]
],
theme=gr.themes.Soft()
)
# Launch the app
if __name__ == "__main__":
demo.launch()