from smolagents import Tool from typing import Any, Optional class SimpleTool(Tool): name = "advanced_web_analyzer" description = "Advanced web content analyzer with ML-powered analysis capabilities." inputs = {"url":{"type":"string","description":"The webpage URL to analyze."},"analysis_mode":{"type":"string","nullable":True,"description":"Analysis mode ('analyze', 'search', 'summarize', 'sentiment', 'topics')."},"query":{"type":"string","nullable":True,"description":"Optional search term for 'search' mode."},"language":{"type":"string","nullable":True,"description":"Content language (default: 'en')."}} output_type = "string" def forward(self, url: str, analysis_mode: str = "analyze", query: Optional[str] = None, language: str = "en") -> str: """Advanced web content analyzer with ML-powered analysis capabilities. Args: url: The webpage URL to analyze. analysis_mode: Analysis mode ('analyze', 'search', 'summarize', 'sentiment', 'topics'). query: Optional search term for 'search' mode. language: Content language (default: 'en'). Returns: str: Advanced analysis of web content. """ import requests from bs4 import BeautifulSoup from urllib.parse import urlparse import re from collections import Counter from transformers import pipeline try: # Validate URL parsed_url = urlparse(url) if not all([parsed_url.scheme, parsed_url.netloc]): return "Error: Invalid URL format. Please provide a valid URL." # Fetch webpage headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'} response = requests.get(url, headers=headers, timeout=10) response.raise_for_status() # Parse content soup = BeautifulSoup(response.text, 'html.parser') for tag in soup(['script', 'style', 'meta']): tag.decompose() # Extract basic elements title = soup.title.string if soup.title else "No title found" title = re.sub(r'\s+', ' ', title).strip() if title else "No title found" text_content = soup.get_text() text_content = re.sub(r'\s+', ' ', text_content).strip() # Process based on mode if analysis_mode == "analyze": # Initialize ML pipelines summarizer = pipeline("summarization", model="facebook/bart-large-cnn") classifier = pipeline("text-classification", model="nlptown/bert-base-multilingual-uncased-sentiment") # Get word statistics words = text_content.lower().split() word_count = len(words) word_freq = Counter(words).most_common(5) common_words = "" for word, count in word_freq: common_words += f"- {word}: {count} times\n" # Get summary summary = summarizer(text_content[:1024], max_length=100, min_length=30)[0]['summary_text'] # Get sentiment sentiment = classifier(text_content[:512])[0] sentiment_score = int(sentiment['label'][0]) # Convert '5 stars' to number sentiment_desc = ["Very Negative", "Negative", "Neutral", "Positive", "Very Positive"][sentiment_score-1] # Format comprehensive analysis return f"""🔍 Comprehensive Content Analysis 📑 Basic Information: Title: {title} Word Count: {word_count} Reading Time: {word_count // 200} minutes 📝 Quick Summary: {summary} 😊 Content Sentiment: {sentiment_desc} ({sentiment_score}/5 stars) 📊 Most Common Words: {common_words}""" elif analysis_mode == "summarize": # Use BART for better summarization summarizer = pipeline("summarization", model="facebook/bart-large-cnn") # Split into chunks if text is too long chunks = [text_content[i:i+1024] for i in range(0, len(text_content), 1024)] summaries = [] for chunk in chunks[:3]: # Process up to 3 chunks if len(chunk.strip()) > 100: summary = summarizer(chunk, max_length=100, min_length=30)[0]['summary_text'] summaries.append(summary) return f"""📝 Content Summary for: {title} {' '.join(summaries)}""" elif analysis_mode == "sentiment": # Use multilingual sentiment analyzer classifier = pipeline("text-classification", model="nlptown/bert-base-multilingual-uncased-sentiment") # Analyze main content and paragraphs main_sentiment = classifier(text_content[:512])[0] # Analyze individual paragraphs paragraphs = soup.find_all('p') detailed_sentiments = "" para_count = 0 for p in paragraphs: text = p.text.strip() if len(text) > 50: # Only analyze meaningful paragraphs sentiment = classifier(text[:512])[0] score = int(sentiment['label'][0]) mood = ["Very Negative", "Negative", "Neutral", "Positive", "Very Positive"][score-1] detailed_sentiments += f"\nParagraph {para_count + 1}: {mood} ({score}/5)" para_count += 1 if para_count >= 5: # Limit to 5 paragraphs break return f"""😊 Sentiment Analysis Overall Sentiment: {["Very Negative", "Negative", "Neutral", "Positive", "Very Positive"][int(main_sentiment['label'][0])-1]} Overall Score: {main_sentiment['label'][0]}/5 Detailed Analysis:{detailed_sentiments}""" elif analysis_mode == "topics": # Use Zero-shot classification for topic detection classifier = pipeline("zero-shot-classification", model="facebook/bart-large-mnli") # Define potential topics topics = [ "Technology", "Business", "Politics", "Science", "Health", "Entertainment", "Sports", "Education", "Environment", "Culture" ] # Analyze main content topic_results = classifier(text_content[:512], topics) # Format results topic_analysis = "Main Topics:\n" for topic, score in zip(topic_results['labels'], topic_results['scores']): if score > 0.1: # Only show relevant topics topic_analysis += f"- {topic}: {score*100:.1f}% confidence\n" # Get key phrases from keybert import KeyBERT kw_model = KeyBERT() keywords = kw_model.extract_keywords(text_content[:5000], keyphrase_ngram_range=(1, 2), stop_words='english', top_n=5) key_phrases = "\nKey Phrases:\n" for phrase, score in keywords: key_phrases += f"- {phrase}: {score:.2f} relevance\n" return f"""🎯 Topic Analysis {topic_analysis} {key_phrases}""" elif analysis_mode == "search": if not query: return "Error: Search query is required for search mode." # Use transformers for better search qa_pipeline = pipeline("question-answering", model="distilbert-base-cased-distilled-squad") # Search in paragraphs paragraphs = soup.find_all('p') search_results = "" result_count = 0 for p in paragraphs: text = p.text.strip() if len(text) > 50 and query.lower() in text.lower(): # Get AI-enhanced answer qa_result = qa_pipeline(question=query, context=text) search_results += f"\n{result_count + 1}. Found in context: {qa_result['answer']}\n" search_results += f" Confidence: {qa_result['score']:.2%}\n" search_results += f" Full context: {text}\n" result_count += 1 if result_count >= 3: break if not search_results: return f"No matches found for '{query}'" return f"""🔍 AI-Enhanced Search Results for '{query}': {search_results}""" else: return f"Error: Unknown mode '{analysis_mode}'" except Exception as e: return f"Error processing webpage: {str(e)}"