Update app.py

app.py

@@ -2,19 +2,25 @@ import os
 import re
 import time
 import asyncio
+from concurrent.futures import ThreadPoolExecutor
 from typing import List, Optional, Dict, Any
 from urllib.parse import urlparse
-from fastapi import FastAPI, HTTPException, Query, Request
+from fastapi import FastAPI, HTTPException, Query, Request, BackgroundTasks
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import JSONResponse
 from pydantic import BaseModel
-from playwright.async_api import async_playwright, Browser, BrowserContext, Page
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.chrome.options import Options
+from selenium.common.exceptions import TimeoutException, NoSuchElementException, WebDriverException
 import uvicorn
 
 app = FastAPI(
-    title="Threads Media Extractor API - Playwright",
-    description="Fast extraction of media URLs from Threads posts using Playwright",
-    version="3.0.0"
+    title="Threads Media Extractor API",
+    description="Extract media URLs from Threads posts - Optimized version",
+    version="2.1.0"
 )
 
 # Add CORS middleware
@@ -26,8 +32,9 @@ app.add_middleware(
     allow_headers=["*"],
 )
 
-# Global browser instance for reuse
-browser: Optional[Browser] = None
+# Global driver pool for reuse
+driver_pool = []
+executor = ThreadPoolExecutor(max_workers=2)
 
 class ThreadsResponse(BaseModel):
     url: str
@@ -42,32 +49,68 @@ class ErrorResponse(BaseModel):
     error: str
     success: bool = False
 
-async def get_browser() -> Browser:
-    """Get or create browser instance"""
-    global browser
-    if browser is None:
-        playwright = await async_playwright().start()
-        browser = await playwright.chromium.launch(
-            headless=True,
-            args=[
-                '--no-sandbox',
-                '--disable-dev-shm-usage',
-                '--disable-gpu',
-                '--disable-extensions',
-                '--disable-default-apps',
-                '--disable-background-timer-throttling',
-                '--disable-backgrounding-occluded-windows',
-                '--disable-renderer-backgrounding',
-                '--disable-features=TranslateUI',
-                '--memory-pressure-off',
-                '--window-size=1280,720',
-                '--disable-background-networking',
-                '--disable-sync',
-                '--disable-plugins',
-                '--disable-images',  # Speed up by not loading images initially
-            ]
-        )
-    return browser
+def create_optimized_driver():
+    """Create and configure optimized Chrome WebDriver"""
+    options = Options()
+    options.add_argument('--headless=new')  # Use new headless mode
+    options.add_argument('--no-sandbox')
+    options.add_argument('--disable-dev-shm-usage')
+    options.add_argument('--disable-gpu')
+    options.add_argument('--disable-extensions')
+    options.add_argument('--disable-plugins')
+    options.add_argument('--disable-default-apps')
+    options.add_argument('--disable-background-timer-throttling')
+    options.add_argument('--disable-backgrounding-occluded-windows')
+    options.add_argument('--disable-renderer-backgrounding')
+    options.add_argument('--disable-features=TranslateUI')
+    options.add_argument('--disable-ipc-flooding-protection')
+    
+    # Performance optimizations
+    options.add_argument('--memory-pressure-off')
+    options.add_argument('--max_old_space_size=4096')
+    options.add_argument('--window-size=1280,720')  # Smaller window
+    
+    # Network optimizations
+    options.add_argument('--aggressive-cache-discard')
+    options.add_argument('--disable-background-networking')
+    
+    # Disable unnecessary features
+    options.add_experimental_option('useAutomationExtension', False)
+    options.add_experimental_option("excludeSwitches", ["enable-automation"])
+    options.add_argument('--disable-blink-features=AutomationControlled')
+    
+    # User agent
+    options.add_argument('--user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36')
+    
+    try:
+        driver = webdriver.Chrome(options=options)
+        driver.implicitly_wait(5)  # Reduced wait time
+        driver.set_page_load_timeout(15)  # Reduced timeout
+        
+        # Optimize browser settings
+        driver.execute_cdp_cmd('Network.setUserAgentOverride', {
+            "userAgent": 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+        })
+        
+        return driver
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Failed to create browser driver: {str(e)}")
+
+def get_driver():
+    """Get driver from pool or create new one"""
+    if driver_pool:
+        return driver_pool.pop()
+    return create_optimized_driver()
+
+def return_driver(driver):
+    """Return driver to pool for reuse"""
+    if len(driver_pool) < 2:  # Keep max 2 drivers in pool
+        driver_pool.append(driver)
+    else:
+        try:
+            driver.quit()
+        except:
+            pass
 
 def extract_post_id_from_url(url: str) -> Optional[str]:
     """Extract post ID from Threads URL"""
@@ -96,8 +139,8 @@ def is_valid_threads_url(url: str) -> bool:
     except:
         return False
 
-async def extract_media_playwright(page: Page, url: str) -> Dict[str, Any]:
-    """Extract media URLs using Playwright"""
+def fast_extract_media(driver: webdriver.Chrome, url: str) -> Dict[str, Any]:
+    """Optimized media extraction with faster loading"""
     media_urls = []
     post_text = None
     author = None
@@ -105,74 +148,54 @@ async def extract_media_playwright(page: Page, url: str) -> Dict[str, Any]:
     try:
         start_time = time.time()
         
-        # Navigate to the URL with optimized loading
-        await page.goto(url, wait_until='domcontentloaded', timeout=15000)
+        # Navigate to the URL
+        driver.get(url)
         
-        # Wait a bit for dynamic content but not too long
-        await asyncio.sleep(2)
+        # Wait for essential elements only
+        try:
+            WebDriverWait(driver, 8).until(
+                lambda d: d.execute_script("return document.readyState") == "complete"
+            )
+        except TimeoutException:
+            pass  # Continue even if timeout
+        
+        # Quick wait for dynamic content
+        time.sleep(1.5)  # Reduced from 3 seconds
         
-        # Extract videos first (most important for Threads)
-        video_elements = await page.query_selector_all('video')
+        # Extract videos first (most important)
+        video_elements = driver.find_elements(By.TAG_NAME, 'video')
         for video in video_elements:
-            src = await video.get_attribute('src')
+            src = video.get_attribute('src')
             if src and src.startswith('http'):
                 media_urls.append(src)
             
-            # Check source elements within video
-            sources = await video.query_selector_all('source')
+            # Check source elements
+            sources = video.find_elements(By.TAG_NAME, 'source')
             for source in sources:
-                src = await source.get_attribute('src')
+                src = source.get_attribute('src')
                 if src and src.startswith('http'):
                     media_urls.append(src)
         
-        # If no videos, look for images
+        # If no videos found, look for images quickly
         if not media_urls:
-            img_elements = await page.query_selector_all('img')
-            for img in img_elements[:10]:  # Limit to first 10
-                src = await img.get_attribute('src')
+            img_elements = driver.find_elements(By.TAG_NAME, 'img')[:10]  # Limit to first 10 images
+            for img in img_elements:
+                src = img.get_attribute('src')
                 if src and src.startswith('http') and any(ext in src.lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp']):
-                    # Filter out small images, profiles, etc.
                     if not any(exclude in src.lower() for exclude in ['profile', 'avatar', 'icon', 'logo']):
                         media_urls.append(src)
         
-        # Try to extract post text (quick attempt)
+        # Quick text extraction (optional, skip if taking too long)
         try:
-            text_selectors = [
-                'div[role="article"] span',
-                'article span',
-                '[data-testid="post-text"]'
-            ]
-            
-            for selector in text_selectors:
-                elements = await page.query_selector_all(selector)
-                for element in elements[:3]:  # Check only first 3
-                    text = await element.inner_text()
-                    if text and len(text.strip()) > 10:
-                        post_text = text.strip()
-                        break
-                if post_text:
+            text_elements = driver.find_elements(By.CSS_SELECTOR, 'div[role="article"] span, article span')[:5]
+            for element in text_elements:
+                text = element.text.strip()
+                if text and len(text) > 10 and not post_text:
+                    post_text = text
                     break
         except:
             pass
         
-        # Try to extract author (quick attempt)
-        try:
-            author_selectors = [
-                'a[role="link"] span',
-                'header a span',
-                '[data-testid="user-name"]'
-            ]
-            
-            for selector in author_selectors:
-                element = await page.query_selector(selector)
-                if element:
-                    author_text = await element.inner_text()
-                    if author_text and not author_text.startswith('@'):
-                        author = author_text.strip()
-                        break
-        except:
-            pass
-        
         # Remove duplicates
         seen = set()
         unique_media_urls = []
@@ -193,26 +216,25 @@ async def extract_media_playwright(page: Page, url: str) -> Dict[str, Any]:
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error extracting media: {str(e)}")
 
-@app.on_event("startup")
-async def startup_event():
-    """Initialize browser on startup"""
-    await get_browser()
-
-@app.on_event("shutdown")
-async def shutdown_event():
-    """Clean up browser on shutdown"""
-    global browser
-    if browser:
-        await browser.close()
+def extract_media_sync(url: str) -> Dict[str, Any]:
+    """Synchronous wrapper for thread execution"""
+    driver = None
+    try:
+        driver = get_driver()
+        result = fast_extract_media(driver, url)
+        return result
+    finally:
+        if driver:
+            return_driver(driver)
 
 @app.get("/", response_model=Dict[str, str])
 async def root():
     """Root endpoint with API information"""
     return {
-        "message": "Threads Media Extractor API v3.0 - Playwright",
-        "description": "Fast extraction of media URLs from Threads posts using Playwright",
-        "version": "3.0.0",
-        "engine": "Playwright (faster than Selenium)",
+        "message": "Threads Media Extractor API v2.1 - Optimized",
+        "description": "Fast extraction of media URLs from Threads posts",
+        "version": "2.1.0",
+        "optimization": "Driver pooling, reduced timeouts, focused extraction",
         "endpoints": {
             "extract": "/extract?url=<threads_url>",
             "health": "/health"
@@ -222,19 +244,17 @@ async def root():
 @app.get("/health")
 async def health_check():
     """Health check endpoint"""
-    global browser
     return {
         "status": "healthy", 
         "service": "threads-media-extractor", 
-        "version": "3.0.0",
-        "engine": "playwright",
-        "browser_ready": browser is not None
+        "version": "2.1.0",
+        "driver_pool_size": len(driver_pool)
     }
 
 @app.get("/extract", response_model=ThreadsResponse)
 async def extract_media(url: str = Query(..., description="Threads post URL")):
     """
-    Extract media URLs from a Threads post using Playwright
+    Extract media URLs from a Threads post - Optimized version
     
     Args:
         url: The Threads post URL to extract media from
@@ -256,37 +276,35 @@ async def extract_media(url: str = Query(..., description="Threads post URL")):
         raise HTTPException(status_code=400, detail="Could not extract post ID from URL")
     
     try:
-        browser = await get_browser()
+        # Run extraction in thread pool for better async handling
+        loop = asyncio.get_event_loop()
+        extracted_data = await loop.run_in_executor(executor, extract_media_sync, url)
         
-        # Create a new context for each request (isolation)
-        context = await browser.new_context(
-            user_agent='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-            viewport={'width': 1280, 'height': 720}
+        return ThreadsResponse(
+            url=url,
+            media_urls=extracted_data["media_urls"],
+            media_count=len(extracted_data["media_urls"]),
+            post_text=extracted_data["post_text"],
+            author=extracted_data["author"],
+            success=True,
+            processing_time=extracted_data.get("processing_time")
         )
         
-        page = await context.new_page()
-        
-        try:
-            # Extract media URLs and metadata
-            extracted_data = await extract_media_playwright(page, url)
-            
-            return ThreadsResponse(
-                url=url,
-                media_urls=extracted_data["media_urls"],
-                media_count=len(extracted_data["media_urls"]),
-                post_text=extracted_data["post_text"],
-                author=extracted_data["author"],
-                success=True,
-                processing_time=extracted_data.get("processing_time")
-            )
-        finally:
-            await context.close()
-        
     except HTTPException:
         raise
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
 
+@app.on_event("shutdown")
+async def shutdown_event():
+    """Clean up resources on shutdown"""
+    executor.shutdown(wait=False)
+    for driver in driver_pool:
+        try:
+            driver.quit()
+        except:
+            pass
+
 @app.exception_handler(HTTPException)
 async def http_exception_handler(request: Request, exc: HTTPException):
     """Custom HTTP exception handler"""