Update helpers.py

helpers.py

@@ -16,80 +16,77 @@ import requests
 from bs4 import BeautifulSoup
 import os
 
-# os.system("playwright install-deps chromium")
-os.system("playwright install chromium")
-
 # from file_loader import get_vectorstore
 if "GOOGLE_API_KEY" not in os.environ:
     os.environ["GOOGLE_API_KEY"] = "AIzaSyDJ4vIKuIBIPNHATLxnoHlagXWbsAz-vRs"
 key = "AIzaSyDJ4vIKuIBIPNHATLxnoHlagXWbsAz-vRs"
 
-import asyncio
-from urllib.parse import urljoin
-from playwright.async_api import async_playwright
-from langchain_community.document_loaders import AsyncHtmlLoader
-from langchain_community.document_transformers import Html2TextTransformer
-from tqdm.asyncio import tqdm
-
-async def _fetch_urls(base_url):
-    """Extract all links from a JavaScript-rendered webpage."""
-    async with async_playwright() as p:
-        try:
-            browser = await p.chromium.launch(headless=True)
-            page = await browser.new_page()
-            await page.goto(base_url)
-            await page.wait_for_load_state("networkidle")
+# import asyncio
+# from urllib.parse import urljoin
+# from playwright.async_api import async_playwright
+# from langchain_community.document_loaders import AsyncHtmlLoader
+# from langchain_community.document_transformers import Html2TextTransformer
+# from tqdm.asyncio import tqdm
+
+# async def _fetch_urls(base_url):
+#     """Extract all links from a JavaScript-rendered webpage."""
+#     async with async_playwright() as p:
+#         try:
+#             browser = await p.chromium.launch(headless=True)
+#             page = await browser.new_page()
+#             await page.goto(base_url)
+#             await page.wait_for_load_state("networkidle")
     
-            urls = set()
-            links = await page.locator("a").all()
-            for link in links:
-                href = await link.get_attribute("href")
-                if href and "#" not in href:
-                    full_url = urljoin(base_url, href)
-                    if full_url.startswith(base_url):
-                        urls.add(full_url)
+#             urls = set()
+#             links = await page.locator("a").all()
+#             for link in links:
+#                 href = await link.get_attribute("href")
+#                 if href and "#" not in href:
+#                     full_url = urljoin(base_url, href)
+#                     if full_url.startswith(base_url):
+#                         urls.add(full_url)
     
-            await browser.close()
-        except Exception as e:
-            print(f"⚠️ Không thể truy cập {base_url}: {e}")
-        return []  # Trả về danh sách rỗng nếu gặp lỗi
-    return list(urls)
-
-async def _fetch_web_content(urls):
-    """Fetch HTML content and convert it to text, with a progress bar."""
-    docs = []
-    progress_bar = tqdm(total=len(urls), desc="Scraping Pages", unit="page")
-
-    for page_url in urls:
-        try:
-            loader = AsyncHtmlLoader(page_url)
-            html2text = Html2TextTransformer()
-            html = await loader.aload()
-            doc = html2text.transform_documents(html)
-            docs.extend(doc)
-        except Exception as e:
-            print(f"Error loading {page_url}: {e}")
-
-        progress_bar.update(1)  # Update progress bar
-
-    progress_bar.close()
-    return docs
-
-def scrape_website(base_urls):
-    """
-    Scrapes a list of base URLs and extracts their content.
-    Includes a progress bar for tracking.
-    """
-    async def _main():
-        all_urls = []
-        for base_url in base_urls:
-            urls = await _fetch_urls(base_url)
-            all_urls.extend(urls)
-
-        docs = await _fetch_web_content(all_urls)
-        return docs
-
-    return asyncio.run(_main())
+#             await browser.close()
+#         except Exception as e:
+#             print(f"⚠️ Không thể truy cập {base_url}: {e}")
+#         return []  # Trả về danh sách rỗng nếu gặp lỗi
+#     return list(urls)
+
+# async def _fetch_web_content(urls):
+#     """Fetch HTML content and convert it to text, with a progress bar."""
+#     docs = []
+#     progress_bar = tqdm(total=len(urls), desc="Scraping Pages", unit="page")
+
+#     for page_url in urls:
+#         try:
+#             loader = AsyncHtmlLoader(page_url)
+#             html2text = Html2TextTransformer()
+#             html = await loader.aload()
+#             doc = html2text.transform_documents(html)
+#             docs.extend(doc)
+#         except Exception as e:
+#             print(f"Error loading {page_url}: {e}")
+
+#         progress_bar.update(1)  # Update progress bar
+
+#     progress_bar.close()
+#     return docs
+
+# def scrape_website(base_urls):
+#     """
+#     Scrapes a list of base URLs and extracts their content.
+#     Includes a progress bar for tracking.
+#     """
+#     async def _main():
+#         all_urls = []
+#         for base_url in base_urls:
+#             urls = await _fetch_urls(base_url)
+#             all_urls.extend(urls)
+
+#         docs = await _fetch_web_content(all_urls)
+#         return docs
+
+#     return asyncio.run(_main())
     
 def log_message(messages, filename="chat_log.txt"):
     """Ghi lịch sử tin nhắn vào file log"""