:zap: [Enhance] Rename HTMLFetcher to WebpageFetcher, and add output_parent param

apis/search_api.py

@@ -10,7 +10,7 @@ from typing import Union
 from sse_starlette.sse import EventSourceResponse, ServerSentEvent
 from utils.logger import logger
 from networks.google_searcher import GoogleSearcher
-from networks.html_fetcher import HTMLFetcher
+from networks.webpage_fetcher import WebpageFetcher
 from documents.query_results_extractor import QueryResultsExtractor
 from documents.webpage_content_extractor import WebpageContentExtractor
 from utils.logger import logger
@@ -74,14 +74,16 @@ class SearchAPIApp:
         logger.note(queries_search_results)
 
         if item.extract_content:
-            html_fetcher = HTMLFetcher()
+            webpage_fetcher = WebpageFetcher()
             webpage_content_extractor = WebpageContentExtractor()
             for query_idx, query_search_result in enumerate(queries_search_results):
                 for query_result_idx, query_result in enumerate(
                     query_search_result["query_results"]
                 ):
-                    webpage_html_path = html_fetcher.fetch(
-                        query_result["url"], overwrite=item.overwrite_webpage_html
+                    webpage_html_path = webpage_fetcher.fetch(
+                        query_result["url"],
+                        overwrite=item.overwrite_webpage_html,
+                        output_parent=query_search_result["query"],
                     )
                     extracted_content = webpage_content_extractor.extract(
                         webpage_html_path

networks/filepath_converter.py

@@ -1,7 +1,7 @@
 import platform
 import re
 from pathlib import Path
-from urllib.parse import quote
+from urllib.parse import quote, unquote
 
 
 # What characters are forbidden in Windows and Linux directory names?
@@ -42,6 +42,8 @@ class FilepathConverter:
         return input_string
 
     def validate(self, input_string):
+        if not input_string:
+            return input_string
         filename = input_string
         for char in INVALID_FILE_PATH_CHARS:
             filename = filename.replace(char, "_")
@@ -65,6 +67,7 @@ class FilepathConverter:
         filename = self.append_extension(filename)
 
         parent = parent or self.parent
+        parent = self.validate(parent)
         if parent:
             filepath = self.output_root / parent / filename
         else:
@@ -82,7 +85,7 @@ class UrlToFilepathConverter(FilepathConverter):
         self.output_root = self.output_root / "urls"
 
     def preprocess(self, url):
-        filename = url.split("//")[1]
+        filename = unquote(url.split("//")[1])
         return filename
 
 

networks/{html_fetcher.py → webpage_fetcher.py}

@@ -7,7 +7,7 @@ from networks.filepath_converter import UrlToFilepathConverter
 from networks.network_configs import IGNORE_HOSTS, REQUESTS_HEADERS
 
 
-class HTMLFetcher:
+class WebpageFetcher:
     def __init__(self):
         self.enver = enver
         self.enver.set_envs(proxies=True)
@@ -34,10 +34,12 @@ class HTMLFetcher:
         with open(self.output_path, "wb") as wf:
             wf.write(self.request_response.content)
 
-    def fetch(self, url, overwrite=False):
+    def fetch(self, url, overwrite=False, output_parent=None):
         self.url = url
         logger.note(f"Fetching: [{self.url}]")
-        self.output_path = self.filepath_converter.convert(self.url)
+        self.output_path = self.filepath_converter.convert(
+            self.url, parent=output_parent
+        )
 
         if self.is_ignored_host(self.url):
             logger.warn(f"Ignore host: [{self.host}]")
@@ -57,5 +59,5 @@ if __name__ == "__main__":
         # "https://www.liaoxuefeng.com/wiki/1016959663602400/1017495723838528"
         "https://docs.python.org/zh-cn/3/tutorial/interpreter.html"
     )
-    fetcher = HTMLFetcher()
+    fetcher = WebpageFetcher()
     fetcher.fetch(url)