Update app.py

app.py

@@ -143,7 +143,9 @@ def get_serp_results(query):
 def fetch_content(url):
     logger.info(f"Fetching content from URL: {url}")
     try:
-        response = requests.get(url, timeout=10)
+        # Decode URL-encoded characters
+        decoded_url = urllib.parse.unquote(url)
+        response = requests.get(decoded_url, timeout=10)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
         content = soup.get_text(separator=' ', strip=True)
@@ -157,6 +159,10 @@ def fetch_content(url):
 def calculate_relevance_score(page_content, query, co):
     logger.info(f"Calculating relevance score for query: {query}")
     try:
+        if not page_content:
+            logger.warning("Empty page content. Returning score 0.")
+            return 0
+        
         page_embedding = co.embed(texts=[page_content], model='embed-english-v3.0', input_type='search_document').embeddings[0]
         query_embedding = co.embed(texts=[query], model='embed-english-v3.0', input_type='search_query').embeddings[0]
         score = cosine_similarity([query_embedding], [page_embedding])[0][0]
@@ -177,13 +183,12 @@ def analyze_competitors(row, co):
     results = []
     for url in [our_url] + competitor_urls:
         try:
-            logger.debug(f"Fetching content for URL: {url}")
+            logger.debug(f"Processing URL: {url}")
             content = fetch_content(url)
             if not content:
                 logger.warning(f"No content fetched for URL: {url}")
                 continue
             
-            logger.debug(f"Calculating relevance score for URL: {url}")
             score = calculate_relevance_score(content, query, co)
             
             logger.info(f"URL: {url}, Score: {score}")
@@ -209,6 +214,15 @@ def show_competitor_analysis(row, co):
             if our_data.empty:
                 st.error(f"Our page '{row['page']}' is not in the results. This indicates an error in fetching or processing the page.")
                 logger.error(f"Our page '{row['page']}' is missing from the results.")
+                
+                # Additional debugging information
+                st.write("Debugging Information:")
+                st.json({
+                    "our_url": row['page'],
+                    "query": row['query'],
+                    "content_fetched": fetch_content(row['page']),
+                    "urls_processed": results_df['url'].tolist()
+                })
             else:
                 our_rank = our_data.index[0] + 1
                 total_results = len(results_df)
@@ -220,6 +234,14 @@ def show_competitor_analysis(row, co):
                 
                 if our_score == 0:
                     st.warning("Our page's relevancy score is 0. This might indicate an issue with content fetching or score calculation.")
+                    # Additional debugging information
+                    st.write("Debugging Information:")
+                    content = fetch_content(row['page'])
+                    st.json({
+                        "content_length": len(content),
+                        "content_preview": content[:500] if content else "No content fetched",
+                        "query": row['query']
+                    })
                 elif our_rank == 1:
                     st.success("Your page has the highest relevancy score!")
                 elif our_rank <= 3: