Update app.py

app.py

@@ -143,7 +143,7 @@ def get_serp_results(query):
 def fetch_content(url):
     logger.info(f"Fetching content from URL: {url}")
     try:
-        response = requests.get(url)
+        response = requests.get(url, timeout=10)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
         content = soup.get_text(separator=' ', strip=True)
@@ -175,17 +175,28 @@ def analyze_competitors(row, co):
     competitor_urls = get_serp_results(query)
     
     results = []
-    for url in [our_url] + competitor_urls:  # Include our URL at the beginning
-        content = fetch_content(url)
-        score = calculate_relevance_score(content, query, co)
-        results.append({'url': url, 'relevancy_score': score})
+    for url in [our_url] + competitor_urls:
+        try:
+            logger.debug(f"Fetching content for URL: {url}")
+            content = fetch_content(url)
+            if not content:
+                logger.warning(f"No content fetched for URL: {url}")
+                continue
+            
+            logger.debug(f"Calculating relevance score for URL: {url}")
+            score = calculate_relevance_score(content, query, co)
+            
+            logger.info(f"URL: {url}, Score: {score}")
+            results.append({'url': url, 'relevancy_score': score})
+        except Exception as e:
+            logger.error(f"Error processing URL {url}: {str(e)}")
+            st.error(f"Error processing URL {url}: {str(e)}")
     
     results_df = pd.DataFrame(results).sort_values('relevancy_score', ascending=False)
     
     logger.info(f"Competitor analysis completed. {len(results)} results obtained.")
     return results_df
 
-
 def show_competitor_analysis(row, co):
     if st.button("Check Competitors", key=f"comp_{row['page']}"):
         logger.info(f"Competitor analysis requested for page: {row['page']}")
@@ -194,20 +205,27 @@ def show_competitor_analysis(row, co):
             st.write("Relevancy Score Comparison:")
             st.dataframe(results_df)
             
-            our_rank = results_df.index[results_df['url'] == row['page']].tolist()[0] + 1
-            total_results = len(results_df)
-            our_score = results_df.loc[results_df['url'] == row['page'], 'relevancy_score'].values[0]
-            
-            logger.info(f"Our page ranks {our_rank} out of {total_results} in terms of relevancy score.")
-            st.write(f"Our page ('{row['page']}') ranks {our_rank} out of {total_results} in terms of relevancy score.")
-            st.write(f"Our relevancy score: {our_score:.4f}")
-            
-            if our_rank == 1:
-                st.success("Your page has the highest relevancy score!")
-            elif our_rank <= 3:
-                st.info("Your page is among the top 3 most relevant results.")
-            elif our_rank > total_results / 2:
-                st.warning("Your page's relevancy score is in the lower half of the results. Consider optimizing your content.")
+            our_data = results_df[results_df['url'] == row['page']]
+            if our_data.empty:
+                st.error(f"Our page '{row['page']}' is not in the results. This indicates an error in fetching or processing the page.")
+                logger.error(f"Our page '{row['page']}' is missing from the results.")
+            else:
+                our_rank = our_data.index[0] + 1
+                total_results = len(results_df)
+                our_score = our_data['relevancy_score'].values[0]
+                
+                logger.info(f"Our page ranks {our_rank} out of {total_results} in terms of relevancy score.")
+                st.write(f"Our page ('{row['page']}') ranks {our_rank} out of {total_results} in terms of relevancy score.")
+                st.write(f"Our relevancy score: {our_score:.4f}")
+                
+                if our_score == 0:
+                    st.warning("Our page's relevancy score is 0. This might indicate an issue with content fetching or score calculation.")
+                elif our_rank == 1:
+                    st.success("Your page has the highest relevancy score!")
+                elif our_rank <= 3:
+                    st.info("Your page is among the top 3 most relevant results.")
+                elif our_rank > total_results / 2:
+                    st.warning("Your page's relevancy score is in the lower half of the results. Consider optimizing your content.")
 
 
 def analyze_competitors(row, co):
@@ -320,6 +338,10 @@ def fetch_gsc_data(webproperty, search_type, start_date, end_date, dimensions, d
 def calculate_relevance_score(page_content, query, co):
     logger.info(f"Calculating relevance score for query: {query}")
     try:
+        if not page_content:
+            logger.warning("Empty page content. Returning score 0.")
+            return 0
+        
         page_embedding = co.embed(texts=[page_content], model='embed-english-v3.0', input_type='search_document').embeddings[0]
         query_embedding = co.embed(texts=[query], model='embed-english-v3.0', input_type='search_query').embeddings[0]
         score = cosine_similarity([query_embedding], [page_embedding])[0][0]