Update app.py

app.py

@@ -177,26 +177,45 @@ def calculate_relevance_score(page_content, query, co):
         st.error(f"Error calculating relevance score: {str(e)}")
         return 0
 
-def analyze_competitors(row, co, custom_url=None):
+def analyze_competitors(row, co):
    # logger.info(f"Analyzing competitors for query: {row['query']}")
     query = row['query']
     our_url = row['page']
     
-    competitor_urls = get_serp_results(query)
-    competitor_urls = [url for url in competitor_urls if not url.startswith('/search')][:5]  # Get top 5 valid competitors
-    
-    if custom_url and custom_url not in competitor_urls:
-        competitor_urls.append(custom_url)
+    competitor_urls = get_serp_results(query)[:5]  # Get top 5 competitors
     
     results = []
+    
+    # Calculate score for our page first
+    our_content = fetch_content(our_url)
+    print(our_url)
+    print(our_content)
+    if our_content:
+        our_score = calculate_relevance_score(our_content, query, co)
+        results.append({'url': our_url, 'relevancy_score': our_score})
+        #logger.info(f"Our URL: {our_url}, Score: {our_score}")
+        #logger.warning(f"No content fetched for our URL: {our_url}")
+    
+    # Calculate scores for competitor pages
     for url in competitor_urls:
-        content = fetch_content(url)
-        score = calculate_relevance_score(content, query, co)
-        results.append({'url': url, 'relevancy_score': score})
+        try:
+           # logger.debug(f"Processing competitor URL: {url}")
+            content = fetch_content(url)
+            if not content:
+        #        logger.warning(f"No content fetched for competitor URL: {url}")
+                continue
+            
+            score = calculate_relevance_score(content, query, co)
+            
+          #  logger.info(f"Competitor URL: {url}, Score: {score}")
+            results.append({'url': url, 'relevancy_score': score})
+        except Exception as e:
+          #  logger.error(f"Error processing URL {url}: {str(e)}")
+            st.error(f"Error processing URL {url}: {str(e)}")
     
-    results.append({'url': our_url, 'relevancy_score': row['relevancy_score']})
     results_df = pd.DataFrame(results).sort_values('relevancy_score', ascending=False)
     
+  #  logger.info(f"Competitor analysis completed. {len(results)} results obtained.")
     return results_df
 
 def show_competitor_analysis(row, co):
@@ -246,6 +265,24 @@ def show_competitor_analysis(row, co):
                 elif our_rank > total_results / 2:
                     st.warning("Your page's relevancy score is in the lower half of the results. Consider optimizing your content.")
 
+def analyze_competitors(row, co):
+    query = row['query']
+    our_url = row['page']
+    our_score = row['relevancy_score']
+    
+    competitor_urls = get_serp_results(query)
+    competitor_urls = [url for url in competitor_urls if not url.startswith('/search')][:5]  # Get top 5 valid competitors
+    
+    results = []
+    for url in competitor_urls:
+        content = fetch_content(url)
+        score = calculate_relevance_score(content, query, co)
+        results.append({'url': url, 'relevancy_score': score})
+    
+    results.append({'url': our_url, 'relevancy_score': our_score})
+    results_df = pd.DataFrame(results).sort_values('relevancy_score', ascending=False)
+    
+    return results_df
 def process_gsc_data(df):
     #logging.info("Processing GSC data")
     df_sorted = df.sort_values(['impressions'], ascending=[False])
@@ -463,7 +500,7 @@ def show_tabular_data(df, co):
         return score
 
     # Function to analyze competitors
-    def analyze_competitors(row, custom_url=None):
+    def analyze_competitors(row):
         query = row['query']
         our_url = row['page']
         our_score = row['relevancy_score']
@@ -471,9 +508,6 @@ def show_tabular_data(df, co):
         competitor_urls = get_serp_results(query)
         competitor_urls = [url for url in competitor_urls if not url.startswith('/search')][:5]  # Get top 5 valid competitors
         
-        if custom_url and custom_url not in competitor_urls:
-            competitor_urls.append(custom_url)
-        
         results = []
         for url in competitor_urls:
             content = fetch_content(url)
@@ -505,29 +539,21 @@ def show_tabular_data(df, co):
                 st.write(f"Relevancy Score: {row['relevancy_score']:.4f}")
         with col3:
             if st.button(f"Compare Competitors", key=f"comp_{index}"):
-                custom_url = st.text_input("Enter a custom URL to compare (optional):", key=f"custom_url_{index}")
-                
-                if st.button("Analyze", key=f"analyze_{index}"):
-                    with st.spinner('Analyzing competitors...'):
-                        results_df = analyze_competitors(row, custom_url)
-                        st.write("Competitor Comparison:")
-                        st.dataframe(results_df)
-                        
-                        our_rank = results_df.index[results_df['url'] == row['page']].tolist()[0] + 1
-                        total_results = len(results_df)
-                        st.write(f"Our page ranks {our_rank} out of {total_results} in terms of relevancy score.")
-                        
-                        if our_rank == 1:
-                            st.success("Your page has the highest relevancy score!")
-                        elif our_rank <= 3:
-                            st.info("Your page is among the top 3 most relevant results.")
-                        elif our_rank > total_results / 2:
-                            st.warning("Your page's relevancy score is in the lower half of the results. Consider optimizing your content.")
-                        
-                        if custom_url:
-                            custom_rank = results_df.index[results_df['url'] == custom_url].tolist()[0] + 1
-                            custom_score = results_df[results_df['url'] == custom_url]['relevancy_score'].values[0]
-                            st.write(f"Custom URL '{custom_url}' ranks {custom_rank} out of {total_results} with a relevancy score of {custom_score:.4f}")
+                with st.spinner('Analyzing competitors...'):
+                    results_df = analyze_competitors(row)
+                    st.write("Competitor Comparison:")
+                    st.dataframe(results_df)
+                    
+                    our_rank = results_df.index[results_df['url'] == row['page']].tolist()[0] + 1
+                    total_results = len(results_df)
+                    st.write(f"Our page ranks {our_rank} out of {total_results} in terms of relevancy score.")
+                    
+                    if our_rank == 1:
+                        st.success("Your page has the highest relevancy score!")
+                    elif our_rank <= 3:
+                        st.info("Your page is among the top 3 most relevant results.")
+                    elif our_rank > total_results / 2:
+                        st.warning("Your page's relevancy score is in the lower half of the results. Consider optimizing your content.")
 
     return df  # Return the updated dataframe