Update app.py

app.py

@@ -120,8 +120,6 @@ def init_session_state():
         st.session_state.custom_start_date = datetime.date.today() - datetime.timedelta(days=7)
     if 'custom_end_date' not in st.session_state:
         st.session_state.custom_end_date = datetime.date.today()
-    if 'relevancy_scores' not in st.session_state:
-        st.session_state.relevancy_scores = {}
     #logging.info("Session state initialized")
 
 # -------------
@@ -223,21 +221,20 @@ def fetch_content(url, query):
     except requests.RequestException:
         return ""
 
-def calculate_relevance_score(page_content, query, co, model_type='english'):
+def calculate_relevance_score(page_content, query, co):
+   # logger.info(f"Calculating relevance score for query: {query}")
     try:
-        if not page_content.strip():
-            st.warning("Page content is empty. Cannot calculate relevance score.")
-            return 0
-        model = 'embed-english-v3.0' if model_type == 'english' else 'embed-multilingual-v3.0'
-        embeddings = co.embed(texts=[page_content, query], model=model, input_type=['search_document', 'search_query'])
-        page_embedding = embeddings.embeddings[0]
-        query_embedding = embeddings.embeddings[1]
-        if not any(page_embedding) or not any(query_embedding):
-            st.warning("One of the embeddings is empty. Returning a score of 0.")
+        if not page_content:
+           # logger.warning("Empty page content. Returning score 0.")
             return 0
+        
+        page_embedding = co.embed(texts=[page_content], model='embed-english-v3.0', input_type='search_document').embeddings[0]
+        query_embedding = co.embed(texts=[query], model='embed-english-v3.0', input_type='search_query').embeddings[0]
         score = cosine_similarity([query_embedding], [page_embedding])[0][0]
+     #   logger.debug(f"Relevance score calculated: {score}")
         return score
     except Exception as e:
+       # logger.exception(f"Error calculating relevance score: {str(e)}")
         st.error(f"Error calculating relevance score: {str(e)}")
         return 0
 
@@ -247,47 +244,46 @@ def normalize_url(url):
 def analyze_competitors(row, co, custom_url=None, country_code=None):
     query = row['query']
     our_url = normalize_url(row['page'])
-    model_type = row.get('model_type', 'english')  # Assuming you store model_type per row
-
+    
     competitor_data = get_serp_results(query, country_code)
-
+    
     results = []
-    our_url_found = False  # Flag to check if our URL is in the results
-
     for data in competitor_data:
         competitor_url = normalize_url(data['url'])
-        score = calculate_relevance_score(data['content'], query, co, model_type=model_type)
-        is_our = competitor_url == our_url
-        if is_our:
-            our_url_found = True
+        score = calculate_relevance_score(data['content'], query, co)
         results.append({
             'Position': data['position'],
             'URL': competitor_url,
             'Score': score,
-            'is_our_url': is_our
+            'is_our_url': competitor_url == our_url
         })
-
-    # Retrieve "Our Score" from the main data table
-    our_score = st.session_state.relevancy_scores.get(our_url, 0)
-
-    if not our_url_found:
+    
+    our_content = fetch_content(our_url, query)
+    our_score = calculate_relevance_score(our_content, query, co)
+    
+    if not any(r['is_our_url'] for r in results):
         results.append({
             'Position': len(results) + 1,
-            'URL': f"{our_url} (Our URL)",
+            'URL': our_url,
             'Score': our_score,
             'is_our_url': True
         })
-
-    # Sort results by position in ascending order
+    
+    # Sort results by position
     results = sorted(results, key=lambda x: x['Position'])
-
+    
     # Create DataFrame
     results_df = pd.DataFrame(results)
     results_df['Position'] = results_df['Position'].astype(int)
-
+    
+    # Mark our URL
+    results_df['URL'] = results_df.apply(
+        lambda x: f"{x['URL']} (Our URL)" if x['is_our_url'] else x['URL'], axis=1
+    )
+    
     # Keep only the columns we want to display
     results_df = results_df[['Position', 'URL', 'Score']]
-
+    
     return results_df
 
 def show_competitor_analysis(row, co, country_code):
@@ -411,10 +407,19 @@ def fetch_gsc_data(webproperty, search_type, start_date, end_date, dimensions, d
 
     
 def calculate_relevancy_scores(df, model_type):
-    for index, row in df.iterrows():
-        if pd.isna(row['relevancy_score']) or row['relevancy_score'] == 0:
-            score = calculate_single_relevancy(row)
-            df.at[index, 'relevancy_score'] = score
+    #logging.info("Calculating relevancy scores")
+    with st.spinner('Calculating relevancy scores...'):
+        try:
+            page_contents = [fetch_content(url) for url in df['page']]
+            page_embeddings = generate_embeddings(page_contents, model_type)
+            query_embeddings = generate_embeddings(df['query'].tolist(), model_type)
+            relevancy_scores = cosine_similarity(query_embeddings, page_embeddings).diagonal()
+            df = df.assign(relevancy_score=relevancy_scores)
+            #logging.info("Relevancy scores calculated successfully")
+        except Exception as e:
+            #logging.error(f"Error calculating relevancy scores: {e}")
+            st.warning(f"Error calculating relevancy scores: {e}")
+            df = df.assign(relevancy_score=0)
     return df
 
 # -------------
@@ -517,9 +522,7 @@ def show_model_type_selector():
 def calculate_single_relevancy(row):
     page_content = fetch_content(row['page'], row['query'])
     query = row['query']
-    model_type = st.session_state.get('model_type_selector', 'english')  # Retrieve from session state
-    score = calculate_relevance_score(page_content, query, co, model_type=model_type)
-    st.session_state.relevancy_scores[normalize_url(row['page'])] = score  # Ensure score is stored
+    score = calculate_relevance_score(page_content, query, co)
     return score
 
 def compare_with_top_result(row, co, country_code):
@@ -540,11 +543,9 @@ def compare_with_top_result(row, co, country_code):
     our_content = fetch_content(our_url, query)
     top_content = top_result['content']
     
-    # Retrieve "Our Score" from the main data table
-    our_score = st.session_state['relevancy_scores'].get(normalize_url(our_url), 0)
-    
     # Calculate relevancy scores
-    top_score = calculate_relevance_score(top_content, query, co, model_type=row.get('model_type', 'english'))
+    our_score = calculate_relevance_score(our_content, query, co)
+    top_score = calculate_relevance_score(top_content, query, co)
     
     # Prepare prompt for GPT-4
     prompt = f"""
@@ -575,12 +576,12 @@ def compare_with_top_result(row, co, country_code):
         
         # Display results
         st.subheader("Content Comparison Analysis")
-        st.write(f"**Query:** {query}")
-        st.write(f"**Top-ranking URL:** {top_url}")
-        st.write(f"**Our URL:** {our_url}")
-        st.write(f"**Top-ranking score:** {top_score:.4f}")
-        st.write(f"**Our score:** {our_score:.4f}")
-        st.write("**Analysis:**")
+        st.write(f"Query: {query}")
+        st.write(f"Top-ranking URL: {top_url}")
+        st.write(f"Our URL: {our_url}")
+        st.write(f"Top-ranking score: {top_score:.4f}")
+        st.write(f"Our score: {our_score:.4f}")
+        st.write("Analysis:")
         st.write(analysis)
     except Exception as e:
         st.error(f"Error in GPT-4 analysis: {str(e)}")
@@ -628,16 +629,12 @@ def show_tabular_data(df, co, country_code):
     )
     if st.button("Click here to calculate relevancy for selected pages"):
         selected_indices = [i for i, selected in enumerate(st.session_state.selected_rows) if selected]
-        if selected_indices:
-            progress_bar = st.progress(0)
-            for i, index in enumerate(selected_indices):
-                if pd.isna(df.at[index, 'relevancy_score']) or df.at[index, 'relevancy_score'] == 0:
-                    df.at[index, 'relevancy_score'] = calculate_single_relevancy(df.iloc[index])
-                progress_bar.progress((i + 1) / len(selected_indices))
-            st.success(f"Calculated relevancy scores for {len(selected_indices)} selected rows.")
-            st.session_state.report_data = df  # Update the report_data in session state
-        else:
-            st.warning("No rows selected. Please select at least one row to calculate relevancy.")
+        with st.spinner('Calculating relevancy scores...'):
+            for index in selected_indices:
+                if pd.isna(df.iloc[index]['relevancy_score']) or df.iloc[index]['relevancy_score'] == 0:
+                    df.iloc[index, df.columns.get_loc('relevancy_score')] = calculate_single_relevancy(df.iloc[index])
+        st.success(f"Calculated relevancy scores for {len(selected_indices)} selected rows.")
+        st.experimental_rerun()
     
     # Display column headers
     cols = st.columns([0.5, 3, 2, 1, 1, 1, 1, 1, 1])
@@ -650,8 +647,9 @@ def show_tabular_data(df, co, country_code):
         cols = st.columns([0.5, 3, 2, 1, 1, 1, 1, 1, 1])
         
         # Checkbox for row selection
-        cols[0].checkbox("Select", key=f"select_{i}", value=st.session_state.selected_rows[i],
-                         on_change=lambda idx=i: update_selected_rows(idx))
+        cols[0].checkbox("", key=f"select_{i}", value=st.session_state.selected_rows[i],
+                         on_change=lambda idx=i: setattr(st.session_state, 'selected_rows', 
+                                                           [True if j == idx else x for j, x in enumerate(st.session_state.selected_rows)]))
         
         # Truncate and make the URL clickable
         truncated_url = row.page[:30] + '...' if len(row.page) > 30 else row.page
@@ -680,7 +678,7 @@ def show_tabular_data(df, co, country_code):
             if st.session_state[competitor_state_key]:
                 st.write(f"Competitor Analysis for: {row.query}")
                 with st.spinner('Analyzing competitors...'):
-                    results_df = analyze_competitors(row, co, country_code=country_code)
+                    results_df = analyze_competitors(row._asdict(), co, country_code=country_code)
                     
                     # Sort the results by Position in ascending order
                     results_df = results_df.sort_values('Position', ascending=True).reset_index(drop=True)
@@ -720,11 +718,11 @@ def show_tabular_data(df, co, country_code):
                             st.warning("Your page's relevancy score is in the lower half of the results. Consider optimizing your content.")
                     else:
                         st.error(f"Our page '{row.page}' is not in the results. This indicates an error in fetching or processing the page.")
-    
+
                     if compare_state_key not in st.session_state:
                         st.session_state[compare_state_key] = False
                     
-                    if cols[8].button("Compare Your Relevancy Score to the Page In First Place", key=compare_button_key):
+                    if st.button("Compare Your Relevancy Score to the Page In First Place", key=compare_button_key):
                         st.session_state[compare_state_key] = True
                     
                     if st.session_state[compare_state_key]:
@@ -734,9 +732,6 @@ def show_tabular_data(df, co, country_code):
 
     return df  # Return the updated dataframe
 
-def update_selected_rows(idx):
-    st.session_state.selected_rows[idx] = not st.session_state.selected_rows[idx]
-
 def show_date_range_selector():
   #  logging.info("Showing date range selector")
     return st.selectbox(
@@ -889,8 +884,7 @@ def main():
             if st.session_state.report_data is not None and not st.session_state.report_data.empty:
                 st.write("Data fetched successfully.")
                 
-                updated_df = show_tabular_data(st.session_state.report_data, co, country_code)
-                st.session_state.report_data = updated_df  # Update the report_data with the potentially modified dataframe
+                st.session_state.report_data = show_tabular_data(st.session_state.report_data, co, country_code)
                 
                 download_csv_link(st.session_state.report_data)
             elif st.session_state.report_data is not None: