Update app.py

app.py

@@ -1,3 +1,5 @@
+import logging
+
 # Standard library imports
 import datetime
 import base64
@@ -16,12 +18,16 @@ from sklearn.metrics.pairwise import cosine_similarity
 import requests
 from bs4 import BeautifulSoup
 
+# Configure logging
+logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
+
 load_dotenv()
-#test
+logging.info("Environment variables loaded")
 
 # Initialize Cohere client
 COHERE_API_KEY = os.environ["COHERE_API_KEY"]
 co = cohere.Client(COHERE_API_KEY)
+logging.info("Cohere client initialized")
 
 # Configuration: Set to True if running locally, False if running on Streamlit Cloud
 IS_LOCAL = False
@@ -50,6 +56,7 @@ def setup_streamlit():
     st.set_page_config(page_title="Keyword Relevance Test", layout="wide")
     st.title("Keyword Relevance Test Using Vector Embedding")
     st.divider()
+    logging.info("Streamlit app configured")
 
 def init_session_state():
     if 'selected_property' not in st.session_state:
@@ -70,47 +77,55 @@ def init_session_state():
         st.session_state.custom_start_date = datetime.date.today() - datetime.timedelta(days=7)
     if 'custom_end_date' not in st.session_state:
         st.session_state.custom_end_date = datetime.date.today()
+    logging.info("Session state initialized")
 
 # -------------
 # Data Processing Functions
 # -------------
 
 def fetch_content(url):
+    logging.debug(f"Fetching content from URL: {url}")
     try:
         response = requests.get(url)
         response.raise_for_status()
         soup = BeautifulSoup(response.text, 'html.parser')
         content = soup.get_text(separator=' ', strip=True)
+        logging.debug(f"Content fetched successfully from URL: {url}")
         return content
     except requests.RequestException as e:
+        logging.error(f"Error fetching content from URL: {url} - {e}")
         return str(e)
 
 def generate_embeddings(text_list, model_type):
+    logging.debug(f"Generating embeddings for model type: {model_type}")
     if not text_list:
+        logging.warning("Text list is empty, returning empty embeddings")
         return []
     model = 'embed-english-v3.0' if model_type == 'english' else 'embed-multilingual-v3.0'
     input_type = 'search_document'
     response = co.embed(model=model, texts=text_list, input_type=input_type)
     embeddings = response.embeddings
+    logging.debug(f"Embeddings generated successfully for model type: {model_type}")
     return embeddings
 
 def calculate_relevancy_scores(df, model_type):
+    logging.info("Calculating relevancy scores")
     try:
         page_contents = [fetch_content(url) for url in df['page']]
         page_embeddings = generate_embeddings(page_contents, model_type)
         query_embeddings = generate_embeddings(df['query'].tolist(), model_type)
         relevancy_scores = cosine_similarity(query_embeddings, page_embeddings).diagonal()
         df = df.assign(relevancy_score=relevancy_scores)
+        logging.info("Relevancy scores calculated successfully")
     except Exception as e:
+        logging.error(f"Error calculating relevancy scores: {e}")
         st.warning(f"Error calculating relevancy scores: {e}")
         df = df.assign(relevancy_score=0)
     return df
 
 def process_gsc_data(df):
-    # Remove the filter for queries below position 10
+    logging.info("Processing GSC data")
     df_sorted = df.sort_values(['impressions'], ascending=[False])
-    
-    # Keep only the highest impression query for each page
     df_unique = df_sorted.drop_duplicates(subset='page', keep='first')
     
     if 'relevancy_score' not in df_unique.columns:
@@ -119,6 +134,7 @@ def process_gsc_data(df):
         df_unique['relevancy_score'] = df_sorted.groupby('page')['relevancy_score'].first().values
     
     result = df_unique[['page', 'query', 'clicks', 'impressions', 'ctr', 'position', 'relevancy_score']]
+    logging.info("GSC data processed successfully")
     return result
 
 # -------------
@@ -126,6 +142,7 @@ def process_gsc_data(df):
 # -------------
 
 def load_config():
+    logging.info("Loading Google client configuration")
     client_config = {
         "web": {
             "client_id": os.environ["CLIENT_ID"],
@@ -135,23 +152,29 @@ def load_config():
             "redirect_uris": ["https://poemsforaphrodite-gscpro.hf.space/"],
         }
     }
+    logging.info("Google client configuration loaded")
     return client_config
 
 def init_oauth_flow(client_config):
+    logging.info("Initializing OAuth flow")
     scopes = ["https://www.googleapis.com/auth/webmasters.readonly"]
     flow = Flow.from_client_config(
         client_config,
         scopes=scopes,
         redirect_uri=client_config["web"]["redirect_uris"][0]
     )
+    logging.info("OAuth flow initialized")
     return flow
 
 def google_auth(client_config):
+    logging.info("Starting Google authentication")
     flow = init_oauth_flow(client_config)
     auth_url, _ = flow.authorization_url(prompt="consent")
+    logging.info("Google authentication URL generated")
     return flow, auth_url
 
 def auth_search_console(client_config, credentials):
+    logging.info("Authenticating with Google Search Console")
     token = {
         "token": credentials.token,
         "refresh_token": credentials.refresh_token,
@@ -161,6 +184,7 @@ def auth_search_console(client_config, credentials):
         "scopes": credentials.scopes,
         "id_token": getattr(credentials, "id_token", None),
     }
+    logging.info("Google Search Console authenticated")
     return searchconsole.authenticate(client_config=client_config, credentials=token)
 
 # -------------
@@ -168,22 +192,29 @@ def auth_search_console(client_config, credentials):
 # -------------
 
 def list_gsc_properties(credentials):
+    logging.info("Listing GSC properties")
     service = build('webmasters', 'v3', credentials=credentials)
     site_list = service.sites().list().execute()
-    return [site['siteUrl'] for site in site_list.get('siteEntry', [])] or ["No properties found"]
+    properties = [site['siteUrl'] for site in site_list.get('siteEntry', [])] or ["No properties found"]
+    logging.info(f"GSC properties listed: {properties}")
+    return properties
 
 def fetch_gsc_data(webproperty, search_type, start_date, end_date, dimensions, device_type=None):
+    logging.info(f"Fetching GSC data for property: {webproperty}, search_type: {search_type}, date_range: {start_date} to {end_date}, dimensions: {dimensions}, device_type: {device_type}")
     query = webproperty.query.range(start_date, end_date).search_type(search_type).dimension(*dimensions)
     if 'device' in dimensions and device_type and device_type != 'All Devices':
         query = query.filter('device', 'equals', device_type.lower())
     try:
         df = query.limit(MAX_ROWS).get().to_dataframe()
+        logging.info("GSC data fetched successfully")
         return process_gsc_data(df)
     except Exception as e:
+        logging.error(f"Error fetching GSC data: {e}")
         show_error(e)
         return pd.DataFrame()
 
 def calculate_relevancy_scores(df, model_type):
+    logging.info("Calculating relevancy scores")
     with st.spinner('Calculating relevancy scores...'):
         try:
             page_contents = [fetch_content(url) for url in df['page']]
@@ -191,20 +222,23 @@ def calculate_relevancy_scores(df, model_type):
             query_embeddings = generate_embeddings(df['query'].tolist(), model_type)
             relevancy_scores = cosine_similarity(query_embeddings, page_embeddings).diagonal()
             df = df.assign(relevancy_score=relevancy_scores)
+            logging.info("Relevancy scores calculated successfully")
         except Exception as e:
+            logging.error(f"Error calculating relevancy scores: {e}")
             st.warning(f"Error calculating relevancy scores: {e}")
             df = df.assign(relevancy_score=0)
     return df
 
-
 # -------------
 # Utility Functions
 # -------------
 
 def update_dimensions(selected_search_type):
+    logging.debug(f"Updating dimensions for search type: {selected_search_type}")
     return BASE_DIMENSIONS + ['device'] if selected_search_type in SEARCH_TYPES else BASE_DIMENSIONS
 
 def calc_date_range(selection, custom_start=None, custom_end=None):
+    logging.debug(f"Calculating date range for selection: {selection}")
     range_map = {
         'Last 7 Days': 7,
         'Last 30 Days': 30,
@@ -216,15 +250,21 @@ def calc_date_range(selection, custom_start=None, custom_end=None):
     today = datetime.date.today()
     if selection == 'Custom Range':
         if custom_start and custom_end:
+            logging.debug(f"Custom date range: {custom_start} to {custom_end}")
             return custom_start, custom_end
         else:
+            logging.debug("Defaulting custom date range to last 7 days")
             return today - datetime.timedelta(days=7), today
-    return today - datetime.timedelta(days=range_map.get(selection, 0)), today
+    date_range = today - datetime.timedelta(days=range_map.get(selection, 0)), today
+    logging.debug(f"Date range calculated: {date_range}")
+    return date_range
 
 def show_error(e):
+    logging.error(f"An error occurred: {e}")
     st.error(f"An error occurred: {e}")
 
 def property_change():
+    logging.info(f"Property changed to: {st.session_state['selected_property_selector']}")
     st.session_state.selected_property = st.session_state['selected_property_selector']
 
 # -------------
@@ -232,28 +272,33 @@ def property_change():
 # -------------
 
 def show_dataframe(report):
+    logging.info("Showing dataframe preview")
     with st.expander("Preview the First 100 Rows (Unique Pages with Top Query)"):
         st.dataframe(report.head(DF_PREVIEW_ROWS))
 
 def download_csv_link(report):
+    logging.info("Generating CSV download link")
     def to_csv(df):
         return df.to_csv(index=False, encoding='utf-8-sig')
     csv = to_csv(report)
     b64_csv = base64.b64encode(csv.encode()).decode()
     href = f'<a href="data:file/csv;base64,{b64_csv}" download="search_console_data.csv">Download CSV File</a>'
     st.markdown(href, unsafe_allow_html=True)
+    logging.info("CSV download link generated")
 
 # -------------
 # Streamlit UI Components
 # -------------
 
 def show_google_sign_in(auth_url):
+    logging.info("Showing Google sign-in button")
     with st.sidebar:
         if st.button("Sign in with Google"):
             st.write('Please click the link below to sign in:')
             st.markdown(f'[Google Sign-In]({auth_url})', unsafe_allow_html=True)
 
 def show_property_selector(properties, account):
+    logging.info("Showing property selector")
     selected_property = st.selectbox(
         "Select a Search Console Property:",
         properties,
@@ -265,6 +310,7 @@ def show_property_selector(properties, account):
     return account[selected_property]
 
 def show_search_type_selector():
+    logging.info("Showing search type selector")
     return st.selectbox(
         "Select Search Type:",
         SEARCH_TYPES,
@@ -273,6 +319,7 @@ def show_search_type_selector():
     )
 
 def show_model_type_selector():
+    logging.info("Showing model type selector")
     return st.selectbox(
         "Select the embedding model:",
         ["english", "multilingual"],
@@ -280,6 +327,7 @@ def show_model_type_selector():
     )
 
 def show_date_range_selector():
+    logging.info("Showing date range selector")
     return st.selectbox(
         "Select Date Range:",
         DATE_RANGE_OPTIONS,
@@ -288,10 +336,12 @@ def show_date_range_selector():
     )
 
 def show_custom_date_inputs():
+    logging.info("Showing custom date inputs")
     st.session_state.custom_start_date = st.date_input("Start Date", st.session_state.custom_start_date)
     st.session_state.custom_end_date = st.date_input("End Date", st.session_state.custom_end_date)
 
 def show_dimensions_selector(search_type):
+    logging.info("Showing dimensions selector")
     available_dimensions = update_dimensions(search_type)
     return st.multiselect(
         "Select Dimensions:",
@@ -301,57 +351,50 @@ def show_dimensions_selector(search_type):
     )
 
 def show_paginated_dataframe(report, rows_per_page=20):
-    # Convert 'position' column to integer and 'impressions' to numeric
+    logging.info("Showing paginated dataframe")
     report['position'] = report['position'].astype(int)
     report['impressions'] = pd.to_numeric(report['impressions'], errors='coerce')
     
-    # Format CTR as percentage and relevancy_score with two decimal places
     def format_ctr(x):
         try:
             return f"{float(x):.2%}"
         except ValueError:
-            return x  # Return the original value if it can't be converted to float
+            return x
     
     def format_relevancy_score(x):
         try:
             return f"{float(x):.2f}"
         except ValueError:
-            return x  # Return the original value if it can't be converted to float
+            return x
     
     report['ctr'] = report['ctr'].apply(format_ctr)
     report['relevancy_score'] = report['relevancy_score'].apply(format_relevancy_score)
     
-    # Create a clickable URL column
     def make_clickable(url):
         return f'<a href="{url}" target="_blank">{url}</a>'
     
     report['clickable_url'] = report['page'].apply(make_clickable)
     
-    # Reorder columns to put clickable_url first
     columns = ['clickable_url', 'query', 'impressions', 'clicks', 'ctr', 'position', 'relevancy_score']
     report = report[columns]
 
-    # Add sorting functionality
-    sort_column = st.selectbox("Sort by:", columns[1:], index=columns[1:].index('impressions'))  # Set 'impressions' as default
+    sort_column = st.selectbox("Sort by:", columns[1:], index=columns[1:].index('impressions'))
     sort_order = st.radio("Sort order:", ("Descending", "Ascending"))
     
     ascending = sort_order == "Ascending"
     
-    # Convert back to numeric for sorting
     def safe_float_convert(x):
         try:
             return float(x.rstrip('%')) / 100 if isinstance(x, str) and x.endswith('%') else float(x)
         except ValueError:
-            return 0  # Return 0 or another default value if conversion fails
+            return 0
     
     report['ctr_numeric'] = report['ctr'].apply(safe_float_convert)
     report['relevancy_score_numeric'] = report['relevancy_score'].apply(safe_float_convert)
     
-    # Sort using the numeric columns
     sort_column_numeric = sort_column + '_numeric' if sort_column in ['ctr', 'relevancy_score'] else sort_column
     report = report.sort_values(by=sort_column_numeric, ascending=ascending)
     
-    # Remove the temporary numeric columns
     report = report.drop(columns=['ctr_numeric', 'relevancy_score_numeric'])
 
     total_rows = len(report)
@@ -373,26 +416,23 @@ def show_paginated_dataframe(report, rows_per_page=20):
     start_idx = (st.session_state.current_page - 1) * rows_per_page
     end_idx = start_idx + rows_per_page
     
-    # Use st.markdown to display the dataframe with clickable links
     st.markdown(report.iloc[start_idx:end_idx].to_html(escape=False, index=False), unsafe_allow_html=True)
+
 # -------------
 # Main Streamlit App Function
 # -------------
 
 def main():
+    logging.info("Starting main function")
     setup_streamlit()
     client_config = load_config()
     
     if 'auth_flow' not in st.session_state or 'auth_url' not in st.session_state:
         st.session_state.auth_flow, st.session_state.auth_url = google_auth(client_config)
 
-    # Directly access query parameters using st.query_params
     query_params = st.query_params
-
-    # Retrieve the 'code' parameter
     auth_code = query_params.get("code", None)
     
-
     if auth_code and 'credentials' not in st.session_state:
         st.session_state.auth_flow.fetch_token(code=auth_code)
         st.session_state.credentials = st.session_state.auth_flow.credentials
@@ -408,7 +448,7 @@ def main():
             webproperty = show_property_selector(properties, account)
             search_type = show_search_type_selector()
             date_range_selection = show_date_range_selector()
-            model_type = show_model_type_selector()  # Add this line
+            model_type = show_model_type_selector()
             if date_range_selection == 'Custom Range':
                 show_custom_date_inputs()
                 start_date, end_date = st.session_state.custom_start_date, st.session_state.custom_end_date
@@ -433,7 +473,8 @@ def main():
                 download_csv_link(st.session_state.report_data)
             elif st.session_state.report_data is not None:
                 st.warning("No data found for the selected criteria.")
+                logging.warning("No data found for the selected criteria")
 
-                
 if __name__ == "__main__":
+    logging.info("Running main function")
     main()