Update app.py

app.py

@@ -206,6 +206,25 @@ def get_feature_from_index(subject, index):
  feature = next((f for f in subject_data[subject]['feature_analysis'] if f['index'] == index), None)
  return feature
 
+# def visualize_feature(subject, index):
+# feature = next((f for f in subject_data[subject]['feature_analysis'] if f['index'] == index), None)
+# if feature is None:
+# return "Invalid feature index", None, None, None, None, None, None
+
+# output = f"# {feature['label']}\n\n"
+# output += f"* Pearson correlation: {feature['pearson_correlation']:.4f}\n\n"
+# output += f"* Density: {feature['density']:.4f}\n\n"
+
+# # Top m abstracts
+# top_m_abstracts = get_feature_activations(subject, index)
+ 
+# # Create dataframe for top abstracts
+# df_data = [
+# {"Title": m[1].split('\n\n')[0], "Activation value": f"{m[2]:.4f}"}
+# for m in top_m_abstracts
+# ]
+# df_top_abstracts = pd.DataFrame(df_data)
+
 def visualize_feature(subject, index):
  feature = next((f for f in subject_data[subject]['feature_analysis'] if f['index'] == index), None)
  if feature is None:
@@ -218,12 +237,30 @@ def visualize_feature(subject, index):
  # Top m abstracts
  top_m_abstracts = get_feature_activations(subject, index)
  
- # Create dataframe for top abstracts
- df_data = [
- {"Title": m[1].split('\n\n')[0], "Activation value": f"{m[2]:.4f}"}
- for m in top_m_abstracts
- ]
+ # Create dataframe for top abstracts with clickable links
+ df_data = []
+ for doc_id, abstract, activation_value in top_m_abstracts:
+ title = abstract.split('\n\n')[0]
+ title = title.replace('[', '').replace(']', '')
+ title = title.replace("'", "")
+ title = title.replace('"', '')
+ url_id = doc_id.replace('_arXiv.txt', '')
+ if 'astro-ph' in url_id:
+ url_id = url_id.split('astro-ph')[1]
+ url = f"https://arxiv.org/abs/astro-ph/{url_id}"
+ else:
+ if '.' in doc_id:
+ url = f"https://arxiv.org/abs/{url_id}"
+ else:
+ url = f"https://arxiv.org/abs/hep-ph/{url_id}"
+ 
+ linked_title = f"[{title}]({url})"
+ df_data.append({"Title": linked_title, "Activation value": activation_value})
+ 
  df_top_abstracts = pd.DataFrame(df_data)
+ styled_top_abstracts = df_top_abstracts.style.format({
+ "Activation value": "{:.4f}"
+ })
 
  # Activation value distribution
  topk_indices = subject_data[subject]['topk_indices']
@@ -249,23 +286,62 @@ def visualize_feature(subject, index):
  topk_indices_cosine = np.argsort(-cosine_similarities)[:topk]
  topk_values_cosine = cosine_similarities[topk_indices_cosine]
 
- # Create dataframe for top 5 correlated features
- df_top_correlated = pd.DataFrame({
- "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_cosine],
- "Cosine similarity": [f"{v:.4f}" for v in topk_values_cosine]
- })
- df_top_correlated_styled = style_dataframe(df_top_correlated, is_top=True)
+ # # Create dataframe for top 5 correlated features
+ # df_top_correlated = pd.DataFrame({
+ # "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_cosine],
+ # "Cosine similarity": topk_values_cosine
+ # })
+ # df_top_correlated_styled = df_top_correlated.style.format({
+ # "Cosine similarity": "{:.4f}"
+ # })
 
  bottomk = 5
  bottomk_indices_cosine = np.argsort(cosine_similarities)[:bottomk]
  bottomk_values_cosine = cosine_similarities[bottomk_indices_cosine]
+ 
+ # # Create dataframe for bottom 5 correlated features
+ # df_bottom_correlated = pd.DataFrame({
+ # "Feature": [get_feature_from_index(subject, i)['label'] for i in bottomk_indices_cosine],
+ # "Cosine similarity": bottomk_values_cosine
+ # })
+ # df_bottom_correlated_styled = df_bottom_correlated.style.format({
+ # "Cosine similarity": "{:.4f}"
+ # })
+
+ # # Co-occurrences
+ # co_occurrences = calculate_co_occurrences(subject, index)
+ # topk = 5
+ # topk_indices_co_occurrence = np.argsort(-co_occurrences)[:topk]
+ # topk_values_co_occurrence = co_occurrences[topk_indices_co_occurrence]
+
+ # # Create dataframe for top 5 co-occurring features
+ # df_co_occurrences = pd.DataFrame({
+ # "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_co_occurrence],
+ # "Co-occurrences": topk_values_co_occurrence
+ # })
+ # df_co_occurrences_styled = df_co_occurrences.style.format({
+ # "Co-occurrences": "{:.4f}"
+ # })
+
+ # return output, styled_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences_styled, fig2
+
+ # Create dataframe for top 5 correlated features
+ df_top_correlated = pd.DataFrame({
+ "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_cosine],
+ "Cosine similarity": topk_values_cosine
+ })
+ df_top_correlated_styled = df_top_correlated.style.format({
+ "Cosine similarity": "{:.4f}"
+ })
 
  # Create dataframe for bottom 5 correlated features
  df_bottom_correlated = pd.DataFrame({
  "Feature": [get_feature_from_index(subject, i)['label'] for i in bottomk_indices_cosine],
- "Cosine similarity": [f"{v:.4f}" for v in bottomk_values_cosine]
+ "Cosine similarity": bottomk_values_cosine
+ })
+ df_bottom_correlated_styled = df_bottom_correlated.style.format({
+ "Cosine similarity": "{:.4f}"
  })
- df_bottom_correlated_styled = style_dataframe(df_bottom_correlated, is_top=False)
 
  # Co-occurrences
  co_occurrences = calculate_co_occurrences(subject, index)
@@ -278,8 +354,102 @@ def visualize_feature(subject, index):
  "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_co_occurrence],
  "Co-occurrences": topk_values_co_occurrence
  })
+ df_co_occurrences_styled = df_co_occurrences.style.format({
+ "Co-occurrences": "{:.0f}" # Keep as integer
+ })
 
- return output, df_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences, fig2
+ return output, styled_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences_styled, fig2
+
+# def visualize_feature(subject, index):
+# feature = next((f for f in subject_data[subject]['feature_analysis'] if f['index'] == index), None)
+# if feature is None:
+# return "Invalid feature index", None, None, None, None, None, None
+
+# output = f"# {feature['label']}\n\n"
+# output += f"* Pearson correlation: {feature['pearson_correlation']:.4f}\n\n"
+# output += f"* Density: {feature['density']:.4f}\n\n"
+
+# # Top m abstracts
+# top_m_abstracts = get_feature_activations(subject, index)
+ 
+# # Create dataframe for top abstracts with clickable links
+# df_data = []
+# for doc_id, abstract, activation_value in top_m_abstracts:
+# title = abstract.split('\n\n')[0]
+# title = title.replace('[', '').replace(']', '')
+# title = title.replace("'", "")
+# title = title.replace('"', '')
+# url_id = doc_id.replace('_arXiv.txt', '')
+# if 'astro-ph' in url_id:
+# url_id = url_id.split('astro-ph')[1]
+# url = f"https://arxiv.org/abs/astro-ph/{url_id}"
+# else:
+# if '.' in doc_id:
+# url = f"https://arxiv.org/abs/{url_id}"
+# else:
+# url = f"https://arxiv.org/abs/hep-ph/{url_id}"
+ 
+# linked_title = f"[{title}]({url})"
+# df_data.append({"Title": linked_title, "Activation value": activation_value})
+ 
+# df_top_abstracts = pd.DataFrame(df_data)
+
+# # Activation value distribution
+# topk_indices = subject_data[subject]['topk_indices']
+# topk_values = subject_data[subject]['topk_values']
+
+# activation_values = np.where(topk_indices == index, topk_values, 0).max(axis=1)
+# fig2 = px.histogram(x=activation_values, nbins=50)
+# fig2.update_layout(
+# #title=f'{feature["label"]}',
+# xaxis_title='Activation value',
+# yaxis_title=None,
+# yaxis_type='log',
+# height=220,
+# )
+
+# # Correlated features
+# decoder = subject_data[subject]['decoder']
+# feature_vector = decoder[:, index]
+# decoder_without_feature = np.delete(decoder, index, axis=1)
+# cosine_similarities = np.dot(feature_vector, decoder_without_feature) / (np.linalg.norm(decoder_without_feature, axis=0) * np.linalg.norm(feature_vector))
+
+# topk = 5
+# topk_indices_cosine = np.argsort(-cosine_similarities)[:topk]
+# topk_values_cosine = cosine_similarities[topk_indices_cosine]
+
+# # Create dataframe for top 5 correlated features
+# df_top_correlated = pd.DataFrame({
+# "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_cosine],
+# "Cosine similarity": [f"{v:.4f}" for v in topk_values_cosine]
+# })
+# df_top_correlated_styled = style_dataframe(df_top_correlated, is_top=True)
+
+# bottomk = 5
+# bottomk_indices_cosine = np.argsort(cosine_similarities)[:bottomk]
+# bottomk_values_cosine = cosine_similarities[bottomk_indices_cosine]
+
+# # Create dataframe for bottom 5 correlated features
+# df_bottom_correlated = pd.DataFrame({
+# "Feature": [get_feature_from_index(subject, i)['label'] for i in bottomk_indices_cosine],
+# "Cosine similarity": [f"{v:.4f}" for v in bottomk_values_cosine]
+# })
+# df_bottom_correlated_styled = style_dataframe(df_bottom_correlated, is_top=False)
+
+# # Co-occurrences
+# co_occurrences = calculate_co_occurrences(subject, index)
+# topk = 5
+# topk_indices_co_occurrence = np.argsort(-co_occurrences)[:topk]
+# topk_values_co_occurrence = co_occurrences[topk_indices_co_occurrence]
+
+# # Create dataframe for top 5 co-occurring features
+# df_co_occurrences = pd.DataFrame({
+# "Feature": [get_feature_from_index(subject, i)['label'] for i in topk_indices_co_occurrence],
+# "Co-occurrences": topk_values_co_occurrence
+# })
+
+# #return output, df_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences, fig2
+# return output, df_top_abstracts, df_top_correlated_styled, df_bottom_correlated_styled, df_co_occurrences, fig2
 
 # Modify the main interface function
 def create_interface():
@@ -334,11 +504,16 @@ def create_interface():
  manually_added_features_state = gr.State([])
 
  def update_search_results(feature_values, feature_indices, manually_added_features, current_subject):
+ # ae = subject_data[current_subject]['ae']
+ # abstract_embeddings = subject_data[current_subject]['abstract_embeddings']
+ # abstract_texts = subject_data[current_subject]['abstract_texts']
+ # df_metadata = subject_data[current_subject]['df_metadata']
  ae = subject_data[current_subject]['ae']
  abstract_embeddings = subject_data[current_subject]['abstract_embeddings']
  abstract_texts = subject_data[current_subject]['abstract_texts']
  df_metadata = subject_data[current_subject]['df_metadata']
 
+
  # Combine manually added features with query-generated features
  all_indices = []
  all_values = []
@@ -368,12 +543,40 @@ def create_interface():
  doc_ids = abstract_texts['doc_ids']
  topk_doc_ids = [doc_ids[i] for i in topk_indices_search]
 
+ # # Prepare search results
+ # search_results = []
+ # for doc_id in topk_doc_ids:
+ # metadata = df_metadata[df_metadata['arxiv_id'] == doc_id].iloc[0]
+ # title = metadata['title'].replace('[', '').replace(']', '')
+ # # Remove single quotes from title
+ # title = title.replace("'", "")
+
+ # url_id = doc_id.replace('_arXiv.txt', '')
+ # if 'astro-ph' in url_id:
+ # url_id = url_id.split('astro-ph')[1]
+ # url = f"https://arxiv.org/abs/astro-ph/{url_id}"
+ # else:
+ # # Create the clickable link based on the doc_id
+ # if '.' in doc_id:
+ # url = f"https://arxiv.org/abs/{doc_id.replace('_arXiv.txt', '')}"
+ # else:
+ # url = f"https://arxiv.org/abs/hep-ph/{doc_id.replace('_arXiv.txt', '')}"
+ 
+ # linked_title = f"[{title}]({url})"
+ 
+ # search_results.append([
+ # linked_title,
+ # int(metadata['citation_count']),
+ # int(metadata['year'])
+ # ])
+
+ # return search_results, all_values, all_indices
+
  # Prepare search results
  search_results = []
  for doc_id in topk_doc_ids:
  metadata = df_metadata[df_metadata['arxiv_id'] == doc_id].iloc[0]
  title = metadata['title'].replace('[', '').replace(']', '')
- # Remove single quotes from title
  title = title.replace("'", "")
 
  url_id = doc_id.replace('_arXiv.txt', '')
@@ -381,7 +584,6 @@ def create_interface():
  url_id = url_id.split('astro-ph')[1]
  url = f"https://arxiv.org/abs/astro-ph/{url_id}"
  else:
- # Create the clickable link based on the doc_id
  if '.' in doc_id:
  url = f"https://arxiv.org/abs/{doc_id.replace('_arXiv.txt', '')}"
  else:
@@ -395,7 +597,14 @@ def create_interface():
  int(metadata['year'])
  ])
 
- return search_results, all_values, all_indices
+ # Convert search_results to a DataFrame and apply styling
+ df_search_results = pd.DataFrame(search_results, columns=["Title", "Citation Count", "Year"])
+ styled_search_results = df_search_results.style.format({
+ "Citation Count": "{:.0f}", # Keep as integer
+ "Year": "{:.0f}" # Keep as integer
+ })
+
+ return styled_search_results, all_values, all_indices
 
  @gr.render(inputs=[input_text, search_results_state, feature_values_state, feature_indices_state, manually_added_features_state, subject])
  def show_components(text, search_results, feature_values, feature_indices, manually_added_features, current_subject):
@@ -418,11 +627,6 @@ def create_interface():
 
  with gr.Row():
  with gr.Column(scale=2):
- # df = gr.Dataframe(
- # headers=["Title", "Citation Count", "Year"],
- # value=search_results,
- # label="Top 10 Search Results"
- # )
  df = gr.Dataframe(
  headers=["Title", "Citation Count", "Year"],
  value=search_results,
@@ -462,18 +666,7 @@ def create_interface():
  with gr.Column(scale=1):
  update_button = gr.Button("Update Results")
  sliders = []
- # for i, (value, index) in enumerate(zip(feature_values, feature_indices)):
- # feature = next((f for f in subject_data[current_subject]['feature_analysis'] if f['index'] == index), None)
- # label = f"{feature['label']} ({index})" if feature else f"Feature {index}"
- 
- # # Add prefix and change color for manually added features
- # if index in manually_added_features:
- # label = f"[Custom] {label}"
- # slider = gr.Slider(minimum=0, maximum=1, step=0.01, value=value, label=label, key=f"slider-{index}", elem_id=f"custom-slider-{index}")
- # else:
- # slider = gr.Slider(minimum=0, maximum=1, step=0.01, value=value, label=label, key=f"slider-{index}")
- 
- # sliders.append(slider)
+
  for i, (value, index) in enumerate(zip(feature_values, feature_indices)):
  feature = next((f for f in subject_data[current_subject]['feature_analysis'] if f['index'] == index), None)
  label = f"{feature['label']} ({index})" if feature else f"Feature {index}"
@@ -491,16 +684,6 @@ def create_interface():
  
  sliders.append(slider)
 
- # def on_slider_change(*values):
- # manually_added_features = values[-1]
- # slider_values = list(values[:-1])
- 
- # # Reconstruct feature_indices based on the order of sliders
- # reconstructed_indices = [int(slider.label.split('(')[-1].split(')')[0]) for slider in sliders]
- 
- # new_results, new_values, new_indices = update_search_results(slider_values, reconstructed_indices, manually_added_features, current_subject)
- # return new_results, new_values, new_indices, manually_added_features
-
  def on_slider_change(*values):
  manually_added_features = values[-1]
  slider_values = list(values[:-1])
@@ -530,10 +713,18 @@ def create_interface():
  visualize_button = gr.Button("Visualize Feature")
  
  feature_info = gr.Markdown()
+ # abstracts_heading = gr.Markdown("## Top 5 Abstracts")
+ # top_abstracts = gr.Dataframe(
+ # headers=["Title", "Activation value"],
+ # interactive=False
+ # )
+
  abstracts_heading = gr.Markdown("## Top 5 Abstracts")
  top_abstracts = gr.Dataframe(
  headers=["Title", "Activation value"],
- interactive=False
+ datatype=["markdown", "number"],
+ interactive=False,
+ wrap=True
  )
  
  gr.Markdown("## Correlated Features")