multi error clusters

app.py

@@ -3,7 +3,7 @@
 import numpy as np
 import pandas as pd
 import torch
-import json
+import math
 from tqdm import tqdm
 from math import floor
 from datasets import load_dataset
@@ -104,7 +104,8 @@ def quant_panel(embedding_df):
  st.markdown("* Each **point** is an input example.")
  st.markdown("* Gray points have low-loss and the colored have high-loss. High-loss instances are clustered using **kmeans** and each color represents a cluster.")
  st.markdown("* The **shape** of each point reflects the label category -- positive (diamond) or negative sentiment (circle).")
- st.altair_chart(data_comparison(down_samp(embedding_df)), use_container_width=True)
+ #st.altair_chart(data_comparison(down_samp(embedding_df)), use_container_width=True)
+ st.altair_chart(data_comparison(embedding_df), use_container_width=True)
 
 
 def frequent_tokens(data, tokenizer, loss_quantile=0.95, top_k=200, smoothing=0.005):
@@ -136,7 +137,7 @@ def frequent_tokens(data, tokenizer, loss_quantile=0.95, top_k=200, smoothing=0.
  for i, (token) in enumerate(tokens_sorted[:top_k]):
  top_tokens.append(['%10s' % (tokenizer.decode(token)), '%.4f' % (token_frequencies[token]), '%.4f' % (
  token_frequencies_error[token]), '%4.2f' % (token_lrs[token])])
- return pd.DataFrame(top_tokens, columns=['Token', 'Freq', 'Freq error slice', 'lrs'])
+ return pd.DataFrame(top_tokens, columns=['Token', 'Freq', 'Freq error slice', 'Ratio w/ smoothing'])
 
 
 @st.cache(ttl=600)
@@ -160,12 +161,12 @@ def clustering(data,num_clusters):
  return data, assigned_clusters
 
 def kmeans(df, num_clusters=3):
- data_hl = df.loc[df['slice'] == 'high-loss']
- data_kmeans,clusters = clustering(data_hl,num_clusters)
- merged = pd.merge(df, data_kmeans, left_index=True, right_index=True, how='outer', suffixes=('', '_y'))
- merged.drop(merged.filter(regex='_y$').columns.tolist(),axis=1,inplace=True)
- merged['cluster'] = merged['cluster'].fillna(num_clusters).astype('int')
- return merged
+ #data_hl = df.loc[df['slice'] == 'high-loss']
+ data_kmeans,clusters = clustering(df,num_clusters)
+ #merged = pd.merge(df, data_kmeans, left_index=True, right_index=True, how='outer', suffixes=('', '_y'))
+ #merged.drop(merged.filter(regex='_y$').columns.tolist(),axis=1,inplace=True)
+ #merged['cluster'] = merged['cluster'].fillna(num_clusters).astype('int')
+ return data_kmeans
 
 def distance_from_centroid(row):
  return sdist.norm(row['embedding'] - row['centroid'].tolist())
@@ -173,16 +174,16 @@ def distance_from_centroid(row):
 @st.cache(ttl=600)
 def topic_distribution(weights, smoothing=0.01):
  topic_frequencies = defaultdict(float)
- topic_frequencies_spotlight = defaultdict(float)
+ topic_frequencies_error= defaultdict(float)
  weights_uniform = np.full_like(weights, 1 / len(weights))
  num_examples = len(weights)
  for i in range(num_examples):
  example = dataset[i]
  category = example['title']
  topic_frequencies[category] += weights_uniform[i]
- topic_frequencies_spotlight[category] += weights[i]
+ topic_frequencies_error[category] += weights[i]
 
- topic_ratios = {c: (smoothing + topic_frequencies_spotlight[c]) / (
+ topic_ratios = {c: (smoothing + topic_frequencies_error[c]) / (
  smoothing + topic_frequencies[c]) for c in topic_frequencies}
 
  categories_sorted = map(lambda x: x[0], sorted(
@@ -191,11 +192,9 @@ def topic_distribution(weights, smoothing=0.01):
  topic_distr = []
  for category in categories_sorted:
  topic_distr.append(['%.3f' % topic_frequencies[category], '%.3f' %
- topic_frequencies_spotlight[category], '%.2f' % topic_ratios[category], '%s' % category])
+ topic_frequencies_error[category], '%.2f' % topic_ratios[category], '%s' % category])
 
  return pd.DataFrame(topic_distr, columns=['Overall frequency', 'Error frequency', 'Ratio', 'Category'])
- # for category in categories_sorted:
- # return(topic_frequencies[category], topic_frequencies_spotlight[category], topic_ratios[category], category)
 
 def populate_session(dataset,model):
  data_df = read_file_to_df('./assets/data/'+dataset+ '_'+ model+'.parquet')
@@ -239,13 +238,17 @@ if __name__ == "__main__":
  #populate_session(dataset, model)
  data_df = read_file_to_df('./assets/data/'+dataset+ '_'+ model+'.parquet')
  loss_quantile = st.sidebar.slider(
- "Loss Quantile", min_value=0.5, max_value=1.0,step=0.01,value=0.95
+ "Loss Quantile", min_value=0.5, max_value=1.0,step=0.01,value=0.99
  )
+ data_df = data_df.drop(data_df[data_df.pred == data_df.label].index) #drop rows that are not errors
  data_df['loss'] = data_df['loss'].astype(float)
  losses = data_df['loss']
  high_loss = losses.quantile(loss_quantile)
  data_df['slice'] = 'high-loss'
  data_df['slice'] = data_df['slice'].where(data_df['loss'] > high_loss, 'low-loss') 
+ data_hl = data_df.drop(data_df[data_df['slice'] == 'low-loss'].index) #drop rows that are not hl
+ data_ll = data_df.drop(data_df[data_df['slice'] == 'high-loss'].index) 
+ df_list = [d for _, d in data_hl.groupby(['label'])] # this is to allow clustering over each error type. fp, fn for binary classification
 
  with lcol:
  st.markdown('<h3>Error Slices</h3>',unsafe_allow_html=True)
@@ -279,7 +282,16 @@ if __name__ == "__main__":
 
  if run_kmeans == 'True':
  with st.spinner(text='running kmeans...'):
- merged = kmeans(data_df,num_clusters=num_clusters)
+ merged = pd.DataFrame()
+ ind=0
+ for df in df_list:
+ #num_clusters= int(math.sqrt(len(df)/2))
+ kmeans_df = kmeans(df,num_clusters=num_clusters)
+ #print(kmeans_df.loc[kmeans_df['cluster'].idxmax()])
+ kmeans_df['cluster'] = kmeans_df['cluster'] + ind*num_clusters
+ ind = ind+1
+ merged = pd.concat([merged, kmeans_df])
+ merged = pd.concat([merged, data_ll])
 
  with st.spinner(text='loading visualization...'):
  quant_panel(merged)