Classification Conformal

app.py

@@ -8,24 +8,16 @@ from sklearn.linear_model import LinearRegression
 from sklearn.metrics import mean_squared_error,mean_absolute_error,r2_score,mean_absolute_percentage_error
 from matplotlib import pyplot as plt
 import matplotlib.image as mpimg
-import pickle
+import torch
+
 
 load_reg_data = False
 load_class_data = False
-california_img=plt.imread("./california.png")
-
-
-# plt.rcParams['axes.spines.right'] = False
-# plt.rcParams['axes.spines.top'] = False
-# plt.rcParams['axes.spines.left'] = False
-# plt.rcParams['axes.spines.bottom'] = False
 
 
 
 def conformal_Predict(cal_err,alpha = 0.8):
-
     assert alpha != None, " Provide a value of alpha "
-    # assert cal_err!= [] or None, "Provide the caliberation errors"
     idx = int(alpha*len(cal_err))
     return cal_err[idx]
     
@@ -40,13 +32,17 @@ if __name__ == '__main__':
         y_test = np.load("./Reg_Test_y.npy")
         err_calib = np.load("./Reg_calib_err.npy")
         y_pred = np.load("./Reg_y_pred.npy")
+        california_img=plt.imread("./california.png")
         load_reg_data = True
 
 
     if not(load_class_data):
-        pass
-
+        img = np.load("./final_images.npy")
+        pred = np.load("./final_pred.npy")
+        cls_calib = np.load("./cerr.npy")
+        load_class_data = True
 
+    
 
     st.title("Conformal Prediction")
     intro_tab , reg_tab , class_tab = st.tabs(["Introduction","Regression", "Classification"])
@@ -117,8 +113,112 @@ if __name__ == '__main__':
 
 
     with class_tab:
-        st.subheader("Classification")
-        st.write("This is the classification tab")
+        
+        st.write("For Classification we are using Fashion-MNIST dataset. Fashion-MNIST is a dataset of Zalando's article images. Zalando intends Fashion-MNIST to serve as a direct drop-in for benchmarking machine learning algorithms. Each example is assigned to one of the following labels: 0 T-shirt/top, 1 Trouser,2 Pullover, 3 Dress, 4 Coat, 5 Sandal, 6 Shirt, 7 Sneaker, 8 Bag, 9 Ankle boot") 
+
+        st.write("---")
+
+        st.write("Lets assume you have a model trained for Object detection but you cant just rely on the softmax output for that model. This is where conformal prediction comes into play. We can use the alpha value to pick up a threshold. When softmax scores go beyond this threshold score then onlt that label is considered as the predicted class.") 
+
+        st.write("The higher the value of alpha more the model is certain about its prediction")
+
+
+
+        c1,c2,c3 = st.columns(3)
+        with c2:
+            alpha1 = st.slider('Select a value of alpha for the Model',min_value=0.1,max_value=.99,value=0.5,step=0.05)
+        sigma = conformal_Predict(cls_calib,alpha1)
+
+        labels = np.array(['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat', 'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot'])   
+
+        with st.container():
+            
+            c1,col1, col2, col3,c2 = st.columns([0.2,0.3,0.3,0.3,0.2])
+
+            with col1:
+                fig1, ax1 = plt.figure(), plt.gca()
+                ax1.imshow(torch.tensor(img[0]).permute(1,2,0),cmap='gray')
+                ax1.spines['top'].set_visible(False)
+                ax1.spines['bottom'].set_visible(False)
+                ax1.spines['right'].set_visible(False)
+                ax1.spines['left'].set_visible(False)
+                ax1.set_xticks([])
+                ax1.set_yticks([])
+                st.pyplot(fig1)
+
+                fig1, ax1 = plt.figure(), plt.gca()
+                ax1.bar(range(10),pred[0])
+                ax1.axhline(y=sigma,linestyle='dashed',c='r')
+                ax1.set_xlabel("Classe Labels")
+                ax1.set_ylabel("SoftMax Probabilities")
+                ax1.set_title("Class Scores with Threshold")
+                ax1.set_xticks([i for i in range(10)])
+                st.pyplot(fig1)
+
+                out_labels = labels[pred[0]>sigma]
+                if len(out_labels)==0:
+                    out_labels = ["None"]
+                out_labels = ",".join(out_labels)
+                st.write("Ouput Labels : "+out_labels)
+                st.write("True Label : Coat")
+
+            
+            with col2:
+                fig1, ax1 = plt.figure(), plt.gca()
+                ax1.imshow(torch.tensor(img[1]).permute(1,2,0),cmap='gray')
+                ax1.spines['top'].set_visible(False)
+                ax1.spines['bottom'].set_visible(False)
+                ax1.spines['right'].set_visible(False)
+                ax1.spines['left'].set_visible(False)
+                ax1.set_xticks([])
+                ax1.set_yticks([])
+                st.pyplot(fig1)
+
+                fig1, ax1 = plt.figure(), plt.gca()
+                ax1.bar(range(10),pred[1])
+                ax1.axhline(y=sigma,linestyle='dashed',c='r')
+                ax1.set_xlabel("Classe Labels")
+                ax1.set_ylabel("SoftMax Probabilities")
+                ax1.set_title("Class Scores with Threshold")
+                ax1.set_xticks([i for i in range(10)])
+                st.pyplot(fig1)
+
+                out_labels = labels[pred[1]>sigma]
+                if len(out_labels)==0:
+                    out_labels = ["None"]
+                out_labels = ",".join(out_labels)
+                st.write("Ouput Labels : "+out_labels)
+                st.write("True Label : Ankle Boot")
+
+            with col3:
+                fig1, ax1 = plt.figure(), plt.gca()
+                ax1.imshow(torch.tensor(img[2]).permute(1,2,0),cmap='gray')
+                ax1.spines['top'].set_visible(False)
+                ax1.spines['bottom'].set_visible(False)
+                ax1.spines['right'].set_visible(False)
+                ax1.spines['left'].set_visible(False)
+                ax1.set_xticks([])
+                ax1.set_yticks([])
+                st.pyplot(fig1)
+
+                fig1, ax1 = plt.figure(), plt.gca()
+                ax1.bar(range(10),pred[2])
+                ax1.axhline(y=sigma,linestyle='dashed',c='r')
+                ax1.set_xlabel("Classe Labels")
+                ax1.set_ylabel("SoftMax Probabilities")
+                ax1.set_title("Class Scores with Threshold")
+                ax1.set_xticks([i for i in range(10)])
+                st.pyplot(fig1)
+
+                out_labels = labels[pred[2]>sigma]
+                if len(out_labels)==0:
+                    out_labels = ["None"]
+                out_labels = ",".join(out_labels)
+                st.write("Ouput Labels : "+out_labels)
+                st.write("True Label : Bag")
+
+
+
 
 
 

cerr.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:25367f293f8667889b42da0c304630cee93f1dc3f6d5faebcb3211489c500613
+size 4128

final_images.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa9347527bfb8edfa7a17e3e07e6aa6f8c40a22cfb0b7fdc29b2b3b6c01066d9
+size 9536

final_pred.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4dab1a251533ac80f3125474ac74ab3ea6088be558d8bffab4774ee3a6e7f304
+size 248