chore: Refactor RTUAnomalizer1.py and lstm_vav_rtu2.ipynb

dashboard.py

@@ -1,3 +1,4 @@
+from collections import deque
 import streamlit as st
 import pandas as pd
 import numpy as np
@@ -26,8 +27,26 @@ rtu_anomalizer1 = RTUAnomalizer1(
     num_outputs=rtu_data_pipeline.num_outputs,
 )
 
-print("Kmeans models:")
-print(rtu_anomalizer1.kmeans_models)
+
+rtu_anomalizer2 = RTUAnomalizer1(
+    prediction_model_path="src/rtu/models/lstm_2rtu_smooth_04.keras",
+    clustering_model_paths=[
+        "src/rtu/models/kmeans_rtu_1.pkl",
+        "src/rtu/models/kmeans_rtu_2.pkl",
+    ],
+    pca_model_paths=[
+        "src/rtu/models/pca_rtu_1.pkl",
+        "src/rtu/models/pca_rtu_2.pkl",
+    ],
+    num_inputs=rtu_data_pipeline.num_inputs,
+    num_outputs=rtu_data_pipeline.num_outputs,
+)
+
+rtu_1_thresholds = deque(maxlen=60)
+rtu_1_fault = False
+for i in range(60):
+    rtu_1_thresholds.append(0)
+
 
 # Set the layout of the page to 'wide'
 st.set_page_config(layout="wide")
@@ -263,21 +282,11 @@ with st.container():
 
         with row2_row2_col1:
             st.subheader("Energy Usage - North Wing")
-            df_energy = generate_energy_data()  # ---- REPLACE WITH ACTUAL DATA ----
-            fig, ax = plt.subplots(figsize=(5, 1.5))
-            ax.plot(df_energy["Time"], df_energy["Energy"])
-            ax.set_xlabel("Time")
-            ax.set_ylabel("Energy (kWh)")
-            st.pyplot(fig)
+            north_wing_energy_container = st.empty()
 
             # with row2_row2_col2:
             st.subheader("Energy Usage - South Wing")
-            df_energy = generate_energy_data()  # ---- REPLACE WITH ACTUAL DATA ----
-            fig, ax = plt.subplots(figsize=(5, 1.5))
-            ax.plot(df_energy["Time"], df_energy["Energy"])
-            ax.set_xlabel("Time")
-            ax.set_ylabel("Energy (kWh)")
-            st.pyplot(fig)
+            south_wing_energy_container = st.empty()
 
             # Energy Comsumption Statistics
             with row2_row2_col2:
@@ -287,16 +296,56 @@ with st.container():
                 )  # ---- REPLACE WITH ACTUAL DATA ----
 
 
-distance_placeholder = st.empty()
+distances = []
+
+
+def create_residual_plot(resid_pca_list, rtu_id):
+    if rtu_id % 2 == 1:
+        ax1 = 0
+        ax2 = 1
+    elif rtu_id % 2 == 0:
+        ax1 = 2
+        ax2 = 3
+    fig = px.scatter(
+        x=resid_pca_list[:, ax1],
+        y=resid_pca_list[:, ax2],
+        labels={"x": "Time", "y": "Residual"},
+        width=500,
+        height=500,
+    )
+    fig.update_layout(
+        xaxis_range=[-8, 8],
+        yaxis_range=[-8, 8],
+        xaxis=dict(showgrid=True, gridwidth=1, gridcolor="lightgray"),
+        yaxis=dict(showgrid=True, gridwidth=1, gridcolor="lightgray"),
+        margin=dict(l=20, r=20, t=20, b=20),
+        hovermode="closest",
+        showlegend=False,
+        autosize=False,
+        hoverlabel=dict(bgcolor="white", font_size=12),
+        hoverlabel_align="left",
+        hoverlabel_font_color="black",
+        hoverlabel_bordercolor="lightgray",
+    )
+    fig.update_traces(marker=dict(size=5, color="blue"))
+
+    return fig
+
+
 resid_placeholder = st.empty()
 
-distances = []
+
 while True:
 
     if mqtt_client.data_list:
         all_data.extend(mqtt_client.data_list)
+        if len(all_data) > 100:
+            all_data.pop(0)
         df = pd.DataFrame(all_data)
 
+        if sum(list(rtu_1_thresholds)) > 50:
+            rtu_1_fault = True
+
         df_time = df["date"].iloc[-1]  # Obtain the latest datetime of data
 
         with placeholder_header_time:
@@ -312,57 +361,72 @@ while True:
 
         dist = None
         resid_pca_list = None
-        df_new1, df_trans1, df_new2, df_trans2 = rtu_data_pipeline.fit(df)
+        resid_pca_list_2 = None
+
+        df_new1, df_trans1, df_new2, df_trans2 = rtu_data_pipeline.fit(
+            pd.DataFrame(mqtt_client.data_list)
+        )
         if (
             not df_new1 is None
             and not df_trans1 is None
             and not df_new2 is None
             and not df_trans2 is None
         ):
-            actual_list, pred_list, resid_list, resid_pca_list, dist = (
+            actual_list, pred_list, resid_list, resid_pca_list, dist, over_threshold = (
                 rtu_anomalizer1.pipeline(df_new1, df_trans1, rtu_data_pipeline.scaler1)
             )
+            (
+                actual_list_2,
+                pred_list_2,
+                resid_list_2,
+                resid_pca_list_2,
+                dist_2,
+                over_threshold_2,
+            ) = rtu_anomalizer1.pipeline(df_new1, df_trans1, rtu_data_pipeline.scaler1)
 
         if resid_pca_list is not None:
+            rtu_1_thresholds.append(over_threshold[0])
             resid_pca_list = np.array(resid_pca_list)
+            resid_pca_list_2 = np.array(resid_pca_list_2)
 
-        # plot the distances in a scatter chart in streamlit plotly express
-        with distance_placeholder:
-            if dist is not None:
-                distances.append(float(dist[0][0]))
-                fig = px.line(
-                    x=range(len(distances)),
-                    y=distances,
-                    labels={"x": "Time", "y": "Distance"},
-                    title="Distance from Cluster Center",
-                )
-                st.plotly_chart(fig)
-
-        with resid_placeholder:
-            if resid_pca_list is not None:
-                fig = px.scatter(
-                    x=resid_pca_list[:, 0],
-                    y=resid_pca_list[:, 1],
-                    labels={"x": "Time", "y": "Residual"},
-                    title="Residuals",
-                    width=800,
-                    height=800,
-                )
-                fig.update_layout(
-                    xaxis_range=[-1, 1],
-                    yaxis_range=[-1, 1],
-                    xaxis=dict(showgrid=True, gridwidth=1, gridcolor="lightgray"),
-                    yaxis=dict(showgrid=True, gridwidth=1, gridcolor="lightgray"),
-                    margin=dict(l=20, r=20, t=20, b=20),
-                    hovermode="closest",
-                    showlegend=False,
-                    autosize=False,
-                    hoverlabel=dict(bgcolor="white", font_size=12),
-                    hoverlabel_align="left",
-                    hoverlabel_font_color="black",
-                    hoverlabel_bordercolor="lightgray",
-                )
-                fig.update_traces(marker=dict(size=5, color="blue"))
-                st.plotly_chart(fig)
+        if resid_pca_list is not None:
+            with resid_placeholder.container():
+                resid_rtu1_placeholder, resid_rtu2_placeholder = st.columns(2)
+                with resid_rtu1_placeholder:
+                    st.subheader("RTU 1 Residuals")
+                    fig = create_residual_plot(resid_pca_list, rtu_id=1)
+                    st.plotly_chart(fig)
+
+                with resid_rtu2_placeholder:
+                    st.subheader("RTU 2 Residuals")
+                    fig = create_residual_plot(resid_pca_list, rtu_id=2)
+                    st.plotly_chart(fig)
+
+                resid_rtu3_placeholder, resid_rtu4_placeholder = st.columns(2)
+                with resid_rtu3_placeholder:
+                    st.subheader("RTU 3 Residuals")
+                    fig = create_residual_plot(resid_pca_list, rtu_id=3)
+                    st.plotly_chart(fig)
+
+                with resid_rtu4_placeholder:
+                    st.subheader("RTU 4 Residuals")
+                    fig = create_residual_plot(resid_pca_list, rtu_id=4)
+                    st.plotly_chart(fig)
+
+        # with north_wing_energy_container:
+        #     df_energy = generate_energy_data()  # ---- REPLACE WITH ACTUAL DATA ----
+        #     fig, ax = plt.subplots(figsize=(5, 1.5))
+        #     ax.plot(df_energy["Time"], df_energy["Energy"])
+        #     ax.set_xlabel("Time")
+        #     ax.set_ylabel("Energy (kWh)")
+        #     st.pyplot(fig)
+
+        # with south_wing_energy_container:
+        #     df_energy = generate_energy_data()  # ---- REPLACE WITH ACTUAL DATA ----
+        #     fig, ax = plt.subplots(figsize=(5, 1.5))
+        #     ax.plot(df_energy["Time"], df_energy["Energy"])
+        #     ax.set_xlabel("Time")
+        #     ax.set_ylabel("Energy (kWh)")
+        #     st.pyplot(fig)
 
         mqtt_client.data_list.clear()

physLSTM/lstm_vav_rtu2.ipynb

@@ -808,6 +808,257 @@
    "source": [
     "sum(labels==0)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "df = pd.DataFrame()\n",
+    "\n",
+    "# create a dataframe with random data with 4 columns with random names\n",
+    "\n",
+    "df['col1'] = np.random.rand(100)\n",
+    "df['col2'] = np.random.rand(100)\n",
+    "df['col3'] = np.random.rand(100)\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>col1</th>\n",
+       "      <th>col2</th>\n",
+       "      <th>col3</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.766087</td>\n",
+       "      <td>0.869167</td>\n",
+       "      <td>0.189766</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>0.663337</td>\n",
+       "      <td>0.732319</td>\n",
+       "      <td>0.916248</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>0.691630</td>\n",
+       "      <td>0.013371</td>\n",
+       "      <td>0.542655</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>0.184282</td>\n",
+       "      <td>0.222953</td>\n",
+       "      <td>0.066323</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>0.522267</td>\n",
+       "      <td>0.938941</td>\n",
+       "      <td>0.147908</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>95</th>\n",
+       "      <td>0.179310</td>\n",
+       "      <td>0.093981</td>\n",
+       "      <td>0.248775</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>96</th>\n",
+       "      <td>0.097265</td>\n",
+       "      <td>0.834429</td>\n",
+       "      <td>0.612038</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>97</th>\n",
+       "      <td>0.385221</td>\n",
+       "      <td>0.788180</td>\n",
+       "      <td>0.324235</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>98</th>\n",
+       "      <td>0.429241</td>\n",
+       "      <td>0.834097</td>\n",
+       "      <td>0.065782</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>99</th>\n",
+       "      <td>0.267903</td>\n",
+       "      <td>0.639509</td>\n",
+       "      <td>0.804702</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>100 rows × 3 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        col1      col2      col3\n",
+       "0   0.766087  0.869167  0.189766\n",
+       "1   0.663337  0.732319  0.916248\n",
+       "2   0.691630  0.013371  0.542655\n",
+       "3   0.184282  0.222953  0.066323\n",
+       "4   0.522267  0.938941  0.147908\n",
+       "..       ...       ...       ...\n",
+       "95  0.179310  0.093981  0.248775\n",
+       "96  0.097265  0.834429  0.612038\n",
+       "97  0.385221  0.788180  0.324235\n",
+       "98  0.429241  0.834097  0.065782\n",
+       "99  0.267903  0.639509  0.804702\n",
+       "\n",
+       "[100 rows x 3 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df2 = pd.DataFrame()\n",
+    "df2['col1'] = np.random.rand(1)\n",
+    "df2['col2'] = np.random.rand(1)\n",
+    "df2['col3'] = np.random.rand(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>col1</th>\n",
+       "      <th>col2</th>\n",
+       "      <th>col3</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0.474836</td>\n",
+       "      <td>0.927463</td>\n",
+       "      <td>0.761931</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "       col1      col2      col3\n",
+       "0  0.474836  0.927463  0.761931"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df2"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "c = []\n",
+    "a = [2,3]\n",
+    "b = [3,4]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "c = c+a"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "c = c+b"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

src/rtu/RTUAnomalizer1.py

@@ -217,4 +217,12 @@ class RTUAnomalizer1:
         actual, pred = self.inverse_transform(scaler, pred, df_trans)
         actual_list, pred_list, resid_list = self.update_lists(actual, pred, resid)
         dist = self.calculate_distances(resid)
-        return actual_list, pred_list, resid_list, self.resid_pca_list, dist
+        over_threshold = dist > 0.5
+        return (
+            actual_list,
+            pred_list,
+            resid_list,
+            self.resid_pca_list,
+            dist,
+            over_threshold,
+        )