Merge branch 'lstm_pipeline' of hf.co:spaces/smartbuildings/smart-buildings into lstm_pipeline

physLSTM/lstm_vav_rtu1.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 35,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -23,7 +23,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -32,7 +32,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 31,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -51,12 +51,6 @@
     "        ):\n",
     "            cols.append(column)\n",
     "\n",
-    "\n",
-    "# for rtu in rtus:\n",
-    "#     for column in merged.columns:\n",
-    "#         if f\"rtu_00{rtu}_fltrd_sa\" or f\"rtu_00{rtu}_sa_temp\" in column:\n",
-    "#                 cols.append(column)\n",
-    "\n",
     "cols = (\n",
     "    [\"date\"]\n",
     "    + cols\n",
@@ -82,14 +76,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 32,
+   "execution_count": 12,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "C:\\Users\\arbal\\AppData\\Local\\Temp\\ipykernel_29192\\4293840618.py:1: SettingWithCopyWarning: \n",
+      "C:\\Users\\arbal\\AppData\\Local\\Temp\\ipykernel_368\\4293840618.py:1: SettingWithCopyWarning: \n",
       "A value is trying to be set on a copy of a slice from a DataFrame.\n",
       "Try using .loc[row_indexer,col_indexer] = value instead\n",
       "\n",
@@ -115,7 +109,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 36,
+   "execution_count": 13,
    "metadata": {},
    "outputs": [
     {
@@ -124,7 +118,7 @@
        "[]"
       ]
      },
-     "execution_count": 36,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -144,7 +138,46 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 37,
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['date', 'zone_069_temp', 'zone_069_fan_spd', 'zone_068_temp',\n",
+       "       'zone_068_fan_spd', 'zone_067_temp', 'zone_067_fan_spd',\n",
+       "       'zone_066_temp', 'zone_066_fan_spd', 'zone_065_temp',\n",
+       "       'zone_065_fan_spd', 'zone_064_temp', 'zone_064_fan_spd',\n",
+       "       'zone_042_temp', 'zone_042_fan_spd', 'zone_041_temp',\n",
+       "       'zone_041_fan_spd', 'zone_040_temp', 'zone_040_fan_spd',\n",
+       "       'zone_039_temp', 'zone_039_fan_spd', 'zone_038_temp',\n",
+       "       'zone_038_fan_spd', 'zone_037_temp', 'zone_037_fan_spd',\n",
+       "       'zone_036_temp', 'zone_036_fan_spd', 'rtu_001_fltrd_sa_flow_tn',\n",
+       "       'rtu_001_sa_temp', 'air_temp_set_1', 'air_temp_set_2',\n",
+       "       'dew_point_temperature_set_1d', 'relative_humidity_set_1',\n",
+       "       'solar_radiation_set_1', 'zone_069_cooling_sp', 'zone_069_heating_sp',\n",
+       "       'zone_067_cooling_sp', 'zone_067_heating_sp', 'zone_066_cooling_sp',\n",
+       "       'zone_066_heating_sp', 'zone_065_cooling_sp', 'zone_065_heating_sp',\n",
+       "       'zone_064_cooling_sp', 'zone_064_heating_sp', 'zone_042_cooling_sp',\n",
+       "       'zone_042_heating_sp', 'zone_041_cooling_sp', 'zone_041_heating_sp',\n",
+       "       'zone_039_cooling_sp', 'zone_039_heating_sp', 'zone_038_cooling_sp',\n",
+       "       'zone_038_heating_sp', 'zone_037_cooling_sp', 'zone_037_heating_sp',\n",
+       "       'zone_036_cooling_sp', 'zone_036_heating_sp'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "traindataset_df.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
    "metadata": {},
    "outputs": [
     {
@@ -161,7 +194,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 38,
+   "execution_count": 16,
    "metadata": {},
    "outputs": [
     {
@@ -170,7 +203,7 @@
        "(1073512, 391818)"
       ]
      },
-     "execution_count": 38,
+     "execution_count": 16,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -181,7 +214,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 39,
+   "execution_count": 18,
    "metadata": {},
    "outputs": [
     {
@@ -190,7 +223,7 @@
        "['scaler_vav_1.pkl']"
       ]
      },
-     "execution_count": 39,
+     "execution_count": 18,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -208,7 +241,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 51,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -231,7 +264,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 52,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -251,7 +284,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 54,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -304,6 +337,18 @@
     "model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=3, batch_size=128, verbose=1, callbacks=[checkpoint_callback])"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import keras\n",
+    "checkpoint_path = \"lstm_vav_01.keras\"\n",
+    "\n",
+    "model = keras.models.load_model(checkpoint_path)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 55,

src/main.py

@@ -30,25 +30,39 @@ def main():
         num_outputs=rtu_data_pipeline.num_outputs,
     )
 
-    # vav_pipeline = VAVPipeline(rtu_id=1, scaler_path="src/vav/models/scaler_vav_1.pkl")
+    vav_pipeline = VAVPipeline(rtu_id=1, scaler_path="src/vav/models/scaler_vav_1.pkl")
 
-    # vav_anomalizer = VAVAnomalizer(prediction_model_path="src/vav/models/lstm__vav_01")
+    vav_anomalizer = VAVAnomalizer(
+        rtu_id=1,
+        prediction_model_path="src/vav/models/lstm_vav_01.keras",
+        clustering_model_path="src/vav/models/kmeans_vav_1.pkl",
+        num_inputs=vav_pipeline.num_inputs,
+        num_outputs=vav_pipeline.num_outputs,
+    )
     # print(vav_pipeline.input_col_names)
 
     # print(len(vav_pipeline.output_col_names))
 
     def on_message(client, userdata, message):
-        # print(json.loads(message.payload.decode()))
+        df_new_vav, df_trans_vav = vav_pipeline.fit(message)
+        vav_anomalizer.num_inputs = vav_pipeline.num_inputs
+        vav_anomalizer.num_outputs = vav_pipeline.num_outputs
+        if not df_new_vav is None and not df_trans_vav is None:
+            out_vav = vav_anomalizer.pipeline(
+                df_new_vav, df_trans_vav, vav_pipeline.scaler
+            )
+            
         df_new1, df_trans1, df_new2, df_trans2 = rtu_data_pipeline.fit(message)    
         if not df_new1 is None and not df_trans1 is None and not df_new2 is None and not df_trans2 is None:
             out1,out2,out3,out4 = rtu_anomalizer1.pipeline(df_new1, df_trans1, rtu_data_pipeline.scaler1)
             out5,out6,out7,out8 = rtu_anomalizer2.pipeline(df_new2, df_trans2, rtu_data_pipeline.scaler2)
-            print(out2)
+            #print(out2)
 
     broker_address = "localhost"
     broker_port = 1883
     topic = "sensor_data"
     client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION1)
+    print("Connecting to broker")
     client.on_message = on_message
     client.connect(broker_address, broker_port)
     client.subscribe(topic)

src/vav/VAVPipeline.py

@@ -1,15 +1,41 @@
 import json
+import joblib
+import pandas as pd
 from sklearn.preprocessing import StandardScaler
-from pickle import load
 import numpy as np
 
 
 class VAVPipeline:
+    """
+    A class representing a Variable Air Volume (VAV) pipeline.
+
+    Attributes:
+        rtu_id (int): The ID of the RTU (Roof Top Unit).
+        scaler_path (str): The path to the scaler file.
+        window_size (int): The size of the sliding window.
+
+    Methods:
+        get_scaler(scaler_path): Loads the scaler from the given path.
+        get_window(df): Returns the sliding window of the given dataframe.
+        transform_window(df_window): Transforms the values of the dataframe using the scaler.
+        prepare_input(df_trans): Prepares the input for the model.
+        get_input_output(df): Extracts the input and output column names from the dataframe.
+        extract_data_from_message(message): Extracts data from the message payload and returns a dataframe.
+        fit(message): Fits the model with the extracted data and returns the prepared input and transformed data.
+    """
 
     def __init__(self, rtu_id, scaler_path=None, window_size=30):
-
+        """
+        Initializes a VAVPipeline object.
+
+        Args:
+            rtu_id (int): The ID of the RTU (Roof Top Unit).
+            scaler_path (str, optional): The path to the scaler file. Defaults to None.
+            window_size (int, optional): The size of the sliding window. Defaults to 30.
+        """
+        self.get_cols = True
         self.window_size = window_size
-
+        self.rtu_id = rtu_id
         if rtu_id == 1:
             self.zones = [69, 68, 67, 66, 65, 64, 42, 41, 40, 39, 38, 37, 36]
         if rtu_id == 2:
@@ -36,8 +62,6 @@ class VAVPipeline:
                 28,
             ]
 
-        outputs = ["temp", "fan_speed"]
-        inputs = ["cooling_sp", "heating_sp"]
         self.output_col_names = []
         self.input_col_names = [
             f"rtu_00{rtu_id}_fltrd_sa_flow_tn",
@@ -48,21 +72,37 @@ class VAVPipeline:
             "relative_humidity_set_1",
             "solar_radiation_set_1",
         ]
-        for zone in self.zones:
-            for output in outputs:
-                self.output_col_names.append(f"zone_0{zone}_{output}")
-            for input in inputs:
-                self.input_col_names.append(f"zone_0{zone}_{input}")
 
         self.column_names = self.output_col_names + self.input_col_names
 
+        self.num_inputs = len(self.input_col_names)
+        self.num_outputs = len(self.output_col_names)
+
         if scaler_path:
             self.scaler = self.get_scaler(scaler_path)
 
     def get_scaler(self, scaler_path):
-        return load(scaler_path)
+        """
+        Loads the scaler from the given path.
+
+        Args:
+            scaler_path (str): The path to the scaler file.
+
+        Returns:
+            StandardScaler: The loaded scaler object.
+        """
+        return joblib.load(scaler_path)
 
     def get_window(self, df):
+        """
+        Returns the sliding window of the given dataframe.
+
+        Args:
+            df (pd.DataFrame): The dataframe.
+
+        Returns:
+            pd.DataFrame: The sliding window dataframe.
+        """
         len_df = len(df)
         if len_df > self.window_size:
             return df[len_df - (self.window_size + 1) : len_df].astype("float32")
@@ -70,26 +110,99 @@ class VAVPipeline:
             return None
 
     def transform_window(self, df_window):
-        return self.scaler.transform(df_window)
+        """
+        Transforms the values of the dataframe using the scaler.
+
+        Args:
+            df_window (pd.DataFrame): The dataframe.
+
+        Returns:
+            np.ndarray: The transformed values.
+        """
+        return self.scaler.transform(df_window.values)
 
     def prepare_input(self, df_trans):
+        """
+        Prepares the input for the model.
+
+        Args:
+            df_trans (np.ndarray): The transformed values.
+
+        Returns:
+            np.ndarray: The prepared input.
+        """
         return df_trans[: self.window_size, :].reshape(
             (1, self.window_size, len(self.column_names))
         )
 
+    def get_input_output(self, df: pd.DataFrame):
+        """
+        Extracts the input and output column names from the dataframe.
+
+        Args:
+            df (pd.DataFrame): The dataframe.
+        """
+        for zone in self.zones:
+            for column in df.columns:
+                if (
+                    f"zone_0{zone}" in column
+                    and "co2" not in column
+                    and "hw_valve" not in column
+                    and "cooling_sp" not in column
+                    and "heating_sp" not in column
+                ):
+                    self.output_col_names.append(column)
+        self.input_col_names = [
+            f"rtu_00{self.rtu_id}_fltrd_sa_flow_tn",
+            f"rtu_00{self.rtu_id}_sa_temp",
+            "air_temp_set_1",
+            "air_temp_set_2",
+            "dew_point_temperature_set_1d",
+            "relative_humidity_set_1",
+            "solar_radiation_set_1",
+        ]
+        for zone in self.zones:
+            for column in df.columns:
+                if f"zone_0{zone}" in column:
+                    if "cooling_sp" in column or "heating_sp" in column:
+                        self.input_col_names.append(column)
+        self.column_names = self.output_col_names + self.input_col_names
+        self.num_inputs = len(self.input_col_names)
+        self.num_outputs = len(self.output_col_names)
+        self.df = pd.DataFrame(columns=self.column_names)
+
     def extract_data_from_message(self, message):
-        payload = json.loads(message.payload.decode())
+        """
+        Extracts data from the message payload and returns a dataframe.
 
-        len_df = len(self.df)
+        Args:
+            message: The message containing the payload.
+
+        Returns:
+            pd.DataFrame: The extracted data as a dataframe.
+        """
+        payload = json.loads(message.payload.decode())
+        df = pd.DataFrame.from_dict(payload, orient="index").T
+        if self.get_cols == True:
+            self.get_input_output(df)
+            self.get_cols = False
+        df = df[self.column_names]
+        self.df.loc[len(self.df)] = df.values[0]
 
-        k = {}
-        for col in self.column_names:
-            k[col] = payload[col]
-        self.df.loc[len_df] = k
         return self.df
 
     def fit(self, message):
+        """
+        Fits the model with the extracted data and returns the prepared input and transformed data.
+
+        Args:
+            message: The message containing the data.
+
+        Returns:
+            tuple: A tuple containing the prepared input and transformed data.
+        """
         df = self.extract_data_from_message(message)
+
         df_window = self.get_window(df)
         if df_window is not None:
             df_trans = self.transform_window(df_window)