Add dataset in preparation for the data module

models/notebooks/1.0-hfk-datamodules-exploration.ipynb

@@ -0,0 +1,394 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Exploring Code for Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "metadata = pd.read_csv(\"../data/focus/metadata.csv\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>Unnamed: 0</th>\n",
+       "      <th>image_path</th>\n",
+       "      <th>original_filename</th>\n",
+       "      <th>study_id</th>\n",
+       "      <th>scan_uuid</th>\n",
+       "      <th>focus_value</th>\n",
+       "      <th>stack_id</th>\n",
+       "      <th>obj_name</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>0</td>\n",
+       "      <td>31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01631...</td>\n",
+       "      <td>I01631_X013_Y012_Z5107.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>fba56d57-656e-4b6f-ba63-e4ba3ad083f5</td>\n",
+       "      <td>-2.82953</td>\n",
+       "      <td>1658220</td>\n",
+       "      <td>133</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1</td>\n",
+       "      <td>31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01632...</td>\n",
+       "      <td>I01632_X013_Y012_Z5175.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>fba56d57-656e-4b6f-ba63-e4ba3ad083f5</td>\n",
+       "      <td>-2.70408</td>\n",
+       "      <td>1658220</td>\n",
+       "      <td>133</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>2</td>\n",
+       "      <td>31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01633...</td>\n",
+       "      <td>I01633_X013_Y012_Z5722.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>fba56d57-656e-4b6f-ba63-e4ba3ad083f5</td>\n",
+       "      <td>-2.69918</td>\n",
+       "      <td>1658220</td>\n",
+       "      <td>133</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3</td>\n",
+       "      <td>31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01634...</td>\n",
+       "      <td>I01634_X013_Y012_Z5244.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>fba56d57-656e-4b6f-ba63-e4ba3ad083f5</td>\n",
+       "      <td>-2.50266</td>\n",
+       "      <td>1658220</td>\n",
+       "      <td>133</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>4</td>\n",
+       "      <td>31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01635...</td>\n",
+       "      <td>I01635_X013_Y012_Z5654.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>fba56d57-656e-4b6f-ba63-e4ba3ad083f5</td>\n",
+       "      <td>-2.36450</td>\n",
+       "      <td>1658220</td>\n",
+       "      <td>133</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>565</th>\n",
+       "      <td>565</td>\n",
+       "      <td>31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01406...</td>\n",
+       "      <td>I01406_X016_Y009_Z5361.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>4c7e9e66-61a1-47ca-aa4e-340b0eef8db1</td>\n",
+       "      <td>-3.41147</td>\n",
+       "      <td>1674918</td>\n",
+       "      <td>217</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>566</th>\n",
+       "      <td>566</td>\n",
+       "      <td>31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01407...</td>\n",
+       "      <td>I01407_X016_Y009_Z5087.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>4c7e9e66-61a1-47ca-aa4e-340b0eef8db1</td>\n",
+       "      <td>-3.05424</td>\n",
+       "      <td>1674918</td>\n",
+       "      <td>217</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>567</th>\n",
+       "      <td>567</td>\n",
+       "      <td>31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01408...</td>\n",
+       "      <td>I01408_X016_Y009_Z5292.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>4c7e9e66-61a1-47ca-aa4e-340b0eef8db1</td>\n",
+       "      <td>-1.48608</td>\n",
+       "      <td>1674918</td>\n",
+       "      <td>217</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>568</th>\n",
+       "      <td>568</td>\n",
+       "      <td>31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01409...</td>\n",
+       "      <td>I01409_X016_Y009_Z5156.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>4c7e9e66-61a1-47ca-aa4e-340b0eef8db1</td>\n",
+       "      <td>-0.52804</td>\n",
+       "      <td>1674918</td>\n",
+       "      <td>217</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>569</th>\n",
+       "      <td>569</td>\n",
+       "      <td>31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01410...</td>\n",
+       "      <td>I01410_X016_Y009_Z5224.jpg</td>\n",
+       "      <td>31</td>\n",
+       "      <td>4c7e9e66-61a1-47ca-aa4e-340b0eef8db1</td>\n",
+       "      <td>0.00000</td>\n",
+       "      <td>1674918</td>\n",
+       "      <td>217</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>570 rows × 8 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "     Unnamed: 0                                         image_path  \\\n",
+       "0             0  31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01631...   \n",
+       "1             1  31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01632...   \n",
+       "2             2  31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01633...   \n",
+       "3             3  31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01634...   \n",
+       "4             4  31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01635...   \n",
+       "..          ...                                                ...   \n",
+       "565         565  31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01406...   \n",
+       "566         566  31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01407...   \n",
+       "567         567  31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01408...   \n",
+       "568         568  31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01409...   \n",
+       "569         569  31/4c7e9e66-61a1-47ca-aa4e-340b0eef8db1/I01410...   \n",
+       "\n",
+       "              original_filename  study_id  \\\n",
+       "0    I01631_X013_Y012_Z5107.jpg        31   \n",
+       "1    I01632_X013_Y012_Z5175.jpg        31   \n",
+       "2    I01633_X013_Y012_Z5722.jpg        31   \n",
+       "3    I01634_X013_Y012_Z5244.jpg        31   \n",
+       "4    I01635_X013_Y012_Z5654.jpg        31   \n",
+       "..                          ...       ...   \n",
+       "565  I01406_X016_Y009_Z5361.jpg        31   \n",
+       "566  I01407_X016_Y009_Z5087.jpg        31   \n",
+       "567  I01408_X016_Y009_Z5292.jpg        31   \n",
+       "568  I01409_X016_Y009_Z5156.jpg        31   \n",
+       "569  I01410_X016_Y009_Z5224.jpg        31   \n",
+       "\n",
+       "                                scan_uuid  focus_value  stack_id  obj_name  \n",
+       "0    fba56d57-656e-4b6f-ba63-e4ba3ad083f5     -2.82953   1658220       133  \n",
+       "1    fba56d57-656e-4b6f-ba63-e4ba3ad083f5     -2.70408   1658220       133  \n",
+       "2    fba56d57-656e-4b6f-ba63-e4ba3ad083f5     -2.69918   1658220       133  \n",
+       "3    fba56d57-656e-4b6f-ba63-e4ba3ad083f5     -2.50266   1658220       133  \n",
+       "4    fba56d57-656e-4b6f-ba63-e4ba3ad083f5     -2.36450   1658220       133  \n",
+       "..                                    ...          ...       ...       ...  \n",
+       "565  4c7e9e66-61a1-47ca-aa4e-340b0eef8db1     -3.41147   1674918       217  \n",
+       "566  4c7e9e66-61a1-47ca-aa4e-340b0eef8db1     -3.05424   1674918       217  \n",
+       "567  4c7e9e66-61a1-47ca-aa4e-340b0eef8db1     -1.48608   1674918       217  \n",
+       "568  4c7e9e66-61a1-47ca-aa4e-340b0eef8db1     -0.52804   1674918       217  \n",
+       "569  4c7e9e66-61a1-47ca-aa4e-340b0eef8db1      0.00000   1674918       217  \n",
+       "\n",
+       "[570 rows x 8 columns]"
+      ]
+     },
+     "execution_count": 3,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "metadata"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'31/fba56d57-656e-4b6f-ba63-e4ba3ad083f5/I01631_X013_Y012_Z5107_600_375.jpg'"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "idx = 0\n",
+    "# File Path\n",
+    "metadata.iloc[idx, 1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "-2.82953"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# Focus Value\n",
+    "metadata.iloc[idx, 5]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Testing FocusDataSet"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'image': array([[[181, 190, 171],\n",
+       "         [180, 189, 170],\n",
+       "         [180, 186, 172],\n",
+       "         ...,\n",
+       "         [172, 176, 177],\n",
+       "         [171, 176, 179],\n",
+       "         [170, 178, 180]],\n",
+       " \n",
+       "        [[181, 190, 173],\n",
+       "         [181, 190, 173],\n",
+       "         [180, 188, 175],\n",
+       "         ...,\n",
+       "         [169, 173, 174],\n",
+       "         [169, 175, 175],\n",
+       "         [170, 176, 176]],\n",
+       " \n",
+       "        [[179, 190, 176],\n",
+       "         [179, 190, 176],\n",
+       "         [179, 189, 180],\n",
+       "         ...,\n",
+       "         [169, 169, 167],\n",
+       "         [169, 171, 170],\n",
+       "         [169, 171, 170]],\n",
+       " \n",
+       "        ...,\n",
+       " \n",
+       "        [[195, 201, 197],\n",
+       "         [195, 201, 197],\n",
+       "         [195, 201, 197],\n",
+       "         ...,\n",
+       "         [198, 195, 188],\n",
+       "         [199, 198, 196],\n",
+       "         [202, 200, 205]],\n",
+       " \n",
+       "        [[195, 201, 197],\n",
+       "         [195, 201, 197],\n",
+       "         [195, 201, 197],\n",
+       "         ...,\n",
+       "         [198, 195, 188],\n",
+       "         [199, 198, 196],\n",
+       "         [202, 200, 205]],\n",
+       " \n",
+       "        [[195, 201, 197],\n",
+       "         [195, 201, 197],\n",
+       "         [195, 201, 197],\n",
+       "         ...,\n",
+       "         [198, 195, 188],\n",
+       "         [199, 198, 196],\n",
+       "         [202, 200, 203]]], dtype=uint8),\n",
+       " 'focus_value': -2.70408}"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "from importlib.machinery import SourceFileLoader\n",
+    "\n",
+    "focus_datamodule = SourceFileLoader(\"focus_datamodule\", \"../src/datamodules/focus_datamodule.py\").load_module()\n",
+    "from focus_datamodule import FocusDataSet\n",
+    "\n",
+    "ds = FocusDataSet(\"../data/focus/metadata.csv\", \"../data/focus/\")\n",
+    "ds[1]"
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac"
+  },
+  "kernelspec": {
+   "display_name": "Python 3.9.7 64-bit",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.10"
+  },
+  "orig_nbformat": 4
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

models/requirements.txt

@@ -4,6 +4,9 @@ torchvision>=0.11.0
 pytorch-lightning>=1.5.10
 torchmetrics>=0.7.0
 
+# --------- data and model dependencies --------- #
+scikit-image
+
 # --------- hydra --------- #
 hydra-core>=1.1.0
 hydra-colorlog>=1.1.0

models/src/datamodules/focus_datamodule.py

@@ -0,0 +1,60 @@
+import os
+from typing import Any, Optional, Tuple, Union
+from typing_extensions import Self
+import numpy as np
+import pandas as pd
+from skimage import io
+
+import torch
+from pytorch_lightning import LightningDataModule
+from torch.utils.data import ConcatDataset, DataLoader, Dataset, random_split
+from torchvision.datasets import MNIST
+from torchvision.transforms import transforms
+
+
+class FocusDataSet(Dataset):
+    """Dataset for z-stacked images of neglected tropical diseaeses."""
+
+    def __init__(self, csv_file, root_dir, transform=None):
+        """Initialize focus satck dataset.
+
+        Args:
+            csv_file (string): Path to the csv file with annotations.
+            root_dir (string): Directory with all the images.
+            transform (callable, optional): Optional transform to be applied
+                on a sample.
+        """
+        self.metadata = pd.read_csv(csv_file)
+        self.root_dir = root_dir
+        self.transform = transform
+
+    def __len__(self) -> int:
+        """Get the length of the dataset.
+
+        Returns:
+            int: the length
+        """
+        return len(self.metadata)
+
+    def __getitem__(self, idx):
+        """Get one items from the dataset.
+
+        Args:
+            idx (int) The index of the sample that is to be retrieved
+
+        Returns:
+            Item/Items which is a dictionary containing "image" and "focus_value"
+        """
+        if torch.is_tensor(idx):
+            idx = idx.tolist()
+
+        img_name = os.path.join(self.root_dir, self.metadata.iloc[idx, 1])
+        image = io.imread(img_name)
+        focus_value = self.metadata.iloc[idx, 5]
+        sample = {"image": image, "focus_value": focus_value}
+
+        if self.transform:
+            sample = self.transform(sample)
+
+        return sample
+