{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loaded variable 'df' from URI: /home/james/CodingProjects/ArcPostDoc/HeDAP-imagesearch/Durham_University_Museums_data (1).xlsx\n",
    "df = pd.read_excel(\n",
    "    r\"/home/james/CodingProjects/ArcPostDoc/HeDAP-imagesearch/Durham_University_Museums_data (1).xlsx\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>object_number</th>\n",
       "      <th>object_name</th>\n",
       "      <th>other_name</th>\n",
       "      <th>reproduction.reference</th>\n",
       "      <th>description</th>\n",
       "      <th>label.text</th>\n",
       "      <th>material</th>\n",
       "      <th>technique</th>\n",
       "      <th>physical_description</th>\n",
       "      <th>number_of_parts</th>\n",
       "      <th>...</th>\n",
       "      <th>Unnamed: 25</th>\n",
       "      <th>Unnamed: 26</th>\n",
       "      <th>Unnamed: 27</th>\n",
       "      <th>Unnamed: 28</th>\n",
       "      <th>Unnamed: 29</th>\n",
       "      <th>Unnamed: 30</th>\n",
       "      <th>Unnamed: 31</th>\n",
       "      <th>Unnamed: 32</th>\n",
       "      <th>Unnamed: 33</th>\n",
       "      <th>Unnamed: 34</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>EG1</td>\n",
       "      <td>Kohl Pot</td>\n",
       "      <td>jar</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Shouldered, squat, incised kohl jar, blackened...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>limestone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EG2</td>\n",
       "      <td>cups</td>\n",
       "      <td>beaker</td>\n",
       "      <td>NaN</td>\n",
       "      <td>slightly concave beaker with flaring rim and c...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>travertine</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>EG3</td>\n",
       "      <td>bowls</td>\n",
       "      <td>bowl</td>\n",
       "      <td>../images/egyptian/eg/eg3-409-d1.jpg</td>\n",
       "      <td>squat shouldered jar, no rim</td>\n",
       "      <td>&lt;SPAN lang=en-GB style='FONT-SIZE: 12pt; FONT-...</td>\n",
       "      <td>limestone</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>EG4</td>\n",
       "      <td>bottles</td>\n",
       "      <td>jar</td>\n",
       "      <td>NaN</td>\n",
       "      <td>necked, globular jar with flared rim</td>\n",
       "      <td>NaN</td>\n",
       "      <td>travertine</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>EG5</td>\n",
       "      <td>bottles</td>\n",
       "      <td>jar</td>\n",
       "      <td>NaN</td>\n",
       "      <td>necked, globular jar with narrow rim, plus sto...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>travertine</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60081</th>\n",
       "      <td>DURMA.2020.3.2072</td>\n",
       "      <td>coins</td>\n",
       "      <td>NaN</td>\n",
       "      <td>../images/fulling_mill/2020/DURMA.2020.3.2072-...</td>\n",
       "      <td>A silver Roman coin which is a part of the Pie...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>metal</td>\n",
       "      <td>hammering</td>\n",
       "      <td>A silver denarius of Elagabalus dating to the ...</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60082</th>\n",
       "      <td>DUROM.2021.286</td>\n",
       "      <td>postcards</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Portrait orientation postcard for the 1996 Ind...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>paper</td>\n",
       "      <td>printing</td>\n",
       "      <td>Digital printed onto paper/card</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60083</th>\n",
       "      <td>DUROM.2021.287</td>\n",
       "      <td>postcards</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Pair of landscape orientation postcard sized p...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>paper</td>\n",
       "      <td>printing</td>\n",
       "      <td>Digital print on card</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60084</th>\n",
       "      <td>DUROM.2021.289</td>\n",
       "      <td>posters</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Portrait orientation poster for the 1996 India...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>paper</td>\n",
       "      <td>printing</td>\n",
       "      <td>digital print on gloss paper</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60085</th>\n",
       "      <td>DUROM.2021.288</td>\n",
       "      <td>posters</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Portrait orientation poster for the 1996 India...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>paper</td>\n",
       "      <td>printing</td>\n",
       "      <td>Digital print on paper</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>60086 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           object_number object_name other_name  \\\n",
       "0                    EG1    Kohl Pot        jar   \n",
       "1                    EG2        cups     beaker   \n",
       "2                    EG3       bowls       bowl   \n",
       "3                    EG4     bottles        jar   \n",
       "4                    EG5     bottles        jar   \n",
       "...                  ...         ...        ...   \n",
       "60081  DURMA.2020.3.2072       coins        NaN   \n",
       "60082     DUROM.2021.286   postcards        NaN   \n",
       "60083     DUROM.2021.287   postcards        NaN   \n",
       "60084     DUROM.2021.289     posters        NaN   \n",
       "60085     DUROM.2021.288     posters        NaN   \n",
       "\n",
       "                                  reproduction.reference  \\\n",
       "0                                                    NaN   \n",
       "1                                                    NaN   \n",
       "2                   ../images/egyptian/eg/eg3-409-d1.jpg   \n",
       "3                                                    NaN   \n",
       "4                                                    NaN   \n",
       "...                                                  ...   \n",
       "60081  ../images/fulling_mill/2020/DURMA.2020.3.2072-...   \n",
       "60082                                                NaN   \n",
       "60083                                                NaN   \n",
       "60084                                                NaN   \n",
       "60085                                                NaN   \n",
       "\n",
       "                                             description  \\\n",
       "0      Shouldered, squat, incised kohl jar, blackened...   \n",
       "1      slightly concave beaker with flaring rim and c...   \n",
       "2                           squat shouldered jar, no rim   \n",
       "3                   necked, globular jar with flared rim   \n",
       "4      necked, globular jar with narrow rim, plus sto...   \n",
       "...                                                  ...   \n",
       "60081  A silver Roman coin which is a part of the Pie...   \n",
       "60082  Portrait orientation postcard for the 1996 Ind...   \n",
       "60083  Pair of landscape orientation postcard sized p...   \n",
       "60084  Portrait orientation poster for the 1996 India...   \n",
       "60085  Portrait orientation poster for the 1996 India...   \n",
       "\n",
       "                                              label.text    material  \\\n",
       "0                                                    NaN   limestone   \n",
       "1                                                    NaN  travertine   \n",
       "2      <SPAN lang=en-GB style='FONT-SIZE: 12pt; FONT-...   limestone   \n",
       "3                                                    NaN  travertine   \n",
       "4                                                    NaN  travertine   \n",
       "...                                                  ...         ...   \n",
       "60081                                                NaN       metal   \n",
       "60082                                                NaN       paper   \n",
       "60083                                                NaN       paper   \n",
       "60084                                                NaN       paper   \n",
       "60085                                                NaN       paper   \n",
       "\n",
       "       technique                               physical_description  \\\n",
       "0            NaN                                                NaN   \n",
       "1            NaN                                                NaN   \n",
       "2            NaN                                                NaN   \n",
       "3            NaN                                                NaN   \n",
       "4            NaN                                                NaN   \n",
       "...          ...                                                ...   \n",
       "60081  hammering  A silver denarius of Elagabalus dating to the ...   \n",
       "60082   printing                    Digital printed onto paper/card   \n",
       "60083   printing                              Digital print on card   \n",
       "60084   printing                       digital print on gloss paper   \n",
       "60085   printing                             Digital print on paper   \n",
       "\n",
       "      number_of_parts  ... Unnamed: 25 Unnamed: 26 Unnamed: 27 Unnamed: 28  \\\n",
       "0                 NaN  ...         NaN         NaN         NaN         NaN   \n",
       "1                 NaN  ...         NaN         NaN         NaN         NaN   \n",
       "2                 NaN  ...         NaN         NaN         NaN         NaN   \n",
       "3                 NaN  ...         NaN         NaN         NaN         NaN   \n",
       "4                 NaN  ...         NaN         NaN         NaN         NaN   \n",
       "...               ...  ...         ...         ...         ...         ...   \n",
       "60081               1  ...         NaN         NaN         NaN         NaN   \n",
       "60082               1  ...         NaN         NaN         NaN         NaN   \n",
       "60083               2  ...         NaN         NaN         NaN         NaN   \n",
       "60084               1  ...         NaN         NaN         NaN         NaN   \n",
       "60085               1  ...         NaN         NaN         NaN         NaN   \n",
       "\n",
       "      Unnamed: 29 Unnamed: 30 Unnamed: 31 Unnamed: 32 Unnamed: 33 Unnamed: 34  \n",
       "0             NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "1             NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "2             NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "3             NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "4             NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "...           ...         ...         ...         ...         ...         ...  \n",
       "60081         NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "60082         NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "60083         NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "60084         NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "60085         NaN         NaN         NaN         NaN         NaN         NaN  \n",
       "\n",
       "[60086 rows x 35 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6625"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"description\"].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 60086 records in the dataset\n",
      "26809 records have an image location\n",
      "53461 records have an description\n",
      "There are 60067 unique museum numbers\n",
      "There are 46166 unique descriptions\n"
     ]
    }
   ],
   "source": [
    "print(f\"There are {len(df)} records in the dataset\")\n",
    "print(f\"{df['reproduction.reference'].notna().sum()} records have an image location\")\n",
    "print(f\"{df['description'].notna().sum()} records have an description\")\n",
    "print(f\"There are {len(df['object_number'].unique())} unique museum numbers\")\n",
    "print(f\"There are {len(df['description'].unique())} unique descriptions\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of images in ../data/raw/images/: 39200\n",
      "Total number of files in ../data/raw/images/: 39244\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "image_count = 0\n",
    "file_count = 0\n",
    "\n",
    "# Define the root directory\n",
    "root_dir = \"../data/raw/images/\"\n",
    "\n",
    "# Iterate through all subdirectories and files\n",
    "for root, dirs, files in os.walk(root_dir):\n",
    "    for file in files:\n",
    "        file_count += 1\n",
    "        # Check if the file is an image file\n",
    "        if file.endswith((\".jpg\", \".jpeg\", \".png\", \".gif\", \".JPG\", \".JPEG\", \".PNG\", \".GIF\")):\n",
    "            # Increment the image count\n",
    "            image_count += 1\n",
    "\n",
    "print(f\"Total number of images in {root_dir}: {image_count}\")\n",
    "print(f\"Total number of files in {root_dir}: {file_count}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}