{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0bb77379",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-17T05:48:17.537446Z",
     "iopub.status.busy": "2023-04-17T05:48:17.536424Z",
     "iopub.status.idle": "2023-04-17T05:48:31.651358Z",
     "shell.execute_reply": "2023-04-17T05:48:31.649651Z"
    },
    "papermill": {
     "duration": 14.123673,
     "end_time": "2023-04-17T05:48:31.654609",
     "exception": false,
     "start_time": "2023-04-17T05:48:17.530936",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting faiss-cpu\r\n",
      "  Downloading faiss_cpu-1.7.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (17.0 MB)\r\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m17.0/17.0 MB\u001b[0m \u001b[31m42.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n",
      "\u001b[?25hInstalling collected packages: faiss-cpu\r\n",
      "Successfully installed faiss-cpu-1.7.3\r\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install faiss-cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3c9019a6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-17T05:48:31.663539Z",
     "iopub.status.busy": "2023-04-17T05:48:31.662590Z",
     "iopub.status.idle": "2023-04-17T05:48:32.984952Z",
     "shell.execute_reply": "2023-04-17T05:48:32.983413Z"
    },
    "papermill": {
     "duration": 1.330221,
     "end_time": "2023-04-17T05:48:32.988111",
     "exception": false,
     "start_time": "2023-04-17T05:48:31.657890",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import datasets\n",
    "from datasets import Dataset\n",
    "import numpy as np\n",
    "import json\n",
    "import os\n",
    "from tqdm.notebook import tqdm as tqdm\n",
    "import faiss\n",
    "\n",
    "ARTICLES_PATH = '/kaggle/input/ysda-ml-02-05-process-json/articles.hf'\n",
    "EMBEDDINGS_PATH = '/kaggle/input/ysda-ml-02-05-generate-embeddings/embeddings.npy'\n",
    "OUTPUT_PATH = '/kaggle/working/articles.index'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0271d205",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-17T05:48:32.997788Z",
     "iopub.status.busy": "2023-04-17T05:48:32.996997Z",
     "iopub.status.idle": "2023-04-17T05:48:47.446365Z",
     "shell.execute_reply": "2023-04-17T05:48:47.445122Z"
    },
    "papermill": {
     "duration": 14.457709,
     "end_time": "2023-04-17T05:48:47.449394",
     "exception": false,
     "start_time": "2023-04-17T05:48:32.991685",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "articles = Dataset.load_from_disk(ARTICLES_PATH)\n",
    "embeddings = np.load(EMBEDDINGS_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7467da22",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-17T05:48:47.458217Z",
     "iopub.status.busy": "2023-04-17T05:48:47.457678Z",
     "iopub.status.idle": "2023-04-17T05:49:05.400026Z",
     "shell.execute_reply": "2023-04-17T05:49:05.398641Z"
    },
    "papermill": {
     "duration": 17.95023,
     "end_time": "2023-04-17T05:49:05.402995",
     "exception": false,
     "start_time": "2023-04-17T05:48:47.452765",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df6dfb9da874419d9783f7f156cd9963",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2239 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "articles.add_faiss_index_from_external_arrays(\n",
    "    embeddings, \n",
    "    'embedding', \n",
    "    metric_type=faiss.METRIC_INNER_PRODUCT\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "61bb1bd4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-04-17T05:49:05.412042Z",
     "iopub.status.busy": "2023-04-17T05:49:05.411098Z",
     "iopub.status.idle": "2023-04-17T05:49:08.841963Z",
     "shell.execute_reply": "2023-04-17T05:49:08.840974Z"
    },
    "papermill": {
     "duration": 3.438676,
     "end_time": "2023-04-17T05:49:08.845080",
     "exception": false,
     "start_time": "2023-04-17T05:49:05.406404",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "articles.save_faiss_index('embedding', OUTPUT_PATH)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 63.431822,
   "end_time": "2023-04-17T05:49:09.990990",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-04-17T05:48:06.559168",
   "version": "2.4.0"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0814242ff41d45b295efc66b88f34057": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a317d1fc94a34dc6a569ef9655c2c5e6",
       "placeholder": "​",
       "style": "IPY_MODEL_0d730835a72d403f82a5c216ad4d13b4",
       "value": " 2239/2239 [00:13&lt;00:00, 71.44it/s]"
      }
     },
     "0d730835a72d403f82a5c216ad4d13b4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "16076bcc1f6644889fc4f2e3fce8bd88": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_21235b1f64e1452fa06a37afc94e37df",
       "placeholder": "​",
       "style": "IPY_MODEL_168f36f0d10a46d1b7e635237f18d601",
       "value": "100%"
      }
     },
     "168f36f0d10a46d1b7e635237f18d601": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "1a5ebfbcfaa34421aa36d8139763c641": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "21235b1f64e1452fa06a37afc94e37df": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6c34fbcfee584e5a840c617c8be1dd62": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a0f7c5c2d54945458130ff18c60a4f3b",
       "max": 2239.0,
       "min": 0.0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_8cf935e4c2f24f77a45173536fa142a7",
       "value": 2239.0
      }
     },
     "8cf935e4c2f24f77a45173536fa142a7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "a0f7c5c2d54945458130ff18c60a4f3b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a317d1fc94a34dc6a569ef9655c2c5e6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "df6dfb9da874419d9783f7f156cd9963": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_16076bcc1f6644889fc4f2e3fce8bd88",
        "IPY_MODEL_6c34fbcfee584e5a840c617c8be1dd62",
        "IPY_MODEL_0814242ff41d45b295efc66b88f34057"
       ],
       "layout": "IPY_MODEL_1a5ebfbcfaa34421aa36d8139763c641"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}