{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4" }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "source": [ "# osiria/bert-italian-cased-question-answering\n", "## test set metrics" ], "metadata": { "id": "oub2ir8ZDkrH" } }, { "cell_type": "code", "execution_count": 1, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "02K9xO6Gvto1", "outputId": "6adbc718-df6a-437d-eb07-d67404f9cd2b" }, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Collecting git+https://github.com/huggingface/transformers\n", " Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-m860w1jf\n", " Running command git clone --filter=blob:none --quiet https://github.com/huggingface/transformers /tmp/pip-req-build-m860w1jf\n", " Resolved https://github.com/huggingface/transformers to commit df5c5c62ae253055336f5bb0828ca8e3e15ab6bd\n", " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (3.13.1)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.19.3 in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (0.19.4)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (1.23.5)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (23.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (6.0.1)\n", "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (2023.6.3)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (2.31.0)\n", "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (0.15.0)\n", "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (0.4.1)\n", "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers==4.36.0.dev0) (4.66.1)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.0.dev0) (2023.6.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.19.3->transformers==4.36.0.dev0) (4.5.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.36.0.dev0) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.36.0.dev0) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.36.0.dev0) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers==4.36.0.dev0) (2023.11.17)\n", "Building wheels for collected packages: transformers\n", " Building wheel for transformers (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for transformers: filename=transformers-4.36.0.dev0-py3-none-any.whl size=8195433 sha256=c5e14f23bb3ae5edc677f78f448f1a67a961992c880e0441c99cb6f4db9469cf\n", " Stored in directory: /tmp/pip-ephem-wheel-cache-ytzg25y9/wheels/c0/14/d6/6c9a5582d2ac191ec0a483be151a4495fe1eb2a6706ca49f1b\n", "Successfully built transformers\n", "Installing collected packages: transformers\n", " Attempting uninstall: transformers\n", " Found existing installation: transformers 4.35.2\n", " Uninstalling transformers-4.35.2:\n", " Successfully uninstalled transformers-4.35.2\n", "Successfully installed transformers-4.36.0.dev0\n" ] } ], "source": [ "!pip install git+https://github.com/huggingface/transformers" ] }, { "cell_type": "code", "source": [ "!pip install datasets" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "VKgybrUswa1q", "outputId": "5c5b315c-6a81-4ce8-e1a8-43f78c084fdc" }, "execution_count": 2, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Collecting datasets\n", " Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n", "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n", "Collecting pyarrow-hotfix (from datasets)\n", " Downloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n", "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n", " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n", "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n", "Collecting multiprocess (from datasets)\n", " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m7.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.1)\n", "Requirement already satisfied: huggingface-hub>=0.18.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.19.4)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.3)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.18.0->datasets) (3.13.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.18.0->datasets) (4.5.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.11.17)\n", "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n", "Installing collected packages: pyarrow-hotfix, dill, multiprocess, datasets\n", "Successfully installed datasets-2.15.0 dill-0.3.7 multiprocess-0.70.15 pyarrow-hotfix-0.6\n" ] } ] }, { "cell_type": "code", "source": [ "!pip install accelerate" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "xYEW0Hjqz2lo", "outputId": "b37614da-516f-42ce-da0b-16946bd26424" }, "execution_count": 3, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Collecting accelerate\n", " Downloading accelerate-0.25.0-py3-none-any.whl (265 kB)\n", "\u001b[?25l \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/265.7 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━\u001b[0m \u001b[32m174.1/265.7 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m\r\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m265.7/265.7 kB\u001b[0m \u001b[31m6.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.23.5)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n", "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.1.0+cu118)\n", "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.19.4)\n", "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.4.1)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.13.1)\n", "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.5.0)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.2.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2023.6.0)\n", "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.1.0)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub->accelerate) (4.66.1)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub->accelerate) (2023.11.17)\n", "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n", "Installing collected packages: accelerate\n", "Successfully installed accelerate-0.25.0\n" ] } ] }, { "cell_type": "code", "source": [ "!pip install evaluate" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "9OFz3JdR0cQ_", "outputId": "7d3e65fc-462d-497c-c48e-e01af07be572" }, "execution_count": 4, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Collecting evaluate\n", " Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: datasets>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.15.0)\n", "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.23.5)\n", "Requirement already satisfied: dill in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.3.7)\n", "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.5.3)\n", "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.31.0)\n", "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from evaluate) (4.66.1)\n", "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from evaluate) (3.4.1)\n", "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.70.15)\n", "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2023.6.0)\n", "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.19.4)\n", "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from evaluate) (23.2)\n", "Collecting responses<0.19 (from evaluate)\n", " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n", "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (9.0.0)\n", "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (0.6)\n", "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (3.9.1)\n", "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (6.0.1)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (3.13.1)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (4.5.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (3.6)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (2023.11.17)\n", "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2.8.2)\n", "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2023.3.post1)\n", "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\n", "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n", "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.3)\n", "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.0)\n", "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n", "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->evaluate) (1.16.0)\n", "Installing collected packages: responses, evaluate\n", "Successfully installed evaluate-0.4.1 responses-0.18.0\n" ] } ] }, { "cell_type": "code", "source": [ "!wget https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/run_qa.py\n", "!wget https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/trainer_qa.py\n", "!wget https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/utils_qa.py" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "JJhpBNwpxOTK", "outputId": "98ad66fd-6030-4829-dada-cf3b3d443259" }, "execution_count": 5, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "--2023-12-09 12:05:19-- https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/run_qa.py\n", "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.110.133, 185.199.109.133, 185.199.111.133, ...\n", "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.110.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 32883 (32K) [text/plain]\n", "Saving to: ‘run_qa.py’\n", "\n", "\rrun_qa.py 0%[ ] 0 --.-KB/s \rrun_qa.py 100%[===================>] 32.11K --.-KB/s in 0.01s \n", "\n", "2023-12-09 12:05:20 (3.22 MB/s) - ‘run_qa.py’ saved [32883/32883]\n", "\n", "--2023-12-09 12:05:20-- https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/trainer_qa.py\n", "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.111.133, 185.199.110.133, ...\n", "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 6253 (6.1K) [text/plain]\n", "Saving to: ‘trainer_qa.py’\n", "\n", "trainer_qa.py 100%[===================>] 6.11K --.-KB/s in 0s \n", "\n", "2023-12-09 12:05:20 (85.6 MB/s) - ‘trainer_qa.py’ saved [6253/6253]\n", "\n", "--2023-12-09 12:05:20-- https://raw.githubusercontent.com/huggingface/transformers/main/examples/pytorch/question-answering/utils_qa.py\n", "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.110.133, 185.199.109.133, ...\n", "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 22777 (22K) [text/plain]\n", "Saving to: ‘utils_qa.py’\n", "\n", "utils_qa.py 100%[===================>] 22.24K --.-KB/s in 0.001s \n", "\n", "2023-12-09 12:05:20 (14.8 MB/s) - ‘utils_qa.py’ saved [22777/22777]\n", "\n" ] } ] }, { "cell_type": "code", "source": [ "!ls" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "Loa6y9XqzXJ4", "outputId": "8bd0088d-b009-4d08-c2a7-dd123598432a" }, "execution_count": 6, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "run_qa.py sample_data\ttrainer_qa.py utils_qa.py\n" ] } ] }, { "cell_type": "code", "source": [ "!mkdir eval_results" ], "metadata": { "id": "QDNxV31P0reW" }, "execution_count": 7, "outputs": [] }, { "cell_type": "code", "source": [ "!python -m run_qa --model_name_or_path osiria/bert-italian-uncased-question-answering --dataset_name squad_it --do_predict --per_device_eval_batch_size=5 --output_dir eval_results" ], "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "id": "B5O5Lom8x_c4", "outputId": "8d066e37-8407-4232-de44-6290495776f9" }, "execution_count": 8, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "2023-12-09 12:05:29.016992: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", "2023-12-09 12:05:29.017054: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", "2023-12-09 12:05:29.017091: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", "2023-12-09 12:05:30.116878: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", "12/09/2023 12:05:33 - WARNING - __main__ - Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: False, 16-bits training: False\n", "12/09/2023 12:05:33 - INFO - __main__ - Training/evaluation parameters TrainingArguments(\n", "_n_gpu=1,\n", "adafactor=False,\n", "adam_beta1=0.9,\n", "adam_beta2=0.999,\n", "adam_epsilon=1e-08,\n", "auto_find_batch_size=False,\n", "bf16=False,\n", "bf16_full_eval=False,\n", "data_seed=None,\n", "dataloader_drop_last=False,\n", "dataloader_num_workers=0,\n", "dataloader_persistent_workers=False,\n", "dataloader_pin_memory=True,\n", "ddp_backend=None,\n", "ddp_broadcast_buffers=None,\n", "ddp_bucket_cap_mb=None,\n", "ddp_find_unused_parameters=None,\n", "ddp_timeout=1800,\n", "debug=[],\n", "deepspeed=None,\n", "disable_tqdm=False,\n", "dispatch_batches=None,\n", "do_eval=False,\n", "do_predict=True,\n", "do_train=False,\n", "eval_accumulation_steps=None,\n", "eval_delay=0,\n", "eval_steps=None,\n", "evaluation_strategy=no,\n", "fp16=False,\n", "fp16_backend=auto,\n", "fp16_full_eval=False,\n", "fp16_opt_level=O1,\n", "fsdp=[],\n", "fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False},\n", "fsdp_min_num_params=0,\n", "fsdp_transformer_layer_cls_to_wrap=None,\n", "full_determinism=False,\n", "gradient_accumulation_steps=1,\n", "gradient_checkpointing=False,\n", "gradient_checkpointing_kwargs=None,\n", "greater_is_better=None,\n", "group_by_length=False,\n", "half_precision_backend=auto,\n", "hub_always_push=False,\n", "hub_model_id=None,\n", "hub_private_repo=False,\n", "hub_strategy=every_save,\n", "hub_token=,\n", "ignore_data_skip=False,\n", "include_inputs_for_metrics=False,\n", "include_num_input_tokens_seen=False,\n", "include_tokens_per_second=False,\n", "jit_mode_eval=False,\n", "label_names=None,\n", "label_smoothing_factor=0.0,\n", "learning_rate=5e-05,\n", "length_column_name=length,\n", "load_best_model_at_end=False,\n", "local_rank=0,\n", "log_level=passive,\n", "log_level_replica=warning,\n", "log_on_each_node=True,\n", "logging_dir=eval_results/runs/Dec09_12-05-33_4da00120f505,\n", "logging_first_step=False,\n", "logging_nan_inf_filter=True,\n", "logging_steps=500,\n", "logging_strategy=steps,\n", "lr_scheduler_kwargs={},\n", "lr_scheduler_type=linear,\n", "max_grad_norm=1.0,\n", "max_steps=-1,\n", "metric_for_best_model=None,\n", "mp_parameters=,\n", "neftune_noise_alpha=None,\n", "no_cuda=False,\n", "num_train_epochs=3.0,\n", "optim=adamw_torch,\n", "optim_args=None,\n", "output_dir=eval_results,\n", "overwrite_output_dir=False,\n", "past_index=-1,\n", "per_device_eval_batch_size=5,\n", "per_device_train_batch_size=8,\n", "prediction_loss_only=False,\n", "push_to_hub=False,\n", "push_to_hub_model_id=None,\n", "push_to_hub_organization=None,\n", "push_to_hub_token=,\n", "ray_scope=last,\n", "remove_unused_columns=True,\n", "report_to=['tensorboard'],\n", "resume_from_checkpoint=None,\n", "run_name=eval_results,\n", "save_on_each_node=False,\n", "save_only_model=False,\n", "save_safetensors=True,\n", "save_steps=500,\n", "save_strategy=steps,\n", "save_total_limit=None,\n", "seed=42,\n", "skip_memory_metrics=True,\n", "split_batches=False,\n", "tf32=None,\n", "torch_compile=False,\n", "torch_compile_backend=None,\n", "torch_compile_mode=None,\n", "torchdynamo=None,\n", "tpu_metrics_debug=False,\n", "tpu_num_cores=None,\n", "use_cpu=False,\n", "use_ipex=False,\n", "use_legacy_prediction_loop=False,\n", "use_mps_device=False,\n", "warmup_ratio=0.0,\n", "warmup_steps=0,\n", "weight_decay=0.0,\n", ")\n", "https://huggingface.co/datasets/squad_it/resolve/main/squad_it.py not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/2f7a246f661e6acb3a9ad072ed14241ea343265e7d863b88717897941c3ab1dd.671c3c3daf2f05e0355b498b0f5a7f2167fa94dc6de36195fe1ba0503e6dcc30.py.incomplete\n", "12/09/2023 12:05:33 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/squad_it/resolve/main/squad_it.py not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/2f7a246f661e6acb3a9ad072ed14241ea343265e7d863b88717897941c3ab1dd.671c3c3daf2f05e0355b498b0f5a7f2167fa94dc6de36195fe1ba0503e6dcc30.py.incomplete\n", "Downloading builder script: 100% 4.74k/4.74k [00:00<00:00, 21.2MB/s]\n", "storing https://huggingface.co/datasets/squad_it/resolve/main/squad_it.py in cache at /root/.cache/huggingface/datasets/downloads/2f7a246f661e6acb3a9ad072ed14241ea343265e7d863b88717897941c3ab1dd.671c3c3daf2f05e0355b498b0f5a7f2167fa94dc6de36195fe1ba0503e6dcc30.py\n", "12/09/2023 12:05:34 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/squad_it/resolve/main/squad_it.py in cache at /root/.cache/huggingface/datasets/downloads/2f7a246f661e6acb3a9ad072ed14241ea343265e7d863b88717897941c3ab1dd.671c3c3daf2f05e0355b498b0f5a7f2167fa94dc6de36195fe1ba0503e6dcc30.py\n", "creating metadata file for /root/.cache/huggingface/datasets/downloads/2f7a246f661e6acb3a9ad072ed14241ea343265e7d863b88717897941c3ab1dd.671c3c3daf2f05e0355b498b0f5a7f2167fa94dc6de36195fe1ba0503e6dcc30.py\n", "12/09/2023 12:05:34 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/2f7a246f661e6acb3a9ad072ed14241ea343265e7d863b88717897941c3ab1dd.671c3c3daf2f05e0355b498b0f5a7f2167fa94dc6de36195fe1ba0503e6dcc30.py\n", "https://huggingface.co/datasets/squad_it/resolve/main/dataset_infos.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/f6d7650a1e4a9f0e42b54f34ac614f7c1142ad26c1bd19a448eb606fe09f7515.b459489a7b1a4d859decbeef851034c7846ef6cce8ec2cf192cf4b4f40d0865d.incomplete\n", "12/09/2023 12:05:34 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/squad_it/resolve/main/dataset_infos.json not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/f6d7650a1e4a9f0e42b54f34ac614f7c1142ad26c1bd19a448eb606fe09f7515.b459489a7b1a4d859decbeef851034c7846ef6cce8ec2cf192cf4b4f40d0865d.incomplete\n", "Downloading metadata: 100% 2.51k/2.51k [00:00<00:00, 13.1MB/s]\n", "storing https://huggingface.co/datasets/squad_it/resolve/main/dataset_infos.json in cache at /root/.cache/huggingface/datasets/downloads/f6d7650a1e4a9f0e42b54f34ac614f7c1142ad26c1bd19a448eb606fe09f7515.b459489a7b1a4d859decbeef851034c7846ef6cce8ec2cf192cf4b4f40d0865d\n", "12/09/2023 12:05:34 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/squad_it/resolve/main/dataset_infos.json in cache at /root/.cache/huggingface/datasets/downloads/f6d7650a1e4a9f0e42b54f34ac614f7c1142ad26c1bd19a448eb606fe09f7515.b459489a7b1a4d859decbeef851034c7846ef6cce8ec2cf192cf4b4f40d0865d\n", "creating metadata file for /root/.cache/huggingface/datasets/downloads/f6d7650a1e4a9f0e42b54f34ac614f7c1142ad26c1bd19a448eb606fe09f7515.b459489a7b1a4d859decbeef851034c7846ef6cce8ec2cf192cf4b4f40d0865d\n", "12/09/2023 12:05:34 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/f6d7650a1e4a9f0e42b54f34ac614f7c1142ad26c1bd19a448eb606fe09f7515.b459489a7b1a4d859decbeef851034c7846ef6cce8ec2cf192cf4b4f40d0865d\n", "https://huggingface.co/datasets/squad_it/resolve/main/README.md not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/b664892e5965a1ef2e173eadc7e9cbf6ad4aeb6240d7b3bb6ab9f32850363a6f.71fe7dbf5f3b574d85b57b5a722ebc49fe8ec8b03d655344891fb1b05c4634fb.incomplete\n", "12/09/2023 12:05:35 - INFO - datasets.utils.file_utils - https://huggingface.co/datasets/squad_it/resolve/main/README.md not found in cache or force_download set to True, downloading to /root/.cache/huggingface/datasets/downloads/b664892e5965a1ef2e173eadc7e9cbf6ad4aeb6240d7b3bb6ab9f32850363a6f.71fe7dbf5f3b574d85b57b5a722ebc49fe8ec8b03d655344891fb1b05c4634fb.incomplete\n", "Downloading readme: 100% 7.27k/7.27k [00:00<00:00, 26.1MB/s]\n", "storing https://huggingface.co/datasets/squad_it/resolve/main/README.md in cache at /root/.cache/huggingface/datasets/downloads/b664892e5965a1ef2e173eadc7e9cbf6ad4aeb6240d7b3bb6ab9f32850363a6f.71fe7dbf5f3b574d85b57b5a722ebc49fe8ec8b03d655344891fb1b05c4634fb\n", "12/09/2023 12:05:35 - INFO - datasets.utils.file_utils - storing https://huggingface.co/datasets/squad_it/resolve/main/README.md in cache at /root/.cache/huggingface/datasets/downloads/b664892e5965a1ef2e173eadc7e9cbf6ad4aeb6240d7b3bb6ab9f32850363a6f.71fe7dbf5f3b574d85b57b5a722ebc49fe8ec8b03d655344891fb1b05c4634fb\n", "creating metadata file for /root/.cache/huggingface/datasets/downloads/b664892e5965a1ef2e173eadc7e9cbf6ad4aeb6240d7b3bb6ab9f32850363a6f.71fe7dbf5f3b574d85b57b5a722ebc49fe8ec8b03d655344891fb1b05c4634fb\n", "12/09/2023 12:05:35 - INFO - datasets.utils.file_utils - creating metadata file for /root/.cache/huggingface/datasets/downloads/b664892e5965a1ef2e173eadc7e9cbf6ad4aeb6240d7b3bb6ab9f32850363a6f.71fe7dbf5f3b574d85b57b5a722ebc49fe8ec8b03d655344891fb1b05c4634fb\n", "Loading Dataset Infos from /root/.cache/huggingface/modules/datasets_modules/datasets/squad_it/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71\n", "12/09/2023 12:05:35 - INFO - datasets.info - Loading Dataset Infos from /root/.cache/huggingface/modules/datasets_modules/datasets/squad_it/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71\n", "Generating dataset squad_it (/root/.cache/huggingface/datasets/squad_it/default/0.1.0/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71)\n", "12/09/2023 12:05:35 - INFO - datasets.builder - Generating dataset squad_it (/root/.cache/huggingface/datasets/squad_it/default/0.1.0/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71)\n", "Downloading and preparing dataset squad_it/default to /root/.cache/huggingface/datasets/squad_it/default/0.1.0/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71...\n", "12/09/2023 12:05:35 - INFO - datasets.builder - Downloading and preparing dataset squad_it/default to /root/.cache/huggingface/datasets/squad_it/default/0.1.0/d442bdb4794b4bae227ab19105b76d706ed7cf2ac342e4c9da4a5c36bde19d71...\n", "Dataset not on Hf google storage. Downloading and preparing it from source\n", "12/09/2023 12:05:35 - INFO - datasets.builder - Dataset not on Hf google storage. Downloading and preparing it from source\n", "Downloading data files: 0% 0/2 [00:00> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--osiria--bert-italian-uncased-question-answering/snapshots/9985f20cde546f78a743c4d4f1efd605519b1c41/config.json\n", "[INFO|configuration_utils.py:802] 2023-12-09 12:05:45,159 >> Model config BertConfig {\n", " \"_name_or_path\": \"osiria/bert-italian-uncased-question-answering\",\n", " \"architectures\": [\n", " \"BertForQuestionAnswering\"\n", " ],\n", " \"attention_probs_dropout_prob\": 0.1,\n", " \"classifier_dropout\": null,\n", " \"directionality\": \"bidi\",\n", " \"hidden_act\": \"gelu\",\n", " \"hidden_dropout_prob\": 0.1,\n", " \"hidden_size\": 768,\n", " \"initializer_range\": 0.02,\n", " \"intermediate_size\": 3072,\n", " \"layer_norm_eps\": 1e-12,\n", " \"max_position_embeddings\": 512,\n", " \"model_type\": \"bert\",\n", " \"num_attention_heads\": 12,\n", " \"num_hidden_layers\": 12,\n", " \"pad_token_id\": 0,\n", " \"pooler_fc_size\": 768,\n", " \"pooler_num_attention_heads\": 12,\n", " \"pooler_num_fc_layers\": 3,\n", " \"pooler_size_per_head\": 128,\n", " \"pooler_type\": \"first_token_transform\",\n", " \"position_embedding_type\": \"absolute\",\n", " \"torch_dtype\": \"float32\",\n", " \"transformers_version\": \"4.36.0.dev0\",\n", " \"type_vocab_size\": 2,\n", " \"use_cache\": true,\n", " \"vocab_size\": 30154\n", "}\n", "\n", "tokenizer_config.json: 100% 314/314 [00:00<00:00, 1.91MB/s]\n", "vocab.txt: 100% 202k/202k [00:00<00:00, 5.01MB/s]\n", "[INFO|tokenization_utils_base.py:2026] 2023-12-09 12:05:45,575 >> loading file vocab.txt from cache at /root/.cache/huggingface/hub/models--osiria--bert-italian-uncased-question-answering/snapshots/9985f20cde546f78a743c4d4f1efd605519b1c41/vocab.txt\n", "[INFO|tokenization_utils_base.py:2026] 2023-12-09 12:05:45,575 >> loading file tokenizer.json from cache at None\n", "[INFO|tokenization_utils_base.py:2026] 2023-12-09 12:05:45,575 >> loading file added_tokens.json from cache at None\n", "[INFO|tokenization_utils_base.py:2026] 2023-12-09 12:05:45,575 >> loading file special_tokens_map.json from cache at None\n", "[INFO|tokenization_utils_base.py:2026] 2023-12-09 12:05:45,575 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--osiria--bert-italian-uncased-question-answering/snapshots/9985f20cde546f78a743c4d4f1efd605519b1c41/tokenizer_config.json\n", "pytorch_model.bin: 100% 435M/435M [00:09<00:00, 46.2MB/s]\n", "[INFO|modeling_utils.py:3299] 2023-12-09 12:05:55,532 >> loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--osiria--bert-italian-uncased-question-answering/snapshots/9985f20cde546f78a743c4d4f1efd605519b1c41/pytorch_model.bin\n", "[INFO|modeling_utils.py:4139] 2023-12-09 12:05:56,828 >> All model checkpoint weights were used when initializing BertForQuestionAnswering.\n", "\n", "[INFO|modeling_utils.py:4147] 2023-12-09 12:05:56,828 >> All the weights of BertForQuestionAnswering were initialized from the model checkpoint at osiria/bert-italian-uncased-question-answering.\n", "If your task is similar to the task the model of the checkpoint was trained on, you can already use BertForQuestionAnswering for predictions without further training.\n", "Running tokenizer on prediction dataset: 0% 0/7609 [00:00> The following columns in the test set don't have a corresponding argument in `BertForQuestionAnswering.forward` and have been ignored: example_id, offset_mapping. If example_id, offset_mapping are not expected by `BertForQuestionAnswering.forward`, you can safely ignore this message.\n", "[INFO|trainer.py:3154] 2023-12-09 12:06:10,622 >> ***** Running Prediction *****\n", "[INFO|trainer.py:3156] 2023-12-09 12:06:10,622 >> Num examples = 7942\n", "[INFO|trainer.py:3159] 2023-12-09 12:06:10,622 >> Batch size = 5\n", "100% 1588/1589 [03:21<00:00, 7.77it/s]12/09/2023 12:09:44 - INFO - utils_qa - Post-processing 7609 example predictions split into 7942 features.\n", "\n", " 0% 0/7609 [00:00> Dropping the following result as it does not have all the necessary fields:\n", "{'task': {'name': 'Question Answering', 'type': 'question-answering'}, 'dataset': {'name': 'squad_it', 'type': 'squad_it'}}\n", "100% 1589/1589 [04:02<00:00, 6.56it/s]\n" ] } ] }, { "cell_type": "code", "source": [], "metadata": { "id": "rMyhUFpnItIQ" }, "execution_count": null, "outputs": [] } ] }