{ "cells": [ { "cell_type": "markdown", "id": "d3092ed4", "metadata": {}, "source": [ "# NLP demo software by HyperbeeAI\n", "\n", "Copyrights © 2023 Hyperbee.AI Inc. All rights reserved. hello@hyperbee.ai \n", "\n", "### Deployment\n", "\n", "This notebook acts as the serial terminal that we use in the ai85 translation demo.\n", "\n", "- load parameter set\n", "- run a test on the PC to determine what to expect from the chip\n", "- run test on the chip via serial terminal on PC" ] }, { "cell_type": "markdown", "id": "e6208384", "metadata": {}, "source": [ "### Initialization" ] }, { "cell_type": "code", "execution_count": 1, "id": "6c10cb53", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "imported utils.py\n", "NLP demo software by HyperbeeAI. Copyrights © 2023 Hyperbee.AI Inc. All rights reserved. hello@hyperbee.ai\n", "\n", "imported layers.py\n", "NLP demo software by HyperbeeAI. Copyrights © 2023 Hyperbee.AI Inc. All rights reserved. hello@hyperbee.ai\n", "\n", "imported functions.py\n", "NLP demo software by HyperbeeAI. Copyrights © 2023 Hyperbee.AI Inc. All rights reserved. hello@hyperbee.ai\n", "\n", "imported models.py\n", "NLP demo software by HyperbeeAI. Copyrights © 2023 Hyperbee.AI Inc. All rights reserved. hello@hyperbee.ai\n", "\n", "imported dataloader.py\n", "NLP demo software by HyperbeeAI. Copyrights © 2023 Hyperbee.AI Inc. All rights reserved. hello@hyperbee.ai\n", "\n" ] } ], "source": [ "import torch, random\n", "import numpy as np\n", "import torch.nn as nn\n", "from torchtext.legacy.datasets import TranslationDataset\n", "from torchtext.legacy.data import Field, BucketIterator\n", "from utils import tokenize_es, tokenize_en, tokenizer_es, tokenizer_en, TRG_PAD_IDX, \\\n", " translate_sentence, calculate_bleu, license_statement\n", "from models import encoder, decoder, seq2seq\n", "from dataloader import NewsDataset\n", "\n", "import serial" ] }, { "cell_type": "code", "execution_count": 2, "id": "9966ccad", "metadata": {}, "outputs": [], "source": [ "SEED = 1234\n", "random.seed(SEED)\n", "torch.manual_seed(SEED)\n", "torch.cuda.manual_seed(SEED)\n", "torch.backends.cudnn.deterministic = True\n", "BATCH_SIZE = 48" ] }, { "cell_type": "code", "execution_count": 3, "id": "6d864c26", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Working with device: cuda\n" ] } ], "source": [ "SRC = Field(tokenize = tokenize_es, \n", " init_token = tokenizer_es.token_to_id(\"\"), \n", " eos_token = tokenizer_es.token_to_id(\"\"), \n", " pad_token = tokenizer_es.token_to_id(\"\"),\n", " unk_token = tokenizer_es.token_to_id(\"\"),\n", " use_vocab = False,\n", " batch_first = True)\n", "\n", "TRG = Field(tokenize = tokenize_en, \n", " init_token = tokenizer_en.token_to_id(\"\"), \n", " eos_token = tokenizer_en.token_to_id(\"\"), \n", " pad_token = tokenizer_en.token_to_id(\"\"),\n", " unk_token = tokenizer_en.token_to_id(\"\"),\n", " use_vocab = False,\n", " batch_first = True)\n", "\n", "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n", "#device = 'cpu'\n", "print(\"Working with device:\", device)" ] }, { "cell_type": "code", "execution_count": 4, "id": "7f1f2efb", "metadata": {}, "outputs": [], "source": [ "train_data, valid_data, test_data = NewsDataset.splits(exts=('.es', '.en'), fields=(SRC, TRG))\n", "train_iterator, valid_iterator, test_iterator = BucketIterator.splits(\n", " (train_data, valid_data, test_data),\n", " batch_size = BATCH_SIZE,\n", " device = device)" ] }, { "cell_type": "code", "execution_count": 5, "id": "ccd6c1fc", "metadata": {}, "outputs": [], "source": [ "enc = encoder(device)\n", "dec = decoder(device, TRG_PAD_IDX)\n", "model = seq2seq(enc, dec)" ] }, { "cell_type": "code", "execution_count": 6, "id": "6ae348e3", "metadata": {}, "outputs": [], "source": [ "trained_checkpoint = \"assets/es2en_hw_cp6.pt\"\n", "model.load_state_dict(torch.load(trained_checkpoint, map_location=device), strict=False);\n", "model.to(device);" ] }, { "cell_type": "markdown", "id": "ddb1a23b", "metadata": {}, "source": [ "### serial conversion functions" ] }, { "cell_type": "code", "execution_count": 7, "id": "534e72f2", "metadata": {}, "outputs": [], "source": [ "def singlepass64_tensor2serial(seq_length, tensor):\n", " data = tensor.cpu().detach().numpy();\n", " char_array = '';\n", "\n", " i=0;\n", " while i < 64:\n", " for j in range(0,seq_length):\n", " ch3 = data[0,i+3,j].astype('int8')\n", " ch2 = data[0,i+2,j].astype('int8')\n", " ch1 = data[0,i+1,j].astype('int8')\n", " ch0 = data[0,i+0,j].astype('int8')\n", "\n", " # 2s complements\n", " val3 = \"{0:#0{1}x}\".format(int(np.binary_repr(ch3, width=8), 2),4)\n", " val2 = \"{0:#0{1}x}\".format(int(np.binary_repr(ch2, width=8), 2),4)\n", " val1 = \"{0:#0{1}x}\".format(int(np.binary_repr(ch1, width=8), 2),4)\n", " val0 = \"{0:#0{1}x}\".format(int(np.binary_repr(ch0, width=8), 2),4)\n", "\n", " char_array += val3[2:] + val2[2:] + val1[2:] + val0[2:]\n", "\n", " i=i+4\n", " \n", " return char_array\n", "\n", "def twos_comp(val, bits):\n", " if (val & (1 << (bits - 1))) != 0:\n", " val = val - (1 << bits)\n", " return val\n", "\n", "def tensor_fromserial_singlepass64(char_array, seq_length, typetensor):\n", " out_tensor = torch.zeros_like(typetensor)\n", " i=0;\n", " while i < 64:\n", " for j in range(0, seq_length):\n", " cursor = (i*seq_length*2 + j*8); # seq_length*2 because we use 2 characters per element due to pyserial \\CR \\LF issue\n", " word = char_array[cursor : cursor+8];\n", " \n", " # 2s complements\n", " val3 = twos_comp(int(word[0:2],16), 8)\n", " val2 = twos_comp(int(word[2:4],16), 8)\n", " val1 = twos_comp(int(word[4:6],16), 8)\n", " val0 = twos_comp(int(word[6:8],16), 8)\n", " \n", " out_tensor[0,i+3,j] = val3;\n", " out_tensor[0,i+2,j] = val2;\n", " out_tensor[0,i+1,j] = val1;\n", " out_tensor[0,i+0,j] = val0;\n", " \n", " i=i+4\n", "\n", " return out_tensor\n", "\n", "def widemode_twos_comp(val, bits):\n", " if (val & (1 << (bits - 1))) != 0:\n", " val = ((val - (1 << bits)) >> 5) + 1\n", " return (val >> 5)\n", "\n", "def tensor_fromserial_widemode64(char_array, seq_length, typetensor):\n", " out_tensor = torch.zeros_like(typetensor)\n", " i=0;\n", " while i < 64:\n", " for j in range(0, seq_length):\n", " cursor = (i*seq_length*8 + j*32); # seq_length*8 now because we use 8 characters per element, same pyserial issue\n", " word = char_array[cursor : cursor+32];\n", " \n", " # 2s complements\n", " val0 = twos_comp(int(word[0:8],16), 32)\n", " val1 = twos_comp(int(word[8:16],16), 32)\n", " val2 = twos_comp(int(word[16:24],16), 32)\n", " val3 = twos_comp(int(word[24:32],16), 32)\n", " \n", " out_tensor[0,i+0,j] = val0;\n", " out_tensor[0,i+1,j] = val1;\n", " out_tensor[0,i+2,j] = val2;\n", " out_tensor[0,i+3,j] = val3;\n", " \n", " i=i+4\n", "\n", " return out_tensor" ] }, { "cell_type": "markdown", "id": "f248bc1d", "metadata": {}, "source": [ "## Test" ] }, { "cell_type": "markdown", "id": "76d11d80", "metadata": {}, "source": [ "### choose id of example" ] }, { "cell_type": "code", "execution_count": 8, "id": "cdbfd418", "metadata": {}, "outputs": [], "source": [ "example_idx = 120" ] }, { "cell_type": "markdown", "id": "26e82b50", "metadata": {}, "source": [ "### on PC" ] }, { "cell_type": "code", "execution_count": 9, "id": "250dcc52", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "trg = but this won ’ t be the last answer , although for the time being it will drive corporate restructuring and the managerial mind .\n", "\n", "predicted trg = but this will not be the latest response , though it will now be the central force of corporate restructuring and managerial thinking .\n", "\n", "src = pero esto no será la última respuesta , aunque por ahora será la fuerza central de la reestructuración corporativa y el pensamiento gerencial .\n", "\n" ] } ], "source": [ "model.to(device)\n", "src = vars(test_data.examples[example_idx])['src']\n", "trg = tokenizer_en.decode(vars(test_data.examples[example_idx])['trg'], skip_special_tokens=False)\n", "print(f'trg = {trg}')\n", "print(\"\")\n", "translation = translate_sentence(src, SRC, TRG, model, device)\n", "print(f'predicted trg = {translation}')\n", "print(\"\")\n", "src = tokenizer_es.decode(src, skip_special_tokens=False)\n", "print(f'src = {src}')\n", "print(\"\")" ] }, { "cell_type": "markdown", "id": "10e43fe8", "metadata": {}, "source": [ "### on chip" ] }, { "cell_type": "code", "execution_count": 10, "id": "b7aa9adc", "metadata": {}, "outputs": [], "source": [ "enc_pre = model.encoder.pre.to(device)\n", "dec_pre = model.decoder.pre.to(device)\n", "dec_i2w = model.decoder.fff.to(device)\n", "\n", "src = vars(test_data.examples[example_idx])['src']\n", "trg = tokenizer_en.decode(vars(test_data.examples[example_idx])['trg'], skip_special_tokens=False)" ] }, { "cell_type": "markdown", "id": "738e668a", "metadata": {}, "source": [ "**MARK**\n", "\n", "The below cell starts running a serial terminal on this notebook. First run this cell, and when it says \"waiting for ai85\", load the \"assets/demo.elf\" program onto the ai85 chip, and start running it (type c in gdb). This should trigger the terminal here, and operation should resume normally.\n", "\n", "The cell is designed to translate a single sentence." ] }, { "cell_type": "code", "execution_count": 11, "id": "0f5a5628", "metadata": {}, "outputs": [], "source": [ "def ai85_demo_function():\n", " \n", " print(\"Please enter a Spanish sentence\")\n", " textinput = input()\n", " print(\"\")\n", " print(\"\")\n", "\n", " src = (tokenizer_es.encode(textinput)).ids\n", " trg = tokenizer_en.decode(vars(test_data.examples[example_idx])['trg'], skip_special_tokens=False)\n", " with serial.Serial('/dev/ttyACM0', 115200) as ser: # , timeout=5 (not necessary, just for info)\n", " tokens = src\n", " tokens = [SRC.init_token] + tokens + [SRC.eos_token] + [SRC.pad_token] * (48 - 2 - len(tokens)) \n", " src_tensor = torch.LongTensor(tokens).unsqueeze(0).to(device)\n", "\n", " batch_size = src_tensor.shape[0];\n", " src_len = src_tensor.shape[1];\n", " enc_pre_d = enc_pre(src_tensor, 0, src_len, batch_size);\n", " encarray = singlepass64_tensor2serial(48, enc_pre_d);\n", "\n", " #### to chip\n", " print(\"** shallow.AI ai85 demo **\")\n", " print(\"** loading demo to ai85 **\")\n", " line = ser.readline()\n", " while(line != b''):\n", " line = ser.readline()\n", " if(line == b'GJcav7Wf2kmhaXJdsO0QVzX3slsv96Ck\\r\\n'):\n", " ser.write(encarray.encode(encoding=\"ascii\"))\n", " line = ser.readline()\n", " break\n", "\n", " trg_indexes = [TRG.init_token, ] + [TRG.pad_token] * (48 - 1) \n", "\n", " done_decoding_flag = False\n", " for i in range(47):\n", " start_idx = max(0, i - 7)\n", " trg_tensor = torch.LongTensor(trg_indexes[start_idx:start_idx + 8]).unsqueeze(0).to(device)\n", " batch_size = trg_tensor.shape[0]\n", " trg_len = trg_tensor.shape[1]\n", " pos_start = max(0, i - 7)\n", " dec_pre_d = dec_pre(trg_tensor, pos_start, trg_len + pos_start, batch_size)\n", " decarray = singlepass64_tensor2serial(8, dec_pre_d);\n", " while(line != b''):\n", " line = ser.readline()\n", " if(line == b'gZMFxLf6muLVf9P6Iyea56VbA4qktpUR\\r\\n'):\n", " if(done_decoding_flag):\n", " print(\"****** ai85 is done ******\")\n", " decarray = \"done\" + decarray[4:]\n", " ser.write(decarray.encode(encoding=\"ascii\"))\n", " line = ser.readline()\n", " break\n", "\n", " if(done_decoding_flag):\n", " break\n", "\n", " line = ser.readline()\n", " h2e_out = tensor_fromserial_widemode64(line, 1, dec_pre_d[:,:,0:1]) / (128.0 * 2**(5+1))\n", " output = dec_i2w(h2e_out.permute(0, 2, 1))\n", " pred_token = output.argmax(2)\n", " trg_indexes[i + 1] = pred_token\n", " if pred_token == TRG.eos_token:\n", " done_decoding_flag = True\n", " \n", " try:\n", " trg_indexes = trg_indexes[1:trg_indexes.index(TRG.eos_token)]\n", " except ValueError: \n", " trg_indexes = trg_indexes[1:]\n", "\n", " trg_tokens = tokenizer_en.decode(trg_indexes, skip_special_tokens=False)\n", " \n", " print(\"\")\n", " print(\"\")\n", " print(\"English translation on ai85:\")\n", " print(f'{trg_tokens}')" ] }, { "cell_type": "markdown", "id": "af1aa370", "metadata": {}, "source": [ "# NLP demo software by HyperbeeAI\n", "\n", "Copyrights © 2023 Hyperbee.AI Inc. All rights reserved. hello@hyperbee.ai " ] }, { "cell_type": "code", "execution_count": 12, "id": "7df357a0", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Please enter a Spanish sentence\n", "La vinculación entre el crecimiento económico y el bienestar humano parece evidente.\n", "\n", "\n", "** shallow.AI ai85 demo **\n", "** loading demo to ai85 **\n", "****** ai85 is done ******\n", "\n", "\n", "English translation on ai85:\n", "the link between economic growth and human welfare seems clear .\n" ] } ], "source": [ "ai85_demo_function()" ] }, { "cell_type": "code", "execution_count": null, "id": "3e7577a0", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "52a397de", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "96f7b68e", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "3fae6816", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "0a92e88d", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "e60ac632", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "9f982aec", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "bfbc6cfc", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "b59b5243", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "61b8c8d3", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "459a0550", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "82cc8933", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "d9e43f05", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "04c6aee2", "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, "id": "de644855", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.8.10" } }, "nbformat": 4, "nbformat_minor": 5 }