chore: Add audio search functionality with Pinecone embeddings

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text

app.py

@@ -0,0 +1,128 @@
+import embed_anything
+from embed_anything import EmbedData
+from tqdm.autonotebook import tqdm
+from pinecone import Pinecone, ServerlessSpec
+import numpy as np
+import os
+from pinecone import PineconeApiException
+import uuid
+import re
+import gradio as gr
+
+
+audio_files = ["examples/samples_hp0.wav", "examples/samples_gb0.wav"]
+
+embeddings: list[list[EmbedData]] = []
+
+for file in audio_files:
+    embedding = embed_anything.embed_file(file, "Whisper-Jina")
+    embeddings.append(embedding)
+
+pc = Pinecone(api_key=os.environ.get("PINECONE_API_KEY"))
+pc.delete_index("search-in-audio")
+try:
+    index = pc.create_index(
+        name="search-in-audio",
+        dimension=768,  # Replace with your model dimensions
+        metric="cosine",  # Replace with your model metric
+        spec=ServerlessSpec(cloud="aws", region="us-east-1"),
+    )
+    index = pc.Index("search-in-audio")
+except PineconeApiException as e:
+    index = pc.Index("search-in-audio")
+    if e.status == 409:
+        print("Index already exists")
+
+    else:
+        print(e)
+
+
+## convert embeddings which is of the form EmbedData : text, embedding, metadata to the form required by pinecone which is id, values, metadata
+def convert_to_pinecone_format(embeddings: list[list[EmbedData]]):
+    data = []
+    for i, embedding in enumerate(embeddings):
+        for j, emb in enumerate(embedding):
+            data.append(
+                {
+                    "id": str(uuid.uuid4()),
+                    "values": emb.embedding,
+                    "metadata": {
+                        "text": emb.text,
+                        "start": emb.metadata["start"],
+                        "end": emb.metadata["end"],
+                        "file": re.split(r"/|\\", emb.metadata["file_name"])[-1],
+                    },
+                }
+            )
+    return data
+
+
+data = convert_to_pinecone_format(embeddings)
+index.upsert(data)
+
+
+files = ["samples_hp0.wav", "samples_gb0.wav"]
+
+
+def search(query, audio):
+
+    results = []
+    query = embed_anything.embed_query([query], "Jina")[0]
+
+    if re.split(r"/|\\", audio)[-1] not in files:
+        print(file, re.split(r"/|\\", audio)[-1])
+        embeddings = embed_anything.embed_file(audio, "Whisper-Jina")
+        embeddings = convert_to_pinecone_format([embeddings])
+        index.upsert(embeddings)
+
+    files.append(re.split(r"/|\\", audio)[-1])
+
+    result = index.query(
+        vector=query.embedding,
+        top_k=5,
+        include_metadata=True,
+    )
+    for res in result.matches:
+        results.append(res.metadata)
+
+    formatted_results = []
+    for result in results:
+        display_text = f"""
+        
+        `File: {result['file']}`
+        
+        `Start: {result['start']}`
+        
+        `End: {result['end']}`
+        
+        Text: {result['text']}"""
+        formatted_results.append(display_text)
+
+    return (
+        formatted_results[0],
+        results[0]["file"],
+        formatted_results[1],
+        results[1]["file"],
+        formatted_results[2],
+        results[2]["file"],
+    )
+
+
+demo = gr.Interface(
+    fn=search,
+    inputs=["text", gr.Audio(label="Audio", type="filepath")],
+    outputs=[
+        gr.Markdown(label="Text"),
+        gr.Audio(label="Audio", type="filepath"),
+        gr.Markdown(label="Text"),
+        gr.Audio(label="Audio", type="filepath"),
+        gr.Markdown(label="Text"),
+        gr.Audio(label="Audio", type="filepath"),
+    ],
+    examples=[
+        ["screwdriver", "examples/samples_hp0.wav"],
+        ["united states", "examples/samples_gb0.wav"],
+        ["united states", "examples/samples_hp0.wav"],
+    ],
+)
+demo.launch()

examples/samples_gb0.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33e02d2a4eacf9e5c90053ffffd95a5fac40ff1009dd443f2a053287ed0874f7
+size 4075598

examples/samples_hp0.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12d6f3a901baf48c878bbe2a8ba7148ef8592ab5569df9fb7b0a3cd0d82b8140
+size 8745688

examples/samples_jfk.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59dfb9a4acb36fe2a2affc14bacbee2920ff435cb13cc314a08c13f66ba7860e
+size 352078

requirements.txt

@@ -0,0 +1,104 @@
+aiofiles==23.2.1
+altair==5.3.0
+annotated-types==0.7.0
+anyio==4.4.0
+asttokens==2.4.1
+attrs==23.2.0
+certifi==2024.7.4
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+comm==0.2.2
+contourpy==1.2.1
+cycler==0.12.1
+debugpy==1.8.2
+decorator==5.1.1
+dnspython==2.6.1
+email_validator==2.2.0
+embed_anything==0.1.21
+exceptiongroup==1.2.2
+executing==2.0.1
+fastapi==0.111.1
+fastapi-cli==0.0.4
+ffmpy==0.3.2
+filelock==3.15.4
+fonttools==4.53.1
+fsspec==2024.6.1
+gradio==4.38.1
+gradio_client==1.1.0
+h11==0.14.0
+httpcore==1.0.5
+httptools==0.6.1
+httpx==0.27.0
+huggingface-hub==0.23.5
+idna==3.7
+importlib_metadata==8.0.0
+importlib_resources==6.4.0
+ipykernel==6.29.5
+ipython==8.26.0
+jedi==0.19.1
+Jinja2==3.1.4
+jsonschema==4.23.0
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.2
+jupyter_core==5.7.2
+kiwisolver==1.4.5
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.9.1
+matplotlib-inline==0.1.7
+mdurl==0.1.2
+nest_asyncio==1.6.0
+numpy==2.0.0
+orjson==3.10.6
+packaging==24.1
+pandas==2.2.2
+parso==0.8.4
+pickleshare==0.7.5
+pillow==10.4.0
+pinecone-client==4.1.2
+pinecone-plugin-interface==0.0.7
+pip==24.0
+platformdirs==4.2.2
+prompt_toolkit==3.0.47
+psutil==6.0.0
+pure-eval==0.2.2
+pydantic==2.8.2
+pydantic_core==2.20.1
+pydub==0.25.1
+Pygments==2.18.0
+pyparsing==3.1.2
+python-dateutil==2.9.0
+python-dotenv==1.0.1
+python-multipart==0.0.9
+pytz==2024.1
+pywin32==306
+PyYAML==6.0.1
+pyzmq==26.0.3
+referencing==0.35.1
+requests==2.32.3
+rich==13.7.1
+rpds-py==0.19.0
+ruff==0.5.2
+semantic-version==2.10.0
+setuptools==70.3.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.1
+stack-data==0.6.2
+starlette==0.37.2
+tomlkit==0.12.0
+toolz==0.12.1
+tornado==6.4.1
+tqdm==4.66.4
+traitlets==5.14.3
+typer==0.12.3
+typing_extensions==4.12.2
+tzdata==2024.1
+urllib3==2.2.2
+uvicorn==0.30.1
+watchfiles==0.22.0
+wcwidth==0.2.13
+websockets==11.0.3
+wheel==0.43.0
+zipp==3.19.2

test.ipynb

@@ -0,0 +1,334 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\Users\\arbal\\AppData\\Local\\Temp\\ipykernel_27040\\3276458500.py:3: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
+      "  from tqdm.autonotebook import tqdm\n"
+     ]
+    }
+   ],
+   "source": [
+    "import embed_anything\n",
+    "from embed_anything import EmbedData\n",
+    "from tqdm.autonotebook import tqdm\n",
+    "from pinecone import Pinecone, ServerlessSpec\n",
+    "import numpy as np\n",
+    "import os\n",
+    "from pinecone import PineconeApiException\n",
+    "import uuid"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "audio_files = [\"examples/samples_hp0.wav\", \"examples/samples_gb0.wav\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "embeddings: list[list[EmbedData]] = []"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for file in audio_files:\n",
+    "    embedding = embed_anything.embed_file(file, \"Whisper-Jina\")\n",
+    "    embeddings.append(embedding)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 96,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pc = Pinecone(api_key=os.environ.get(\"PINECONE_API_KEY\"))\n",
+    "pc.delete_index(\"search-in-audio\")\n",
+    "try:\n",
+    "   index =  pc.create_index(\n",
+    "        name=\"search-in-audio\",\n",
+    "        dimension=768, # Replace with your model dimensions\n",
+    "        metric=\"cosine\", # Replace with your model metric\n",
+    "        spec=ServerlessSpec(\n",
+    "            cloud=\"aws\",\n",
+    "            region=\"us-east-1\"\n",
+    "        ) \n",
+    "    )\n",
+    "   index = pc.Index('search-in-audio')\n",
+    "except(PineconeApiException) as e:\n",
+    "    index = pc.Index('search-in-audio')\n",
+    "    if e.status == 409:\n",
+    "        print(\"Index already exists\")\n",
+    "    \n",
+    "    else: print(e)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 97,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "## convert embeddings which is of the form EmbedData : text, embedding, metadata to the form required by pinecone which is id, values, metadata\n",
+    "import re\n",
+    "def convert_to_pinecone_format(embeddings: list[list[EmbedData]]):\n",
+    "    data = []\n",
+    "    for i, embedding in enumerate(embeddings):\n",
+    "        for j, emb in enumerate(embedding):\n",
+    "            data.append({\"id\": str(uuid.uuid4()), \"values\": emb.embedding, \"metadata\": {\"text\": emb.text, \"start\": emb.metadata[\"start\"], \"end\": emb.metadata[\"end\"], \"file\": re.split(r'/|\\\\',emb.metadata[\"file_name\"])[-1]}})\n",
+    "    return data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 98,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'upserted_count': 15}"
+      ]
+     },
+     "execution_count": 98,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "data = convert_to_pinecone_format(embeddings)\n",
+    "index.upsert(data)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 99,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "query = embed_anything.embed_query(['united states'], \"Jina\")[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 100,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result = index.query(vector=query.embedding, top_k=5, include_metadata=True,)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 101,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "{'end': '120', 'file': 'samples_gb0.wav', 'start': '90', 'text': \" Young democracy from Georgia and Ukraine to Afghanistan and Iraq can look to the United States for proof that self-government can endure and nations that still live under tyranny and oppression can find hope and inspiration in our commitment to liberty. For more than two centuries, Americans have demonstrated the ability of free people to choose their own leaders. Our nation has flourished because of its commitment to trusting the wisdom of our citizenry. In this year's election, we will see this tradition continue and we will be reminded once again\"}\n",
+      "{'end': '60', 'file': 'samples_gb0.wav', 'start': '30', 'text': ' The United States was founded on the belief that all men are created equal. Every election day, millions of Americans of all races, religions and backgrounds step into voting booths throughout the nation, whether they are richer, poor, older, young. Each of them has an equal share in choosing the path that our country will take. And every ballot they cast is a reminder that our founding principles are alive and well. Voting is one of the great privileges of a'}\n",
+      "{'end': '30', 'file': 'samples_gb0.wav', 'start': '0', 'text': \" Good morning. This Tuesday is Election Day. After months of spirited debate in vigorous campaigning, the time has come for Americans to make important decisions about our nation's future and encourage all Americans to go to the polls and vote. Election season brings out the spirit of competition between our political parties. And that competition is an essential part of a healthy democracy. But as the campaigns come to a close, Republicans, Democrats, and independents can find common ground on at least one point. Our system of\"}\n",
+      "{'end': '300', 'file': 'samples_hp0.wav', 'start': '270', 'text': ' TML'}\n",
+      "{'end': '30', 'file': 'samples_hp0.wav', 'start': '0', 'text': ' Henry F. Phillips from Wikipedia, the free encyclopedia at en.wickipedia.org.'}\n"
+     ]
+    }
+   ],
+   "source": [
+    "for res in result.matches:\n",
+    "    print(res.metadata)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 104,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7899\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7899/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/plain": []
+     },
+     "execution_count": 104,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "import gradio as gr\n",
+    "files = [\"samples_hp0.wav\", \"samples_gb0.wav\"]\n",
+    "def search(query, audio):\n",
+    "\n",
+    "    results = []\n",
+    "    query = embed_anything.embed_query([query], \"Jina\")[0]\n",
+    "\n",
+    "    if re.split(r'/|\\\\',audio)[-1] not in files:\n",
+    "            print(file, re.split(r'/|\\\\',audio)[-1])\n",
+    "            embeddings = embed_anything.embed_file(audio, \"Whisper-Jina\")\n",
+    "            embeddings = convert_to_pinecone_format([embeddings])\n",
+    "            index.upsert(embeddings)\n",
+    "\n",
+    "    files.append(re.split(r'/|\\\\',audio)[-1])\n",
+    "\n",
+    "\n",
+    "    result = index.query(vector=query.embedding, top_k=5, include_metadata=True,)\n",
+    "    for res in result.matches:\n",
+    "        results.append(res.metadata)\n",
+    "    \n",
+    "    formatted_results = []\n",
+    "    for result in results:\n",
+    "        display_text = f'''\n",
+    "        \n",
+    "        `File: {result['file']}`\n",
+    "        \n",
+    "        `Start: {result['start']}`\n",
+    "        \n",
+    "        `End: {result['end']}`\n",
+    "        \n",
+    "        Text: {result['text']}'''  \n",
+    "        formatted_results.append(display_text)\n",
+    "\n",
+    "    return formatted_results[0], results[0]['file'], formatted_results[1],  results[1]['file'], formatted_results[2], results[2]['file']\n",
+    "\n",
+    "demo = gr.Interface(\n",
+    "    fn=search,\n",
+    "    inputs=[\"text\", gr.Audio(label='Audio', type = 'filepath')],\n",
+    "    outputs=[\n",
+    "        \n",
+    "        gr.Markdown(label=\"Text\"), gr.Audio(label=\"Audio\", type=\"filepath\"),\n",
+    "        gr.Markdown(label=\"Text\"), gr.Audio(label=\"Audio\", type=\"filepath\"),\n",
+    "        gr.Markdown(label=\"Text\"), gr.Audio(label=\"Audio\", type=\"filepath\")\n",
+    "    ],\n",
+    "    examples=[[\"screwdriver\", \"examples/samples_hp0.wav\"], [\"united states\", \"examples/samples_gb0.wav\"], [\"united states\", \"examples/samples_hp0.wav\"]],\n",
+    ")\n",
+    "demo.launch()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 89,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "False"
+      ]
+     },
+     "execution_count": 89,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "audio = \"C:/Users/arbal/AppData/Local/Temp/gradio/c0490405f0fef485a043991c6a9d6b83f006db51/samples_hp0.wav\"\n",
+    "\"samples_hp0.wav\" not in audio"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 94,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "SyntaxError",
+     "evalue": "(unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape (1955374795.py, line 1)",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;36m  Cell \u001b[1;32mIn[94], line 1\u001b[1;36m\u001b[0m\n\u001b[1;33m    if \"samples_hp0.wav\" not in \"C:\\Users\\arbal\\AppData\\Local\\Temp\\gradio\\c0490405f0fef485a043991c6a9d6b83f006db51\\samples_hp0.wav\"\u001b[0m\n\u001b[1;37m                                                                                                                                   ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m (unicode error) 'unicodeescape' codec can't decode bytes in position 2-3: truncated \\UXXXXXXXX escape\n"
+     ]
+    }
+   ],
+   "source": [
+    "if \"samples_hp0.wav\" not in \"C:\\Users\\arbal\\AppData\\Local\\Temp\\gradio\\c0490405f0fef485a043991c6a9d6b83f006db51\\samples_hp0.wav\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 93,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "'samples_hp0.wav'"
+      ]
+     },
+     "execution_count": 93,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "re.split(r'/|\\\\',audio)[-1]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "embedAnything",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}