Merge pull request #7 from terapyon/terada/mt-245-hf-deploy

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,3 +1,16 @@
+---
+title: Podcast Search
+emoji: 🚀
+colorFrom: green
+colorTo: gray
+sdk: streamlit
+sdk_version: 1.41.1
+app_file: src/app.py
+pinned: false
+license: mit
+short_description: terapyon channel の検索
+---
+
 # podcast-search
 
 Podcast terapyon channelを検索する仕組み

src/app.py

@@ -1,13 +1,23 @@
 from datetime import timedelta
+import os
 import streamlit as st
 import duckdb
 from embedding import get_embeddings
-from config import DUCKDB_FILE
+from config import HF_HOST, DUCKDB_FILE, HF_REPO_TYPE, HF_REPO_ID, HF_FILENAME
 
 
 @st.cache_resource
 def get_conn():
-    return duckdb.connect(DUCKDB_FILE)
+    if HF_HOST:
+        os.environ["HUGGINGFACE_TOKEN"] = os.getenv("HF_TOKEN", "")
+        from huggingface_hub import hf_hub_download
+        local_file = hf_hub_download(
+            repo_type=HF_REPO_TYPE,
+            repo_id=HF_REPO_ID,
+            filename=HF_FILENAME)
+        return duckdb.connect(local_file)
+    else:
+        return duckdb.connect(DUCKDB_FILE)
 
 
 title_query = """SELECT id, title FROM podcasts
@@ -20,11 +30,11 @@ query = """WITH filtered_podcasts AS (
         WHERE id in ?
 ),
 ordered_embeddings AS (
-    SELECT embeddings.id, embeddings.part
+    SELECT embeddings.id, embeddings.part, array_distance(embedding, ?::FLOAT[1024]) AS distance
     FROM embeddings
     JOIN filtered_podcasts fp ON embeddings.id = fp.id
-    ORDER BY array_distance(embedding, ?::FLOAT[1024])
-    LIMIT 10
+    ORDER BY distance
+      LIMIT 10
 )
 SELECT
     p.title,
@@ -33,6 +43,7 @@ SELECT
     e.text,
     e.part,
     p.audio,
+    oe.distance,
   FROM
       ordered_embeddings oe
   JOIN
@@ -42,7 +53,8 @@ SELECT
   JOIN
       podcasts p
     ON
-      oe.id = p.id;
+      oe.id = p.id
+  ORDER BY oe.distance;
 """
 
 st.title("terapyon cannel search")
@@ -65,7 +77,7 @@ if word:
     result = conn.execute(query,
                           (selected_ids, word_embedding,)).df()
     selected = st.dataframe(result,
-                            column_order=["title", "date", "part", "start", "text", "audio"],
+                            column_order=["title", "date", "part", "start", "distance", "text", "audio"],
                             on_select="rerun",
                             selection_mode="single-row")
     if selected:

src/config.py

@@ -4,8 +4,14 @@ from pathlib import Path
 # import logging
 
 
+HF_HOST = False
+HF_REPO_TYPE = "dataset"
+HF_REPO_ID = "terapyon/terapyon-podcast"
+HF_FILENAME = "terapyon-podcast-20250104.duckdb"
+
 HERE = Path(__file__).resolve().parent
 DUCKDB_FILE = HERE.parent / "db" / "terapyon-podcast.duckdb"
+
 STORE_DIR = HERE.parent / "store"
 DATA_DIR = HERE.parent / "data"
 PODCAST_TITLE_LIST = str(STORE_DIR / 'title-list-202301-202501.parquet')