create .gitignore in image folder, and include base64 image in parquet, app.py use base64 image to display

pipeline/process_videos.py

@@ -1,3 +1,5 @@
+import base64
+
 import cv2
 import pandas as pd
 from PIL import Image
@@ -12,6 +14,8 @@ DATAFRAME_PATH = DATA_DIR / "dataset.parquet"
 
 
 def process_videos() -> None:
+ IMAGES_DIR.mkdir(exist_ok=True, parents=True)
+ (IMAGES_DIR / ".gitignore").write_text("*")
  "Runs clip on video frames, saves results to a parquet file"
  clip_wrapper = ClipWrapper()
  results = []
@@ -27,19 +31,21 @@ def process_videos() -> None:
  ):
  image_path = extracted_images_dir / f"{frame_idx}.jpg"
  image.save(image_path)
+ with open(image_path, "rb") as f:
+ encoded_image = base64.b64encode(f.read()).decode()
  results.append(
  [
  video_id,
  frame_idx,
  timestamp_secs,
- str(image_path.relative_to(REPO_ROOT)),
+ encoded_image,
  *clip_vector,
  ]
  )
  complete_file.touch()
  df = pd.DataFrame(
  results,
- columns=["video_id", "frame_idx", "timestamp", "image_path"]
+ columns=["video_id", "frame_idx", "timestamp", "base64_image"]
  + [f"dim_{i}" for i in range(MODEL_DIM)],
  )
  print(f"Saving data to {DATAFRAME_PATH}")

video_semantic_search/app.py

@@ -30,6 +30,7 @@ class SemanticSearcher:
  video_id=row["video_id"],
  frame_idx=row["frame_idx"],
  timestamp=row["timestamp"],
+ base64_image=row["base64_image"],
  score=score,
  )
  for score, (_, row) in zip(D[0], self.metadata.iloc[I[0]].iterrows())
@@ -45,10 +46,12 @@ class SearchResult:
  video_id: str
  frame_idx: int
  timestamp: float
+ base64_image: str
  score: float
 
 
 def get_video_url(video_id: str, timestamp: float) -> str:
+ timestamp = max(0, timestamp - 3) # Show 3 seconds before the query
  return f"https://www.youtube.com/watch?v={video_id}&t={int(timestamp)}"
 
 
@@ -87,19 +90,10 @@ def display_search_results(results: list[SearchResult]) -> None:
  """,
  unsafe_allow_html=True,
  )
-
- # Display the embedded YouTube video
- # st.video(get_video_url(result.video_id), start_time=int(result.timestamp))
- # st.image(f"data/images/{result.video_id}/{result.frame_idx}.jpg")
- with open(
- f"data/images/{result.video_id}/{result.frame_idx}.jpg", "rb"
- ) as f:
- image = f.read()
- encoded = base64.b64encode(image).decode()
  st.markdown(
  f"""
  <a href="{get_video_url(result.video_id, result.timestamp)}">
- <img src="data:image/jpeg;base64,{encoded}" alt="frame {result.frame_idx}" width="100%">
+ <img src="data:image/jpeg;base64,{result.base64_image}" alt="frame {result.frame_idx} timestamp {int(result.timestamp)}" width="100%">
  </a>
  """,
  unsafe_allow_html=True,