wip

Dockerfile

@@ -2,8 +2,20 @@ FROM node:20-alpine AS base
 
 # Install dependencies only when needed
 FROM base AS deps
+
 # Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
 RUN apk add --no-cache libc6-compat
+
+# for dev mode
+RUN apk add git git-lfs procps htop vim nano
+
+RUN apk add alpine-sdk pkgconfig
+
+# For FFMPEG and gl concat
+RUN apk add curl python3 python3-dev libx11-dev libsm-dev libxrender libxext-dev mesa-dev xvfb libxi-dev glew-dev
+
+RUN apk add ffmpeg
+
 WORKDIR /app
 
 # Install dependencies based on the preferred package manager

package-lock.json

@@ -66,6 +66,7 @@
         "gsplat": "^1.2.4",
         "hash-wasm": "^4.11.0",
         "jose": "^5.2.4",
+        "js-base64": "^3.7.7",
         "lodash.debounce": "^4.0.8",
         "lucide-react": "^0.260.0",
         "markdown-yaml-metadata-parser": "^3.0.0",
@@ -6087,6 +6088,11 @@
         "url": "https://github.com/sponsors/panva"
       }
     },
+    "node_modules/js-base64": {
+      "version": "3.7.7",
+      "resolved": "https://registry.npmjs.org/js-base64/-/js-base64-3.7.7.tgz",
+      "integrity": "sha512-7rCnleh0z2CkXhH67J8K1Ytz0b2Y+yxTPL+/KOJoa20hfnVQ/3/T6W/KflYI4bRHRagNeXeU2bkNGI3v1oS/lw=="
+    },
     "node_modules/js-sha3": {
       "version": "0.8.0",
       "resolved": "https://registry.npmjs.org/js-sha3/-/js-sha3-0.8.0.tgz",

package.json

@@ -67,6 +67,7 @@
     "gsplat": "^1.2.4",
     "hash-wasm": "^4.11.0",
     "jose": "^5.2.4",
+    "js-base64": "^3.7.7",
     "lodash.debounce": "^4.0.8",
     "lucide-react": "^0.260.0",
     "markdown-yaml-metadata-parser": "^3.0.0",

src/app/api/generators/search/getNewMediaInfo.ts

@@ -80,11 +80,9 @@ export function getNewMediaInfo(params: Partial<MediaInfo> = {}): MediaInfo {
     updatedAt: new Date().toISOString(),
 
     /**
-     * Arbotrary string tags to label the content
+     * Arbitrary string tags to label the content
      */
-    tags: Array.isArray(params.tags) ? [
-      ...params.tags,
-    ] : [],
+    tags: [],
 
     /**
      * Model name
@@ -139,7 +137,9 @@ export function getNewMediaInfo(params: Partial<MediaInfo> = {}): MediaInfo {
     /**
      * Media projection (cartesian by default)
      */
-    projection: "latent"
+    projection: "latent",
+
+    ...params,
   }
 
   return mediaInfo

src/app/api/providers/huggingface/predictWithHuggingFace.ts

@@ -31,7 +31,7 @@ export async function predict({
       }
     })) {
       instructions += output.token.text
-      // process.stdout.write(output.token.text)
+      process.stdout.write(output.token.text)
       if (
         instructions.includes("</s>") || 
         instructions.includes("<s>") ||

src/app/api/utils/parseRawStringToYAML.ts

@@ -8,8 +8,9 @@ export function parseRawStringToYAML<T>(input: any, defaultValue: T) {
     .replaceAll("```yaml\n", "")
     .replaceAll("```yaml", "")
 
-    // we remove everything after the last ```
-    rawString = rawString.split('```')[0].trim()
+    // we remove everything after the last ``` (or ``)
+    rawString = rawString.split(/```?/)[0].trim()
+
 
     const something: any = YAML.parse(rawString)
 

src/app/api/v1/create/route.ts

@@ -25,7 +25,7 @@ export async function POST(req: NextRequest) {
   
   const prompt = `${request?.prompt || ""}`.trim()
 
-  console.log("[api/generate/story] request:", request)
+  console.log("[api/v1/create] request:", request)
 
   if (!prompt.length) { throw new Error(`please provide a prompt`) }
 
@@ -46,11 +46,11 @@ export async function POST(req: NextRequest) {
     prefix: "```yaml\n",
   })
 
-  console.log("[api/generate/story] rawString: ", rawString)
+  console.log("[api/v1/create] rawString: ", rawString)
 
   const shots = parseRawStringToYAML<LatentStory[]>(rawString, [])
 
-  console.log(`[api/generate/story] generated ${shots.length} shots`)
+  console.log(`[api/v1/create] generated ${shots.length} shots`)
 
   // this is approximate - TTS generation will determine the final duration of each shot
   const defaultSegmentDurationInMs = 7000
@@ -76,7 +76,7 @@ export async function POST(req: NextRequest) {
 
   for (const { title, image, voice } of shots) {
 
-    console.log(`[api/generate/story]  - ${title}`)
+    console.log(`[api/v1/create]  - ${title}`)
 
     // note: it would be nice if we could have a convention saying that
     // track 0 is for videos and track 1 storyboards

src/app/api/v1/create/systemPrompt.ts

@@ -35,5 +35,5 @@ but if the user asks for large numbers, it should be ignored (our limit is 32).
   voice: "Now my dog is eating my birtday cake. Please send help."
 \`\`\
 
-# You turn!
+# Your turn:
 `

src/app/api/v1/edit/dialogues/processShot.ts

@@ -40,14 +40,16 @@ export async function processShot({
 
       const { durationInMs, durationInSec, hasAudio } = await getMediaInfo(shotDialogueSegment.assetUrl)
       
-      shotDialogueSegment.assetDurationInMs = durationInMs
-      shotSegment.assetDurationInMs = durationInMs
-
-      // we update the duration of all the segments for this shot
-      // (it is possible that this makes the two previous lines redundant)
-      clap.segments.filter(s => {
-        s.assetDurationInMs = durationInMs
-      })
+      if (hasAudio && durationInMs > 1000) {
+        shotDialogueSegment.assetDurationInMs = durationInMs
+        shotSegment.assetDurationInMs = durationInMs
+
+        // we update the duration of all the segments for this shot
+        // (it is possible that this makes the two previous lines redundant)
+        clap.segments.filter(s => {
+          s.assetDurationInMs = durationInMs
+        })
+      }
 
     } catch (err) {
       console.log(`[api/generate/dialogues] processShot: failed to generate audio: ${err}`)

src/app/api/v1/edit/storyboards/processShot.ts

@@ -41,7 +41,7 @@ export async function processShot({
       clap.segments.push(shotStoryboardSegment)
     }
 
-    console.log(`[api/generate/storyboards] processShot: generated storyboard segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
+    console.log(`[api/v1/edit/storyboards] processShot: generated storyboard segment [${shotSegment.startTimeInMs}:${shotSegment.endTimeInMs}]`)
   }
   if (!shotStoryboardSegment) { throw new Error(`failed to generate a newSegment`) }
 
@@ -49,12 +49,12 @@ export async function processShot({
   if (!shotStoryboardSegment?.prompt) {
     // storyboard is missing, let's generate it
     shotStoryboardSegment.prompt = getVideoPrompt(shotSegments, clap.entityIndex, ["high quality", "crisp", "detailed"])
-    console.log(`[api/generate/storyboards] processShot: generating storyboard prompt: ${shotStoryboardSegment.prompt}`)
+    console.log(`[api/v1/edit/storyboards] processShot: generating storyboard prompt: ${shotStoryboardSegment.prompt}`)
   }
 
   // TASK 3: GENERATE MISSING STORYBOARD BITMAP
   if (!shotStoryboardSegment.assetUrl) {
-    // console.log(`[api/generate/storyboards] generating image..`)
+    // console.log(`[api/v1/edit/storyboards] generating image..`)
 
     try {
       shotStoryboardSegment.assetUrl = await generateStoryboard({
@@ -64,13 +64,13 @@ export async function processShot({
       })
       shotStoryboardSegment.assetSourceType = getClapAssetSourceType(shotStoryboardSegment.assetUrl)
     } catch (err) {
-      console.log(`[api/generate/storyboards] processShot: failed to generate an image: ${err}`)
+      console.log(`[api/v1/edit/storyboards] processShot: failed to generate an image: ${err}`)
       throw err
     }
   
-    console.log(`[api/generate/storyboards] processShot: generated storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
+    console.log(`[api/v1/edit/storyboards] processShot: generated storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
   } else {
-    console.log(`[api/generate/storyboards] processShot: there is already a storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
+    console.log(`[api/v1/edit/storyboards] processShot: there is already a storyboard image: ${shotStoryboardSegment?.assetUrl?.slice?.(0, 50)}...`)
   }
 
 }

src/app/api/v1/edit/storyboards/route.ts

@@ -23,10 +23,10 @@ export async function POST(req: NextRequest) {
 
   if (!clap?.segments) { throw new Error(`no segment found in the provided clap!`) }
   
-  console.log(`[api/generate/storyboards] detected ${clap.segments.length} segments`)
+  console.log(`[api/v1/edit/storyboards] detected ${clap.segments.length} segments`)
   
   const shotsSegments: ClapSegment[] = clap.segments.filter(s => s.category === "camera")
-  console.log(`[api/generate/storyboards] detected ${shotsSegments.length} shots`)
+  console.log(`[api/v1/edit/storyboards] detected ${shotsSegments.length} shots`)
   
   if (shotsSegments.length > 32) {
     throw new Error(`Error, this endpoint being synchronous, it is designed for short stories only (max 32 shots).`)
@@ -40,7 +40,7 @@ export async function POST(req: NextRequest) {
     })
   ))
 
-  // console.log(`[api/generate/storyboards] returning the clap augmented with storyboards`)
+  // console.log(`[api/v1/edit/storyboards] returning the clap augmented with storyboards`)
 
   return new NextResponse(await serializeClap(clap), {
     status: 200,

src/app/api/v1/search/index.ts

@@ -0,0 +1,60 @@
+"use server"
+
+import YAML from "yaml"
+
+import { predict } from "@/app/api/providers/huggingface/predictWithHuggingFace"
+import { parseRawStringToYAML } from "@/app/api/utils/parseRawStringToYAML"
+
+import { systemPromptForBasicSearchResults, systemPromptForExtendedSearchResults } from "./systemPrompt"
+import type { BasicSearchResult, ExtendedSearchResult } from "./types"
+
+export async function search({
+  prompt = "",
+  nbResults = 4
+}: {
+  prompt: string
+  nbResults: number
+}): Promise<BasicSearchResult[]> {
+  const userPrompt = `${
+    Math.max(1, Math.min(8, nbResults))
+  } search results about: ${
+    prompt || "various trending genres"
+  }`
+
+  // TODO use streaming for the Hugging Face prediction
+  const rawString = await predict({
+    systemPrompt: systemPromptForBasicSearchResults,
+    userPrompt,
+    nbMaxNewTokens: nbResults * 80,
+    prefix: "```yaml\n",
+  })
+
+  console.log("rawString: ", rawString)
+
+  const results = parseRawStringToYAML<BasicSearchResult[]>(rawString, [])
+
+  return results
+}
+
+export async function extend({
+  basicResults = [],
+}: {
+  basicResults: BasicSearchResult[]
+}): Promise<ExtendedSearchResult[]> {
+  const userPrompt = YAML.stringify(basicResults)
+
+
+  // TODO use streaming for the Hugging Face prediction
+  const rawString = await predict({
+    systemPrompt: systemPromptForExtendedSearchResults,
+    userPrompt,
+    nbMaxNewTokens: basicResults.length * 200,
+    prefix: "```yaml\n",
+  })
+
+  console.log("rawString: ", rawString)
+
+  const results = parseRawStringToYAML<ExtendedSearchResult[]>(rawString, [])
+
+  return results
+}

src/app/api/v1/search/route.ts

@@ -0,0 +1,72 @@
+import { NextResponse, NextRequest } from "next/server"
+import queryString from "query-string"
+import { BasicSearchResult, ExtendedSearchResult } from "./types"
+import { extend, search } from "."
+
+export type LatentSearchMode =
+  | "basic"
+  | "extended"
+
+// we hide/wrap the micro-service under a unified AiTube API
+export async function GET(req: NextRequest, res: NextResponse) {
+
+  const qs = queryString.parseUrl(req.url || "")
+  const query = (qs || {}).query
+
+  let mode: LatentSearchMode = "basic"
+  try {
+    mode = decodeURIComponent(query?.m?.toString() || "basic").trim() as LatentSearchMode
+  } catch (err) {}
+
+
+  if (mode === "basic") {
+    let prompt = ""
+    try {
+      prompt = decodeURIComponent(query?.p?.toString() || "").trim() as string
+    } catch (err) {}
+    
+    const basicSearchResults: BasicSearchResult[] = await search({
+      prompt,
+      nbResults: 4
+    })
+
+    console.log(`[api/v1/search] found ${basicSearchResults.length} basic search results`)
+    console.log(`[api/v1/search]`, basicSearchResults)
+
+    return NextResponse.json(basicSearchResults, {
+      status: 200,
+      statusText: "OK",
+    })
+  } else if (mode === "extended") {
+
+    let basicResults: BasicSearchResult[] = []
+    try {
+      const rawString = decodeURIComponent(query?.e?.toString() || "").trim() as string
+      const maybeExistingResults = JSON.parse(rawString)
+      if (Array.isArray(maybeExistingResults)) {
+        basicResults = maybeExistingResults
+      }
+    } catch (err) {}
+
+    const extendedSearchResults: ExtendedSearchResult[] = await extend({
+      basicResults
+    })
+
+    console.log(`[api/v1/search] extended ${extendedSearchResults.length} search results`)
+
+    console.log(`[api/v1/search]`, extendedSearchResults)
+
+    return NextResponse.json(extendedSearchResults, {
+      status: 200,
+      statusText: "OK",
+    })
+  } else {
+    /*
+    return NextResponse.json([], {
+      status: 200,
+      statusText: "OK",
+    })
+    */
+    throw new Error(`Please specify the mode.`)
+  }
+}

src/app/api/v1/search/systemPrompt.ts

@@ -0,0 +1,92 @@
+export const systemPromptForBasicSearchResults: string =
+  `# Context
+You are a server-side function generating search results from a single text input (a "prompt").
+You have a database of new, never-seen-before videos, series, tv shows, movies, documentaries, music videos, found footage, news videos, videos created by influencers, game review, trailers of all kind, tutorials, education videos, user-made short videos etc.. anything you could find on both Netflix or YouTube, really.
+You do not contain any existing IP (intellectual property) content.
+
+# Task
+Your mission is to generate a sequence of search results that closely match the demand by the user.
+
+You will be provided a "prompt" (for the story) and max number of results.
+
+Each result object is composed of:
+- title: title of the video
+- tags: to describe the categories and genres
+# Examples
+
+You most reply by writing/completing a YAML list of objects.
+Here is a short example, the prompt was "4 search results about: cows"
+Note how we asked for "4 search results" for previty, but it's possible that it will omitted. You should return 5 results by default.
+but if the user asks for large numbers, it should be ignored (our limit is 8).
+If the user ask for a specific cateogry eg "<something> recipes" obviously you should only show recipes to cook the thing et.
+Try to be imaginative!
+
+\`\`\`
+- title: "The Cows"
+  tags: ["feature film", "drama", "coming of age"]
+- title: "Happy cows 🐮 mooing and grazing 🌱"
+  tags: ["short video", "nature", "community"]
+- title: "Ganja Dog - Got Milk?"
+  tags: ["music", "music video", "hip-hop"]
+- title: "How Cows Work"
+  tags: ["short video", "education", "influencer"]
+\`\`\
+
+# Your turn:
+`
+
+export const systemPromptForExtendedSearchResults: string =
+  `# Context
+You are a server-side function generating search results from partial results.
+You have a database of new, never-seen-before videos, series, tv shows, movies, documentaries, music videos, found footage, news videos, videos created by influencers, game review, trailers of all kind, tutorials, education videos, user-made short videos etc.. anything you could find on both Netflix or YouTube, really.
+You do not contain any existing IP (intellectual property) content.
+
+# Task
+Your mission is to generate a sequence of search results that best complements and extends the partial results.
+
+You will be provided those partial results as YAML, and you need to return YAML.
+
+An extended search result should contain those fields:
+- title: title of the video
+- cover: you must describe it using a Stable Diffusion prompt - about ~300 characters - using simple descriptive words and adjectives. Describe facts about characters, location, lights, texture, camera orientation, colors, clothes, movements etc. But don't give your opinion, don't talk about the emotions it evokes etc.
+- description: in 2 or 3 sentences please describe the genre, category, visual style, synopsis (You must only describe the content, so don't add any info about the director, author, release year.)
+- tags: to describe the categories and genres
+# Examples
+
+You most reply by writing/completing a YAML list of objects.
+Here is a short example, using this data as input:
+
+\`\`\`
+- title: "The Cows"
+  tags: ["feature film", "drama", "coming of age"]
+- title: "Happy cows 🐮 mooing and grazing 🌱"
+  tags: ["short video", "nature", "community"]
+- title: "Ganja Dog - Got Milk?"
+  tags: ["music", "music video", "hip-hop"]
+- title: "How Cows Work"
+  tags: ["short video", "education", "influencer"]
+\`\`\
+
+And here is one of the many possibilities:
+
+\`\`\`
+- title: "The Cows"
+  description: "a drama about Pete, young man growing up in a farm. His mom died, and his dad (who is sick) expects him to take over the cow farm, but Pete doesn't want to, and dreams of a different life, in the city. He often goes to the city at night, meet girls, but when he comes back late, and unable to work properly in the morning (when he makes mistake, forget to feed the cows etc) his dad grows angrier. Pete doesn't know his dad is sick, though. Near the end, the dad dies, and Pete will have to make some difficult life-changing decisions."
+  cover: "poster of a movie called “The Cows”, mysterious image of a farmer, alone in a grass field, in the morning, cows in the background, sad, angry, mist, award-winning, film poster, movie poster"
+  tags: ["feature film", "drama", "coming of age"]
+- title: "Happy cows 🐮 mooing and grazing 🌱"
+  description: various 4K footage of cows grazing, walking, mooing, in other words being happy. 
+  cover: "well-lit photo of cows, grazing in a field, peaceful, instagram, award winning"
+  tags: ["short video", "nature", "community"]
+- title: "Ganja Dog - Got Milk?"
+  description: "a music video by Ganja Dog, a dog who is a famous rapper and influencer. The clip is shot in a farm, with Ganja Dog rapping and dancing, showing off its cars, clothes, with various people. There are various dramatic camera effects. The lyrics should be funny eg “Witch, I'm a Cow, I go Moo..”, “I'm In the Mood For Some Milk Yo” etc"
+  cover: "medium-shot of a dog rapper, showing off his dollars, in a cow farm, rapping, rich, cows, dogs, puppies, trending, influencer, dramatic, music video, rapping"
+  tags: ["music", "music video", "hip-hop"]
+- title: "How Cows Work"
+  description: "a video explaining how cows work, made using cheap stock footage and an AI voice-over. It should show the history of cows, how they evolved, why human use them, how a farm work, how grass gets turned into milk, differences in breeds, the economics etc"
+  cover: "picture of a cow, drawn like a blueprint, cow factory, cow machine, clickbait youtube thumbnail made by an influencer, overly dramatic, unrealistic, fake, dramatic, overplayed, too much"
+  tags: ["short video", "education", "influencer"]
+\`\`\
+
+# Your turn:
+`

src/app/api/v1/search/types.ts

@@ -0,0 +1,16 @@
+
+// for latent search we need to get rough results fast
+export type BasicSearchResult = {
+  // note: the absence of ID means this is a new, latent result
+  id?: string
+  title: string
+  tags: string[]
+}
+
+export type ExtendedSearchResult = {
+  id?: string
+  title: string
+  description: string
+  cover: string
+  tags: string[]
+}

src/app/{dream/spoiler.tsx → latent/search/page.tsx}

@@ -1,13 +1,15 @@
-import { LatentQueryProps } from "@/types/general"
+import { encode, decode } from 'js-base64'
 import { clapToDataUri, generateClapFromSimpleStory } from "@aitube/clap"
 
-import { Main } from "../main"
-import { getNewMediaInfo } from "../api/generators/search/getNewMediaInfo"
-import { getToken } from "../api/auth/getToken"
+import { LatentQueryProps } from "@/types/general"
+
+import { Main } from "../../main"
+import { getNewMediaInfo } from "../../api/generators/search/getNewMediaInfo"
+import { getToken } from "../../api/auth/getToken"
 
 // https://jmswrnr.com/blog/protecting-next-js-api-routes-query-parameters
 
-export default async function DreamPage({
+export default async function LatentSearchPage({
   searchParams: {
     l: latentContent,
   },
@@ -15,7 +17,8 @@ export default async function DreamPage({
 }: LatentQueryProps) {
   const jwtToken = await getToken({ user: "anonymous" })
 
-  // const latentSearchResult = JSON.parse(atob(`${latentContent}`)) as LatentSearchResult
+
+  // const latentSearchResult = JSON.parse(decodee(`${latentContent}`)) as LatentSearchResult
 
   // this will hallucinate the thumbnail on the fly - maybe we should cache it
   // const latentMedia = await searchResultToMediaInfo(latentSearchResult)

src/app/latent/watch/page.tsx

@@ -0,0 +1,63 @@
+import { encode, decode } from 'js-base64'
+
+import { LatentQueryProps } from "@/types/general"
+import { BasicSearchResult, ExtendedSearchResult } from "@/app/api/v1/search/types"
+
+import { Main } from "../../main"
+import { getNewMediaInfo } from "../../api/generators/search/getNewMediaInfo"
+import { getToken } from "../../api/auth/getToken"
+
+import { extend } from "@/app/api/v1/search"
+
+// https://jmswrnr.com/blog/protecting-next-js-api-routes-query-parameters
+
+export default async function DreamPage({
+  searchParams: {
+    p: prompt,
+  },
+  ...rest
+}: LatentQueryProps) {
+  const jwtToken = await getToken({ user: "anonymous" })
+  console.log(`[/latent/watch] prompt =`, prompt)
+  let basicResult = JSON.parse(decode(`${prompt || ""}`)) as BasicSearchResult
+  
+  console.log("[/latent/watch] basicResult:", basicResult)
+
+  // note that we should generate a longer synopsis from the autocomplete result
+  //
+  // however that is a slow process, maybe not great for a server-side rendering task,
+  // so idk
+  const extendedResults: ExtendedSearchResult[] = await extend({
+    basicResults: [ basicResult ]
+  })
+  console.log(`[/latent/watch] extendedResults =`, extendedResults)
+
+  const extendedResult = extendedResults.at(0)
+
+  if (!extendedResult || !Array.isArray(extendedResult.tags)) {
+    console.error(`failed to generated an extended result, aborting`)
+    throw new Error(`Server error`)
+  }
+  // const latentSearchResult = JSON.parse(decode(`${latentContent}`)) as LatentSearchResult
+
+  // TODO: we should hallucinate the thumbnail at this stage, and on the fly
+  // this is useful to do it on the server-side so we can share the link on social media etc
+  //
+  // maybe we should cache the image
+  // const latentMedia = await searchResultToMediaInfo(latentSearchResult)
+
+  const latentMedia = getNewMediaInfo({
+    label: extendedResult.title,
+    description: extendedResult.description,
+    prompt: extendedResult.description,
+    tags: [...extendedResult.tags],
+  })
+
+  console.log(`[/latent/watch] generated media: `, latentMedia)
+  // now, generating the .clap is another story, it will be much more intensive
+  // so we will generate it later in async, in the client-side
+
+  return (
+    <Main latentMedia={latentMedia} jwtToken={jwtToken} />
+  )
+}

src/app/main.tsx

@@ -19,6 +19,7 @@ import { PublicMediaEmbedView } from "./views/public-media-embed-view"
 import { PublicMediaView } from "./views/public-media-view"
 import { PublicLatentMediaEmbedView } from "./views/public-latent-media-embed-view"
 import { PublicLatentMediaView } from "./views/public-latent-media-view"
+import { PublicLatentSearchView } from "./views/public-latent-search-view"
 
 // this is where we transition from the server-side space
 // and the client-side space
@@ -80,6 +81,8 @@ export function Main({
   const setPublicTracks = useStore(s => s.setPublicTracks)
   const setPublicTrack = useStore(s => s.setPublicTrack)
 
+  console.log("[main.tsx] latentMedia = ", latentMedia)
+  
   useEffect(() => {
     if (typeof jwtToken !== "string" && !jwtToken) { return }
     setJwtToken(jwtToken)
@@ -153,13 +156,13 @@ export function Main({
     console.log(latentMedia)
     setPublicLatentMedia(latentMedia)
     if (!latentMedia || !latentMedia?.id) { return }
-    if (pathname === "/dream/embed") { return }
-    if (pathname !== "/dream") {
+    if (pathname === "/latent/embed") { return }
+    if (pathname !== "/latent/watch") {
       // console.log("we are on huggingface apparently!")
       // router.replace(`/watch?v=${publicMedia.id}`)
 
       // TODO: add params in the URL to represent the latent result
-      router.replace(`/dream`)
+      router.replace(`/latent/watch`)
     }
   }, [latentMedia?.id])
 
@@ -196,6 +199,7 @@ export function Main({
       {view === "public_media" && <PublicMediaView />}
 
       {/* latent content is the content that "doesn't exist" (is generated by the AI) */}
+      {view === "public_latent_search" && <PublicLatentSearchView />}
       {view === "public_latent_media_embed" && <PublicLatentMediaEmbedView />}
       {view === "public_latent_media" && <PublicLatentMediaView />}
 

src/app/state/useStore.ts

@@ -3,6 +3,7 @@
 import { create } from "zustand"
 
 import { ChannelInfo, MediaInfo, InterfaceDisplayMode, InterfaceView, InterfaceMenuMode, InterfaceHeaderMode, CommentInfo, UserInfo } from "@/types/general"
+import { BasicSearchResult } from "../api/v1/search/types"
 
 export const useStore = create<{
   displayMode: InterfaceDisplayMode
@@ -31,8 +32,8 @@ export const useStore = create<{
   searchAutocompleteQuery: string
   setSearchAutocompleteQuery: (searchAutocompleteQuery?: string) => void
 
-  searchAutocompleteResults: MediaInfo[]
-  setSearchAutocompleteResults: (searchAutocompleteResults: MediaInfo[]) => void
+  searchAutocompleteResults: BasicSearchResult[]
+  setSearchAutocompleteResults: (searchAutocompleteResults: BasicSearchResult[]) => void
 
   searchResults: MediaInfo[]
   setSearchResults: (searchResults: MediaInfo[]) => void
@@ -118,8 +119,9 @@ export const useStore = create<{
       "/embed": "public_media_embed",
       "/music": "public_music_videos",
       "/channels": "public_channels",
-      "/dream": "public_latent_media",
-      "/dream/embed": "public_latent_media_embed",
+      "/latent/search": "public_latent_search",
+      "/latent/watch": "public_latent_media",
+      "/latent/embed": "public_latent_media_embed",
       "/channel": "public_channel",
 
       // those are reserved for future use
@@ -149,8 +151,8 @@ export const useStore = create<{
     set({ showAutocompleteBox })
   },
 
-  searchAutocompleteResults: [] as MediaInfo[],
-  setSearchAutocompleteResults: (searchAutocompleteResults: MediaInfo[]) => {
+  searchAutocompleteResults: [] as BasicSearchResult[],
+  setSearchAutocompleteResults: (searchAutocompleteResults: BasicSearchResult[]) => {
     set({ searchAutocompleteResults })
   },
 

src/app/views/public-latent-search-view/index.tsx

@@ -0,0 +1,49 @@
+"use client"
+
+import { useEffect, useTransition } from "react"
+
+import { useStore } from "@/app/state/useStore"
+import { cn } from "@/lib/utils/cn"
+import { MediaInfo } from "@/types/general"
+import { getVideos } from "@/app/api/actions/ai-tube-hf/getVideos"
+import { VideoList } from "@/components/interface/video-list"
+
+export function PublicLatentSearchView() {
+  const [_isPending, startTransition] = useTransition()
+  const setView = useStore(s => s.setView)
+  const currentTag = useStore(s => s.currentTag)
+  const setPublicMedias = useStore(s => s.setPublicMedias)
+  const setPublicMedia = useStore(s => s.setPublicMedia)
+  const publicMedias = useStore(s => s.publicMedias)
+
+  useEffect(() => {
+    startTransition(async () => {
+      const medias = await getVideos({
+        sortBy: "date",
+        mandatoryTags: currentTag ? [currentTag] : [],
+        maxNbMedias: 25
+      })
+
+      // due to some caching on the first function.. we update with fresh data!
+      // const updatedVideos = await extendVideosWithStats(medias)
+
+      setPublicMedias(medias)
+    })
+  }, [currentTag])
+
+  const handleSelect = (media: MediaInfo) => {
+    setView("public_latent_media")
+    setPublicMedia(media)
+  }
+
+  return (
+    <div className={cn(
+     `sm:pr-4`
+    )}>
+      <VideoList
+        items={publicMedias}
+        onSelect={handleSelect}
+      />
+    </div>
+  )
+}

src/components/interface/latent-engine/core/engine.tsx

@@ -14,7 +14,7 @@ import { localStorageKeys } from "@/app/state/localStorageKeys"
 import { defaultSettings } from "@/app/state/defaultSettings"
 import { useStore } from "@/app/state/useStore"
 import { ClapProject, generateClapFromSimpleStory, serializeClap } from "@aitube/clap"
-import { theSimps } from "@/app/dream/samples"
+import { theSimps } from "@/app/latent/samples"
 
 function LatentEngine({
   media,
@@ -76,29 +76,27 @@ function LatentEngine({
   const videoLayerRef = useRef<HTMLDivElement>(null)
   const segmentationLayerRef = useRef<HTMLDivElement>(null)
 
-  const mediaUrl = media.clapUrl || media.assetUrlHd || media.assetUrl
-
   useEffect(() => {
-    if (!stateRef.current.isInitialized && mediaUrl) {
+    if (!stateRef.current.isInitialized) {
       stateRef.current.isInitialized = true
 
       const fn = async () => {
-        // TODO julian
-        // there is a bug, we can't unpack the .clap when it's from a data-uri :/
+        // if we have a clapUrl (eg. from the database) then we use that
+        // otherwise we generate it ourselves, chunk by chunk (since we're live)
+
+        // TODO Julian work on the chunk mechanism
         
-        // open(mediaUrl)
         const mockClap: ClapProject = generateClapFromSimpleStory({
-          story: theSimps
+          story: theSimps,
+          showIntroPoweredByEngine: false,
+          showIntroDisclaimerAboutAI: false
         })
-        const mockArchive: Blob = await serializeClap(mockClap)
-        // for some reason conversion to data uri doesn't work
-        // const mockDataUri = await blobToDataUri(mockArchive, "application/x-gzip")
-        // console.log("mockDataUri:", mockDataUri)
-        open(mockArchive)
+
+        open(mockClap)
       }
       fn()
     }
-  }, [mediaUrl])
+  }, [media.id])
 
   const isPlayingRef = useRef(isPlaying)
   isPlayingRef.current = isPlaying

src/components/interface/latent-search-input/index.tsx

@@ -0,0 +1,157 @@
+import { useRef, useState, useTransition } from "react"
+import Link from "next/link"
+import { encode, decode } from 'js-base64'
+// import throttle from "@jcoreio/async-throttle"
+import debounce from "lodash.debounce"
+import { GoSearch } from "react-icons/go"
+
+import { useStore } from "@/app/state/useStore"
+import { cn } from "@/lib/utils/cn"
+import { Input } from "@/components/ui/input"
+import { Button } from "@/components/ui/button"
+import { MediaInfo } from "@/types/general"
+import { search } from "@/app/api/v1/search"
+
+export function LatentSearchInput() {
+  const [_pending, startTransition] = useTransition()
+
+  const setSearchAutocompleteQuery = useStore(s => s.setSearchAutocompleteQuery)
+  const showAutocompleteBox = useStore(s => s.showAutocompleteBox)
+  const setShowAutocompleteBox = useStore(s => s.setShowAutocompleteBox)
+  
+  const searchAutocompleteResults = useStore(s => s.searchAutocompleteResults)
+  const setSearchAutocompleteResults = useStore(s => s.setSearchAutocompleteResults)
+
+  const setSearchQuery = useStore(s => s.setSearchQuery)
+
+  const [searchDraft, setSearchDraft] = useState("")
+
+  const ref = useRef<HTMLInputElement>(null)
+
+
+  // called when pressing enter or clicking on search
+  const debouncedSearch = debounce((query: string) => {
+    startTransition(async () => {
+      console.log(`searching the latent space for "${query}"..`)
+
+      if (query.length < 2) { console.log("search term is too short") }
+      
+      console.log("imaginating medias..")
+      
+      const imaginedMedias = await search({
+        prompt: query,
+        nbResults: 4
+      })
+
+      console.log(`imagined ${imaginedMedias.length} results:`, imaginedMedias)
+
+      setSearchAutocompleteResults(imaginedMedias.map(item => ({
+        title: item.title,
+        tags: item.tags,
+      })))
+      
+      // TODO: only close the show autocomplete box if we found something
+      // setShowAutocompleteBox(false)
+    })
+  }, 500)
+
+  // called when pressing enter or clicking on search
+  const handleSearch = () => {
+    ref.current?.focus()
+    setSearchQuery(searchDraft)
+    setShowAutocompleteBox(true)
+    debouncedSearch(searchDraft)
+  }
+
+  return (
+    <div className="flex flex-row flex-grow w-[380px] lg:w-[600px]">
+
+      <div className="flex flex-row w-full">
+        <Input
+          ref={ref}
+          placeholder="Search the latent space"
+          className={cn(
+            `bg-neutral-900 text-neutral-200 dark:bg-neutral-900 dark:text-neutral-200`,
+            `rounded-l-full rounded-r-none`,
+            
+            // we increase the line height to better catch the clicks
+            `py-0 h-10 leading-7`,
+
+            `border-neutral-700 dark:border-neutral-700 border-r-0`,
+    
+          )}
+          onFocus={() => {
+            handleSearch()
+          }}
+          onChange={(e) => {
+            setSearchDraft(e.target.value)
+            setShowAutocompleteBox(true)
+            // handleSearch()
+          }}
+          onKeyDown={({ key }) => {
+            if (key === 'Enter') {
+              handleSearch()
+            }
+          }}
+          value={searchDraft}
+        />
+        <Button
+        className={cn(
+          `rounded-l-none rounded-r-full border border-neutral-700 dark:border-neutral-700`,
+          `cursor-pointer`,
+          `transition-all duration-200 ease-in-out`,
+          `text-neutral-200 dark:text-neutral-200 bg-neutral-800 dark:bg-neutral-800 hover:bg-neutral-700 disabled:bg-neutral-900`
+          )}
+          onClick={() => {
+            handleSearch()
+            // console.log("submit")
+            // setShowAutocompleteBox(false)
+            // setSearchDraft("")
+          }}
+        disabled={false}
+      >
+        <GoSearch className="w-6 h-6" />
+      </Button>
+    </div>
+    <div
+      className={cn(
+        `absolute z-50 ml-1`,
+
+        // please keep this in sync with the parent
+        `w-[320px] lg:w-[540px]`,
+
+        `text-neutral-200 dark:text-neutral-200 bg-neutral-900 dark:bg-neutral-900`,
+        `border border-neutral-800 dark:border-neutral-800`,
+        `rounded-xl shadow-2xl`,
+        `flex flex-col p-2 space-y-1`,
+        
+        `transition-all duration-200 ease-in-out`,
+        showAutocompleteBox
+          ? `opacity-100 scale-100 mt-11 pointer-events-auto`
+          : `opacity-0 scale-95 mt-6 pointer-events-none`
+      )}
+    >
+      {searchAutocompleteResults.length === 0 ? <div>Nothing to show, type something and press enter</div> : null}
+        {searchAutocompleteResults.map(item => (
+          <Link key={item.id} href={
+            item.id
+            ? `${process.env.NEXT_PUBLIC_DOMAIN}/watch?v=${item.id}`
+            : `${process.env.NEXT_PUBLIC_DOMAIN}/latent/watch?p=${encode(JSON.stringify(item))}`
+            }>
+            <div
+              className={cn(
+                `dark:hover:bg-neutral-800 hover:bg-neutral-800`,
+                `text-sm`,
+                `px-3 py-2`,
+                `rounded-xl`
+              )}
+
+            >
+              {item.title}
+            </div>
+          </Link>
+        ))}
+    </div>
+  </div>
+  )
+}

src/components/interface/media-player/index.tsx

@@ -25,7 +25,6 @@ export function MediaPlayer({
   console.log("MediaPlayer called for \"" + media?.label + "\"")
   
   if (!media) { return null }
-  if (!media?.assetUrl && !media?.clapUrl) { return null }
 
   // uncomment one of those to forcefully test the .clap player from an external .clap file
   // media.assetUrlHd = "https://huggingface.co/datasets/jbilcke/ai-tube-cinema/tree/main/404.clap"
@@ -48,6 +47,8 @@ export function MediaPlayer({
     )
   }
 
+  if (!media?.assetUrl && !media?.assetUrlHd) { return null }
+
   if (projectionType === "gaussian") {
     // note: for AutoSizer to work properly it needs to be inside a normal div with no display: "flex"
     return (

src/components/interface/search-input/index.tsx

@@ -1,5 +1,6 @@
 import { useRef, useState, useTransition } from "react"
 import Link from "next/link"
+import { encode, decode } from 'js-base64'
 // import throttle from "@jcoreio/async-throttle"
 import debounce from "lodash.debounce"
 import { GoSearch } from "react-icons/go"
@@ -32,7 +33,7 @@ export function SearchInput() {
     startTransition(async () => {
       console.log(`searching for "${query}"..`)
 
-      const videos = await getVideos({
+      const medias = await getVideos({
         query,
         sortBy: "match",
         maxNbMedias: 8,
@@ -40,8 +41,14 @@ export function SearchInput() {
         renewCache: false, // bit of optimization
       })
 
-      console.log(`got ${videos.length} results!`)
-      setSearchAutocompleteResults(videos)
+      console.log(`got ${medias.length} results!`)
+      setSearchAutocompleteResults(medias.map(item => ({
+        id: item.id,
+        title: item.label,
+        tags: item.tags,
+      })))
+      
+
 
       // TODO: only close the show autocomplete box if we found something
       // setShowAutocompleteBox(false)
@@ -125,8 +132,12 @@ export function SearchInput() {
       )}
     >
       {searchAutocompleteResults.length === 0 ? <div>Nothing to show, type something and press enter</div> : null}
-        {searchAutocompleteResults.map(media => (
-          <Link key={media.id} href={`${process.env.NEXT_PUBLIC_DOMAIN}/watch?v=${media.id}`}>
+        {searchAutocompleteResults.map(item => (
+          <Link key={item.id} href={
+            item.id
+            ? `${process.env.NEXT_PUBLIC_DOMAIN}/watch?v=${item.id}`
+            : `${process.env.NEXT_PUBLIC_DOMAIN}/latent/watch?p=${encode(JSON.stringify(item))}`
+            }>
             <div
               className={cn(
                 `dark:hover:bg-neutral-800 hover:bg-neutral-800`,
@@ -136,7 +147,7 @@ export function SearchInput() {
               )}
 
             >
-              {media.label}
+              {item.title}
             </div>
           </Link>
         ))}

src/components/interface/top-header/index.tsx

@@ -11,6 +11,7 @@ import Link from 'next/link'
 import { Input } from '@/components/ui/input'
 import { Button } from '@/components/ui/button'
 import { SearchInput } from '../search-input'
+import { LatentSearchInput } from '../latent-search-input'
 import { pathway } from '@/lib/fonts'
 
 export function TopHeader() {
@@ -122,7 +123,11 @@ export function TopHeader() {
           `px-4 py-2 w-max-64`,
           `text-neutral-400 text-2xs sm:text-xs lg:text-sm italic`
         )}>
-          <SearchInput />
+          {view === "public_latent_search" ||
+          view === "public_latent_media" ||
+          view === "public_latent_media_embed" 
+          ? <LatentSearchInput />
+          : <SearchInput />}
         </div>
         <div className={cn("w-32 xl:w-42")}>
           <span>

src/types/general.ts

@@ -636,6 +636,7 @@ export type InterfaceView =
   | "public_media" // public view of an individual media (video, gaussian splat, clap video)
   | "public_media_embed" // for integration into twitter etc
   | "public_music_videos" // public music videos - it's a special category, because music is *cool*
+  | "public_latent_search" // public view of latent search results
   | "public_latent_media" // public view of an individual dream (a latent media, so it's not a "real" file)
   | "public_latent_media_embed" // for integration into twitter etc (which would be hardcore for our server load.. so maybe not)
   | "public_gaming" // for AiTube Gaming
@@ -712,7 +713,7 @@ export type AppQueryProps = {
 export type LatentQueryProps = {
   params: { jwtToken: string }
   searchParams: {
-    l?: string | string[], // latent content (serialized to a base64 json)
+    p?: string | string[], // search result content (serialized to a base64 json)
     [key: string]: string | string[] | undefined
   }
 }