let's use the AiTube API

package-lock.json

@@ -8,6 +8,8 @@
       "name": "@jbilcke-hf/ai-stories-factory",
       "version": "0.0.0",
       "dependencies": {
+        "@aitube/clap": "^0.0.3",
+        "@aitube/client": "^0.0.0",
         "@radix-ui/react-accordion": "^1.1.2",
         "@radix-ui/react-avatar": "^1.0.4",
         "@radix-ui/react-checkbox": "^1.0.4",
@@ -66,6 +68,31 @@
         "node": ">=0.10.0"
       }
     },
+    "node_modules/@aitube/clap": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/@aitube/clap/-/clap-0.0.3.tgz",
+      "integrity": "sha512-lmCUP3ElK+/WFU00ieRVHMSpiTy6X0I1wDngETJ4NZQIso+KVvlVsLrSfuJZj5cIB6fafpN7UGYDcyE956uS7w==",
+      "dependencies": {
+        "uuid": "^9.0.1",
+        "yaml": "^2.4.1"
+      },
+      "peerDependencies": {
+        "typescript": "^5.4.5"
+      }
+    },
+    "node_modules/@aitube/client": {
+      "version": "0.0.0",
+      "resolved": "https://registry.npmjs.org/@aitube/client/-/client-0.0.0.tgz",
+      "integrity": "sha512-xaHZ2SWUD04MHdLv1P1V4i8AVceld6ECyw9X3KoFvOX0H2/qDBySD6NMB/9GrCa+ASebEsQ8WGzXZGQIgdeCVQ==",
+      "dependencies": {
+        "@aitube/clap": "^0.0.3",
+        "uuid": "^9.0.1",
+        "yaml": "^2.4.1"
+      },
+      "peerDependencies": {
+        "typescript": "^5.4.5"
+      }
+    },
     "node_modules/@alloc/quick-lru": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/@alloc/quick-lru/-/quick-lru-5.2.0.tgz",

package.json

@@ -9,6 +9,8 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@aitube/clap": "^0.0.3",
+    "@aitube/client": "^0.0.0",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.4",
     "@radix-ui/react-checkbox": "^1.0.4",

src/app/main.tsx

@@ -1,6 +1,7 @@
 "use client"
 
 import React, { useEffect, useRef, useState, useTransition } from 'react'
+import { ClapProject } from '@aitube/clap'
 
 import { Card, CardContent, CardHeader } from '@/components/ui/card'
 import { Button } from '@/components/ui/button'
@@ -12,7 +13,6 @@ import { TextareaField } from '@/components/form/textarea-field'
 import { DeviceFrameset } from 'react-device-frameset'
 import 'react-device-frameset/styles/marvel-devices.min.css'
 import { generateClap } from './server/aitube/generateClap'
-import { ClapOutputType, ClapProject } from '@/lib/clap/types'
 import { extendClap } from './server/aitube/extendClap'
 import { exportClap } from './server/aitube/exportClap'
 

src/app/server/{config.ts → aitube/config.ts}

@@ -3,8 +3,3 @@ export const serverHuggingfaceApiKey = `${process.env.HF_API_TOKEN || ""}`
 
 export const aitubeUrl = `${process.env.AI_TUBE_URL || "" }`
 export const aitubeApiUrl = aitubeUrl + (aitubeUrl.endsWith("/") ? "" : "/") + "api/"
-
-console.log("debug:", {
-  aitubeUrl,
-  aitubeApiUrl,
-})

src/app/server/aitube/exportClap.ts

@@ -1,35 +1,12 @@
 "use server"
 
-import { ClapProject } from "@/lib/clap/types"
-import { serializeClap } from "@/lib/clap/serializeClap"
-import { blobToDataUri } from "@/lib/base64/blobToDataUri"
-
-import { aitubeApiUrl } from "../config"
+import { ClapProject } from "@aitube/clap"
+import * as aitube from "@aitube/client"
 
 export async function exportClap({
   clap,
 }: {
   clap: ClapProject
 }): Promise<string> {
-  if (!clap) { throw new Error(`please provide a clap`) }
-
-  // remember: a space needs to be public for the classic fetch() to work
-  const res = await fetch(`${aitubeApiUrl}generate/video`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      // TODO pass the JWT so that only the AI Stories Factory can call the API
-      // Authorization: `Bearer ${hfApiToken}`,
-    },
-    body: await serializeClap(clap),
-    cache: "no-store",
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-  })
-
-  const blob = await res.blob()
-
-  const dataURL = await blobToDataUri(blob)
-  
-  return dataURL
+  return aitube.exportClap({ clap })
 }

src/app/server/aitube/extendClap.ts

@@ -1,35 +1,12 @@
 "use server"
 
-import { parseClap } from "@/lib/clap/parseClap"
-import { ClapProject } from "@/lib/clap/types"
-
-import { aitubeApiUrl } from "../config"
-import { serializeClap } from "@/lib/clap/serializeClap"
+import { ClapProject } from "@aitube/clap"
+import * as aitube from "@aitube/client"
 
 export async function extendClap({
   clap,
 }: {
   clap: ClapProject
 }): Promise<ClapProject> {
-  if (!clap) { throw new Error(`please provide a prompt`) }
-
-  // remember: a space needs to be public for the classic fetch() to work
-  const res = await fetch(`${aitubeApiUrl}generate/storyboards`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      // TODO pass the JWT so that only the AI Stories Factory can call the API
-      // Authorization: `Bearer ${hfApiToken}`,
-    },
-    body: await serializeClap(clap),
-    cache: "no-store",
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-  })
-
-  const blob = await res.blob()
-
-  const extendedClap = await parseClap(blob)
-  
-  return extendedClap
+  return aitube.extendClap({ clap })
 }

src/app/server/aitube/generateClap.ts

@@ -1,44 +1,12 @@
 "use server"
 
-import { parseClap } from "@/lib/clap/parseClap"
-import { ClapProject } from "@/lib/clap/types"
-
-import { aitubeApiUrl } from "../config"
+import { ClapProject } from "@aitube/clap"
+import * as aitube from "@aitube/client"
 
 export async function generateClap({
   prompt = "",
 }: {
   prompt: string
 }): Promise<ClapProject> {
-  if (!prompt) { throw new Error(`please provide a prompt`) }
-
-  // AiTube Stories is nice, but we also need to leave some compute for AiTube Live and AiTube Gaming
-  const height = 1024
-  const width = 512
-
-  // console.log(`calling `+ gradioApi + (gradioApi.endsWith("/") ? "" : "/") + "api/predict")
-
-  // remember: a space needs to be public for the classic fetch() to work
-  const res = await fetch(`${aitubeApiUrl}generate/story`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      // TODO pass the JWT so that only the AI Stories Factory can call the API
-      // Authorization: `Bearer ${hfApiToken}`,
-    },
-    body: JSON.stringify({
-      prompt,
-      width,
-      height
-    }),
-    cache: "no-store",
-    // we can also use this (see https://vercel.com/blog/vercel-cache-api-nextjs-cache)
-    // next: { revalidate: 1 }
-  })
-
-  const blob = await res.blob()
-
-  const clap = await parseClap(blob)
-  
-  return clap
+  return aitube.generateClap({ prompt })
 }

src/lib/clap/clap-specification-draft.md

@@ -1,162 +0,0 @@
-# CLAP Format Specification
-
-- Status: DRAFT
-- Document revision: 0.0.1
-- Last updated: Feb 6th, 2024
-- Author(s): Julian BILCKE (@flngr)
-
-## BEFORE YOU READ
-
-The CLAP format spec is experimental and not finished yet!
-There might be inconsistencies, unnecessary redundancies or blatant omissions.
-
-## What are CLAP files?
-
-The CLAP format (.clap) is a file format designed for AI video projects.
-
-It preserves prompts and assets into the same container, making it easier to share an AI video project between different people or applications.
-
-## Structure
-
-A CLAP is an array of objects serialized into a YAML text string, then finally compressed using gzip to a binary file.
-
-The file extension is `.clap`
-The mime type is `application/x-yaml`
-
-There can be 5 different types of objects:
-
-- one HEADER
-- one METADATA
-- zero, one or more MODEL(s)
-- zero, one or more SCENE(s)
-- zero, one or more SEGMENT(s)
-
-This can be represented in javascript like this:
-
-```javascript
-[
-  clapHeader, // one metadata object
-  clapMeta, // one metadata object
-  ...clapModels, // optional array of models
-  ...clapScenes, // optional array of scenes
-  ...clapSegments // optional array of segments
-]
-```
-
-## Header
-
-The HEADER provides information about how to decode a CLAP.
-
-Knowing in advance the number of models, scenes and segments helps the decoder parsing the information,
-and in some implementation, help with debugging, logging, and provisioning memory usage.
-
-However in the future, it is possible that a different scheme is used, in order to support streaming.
-
-Either by recognizing the shape of each object (fields), or by using a specific field eg. a `_type`.
-
-```typescript
-{
-  // used to know which format version is used.
-  // CLAP is still in development and the format is not fully specified yet,
-  // during the period most .clap file will have the "clap-0" format
-  format: "clap-0"
-  
-  numberOfModels: number // integer
-  numberOfScenes: number // integer
-  numberOfSegments: number // integer
-}
-```
-
-## Metadata
-
-```typescript
-{
-  id: string // "<a valid UUID V4>"
-  title: string // "project title"
-  description: string // "project description"
-  licence: string // "information about licensing"
-
-  // this provides information about the image ratio
-  // this might be removed in the final spec, as this
-  // can be re-computed from width and height
-  orientation: "landscape" | "vertical" | "square"
-
-  // the expected duration of the project
-  durationInMs: number
-
-  // the suggested width and height of the video
-  // note that this is just an indicator,
-  // and might be superseeded by the application reading the .clap file
-  width: number // integer between 256 and 8192 (value in pixels)
-  height: number // integer between 256 and 8192 (value in pixels)
-
-  // name of the suggested video model to use
-  // note that this is just an indicator,
-  // and might be superseeded by the application reading the .clap file
-  defaultVideoModel: string
-
-  // additional prompt to use in the video generation
-  // this helps adding some magic touch and flair to the videos,
-  // but perhaps the field should be renamed
-  extraPositivePrompt: string
-
-  // the screenplay (script) of the video
-  screenplay: string
-
-  // whether to loop the content by default or not
-  isLoop: boolean
-  
-  // helper to indicate whether the .clap might contain interactive elements
-  isInteractive: boolean
-}
-```
-
-## Models
-
-Before talking about models, first we should describe the concept of entity:
-
-in a story, an entity is something (person, place, vehicle, animal, robot, alien, object) with a name, a description of the appearance, an age, mileage or quality, an origin, and so on.
-
-An example could be "a giant magical school bus, with appearance of a cat with wheels, and which talks"
-
-The CLAP model would be an instance (an interpretation) of this entity, where we would assign it an identity:
-- a name and age
-- a visual style (a photo of the magic school bus cat)
-- a voice style
-- and maybe other things eg. an origin or background story
-
-As you can see, it can be difficult to create clearly separated categories, like "vehicule", "character", or "location"
-(the magical cat bus could turn into a location in some scene, a speaking character in another etc)
-
-This is why there is a common schema for all models:
-
-```typescript
-{
-  id: string
-  category: ClapSegmentCategory
-  triggerName: string
-  label: string
-  description: string
-  author: string
-  thumbnailUrl: string
-  seed: number
-
-  assetSourceType: ClapAssetSource
-  assetUrl: string
-  
-  age: number
-  gender: ClapModelGender
-  region: ClapModelRegion
-  appearance: ClapModelAppearance
-  voiceVendor: ClapVoiceVendor
-  voiceId: string
-}
-```
-
-## Atomic types
-
-...
-
-## TO BE CONTINUED
-
-(you can read "./types.ts" for more information)

src/lib/clap/clapToDataUri.ts

@@ -1,11 +0,0 @@
-
-import { blobToDataUri } from "@/lib/base64/blobToDataUri"
-
-import { serializeClap } from "./serializeClap"
-import { ClapProject } from "./types"
-
-export async function clapToDataUri(clap: ClapProject): Promise<string> {
-  const archive = await serializeClap(clap)
-  const dataUri = await blobToDataUri(archive, "application/x-gzip")
-  return dataUri
-}

src/lib/clap/emptyClap.ts

@@ -1,18 +0,0 @@
-import { newClap } from "./newClap"
-import { serializeClap } from "./serializeClap"
-
-let globalState: {
-  blob?: Blob
-} = {
-  blob: undefined
-}
-
-export async function getEmptyClap(): Promise<Blob> {
-  if (globalState.blob) { return globalState.blob }
-
-  const clap = newClap()
-
-  globalState.blob = await serializeClap(clap)
-
-  return globalState.blob
-}

src/lib/clap/generateClapFromSimpleStory.ts

@@ -1,149 +0,0 @@
-import { newClap } from "./newClap"
-import { newSegment } from "./newSegment"
-import { ClapProject } from "./types"
-
-let defaultSegmentDurationInMs = 2500 // 2584
-
-const fishDemoStory = [
-  "Siamese fighting fish, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-  
-  // this one is magnificient!
-  "princess parrot fish, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-  
-  "pacific ocean perch, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-  
-  "Queen angelfish, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-  
-  "sea turtle, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-
-  "hippocampus, bokeh, underwater, coral, lively, bubbles, translucency, perfect",
-]
-
-let demoStory = [
-  ...fishDemoStory,
-
-  // "portrait of one man news anchor, 60yo, thin, fit, american, mustache, beard, wearing a suit, medium-shot, central park, outside, serious, bokeh, perfect",
-  
-  // "screenshot from Call of Duty, FPS game, nextgen, videogame screenshot, unreal engine, raytracing, perfect",
-
-  // "screenshot from a flight simulator, nextgen, videogame screenshot, unreal engine, raytracing, perfect",
-  // "screenshot from fallout3, fallout4, western, wasteland, 3rd person RPG, nextgen, videogame screenshot, unreal engine, raytracing, perfect",
-  // "portrait of single influencer woman, 30yo, thin, fit, american, wearing a red tshirt, medium-shot, central park, outside, serious, bokeh, perfect",
-]
-
-
-export function generateClapFromSimpleStory({
-  story = demoStory,
-  showIntroPoweredByEngine = false,
-  showIntroDisclaimerAboutAI = false,
-}: {
-  story?: string[]
-  showIntroPoweredByEngine?: boolean
-  showIntroDisclaimerAboutAI?: boolean
-} = {
-  story: demoStory,
-  showIntroPoweredByEngine: false,
-  showIntroDisclaimerAboutAI: false,
-}): ClapProject {
-
-  const clap = newClap({
-    meta: {
-      title: "Interactive Demo",
-      isInteractive: true,
-      isLoop: true,
-    }
-  })
-
-  let currentElapsedTimeInMs = 0
-  let currentSegmentDurationInMs = defaultSegmentDurationInMs
-
-  if (showIntroPoweredByEngine) {
-    clap.segments.push(newSegment({
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "interface",
-      prompt: "<BUILTIN:POWERED_BY_ENGINE>",
-      label: "disclaimer",
-      outputType: "interface",
-    }))
-    currentElapsedTimeInMs += currentSegmentDurationInMs
-  }
-
-  if (showIntroDisclaimerAboutAI) {
-    clap.segments.push(newSegment({
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "interface",
-      prompt: "<BUILTIN:DISCLAIMER_ABOUT_AI>",
-      label: "disclaimer",
-      outputType: "interface",
-    }))
-    currentElapsedTimeInMs += currentSegmentDurationInMs
-  }
-
-  /*
-  clap.segments.push(
-    newSegment({
-      // id: string
-      // track: number
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "interface",
-      // modelId: string
-      // sceneId: string
-      prompt: "a hello world",
-      label: "hello world",
-      outputType: "interface"
-      // renderId: string
-      // status: ClapSegmentStatus
-      // assetUrl: string
-      // assetDurationInMs: number
-      // createdBy: ClapAuthor
-      // editedBy: ClapAuthor
-      // outputGain: number
-      // seed: number
-    })
-  )
-
-  currentElapsedTimeInMs += currentSegmentDurationInMs
-  */
-  
- 
-
-  for (let prompt of story) {
-
-    clap.segments.push(newSegment({
-      track: 0,
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "video",
-      prompt: "",
-      label: "video",
-      outputType: "video",
-    }))
-    clap.segments.push(newSegment({
-      track: 1,
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "generic",
-      prompt,
-      label: prompt,
-      outputType: "text",
-    }))
-    clap.segments.push(newSegment({
-      track: 2,
-      startTimeInMs: currentElapsedTimeInMs,
-      endTimeInMs: currentSegmentDurationInMs,
-      category: "camera",
-      prompt: "medium-shot",
-      label: "medium-shot",
-      outputType: "text",
-    }))
-
-    currentElapsedTimeInMs += currentSegmentDurationInMs
-  }
-
-  clap.meta.durationInMs = currentElapsedTimeInMs
-
-  return clap
-}

src/lib/clap/getClapAssetSource.ts

@@ -1,25 +0,0 @@
-import { ClapAssetSource } from "./types"
-
-export function getClapAssetSourceType(input: string = ""): ClapAssetSource {
-  
-  const str = `${input || ""}`.trim()
-
-  if (!str || !str.length) {
-    return "EMPTY"
-  }
-
-  if (str.startsWith("https://") || str.startsWith("http://")) {
-    return "REMOTE"
-  }
-
-  // note that "path" assets are potentially a security risk, they need to be treated with care
-  if (str.startsWith("/") || str.startsWith("../") || str.startsWith("./")) {
-    return "PATH"
-  }
-
-  if (str.startsWith("data:")) {
-    return "DATA"
-  }
-
-  return "PROMPT"
-}

src/lib/clap/newClap.ts

@@ -1,37 +0,0 @@
-
-import { v4 as uuidv4 } from "uuid"
-
-import { ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types"
-import { getValidNumber } from "@/lib/utils/getValidNumber"
-
-// generate an empty clap file, or copy one from a source
-export function newClap(clap: {
-    meta?: Partial<ClapMeta>
-    models?: ClapModel[]
-    scenes?: ClapScene[]
-    segments?: ClapSegment[]
-  } = {}): ClapProject {
-
-  const meta: ClapMeta = {
-    id: clap?.meta?.id === "string" ? clap.meta.id : uuidv4(),
-    title: clap?.meta?.title === "string" ? clap.meta.title : "",
-    description: typeof clap?.meta?.description === "string" ? clap.meta.description : "",
-    synopsis: typeof clap?.meta?.synopsis === "string" ? clap.meta.synopsis : "",
-    licence: typeof clap?.meta?.licence === "string" ? clap.meta.licence : "",
-    orientation: clap?.meta?.orientation === "portrait" ? "portrait" : clap?.meta?.orientation === "square" ? "square" : "landscape",
-    durationInMs: getValidNumber(clap?.meta?.durationInMs, 1000, Number.MAX_SAFE_INTEGER, 4000),
-    width: getValidNumber(clap?.meta?.width, 256, 8192, 1024),
-    height: getValidNumber(clap?.meta?.height, 256, 8192, 576),
-    defaultVideoModel: typeof clap?.meta?.defaultVideoModel === "string" ? clap.meta.defaultVideoModel : "SVD",
-    extraPositivePrompt: Array.isArray(clap?.meta?.extraPositivePrompt) ? clap.meta.extraPositivePrompt : [],
-    screenplay: typeof clap?.meta?.screenplay === "string" ? clap.meta.screenplay : "",
-    isLoop: typeof clap?.meta?.isLoop === "boolean" ? clap.meta.isLoop : false,
-    isInteractive: typeof clap?.meta?.isInteractive === "boolean" ? clap.meta.isInteractive : false,
-  }
-
-  const models: ClapModel[] = clap?.models && Array.isArray(clap.models) ? clap.models : []
-  const scenes: ClapScene[] =  clap?.scenes && Array.isArray(clap.scenes) ? clap.scenes : []
-  const segments: ClapSegment[] =  clap?.segments && Array.isArray(clap.segments) ? clap.segments : []
-
-  return { meta, models, scenes, segments }
-}

src/lib/clap/newSegment.ts

@@ -1,46 +0,0 @@
-import { v4 as uuidv4 } from "uuid"
-
-import { generateSeed } from "../utils/generateSeed"
-import { ClapSegment } from "./types"
-import { isValidNumber } from "../utils/isValidNumber"
-
-export function newSegment(maybeSegment?: Partial<ClapSegment>) {
-
-  const startTimeInMs =
-    isValidNumber(maybeSegment?.startTimeInMs)
-    ? (maybeSegment?.startTimeInMs || 0)
-    : 0
-
-  const assetDurationInMs =
-    isValidNumber(maybeSegment?.assetDurationInMs)
-    ? (maybeSegment?.assetDurationInMs || 0)
-    : 1000
-
-  const endTimeInMs =
-    isValidNumber(maybeSegment?.endTimeInMs)
-    ? (maybeSegment?.endTimeInMs || 0)
-    : (startTimeInMs + assetDurationInMs)
-
-  const segment: ClapSegment = {
-    id: typeof maybeSegment?.id === "string" ? maybeSegment.id : uuidv4(),
-    track: isValidNumber(maybeSegment?.track) ? (maybeSegment?.track || 0) : 0,
-    startTimeInMs,
-    endTimeInMs,
-    category: typeof maybeSegment?.category === "string" ? maybeSegment.category : "generic",
-    modelId: typeof maybeSegment?.modelId === "string" ? maybeSegment.modelId : "",
-    sceneId: typeof maybeSegment?.sceneId === "string" ? maybeSegment.sceneId : "",
-    prompt: typeof maybeSegment?.prompt === "string" ? maybeSegment.prompt : "",
-    label: typeof maybeSegment?.label === "string" ? maybeSegment.label : "",
-    outputType: typeof maybeSegment?.outputType === "string" ? maybeSegment.outputType : "text",
-    renderId: typeof maybeSegment?.renderId === "string" ? maybeSegment.renderId : "",
-    status: typeof maybeSegment?.status === "string" ? maybeSegment.status : "to_generate",
-    assetUrl: typeof maybeSegment?.assetUrl === "string" ? maybeSegment.assetUrl : "",
-    assetDurationInMs,
-    createdBy: typeof maybeSegment?.createdBy === "string" ?  maybeSegment.createdBy : "ai",
-    editedBy: typeof maybeSegment?.editedBy === "string" ?  maybeSegment.editedBy : "ai",
-    outputGain: isValidNumber(maybeSegment?.outputGain) ? (maybeSegment?.outputGain || 0) : 0,
-    seed: isValidNumber(maybeSegment?.seed) ? (maybeSegment?.seed || 0) : generateSeed()
-  }
-
-  return segment
-}

src/lib/clap/parseClap.ts

@@ -1,319 +0,0 @@
-import YAML from "yaml"
-import { v4 as uuidv4 } from "uuid"
-
-import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types"
-import { dataUriToBlob } from "@/lib/base64/dataUriToBlob"
-import { getValidNumber } from "@/lib/utils/getValidNumber"
-
-type StringOrBlob = string | Blob
-
-/**
- * Import a clap file from various data sources into an ClapProject
- * 
- * Inputs can be:
- * - a Clap project (which is an object)
- * - an URL to a remote .clap file
- * - a string containing a YAML array
- * - a data uri containing a gzipped YAML array
- * - a Blob containing a gzipped YAML array
- * 
- * note: it is not really async, because for some reason YAML.parse is a blocking call like for JSON,
- * there is no async version although we are now in the 20s not 90s
- */
-export async function parseClap(src?: ClapProject | string | Blob, debug = false): Promise<ClapProject> {
-
-  try {
-    if (
-      typeof src === "object" &&
-      Array.isArray( (src as any)?.scenes) &&
-      Array.isArray((src as any)?.models)
-    ) {
-      if (debug) {
-        console.log("parseClap: input is already a Clap file, nothing to do:", src)
-      }
-      // we can skip verification
-      return src as ClapProject
-    }
-  } catch (err) {
-    // well, this is not a clap project
-  }
-
-  let stringOrBlob = (src || "") as StringOrBlob
-
-  // both should work
-  const dataUriHeader1 = "data:application/x-gzip;base64,"
-  const dataUriHeader2 = "data:application/octet-stream;base64,"
-
-  const inputIsString = typeof stringOrBlob === "string"
-  const inputIsDataUri = typeof stringOrBlob === "string" ? stringOrBlob.startsWith(dataUriHeader1) || stringOrBlob.startsWith(dataUriHeader2) : false
-  const inputIsRemoteFile = typeof stringOrBlob === "string" ? (stringOrBlob.startsWith("http://") || stringOrBlob.startsWith("https://")) : false
-
-  let inputIsBlob = typeof stringOrBlob !== "string"
-
-  let inputYamlArrayString = ""
-
-  if (debug) {
-    console.log(`parseClap: pre-analysis: ${JSON.stringify({
-      inputIsString,
-      inputIsBlob,
-      inputIsDataUri,
-      inputIsRemoteFile
-    }, null, 2)}`)
-  }
-
-  if (typeof stringOrBlob === "string") {
-    if (debug) {
-      console.log("parseClap: input is a string ", stringOrBlob.slice(0, 120))
-    }
-    if (inputIsDataUri) {
-      if (debug) {
-        console.log(`parseClap: input is a data uri archive`)
-      }
-      stringOrBlob = dataUriToBlob(stringOrBlob, "application/x-gzip")
-      if (debug) {
-        console.log(`parseClap: inputBlob = `, stringOrBlob)
-      }
-      inputIsBlob = true
-    } else if (inputIsRemoteFile) {
-      try {
-        if (debug) {
-          console.log(`parseClap: input is a remote .clap file`)
-        }
-        const res = await fetch(stringOrBlob)
-        stringOrBlob = await res.blob()
-        if (!stringOrBlob) { throw new Error("blob is empty") }
-        inputIsBlob = true
-      } catch (err) {
-        // url seems invalid
-        throw new Error(`failed to download the .clap file (${err})`)
-      }
-    } else {
-      if (debug) {
-        console.log("parseClap: input is a text string containing a YAML array")
-      }
-      inputYamlArrayString = stringOrBlob
-      inputIsBlob = false
-    }
-  }
-
-  if (typeof stringOrBlob !== "string" && stringOrBlob) {
-    if (debug) {
-      console.log("parseClap: decompressing the blob..")
-    }
-    // Decompress the input blob using gzip
-    const decompressedStream = stringOrBlob.stream().pipeThrough(new DecompressionStream('gzip'))
-
-    try {
-      // Convert the stream to text using a Response object
-      const decompressedOutput = new Response(decompressedStream)
-      // decompressedOutput.headers.set("Content-Type", "application/x-gzip")
-      if (debug) {
-        console.log("parseClap: decompressedOutput: ", decompressedOutput)
-      }
-      // const blobAgain = await decompressedOutput.blob()
-      inputYamlArrayString = await decompressedOutput.text()
-
-      if (debug && inputYamlArrayString) {
-        console.log("parseClap: successfully decompressed the blob!")
-      }
-    } catch (err) {
-      const message = `parseClap: failed to decompress (${err})`
-      console.error(message)
-      throw new Error(message)
-    }
-  }
-
-  // we don't need this anymore I think
-  // new Blob([inputStringOrBlob], { type: "application/x-yaml" })
-
-  let maybeArray: any = {}
-  try {
-    if (debug) {
-      console.log("parseClap: parsing the YAML array..")
-    }
-    // Parse YAML string to raw data
-    maybeArray = YAML.parse(inputYamlArrayString)
-  } catch (err) {
-    throw new Error("invalid clap file (input string is not YAML)")
-  }
-
-  if (!Array.isArray(maybeArray) || maybeArray.length < 2) {
-    throw new Error("invalid clap file (need a clap format header block and project metadata block)")
-  }
-
-  if (debug) {
-    console.log("parseClap: the YAML seems okay, continuing decoding..")
-  }
-
-  const maybeClapHeader = maybeArray[0] as ClapHeader
-
-  if (maybeClapHeader.format !== "clap-0") {
-    throw new Error("invalid clap file (sorry, but you can't make up version numbers like that)")
-  }
-
-
-  const maybeClapMeta = maybeArray[1] as ClapMeta
-
-  const clapMeta: ClapMeta = {
-    id: typeof maybeClapMeta.title === "string" ? maybeClapMeta.id : uuidv4(),
-    title: typeof maybeClapMeta.title === "string" ? maybeClapMeta.title : "",
-    description: typeof maybeClapMeta.description === "string" ? maybeClapMeta.description : "",
-    synopsis: typeof maybeClapMeta.synopsis === "string" ? maybeClapMeta.synopsis : "",
-    licence: typeof maybeClapMeta.licence === "string" ? maybeClapMeta.licence : "",
-    orientation: maybeClapMeta.orientation === "portrait" ? "portrait" : maybeClapMeta.orientation === "square" ? "square" : "landscape",
-    durationInMs: getValidNumber(maybeClapMeta.durationInMs, 1000, Number.MAX_SAFE_INTEGER, 4000),
-    width: getValidNumber(maybeClapMeta.width, 128, 8192, 1024),
-    height: getValidNumber(maybeClapMeta.height, 128, 8192, 576),
-    defaultVideoModel: typeof maybeClapMeta.defaultVideoModel === "string" ? maybeClapMeta.defaultVideoModel : "SVD",
-    extraPositivePrompt: Array.isArray(maybeClapMeta.extraPositivePrompt) ? maybeClapMeta.extraPositivePrompt : [],
-    screenplay: typeof maybeClapMeta.screenplay === "string" ? maybeClapMeta.screenplay : "",
-    isLoop: typeof maybeClapMeta.isLoop === "boolean" ? maybeClapMeta.isLoop : false,
-    isInteractive: typeof maybeClapMeta.isInteractive === "boolean" ? maybeClapMeta.isInteractive : false,
-  }
-
-  /*
-  in case we want to support streaming (mix of models and segments etc), we could do it this way:
-
-  const maybeModelsOrSegments = rawData.slice(2)
-  maybeModelsOrSegments.forEach((unknownElement: any) => {
-    if (isValidNumber(unknownElement?.track)) {
-      maybeSegments.push(unknownElement as ClapSegment)
-    } else {
-      maybeModels.push(unknownElement as ClapModel)
-    }
-  })
-  */
-
-
-  const expectedNumberOfModels = maybeClapHeader.numberOfModels || 0
-  const expectedNumberOfScenes = maybeClapHeader.numberOfScenes || 0
-  const expectedNumberOfSegments = maybeClapHeader.numberOfSegments || 0
-
-  // note: we assume the order is strictly enforced!
-  // if you implement streaming (mix of models and segments) you will have to rewrite this!
-
-  const afterTheHeaders = 2
-  const afterTheModels = afterTheHeaders + expectedNumberOfModels
-
-  const afterTheScenes = afterTheModels + expectedNumberOfScenes
-
-  // note: if there are no expected models, maybeModels will be empty
-  const maybeModels = maybeArray.slice(afterTheHeaders, afterTheModels) as ClapModel[]
-
-  // note: if there are no expected scenes, maybeScenes will be empty
-  const maybeScenes = maybeArray.slice(afterTheModels, afterTheScenes) as ClapScene[]
-
-  const maybeSegments = maybeArray.slice(afterTheScenes) as ClapSegment[]
-
-  const clapModels: ClapModel[] = maybeModels.map(({
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }) => ({
-    // TODO: we should verify each of those, probably
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }))
-
-  const clapScenes: ClapScene[] = maybeScenes.map(({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events,
-  }) => ({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events: events.map(e => e)
-  }))
-  
-  const clapSegments: ClapSegment[] = maybeSegments.map(({
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }) => ({
-    // TODO: we should verify each of those, probably
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }))
-
-  if (debug) {
-    console.log(`parseClap: successfully parsed ${clapModels.length} models, ${clapScenes.length} scenes and ${clapSegments.length} segments`)
-  }
-  return {
-    meta: clapMeta,
-    models: clapModels,
-    scenes: clapScenes,
-    segments: clapSegments
-  }
-}

src/lib/clap/serializeClap.ts

@@ -1,169 +0,0 @@
-import YAML from "yaml"
-import { v4 as uuidv4 } from "uuid"
-
-import { getValidNumber } from "@/lib/utils/getValidNumber"
-
-import { ClapHeader, ClapMeta, ClapModel, ClapProject, ClapScene, ClapSegment } from "./types"
-
-export async function serializeClap({
-  meta, // ClapMeta
-  models, // ClapModel[]
-  scenes, // ClapScene[]
-  segments, // ClapSegment[]
-}: ClapProject): Promise<Blob> {
-  
-  // we play it safe, and we verify the structure of the parameters,
-  // to make sure we generate a valid clap file
-  const clapModels: ClapModel[] = models.map(({
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }) => ({
-    id,
-    category,
-    triggerName,
-    label,
-    description,
-    author,
-    thumbnailUrl,
-    seed,
-    assetSourceType,
-    assetUrl,
-    age,
-    gender,
-    region,
-    appearance,
-    voiceVendor,
-    voiceId,
-  }))
-
-  const clapScenes: ClapScene[] = scenes.map(({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events,
-  }) => ({
-    id,
-    scene,
-    line,
-    rawLine,
-    sequenceFullText,
-    sequenceStartAtLine,
-    sequenceEndAtLine,
-    startAtLine,
-    endAtLine,
-    events: events.map(e => e)
-  }))
-
-  const clapSegments: ClapSegment[] = segments.map(({
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }) => ({
-    id,
-    track,
-    startTimeInMs,
-    endTimeInMs,
-    category,
-    modelId,
-    sceneId,
-    prompt,
-    label,
-    outputType,
-    renderId,
-    status,
-    assetUrl,
-    assetDurationInMs,
-    createdBy,
-    editedBy,
-    outputGain,
-    seed,
-  }))
-
-  const clapHeader: ClapHeader = {
-    format: "clap-0",
-    numberOfModels: clapModels.length,
-    numberOfScenes: clapScenes.length,
-    numberOfSegments: clapSegments.length,
-  }
-
-  const clapMeta: ClapMeta = {
-    id: meta.id || uuidv4(),
-    title: typeof meta.title === "string" ? meta.title : "Untitled",
-    description: typeof meta.description === "string" ? meta.description : "",
-    synopsis: typeof meta.synopsis === "string" ? meta.synopsis : "",
-    licence: typeof meta.licence === "string" ? meta.licence : "",
-    orientation: meta.orientation === "portrait" ? "portrait" : meta.orientation === "square" ? "square" : "landscape",
-    durationInMs: getValidNumber(meta.durationInMs, 1000, Number.MAX_SAFE_INTEGER, 4000),
-    width: getValidNumber(meta.width, 256, 8192, 1024),
-    height: getValidNumber(meta.height, 256, 8192, 576),
-    defaultVideoModel:  typeof meta.defaultVideoModel === "string" ? meta.defaultVideoModel : "SVD",
-    extraPositivePrompt: Array.isArray(meta.extraPositivePrompt) ? meta.extraPositivePrompt : [],
-    screenplay: typeof meta.screenplay === "string" ? meta.screenplay : "",
-    isLoop: typeof meta.screenplay === "boolean" ? meta.screenplay : false,
-    isInteractive: typeof meta.isInteractive === "boolean" ? meta.isInteractive : false,
-  }
-
-  const entries = [
-    clapHeader,
-    clapMeta,
-    ...clapModels,
-    ...clapScenes,
-    ...clapSegments
-  ]
-
-  const strigifiedResult = YAML.stringify(entries)
-
-  // Convert the string to a Blob
-  const blobResult = new Blob([strigifiedResult], { type: "application/x-yaml" })
-
-   // Create a stream for the blob
-   const readableStream = blobResult.stream()
-
-   // Compress the stream using gzip
-   const compressionStream = new CompressionStream('gzip')
-   const compressedStream = readableStream.pipeThrough(compressionStream)
-
-   // Create a new blob from the compressed stream
-   const response = new Response(compressedStream)
-   
-   response.headers.set("Content-Type", "application/x-gzip")
-
-   const compressedBlob = await response.blob()
-
-  return compressedBlob
-}

src/lib/clap/types.ts

@@ -1,203 +0,0 @@
-
-export type ClapSegmentCategory =
-  | "splat"
-  | "mesh"
-  | "depth"
-  | "event"
-  | "interface"
-  | "phenomenon"
-  | "video"
-  | "storyboard"
-  | "transition"
-  | "characters"
-  | "location"
-  | "time"
-  | "era"
-  | "lighting"
-  | "weather"
-  | "action"
-  | "music"
-  | "sound"
-  | "dialogue"
-  | "style"
-  | "camera"
-  | "generic"
-
-export type ClapOutputType =
-  | "text"
-  | "animation"
-  | "interface"
-  | "event"
-  | "phenomenon"
-  | "transition"
-  | "image"
-  | "video"
-  | "audio"
-
-export type ClapSegmentStatus =
-  | "to_generate"
-  | "to_interpolate"
-  | "to_upscale"
-  | "completed"
-  | "error"
-
-export type ClapAuthor =
-  | "auto" // the element was edited automatically using basic if/else logical rules
-  | "ai" // the element was edited using a large language model
-  | "human" // the element was edited by a human
-
-export type ClapAssetSource =
-  | "REMOTE" // http:// or https://
-
-    // note that "path" assets are potentially a security risk, they need to be treated with care
-  | "PATH" // a file path eg. /path or ./path/to/ or ../path/to/
-
-  | "DATA" // a data URI, starting with data:
-
-  | "PROMPT" // by default, a plain text prompt
-
-  | "EMPTY"
-
-export type ClapModelGender =
-  | "male"
-  | "female"
-  | "person"
-  | "object"
-
-export type ClapModelAppearance = "serious" | "neutral" | "friendly" | "chill"
-
-// this is used for accent, style..
-export type ClapModelRegion =
-  | "american"
-  | "british"
-  | "australian"
-  | "canadian"
-  | "indian"
-  | "french"
-  | "italian"
-  | "german"
-  | "chinese"
-
-// note: this is all very subjective, so please use good judgment
-//
-// "deep" might indicate a deeper voice tone, thicker, rich in harmonics
-// in this context, it is used to indicate voices that could
-// be associated with African American (AADOS) characters
-//
-// "high" could be used for some other countries, eg. asia
-export type ClapModelTimbre = "high" | "neutral" | "deep"
-
-export type ClapVoiceVendor = "ElevenLabs" | "XTTS"
-
-export type ClapVoice = {
-  name: string
-  gender: ClapModelGender
-  age: number
-  region: ClapModelRegion
-  timbre: ClapModelTimbre
-  appearance: ClapModelAppearance
-  voiceVendor: ClapVoiceVendor
-  voiceId: string
-}
-
-export type ClapHeader = {
-  format: "clap-0"
-  numberOfModels: number
-  numberOfScenes: number
-  numberOfSegments: number
-}
-
-export type ClapMeta = {
-  id: string
-  title: string
-  description: string
-  synopsis: string
-  licence: string
-  orientation: string
-
-  // the default duration of the experience
-  // the real one might last longer if made interactive
-  durationInMs: number
-
-  width: number
-  height: number
-  defaultVideoModel: string
-  extraPositivePrompt: string[]
-  screenplay: string
-  isLoop: boolean
-  isInteractive: boolean
-}
-
-export type ClapSceneEvent = {
-  id: string
-  type: "description" | "dialogue" | "action"
-  character?: string
-  description: string
-  behavior: string
-  startAtLine: number
-  endAtLine: number
-}
-
-export type ClapScene = {
-  id: string
-  scene: string
-  line: string
-  rawLine: string
-  sequenceFullText: string
-  sequenceStartAtLine: number
-  sequenceEndAtLine: number
-  startAtLine: number
-  endAtLine: number
-  events: ClapSceneEvent[]
-}
-
-export type ClapSegment = {
-  id: string
-  track: number // usually track 0 is the video, track 1 is the storyboard, track 2 is the camera
-  startTimeInMs: number
-  endTimeInMs: number
-  category: ClapSegmentCategory
-  modelId: string
-  sceneId: string
-  prompt: string
-  label: string
-  outputType: ClapOutputType
-  renderId: string
-  status: ClapSegmentStatus
-  assetUrl: string
-  assetDurationInMs: number
-  createdBy: ClapAuthor
-  editedBy: ClapAuthor
-  outputGain: number
-  seed: number
-}
-
-export type ClapModel = {
-  id: string
-  category: ClapSegmentCategory
-  triggerName: string
-  label: string
-  description: string
-  author: string
-  thumbnailUrl: string
-  seed: number
-
-  assetSourceType: ClapAssetSource
-  assetUrl: string
-  
-  // those are only used by certain types of models
-  age: number
-  gender: ClapModelGender
-  region: ClapModelRegion
-  appearance: ClapModelAppearance
-  voiceVendor: ClapVoiceVendor
-  voiceId: string
-}
-
-export type ClapProject = {
-  meta: ClapMeta
-  models: ClapModel[]
-  scenes: ClapScene[]
-  segments: ClapSegment[]
-  // let's keep room for other stuff (screenplay etc)
-}