update

package-lock.json

@@ -63,6 +63,7 @@
         "eslint-config-next": "13.4.10",
         "fastest-levenshtein": "^1.0.16",
         "fluent-ffmpeg": "^2.1.2",
+        "fun-word-list": "^1.0.1",
         "gsplat": "^1.2.4",
         "hash-wasm": "^4.11.0",
         "jose": "^5.2.4",
@@ -5285,6 +5286,11 @@
         "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
       }
     },
+    "node_modules/fun-word-list": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/fun-word-list/-/fun-word-list-1.0.1.tgz",
+      "integrity": "sha512-WoUxTcjSPLSWhyMQaDiYgIWGOPHS7SnsZL8TVQrX8qdMyHXz8uPWnfMHq/Kx+kxQpQJ0GZkMjKgynv+INhbzBg=="
+    },
     "node_modules/function-bind": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",

package.json

@@ -65,6 +65,7 @@
     "eslint-config-next": "13.4.10",
     "fastest-levenshtein": "^1.0.16",
     "fluent-ffmpeg": "^2.1.2",
+    "fun-word-list": "^1.0.1",
     "gsplat": "^1.2.4",
     "hash-wasm": "^4.11.0",
     "jose": "^5.2.4",

src/app/api/v1/create/generateRandomStory.ts

@@ -0,0 +1,102 @@
+import { pick } from "@/lib/utils/pick"
+import {
+  addons, // Fragments to add to the end of sentences for different moods. Many include preceding commas or periods.
+  adjectives,  // For use with `nouns` or `concepts`
+  concepts, // Complex nouns
+  descriptions, // For use with `locations`
+  gametypes, // Video game genres/playstyles/feels
+  locations, // Nouns that represent places
+  nouns, // Simple things that can act or be acted upon
+  superlatives, // Adjectives that express the extremes of something. For use with `nouns`, `concepts`, or `locations`
+  verbSecondPerson,  // Verbs for use with "you" or "they" as in "you help the aardvark" or "they fight a bird"
+  verbSecondConcepts,  // Verbs for use with "you" or "they" in combination with `concepts` as in "you are helped by the King" or "they fight against traditions"
+  verbThirdPerson, // Verbs for use with `nouns` as in "The aardvark anticipates rain"
+} from "@/lib/fun-words"
+
+
+const moodTypes = [
+  "A fantasy",
+  "An epic",
+  "A beautiful",
+  "A science-fiction",
+  "A boring",
+  "A sad",
+  "A cheerful",
+  "A funny",
+  "A funny",
+  "A trending",
+  "A cute",
+  "An interesting",
+  "An outrageous",
+  "A dark",
+  "A 3D-rendered",
+  "An animated",
+  "Burlesque",
+  "Short",
+  "Long",
+  "An inspiring",
+  "A marketing",
+  "A commercial",
+  "An internet",
+  "A stupid",
+  "A vintage",
+]
+
+const videoTypes = [
+  "fail compilation",
+  "influencer meltdown",
+  "breaking news",
+  "newscast",
+  "politician speech",
+  "found footage",
+  "meme video",
+  "rap video",
+  "animal documentary",
+  "music video",
+  "live video",
+  "advert",
+  "instagram video",
+  "short movie",
+  "documentary",
+  "movie trailer",
+  "TikTok video",
+]
+
+const locationTypes = [
+  "in some",
+  // "set somewhere in",
+  "near some",
+]
+
+export function generateRandomStory() {
+
+  let pickedWords: any[] = []
+
+  function randomize<T>(words: T[]): T {
+    const picked = pick(words, '' as T)
+    pickedWords.push(picked as any)
+    return picked
+  } 
+
+  return `${
+    pick(moodTypes, '')
+  } ${
+    pick(videoTypes, '')
+  }${
+    Math.random() > 0.5 ? "," : ""
+  } ${
+    pick(locationTypes, '')
+  } ${
+    randomize(descriptions)
+  } and ${
+    randomize(descriptions)
+  } ${
+    randomize(locations)
+  }. It is about ${
+    randomize(nouns)[1]
+  }, ${
+    randomize(nouns)[0]
+  } ${
+    randomize(nouns)[1]
+  } and..`
+}

src/app/api/v1/create/generateStoryPrompt.txt

@@ -0,0 +1,72 @@
+
+import YAML from "yaml"
+
+import { predict } from "@/app/api/providers/huggingface/predictWithHuggingFace"
+import { LatentStory } from "@/app/api/v1/types"
+
+import { import { systemPrompt } from "./systemPromptRandomStory"
+} from "./systemPromptRandomStory"
+
+export async function generateStoryPrompt({
+  turbo = false,
+}: {
+  turbo?: boolean
+} = {
+  turbo: false
+}): Promise<string[]> {
+
+  if (!prompt.length) { throw new Error(`please provide a prompt`) }
+  console.log("generateMusicPrompts(): prompt:", prompt)
+
+
+  if (!latentStory.length) { throw new Error(`please provide a story`) }
+
+  // console.log("generateMusicPrompts(): latentStory:", latentStory)
+
+  const userPrompt = `The input story is about: ${prompt}.
+
+The input story is:
+\`\`\`yaml
+${YAML.stringify(
+  // we need to help the LLM by marking the shots with a simple numeric ID
+  latentStory.map((shot, i) => ({
+    shot: i,
+    ...shot,
+  }))
+)}
+\`\`\`
+
+# Output`
+
+  const prefix = "\""
+
+  // we don't need a lot here!
+  const nbMaxNewTokens = 120
+
+  // TODO use streaming for the Hugging Face prediction
+  //
+  // note that a Clap file is actually a YAML stream of documents
+  // so technically we could stream everything from end-to-end
+  // (but I haven't coded the helpers to do this yet)
+  let rawString = await predict({
+    systemPrompt,
+    userPrompt,
+    nbMaxNewTokens,
+    prefix,
+    turbo,
+  })
+
+  // console.log("generateEntityPrompts(): rawString: ", rawString)
+
+  let results: string[] = []
+  
+  // we remove everything after the last ``` (or ``)
+  rawString = rawString.split(/```?/)[0].trim()
+  results.push(rawString)
+
+  if (!Array.isArray(results) || typeof results.at(0) !== "string" || !results) {
+    throw new Error(`failed to generate the output (rawString is: ${rawString})`)
+  }
+
+  return results
+}

src/app/api/v1/create/index.ts

@@ -7,9 +7,10 @@ import { predict } from "@/app/api/providers/huggingface/predictWithHuggingFace"
 import { parseRawStringToYAML } from "@/app/api/parsers/parseRawStringToYAML"
 import { LatentStory } from "@/app/api/v1/types"
 
-import { systemPrompt } from "./systemPrompt"
+import { systemPromptCompleteStory } from "./systemPromptCompleteStory"
 import { generateMusicPrompts } from "../edit/music/generateMusicPrompt"
 import { clapToLatentStory } from "../edit/entities/clapToLatentStory"
+import { generateRandomStory } from "./generateRandomStory"
 
 // a helper to generate Clap stories from a few sentences
 // this is mostly used by external apps such as the Stories Factory
@@ -26,11 +27,14 @@ export async function create(request: {
 }): Promise<ClapProject> {
 
   // we limit to 512 characters
-  const prompt = `${request?.prompt || ""}`.trim().slice(0, 512)
+  let prompt = `${request?.prompt || ""}`.trim().slice(0, 512)
 
   console.log("api/v1/create(): request:", request)
 
-  if (!prompt.length) { throw new Error(`please provide a prompt`) }
+  if (!prompt.length) {
+    // throw new Error(`please provide a prompt`)
+    prompt = generateRandomStory()
+  }
 
   const width = getValidNumber(request?.width, 256, 8192, 1024)
   const height = getValidNumber(request?.height, 256, 8192, 576)
@@ -49,7 +53,7 @@ Output: `
   // so technically we could stream everything from end-to-end
   // (but I haven't coded the helpers to do this yet)
   let rawString = await predict({
-    systemPrompt,
+    systemPrompt: systemPromptCompleteStory,
     userPrompt,
     nbMaxNewTokens,
     prefix,
@@ -68,7 +72,7 @@ Output: `
     await sleep(2000)
 
     rawString = await predict({
-      systemPrompt,
+      systemPrompt: systemPromptCompleteStory,
       userPrompt: userPrompt + ".", // we trick the Hugging Face cache
       nbMaxNewTokens,
       prefix,

src/app/api/v1/create/{systemPrompt.ts → systemPromptCompleteStory.ts}

@@ -1,4 +1,4 @@
-export const systemPrompt: string =
+export const systemPromptCompleteStory: string =
   `# Context
 You are a server-side function generating stories from a single synopsis/brief (a "prompt").
 The videos are meant to be shared on social media platform (Instagram, TikTok, Snapchat, Twitter, YouTube Shorts etc).

src/app/api/v1/create/systemPromptRandomStory.ts

@@ -0,0 +1,13 @@
+export const systemPromptRandomStory: string =
+  `# Context
+You are a server-side function generating story ideas from a single synopsis/brief (a "prompt").
+
+# Task
+Your mission is to invent a RANDOM story, by taking inspiration from a set of keywords.
+
+You only must generate a sommary and description of the story, not the whole thing.
+
+# Expected output
+
+Generate or or two sentences maximum. NEVER generate violent stories, or stories related to some religions or communities.
+`

src/lib/fun-words/declaration.d.ts

@@ -0,0 +1,22 @@
+declare module 'fun-word-list' {
+  import * as funWordList from 'fun-word-list';
+
+  type FunWordList = {
+    addons: string[] // Fragments to add to the end of sentences for different moods. Many include preceding commas or periods.
+    adjectives: string[]  // For use with `nouns` or `concepts`
+    concepts: string[] // Complex nouns
+    descriptions: string[] // For use with `locations`
+    gametypes: string[] // Video game genres/playstyles/feels
+    locations: string[] // Nouns that represent places
+    nouns: string[][]  // Simple things that can act or be acted upon
+
+    superlatives: string[] // Adjectives that express the extremes of something. For use with `nouns`, `concepts`, or `locations`
+    verbs: {  // All verbs in this list are used for acting in relation to another thing.
+      secondPerson: string[]  // Verbs for use with "you" or "they" as in "you help the aardvark" or "they fight a bird"
+      secondConcepts: string[]  // Verbs for use with "you" or "they" in combination with `concepts` as in "you are helped by the King" or "they fight against traditions"
+      thirdPerson: string[] // Verbs for use with `nouns` as in "The aardvark anticipates rain"
+    }
+  }
+
+ export default funWordList as FunWordList
+}

src/lib/fun-words/filterSensitiveContent.ts

@@ -0,0 +1,81 @@
+
+// unfortunately the lib comes with some words
+// that, when combined, could create some bad buzz
+export const sensitiveTopics = [
+  "The Pope",
+  "gag",
+  "sadist",
+  "actress",
+  "razor",
+  "female",
+  "destroyer",
+  "prisonner",
+  "rib",
+  "cultist",
+  "candy",
+  "candies",
+  "skull",
+  "injury",
+  "lungs",
+  "field",
+  "child",
+  "kid",
+  "walk",
+  "stranger",
+  "children",
+  "faker",
+  "fake",
+  "curse",
+  "scissor",
+  "hobo",
+  "maniac",
+  "drug",
+  "gang",
+  "predator",
+  "hostage",
+  "cutthroat",
+  "swallow",
+  "shotgun",
+  "gun",
+  "pickaxe",
+  "clergymen",
+  "clergyman",
+  "clergy",
+  "dictator",
+  "microbe",
+  "convict",
+  "sniper",
+  "skeleton",
+  "aborigine",
+  "skirt",
+  "thug",
+  "thief",
+  "handgun",
+  "executioner",
+  "beast",
+  "nymph",
+  "rabbis",
+  "rabbies",
+  "rabbi",
+  "giblet",
+  "prisoner",
+  "machine gun",
+  "infant",
+  "crucifix",
+  "hatchet",
+  "beggar",
+  "indian",
+  "corrupted",
+  "russian",
+  "italian",
+  "greek",
+  "bandit",
+  "vulture",
+  "scavenger",
+  "hacksaw",
+  "chick"
+]
+
+export function filterSensitiveContent(words: string[], badWords: string[] = sensitiveTopics): string[] {
+  return words.filter(x => !badWords.includes(x.toLowerCase().trim()))
+}

src/lib/fun-words/index.ts

@@ -0,0 +1,35 @@
+import funWordList from "fun-word-list"
+
+const {
+  addons, // Fragments to add to the end of sentences for different moods. Many include preceding commas or periods.
+  adjectives,  // For use with `nouns` or `concepts`
+  concepts, // Complex nouns
+  descriptions, // For use with `locations`
+  gametypes, // Video game genres/playstyles/feels
+  locations, // Nouns that represent places
+  nouns, // Simple things that can act or be acted upon
+  superlatives, // Adjectives that express the extremes of something. For use with `nouns`, `concepts`, or `locations`
+  verbs: {  // All verbs in this list are used for acting in relation to another thing.
+    secondPerson,  // Verbs for use with "you" or "they" as in "you help the aardvark" or "they fight a bird"
+    secondConcepts,  // Verbs for use with "you" or "they" in combination with `concepts` as in "you are helped by the King" or "they fight against traditions"
+    thirdPerson, // Verbs for use with `nouns` as in "The aardvark anticipates rain"
+  }
+} = funWordList
+
+const verbSecondPerson = secondPerson  // Verbs for use with "you" or "they" as in "you help the aardvark" or "they fight a bird"
+const verbSecondConcepts = secondConcepts  // Verbs for use with "you" or "they" in combination with `concepts` as in "you are helped by the King" or "they fight against traditions"
+const verbThirdPerson = thirdPerson // Verbs for use with `nouns` as in "The aardvark anticipates rain"
+
+export {
+  addons, // Fragments to add to the end of sentences for different moods. Many include preceding commas or periods.
+  adjectives,  // For use with `nouns` or `concepts`
+  concepts, // Complex nouns
+  descriptions, // For use with `locations`
+  gametypes, // Video game genres/playstyles/feels
+  locations, // Nouns that represent places
+  nouns, // Simple things that can act or be acted upon
+  superlatives, // Adjectives that express the extremes of something. For use with `nouns`, `concepts`, or `locations`
+  verbSecondPerson,  // Verbs for use with "you" or "they" as in "you help the aardvark" or "they fight a bird"
+  verbSecondConcepts,  // Verbs for use with "you" or "they" in combination with `concepts` as in "you are helped by the King" or "they fight against traditions"
+  verbThirdPerson, // Verbs for use with `nouns` as in "The aardvark anticipates rain"
+}

src/lib/utils/pick.ts

@@ -1,2 +1,22 @@
 
-export const pick = (items: string[]) => items[Math.floor(Math.random()*items.length)]
+export function pick<T>(items: T[], defaultValue: T, {
+  skipList = [],
+  maxRetries = 10
+}: {
+  skipList?: T[]
+  maxRetries?: number
+} = {
+  skipList: [],
+  maxRetries: 10
+}): T {
+  let candidate: T | undefined = undefined
+  for (let i = 0; i < maxRetries; i++) {
+    candidate = items[Math.floor(Math.random() * items.length)] as T
+    if (skipList.includes(candidate)) { continue }
+  }
+  if (typeof candidate === "undefined") {
+    return defaultValue
+  } else {
+    return candidate
+  }
+}