added an upscaler for images

src/production/generateAudio.mts

@@ -37,7 +37,7 @@ export async function generateAudio(prompt: string, audioFileName: string) {
         waitUntil: "networkidle2",
       })
 
-      await new Promise(r => setTimeout(r, 3000))
+      // await new Promise(r => setTimeout(r, 1000))
 
       const firstTextboxInput = await page.$('input[data-testid="textbox"]')
 

src/production/generateVoice.mts

@@ -36,7 +36,7 @@ export async function generateVoice(prompt: string, voiceFileName: string) {
         waitUntil: "networkidle2",
       })
 
-      await new Promise(r => setTimeout(r, 3000))
+      // await new Promise(r => setTimeout(r, 1000))
 
       const firstTextarea = await page.$('textarea[data-testid="textbox"]')
 

src/production/interpolateVideo.mts

@@ -42,7 +42,7 @@ export async function interpolateVideo(fileName: string, steps: number, fps: num
       const page = await browser.newPage()
       await page.goto(instance, { waitUntil: 'networkidle2' })
       
-      await new Promise(r => setTimeout(r, 3000))
+      // await new Promise(r => setTimeout(r, 1000))
 
       const fileField = await page.$('input[type=file]')
 

src/production/renderImageSegmentation.mts

@@ -17,7 +17,6 @@ export async function renderImageSegmentation(
 
   if (actionnables.length > 0) {
     console.log("we have some actionnables:", actionnables)
-    console.log("going to grab the first frame")
 
     const tmpImageFilePath = path.join(tmpDir, `${uuidv4()}.png`)
 
@@ -26,7 +25,7 @@ export async function renderImageSegmentation(
     console.log("wrote the image to ", tmpImageFilePath)
   
     if (!tmpImageFilePath) {
-      console.error("failed to get the image")
+      console.error("failed to segment the image")
       response.error = "failed to segment the image"
       response.status = "error"
     } else {

src/production/renderImageUpscaling.mts

@@ -0,0 +1,26 @@
+import { RenderedScene, RenderRequest } from "../types.mts"
+import { upscaleImage } from "../utils/upscaleImage.mts"
+
+export async function renderImageUpscaling(
+  request: RenderRequest,
+  response: RenderedScene,
+): Promise<RenderedScene> {
+  
+  try {
+    // note: this converts a base64 PNG to a base64 JPG (which is good, actually!)
+    response.assetUrl = await upscaleImage(response.assetUrl)
+    console.log(`upscaling worked on the first try!`)
+  } catch (err) {
+    console.error(`upscaling failed the first time.. let's try again..`)
+    try {
+      response.assetUrl = await upscaleImage(response.assetUrl)
+      console.log(`upscaling worked on the second try!`)
+    } catch (err) {
+      console.error(`upscaling failed on the second attempt.. let's keep the low-res image then :|`)
+      // no need to log a catastrophic failure here, since we still have the original (low-res image)
+      // to work with
+    }
+  }
+
+  return response
+}

src/production/renderPipeline.mts

@@ -5,6 +5,8 @@ import { renderImage } from "./renderImage.mts"
 import { renderVideo } from "./renderVideo.mts"
 import { renderImageSegmentation } from "./renderImageSegmentation.mts"
 import { renderVideoSegmentation } from "./renderVideoSegmentation.mts"
+import { upscaleImage } from "../utils/upscaleImage.mts"
+import { renderImageUpscaling } from "./renderImageUpscaling.mts"
 
 export async function renderPipeline(request: RenderRequest, response: RenderedScene) {
   const isVideo = request?.nbFrames > 1
@@ -18,7 +20,18 @@ export async function renderPipeline(request: RenderRequest, response: RenderedS
     console.log(`rendering an image..`)
   }
   await renderContent(request, response)
-  await renderSegmentation(request, response)
+
+  // we upscale images with esrgan
+  // and for videos, well.. let's just skip this part,
+  // but later we could use Zeroscope V2 XL maybe?
+  const optionalUpscalingStep = isVideo
+    ? Promise.resolve()
+    : renderImageUpscaling(request, response)
+
+  await Promise.all([
+    renderSegmentation(request, response),
+    optionalUpscalingStep
+  ])
 
   /*
   this is the optimized pipeline

src/production/upscaleVideo.mts

@@ -35,7 +35,7 @@ export async function upscaleVideo(fileName: string, prompt: string) {
     const inputFilePath = path.join(pendingFilesDirFilePath, fileName)
     // console.log(`local file to upscale: ${inputFilePath}`)
     
-    await new Promise(r => setTimeout(r, 3000))
+    // await new Promise(r => setTimeout(r, 1000))
 
     const fileField = await page.$('input[type=file]')
 

src/utils/segmentImage.mts

@@ -40,7 +40,7 @@ export async function segmentImage(
     const page = await browser.newPage()
     await page.goto(instance, { waitUntil: 'networkidle2' })
     
-    await new Promise(r => setTimeout(r, 3000))
+    // await new Promise(r => setTimeout(r, 1000))
 
     const fileField = await page.$('input[type="file"]')
 

src/utils/upscaleImage.mts

@@ -0,0 +1,42 @@
+
+import { client } from "@gradio/client"
+
+import { getValidNumber } from "./getValidNumber.mts"
+
+// we don't use replicas yet, because it ain't easy to get their hostname
+const instances: string[] = [
+  `${process.env.VC_UPSCALING_SPACE_API_URL_1 || ""}`,
+  // `${process.env.VC_UPSCALING_SPACE_API_URL_2 || ""}`,
+].filter(instance => instance?.length > 0)
+
+// this doesn't work because of this error.. I think the version of Gradio is too old/young?
+// ReferenceError: addEventListener is not defined
+//    at file:///Users/jbilcke/Projects/VideoChain-API/node_modules/@gradio/client/dist/index.js:551:15
+//    at processTicksAndRejections (node:internal/process/task_queues:95:5)
+export async function upscaleImage(src: string, factor?: number) {
+
+  // bu default we do a 4X scale
+  const scaleFactor = getValidNumber(factor, 2, 10, 4)
+
+  const instance = instances.shift()
+  instances.push(instance)
+
+  const api = await client(instance, {
+    hf_token: `${process.env.VC_HF_API_TOKEN}` as any
+  })
+  
+  const result = await api.predict("/upscale", [
+    src, 	// blob in 'Source Image' Image component		
+    "realesr-general-x4v3", // string (Option from: ['RealESRGAN_x4plus', 'RealESRNet_x4plus', 'RealESRGAN_x4plus_anime_6B', 'RealESRGAN_x2plus', 'realesr-general-x4v3']) in 'Real-ESRGAN inference model to be used' Dropdown component		
+    0.5, // number (numeric value between 0 and 1) in 'Denoise Strength (Used only with the realesr-general-x4v3 model)' Slider component		
+    false, // boolean  in 'Face Enhancement using GFPGAN (Doesn't work for anime images)' Checkbox component		
+    scaleFactor, // number (numeric value between 1 and 10) in 'Image Upscaling Factor' Slider component
+]);
+
+
+  const rawResponse = result as any 
+
+  // console.log("rawResponse:", rawResponse)
+  
+  return rawResponse?.data?.[0] as string
+}