Feature for runners - handle storyboard-generation-job (#7191)

* Implement processing storyboards by runners

* Fixed storyboard generation by runners

* use common code patterns

* fix import

* improve debug logging for storyboard generation

* config option for storyboard processing with remote-runners

* refactor repetitive pattern

* refactor storyboard related code to share common utlities

* Fix test

* Fix storyboard generation config logic

* Improve logging

* Added tests for storyboard generation with runners

* Refactor PR

---------

Co-authored-by: ilfarpro <ilfarpro@ya.ru>
Co-authored-by: Chocobozzz <me@florianbigard.com>

apps/peertube-runner/src/server/process/process.ts

@@ -7,7 +7,13 @@ import {
   RunnerJobVODWebVideoTranscodingPayload
 } from '@peertube/peertube-models'
 import { logger } from '../../shared/index.js'
-import { processAudioMergeTranscoding, processHLSTranscoding, ProcessOptions, processWebVideoTranscoding } from './shared/index.js'
+import {
+  processAudioMergeTranscoding,
+  processGenerateStoryboard,
+  processHLSTranscoding,
+  ProcessOptions,
+  processWebVideoTranscoding
+} from './shared/index.js'
 import { ProcessLiveRTMPHLSTranscoding } from './shared/process-live.js'
 import { processStudioTranscoding } from './shared/process-studio.js'
 import { processVideoTranscription } from './shared/process-transcription.js'
@@ -42,6 +48,10 @@ export async function processJob (options: ProcessOptions) {
       await processVideoTranscription(options as ProcessOptions<RunnerJobTranscriptionPayload>)
       break
 
+    case 'generate-video-storyboard':
+      await processGenerateStoryboard(options as any)
+      break
+
     default:
       logger.error(`Unknown job ${job.type} to process`)
       return

apps/peertube-runner/src/server/process/shared/common.ts

@@ -1,5 +1,12 @@
 import { pick } from '@peertube/peertube-core-utils'
-import { FFmpegEdition, FFmpegLive, FFmpegVOD, getDefaultAvailableEncoders, getDefaultEncodersToTry } from '@peertube/peertube-ffmpeg'
+import {
+  FFmpegEdition,
+  FFmpegImage,
+  FFmpegLive,
+  FFmpegVOD,
+  getDefaultAvailableEncoders,
+  getDefaultEncodersToTry
+} from '@peertube/peertube-ffmpeg'
 import { RunnerJob, RunnerJobPayload } from '@peertube/peertube-models'
 import { buildUUID } from '@peertube/peertube-node-utils'
 import { PeerTubeServer } from '@peertube/peertube-server-commands'
@@ -8,9 +15,9 @@ import { join } from 'path'
 import { ConfigManager, downloadFile, logger } from '../../../shared/index.js'
 import { getWinstonLogger } from './winston-logger.js'
 
-export type JobWithToken <T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string }
+export type JobWithToken<T extends RunnerJobPayload = RunnerJobPayload> = RunnerJob<T> & { jobToken: string }
 
-export type ProcessOptions <T extends RunnerJobPayload = RunnerJobPayload> = {
+export type ProcessOptions<T extends RunnerJobPayload = RunnerJobPayload> = {
   server: PeerTubeServer
   job: JobWithToken<T>
   runnerToken: string
@@ -108,6 +115,10 @@ export function buildFFmpegEdition () {
   return new FFmpegEdition(getCommonFFmpegOptions())
 }
 
+export function buildFFmpegImage () {
+  return new FFmpegImage(getCommonFFmpegOptions())
+}
+
 function getCommonFFmpegOptions () {
   const config = ConfigManager.Instance.getConfig()
 

apps/peertube-runner/src/server/process/shared/index.ts

@@ -1,3 +1,4 @@
 export * from './common.js'
 export * from './process-vod.js'
 export * from './winston-logger.js'
+export * from './process-storyboard.js'

apps/peertube-runner/src/server/process/shared/process-storyboard.ts

@@ -0,0 +1,60 @@
+import { RunnerJobGenerateStoryboardPayload, GenerateStoryboardSuccess } from '@peertube/peertube-models'
+import { buildUUID } from '@peertube/peertube-node-utils'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
+import { ConfigManager } from '../../../shared/config-manager.js'
+import { logger } from '../../../shared/index.js'
+import { buildFFmpegImage, downloadInputFile, ProcessOptions, scheduleTranscodingProgress } from './common.js'
+
+export async function processGenerateStoryboard (options: ProcessOptions<RunnerJobGenerateStoryboardPayload>) {
+  const { server, job, runnerToken } = options
+
+  const payload = job.payload
+
+  let ffmpegProgress: number
+  let videoInputPath: string
+
+  const outputPath = join(ConfigManager.Instance.getStoryboardDirectory(), `storyboard-${buildUUID()}.jpg`)
+
+  const updateProgressInterval = scheduleTranscodingProgress({
+    job,
+    server,
+    runnerToken,
+    progressGetter: () => ffmpegProgress
+  })
+
+  try {
+    logger.info(`Downloading input file ${payload.input.videoFileUrl} for storyboard job ${job.jobToken}`)
+
+    videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+
+    logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Generating storyboard.`)
+
+    const ffmpegImage = buildFFmpegImage()
+
+    await ffmpegImage.generateStoryboardFromVideo({
+      path: videoInputPath,
+      destination: outputPath,
+      inputFileMutexReleaser: () => {},
+      sprites: payload.sprites
+    })
+
+    const successBody: GenerateStoryboardSuccess = {
+      storyboardFile: outputPath
+    }
+
+    await server.runnerJobs.success({
+      jobToken: job.jobToken,
+      jobUUID: job.uuid,
+      runnerToken,
+      payload: successBody,
+      reqPayload: payload
+    })
+  } finally {
+    if (videoInputPath) await remove(videoInputPath)
+    if (outputPath) await remove(outputPath)
+    if (updateProgressInterval) clearInterval(updateProgressInterval)
+  }
+}
+
+

apps/peertube-runner/src/server/server.ts

@@ -74,9 +74,11 @@ export class RunnerServer {
 
     // Process jobs
     await ensureDir(ConfigManager.Instance.getTranscodingDirectory())
+    await ensureDir(ConfigManager.Instance.getStoryboardDirectory())
     await this.cleanupTMP()
 
     logger.info(`Using ${ConfigManager.Instance.getTranscodingDirectory()} for transcoding directory`)
+    logger.info(`Using ${ConfigManager.Instance.getStoryboardDirectory()} for storyboard directory`)
 
     this.initialized = true
     await this.checkAvailableJobs()

apps/peertube-runner/src/server/shared/supported-job.ts

@@ -7,7 +7,8 @@ import {
   RunnerJobVODAudioMergeTranscodingPayload,
   RunnerJobVODHLSTranscodingPayload,
   RunnerJobVODWebVideoTranscodingPayload,
-  VideoStudioTaskPayload
+  VideoStudioTaskPayload,
+  RunnerJobGenerateStoryboardPayload
 } from '@peertube/peertube-models'
 
 const supportedMatrix: { [ id in RunnerJobType ]: (payload: RunnerJobPayload) => boolean } = {
@@ -33,7 +34,8 @@ const supportedMatrix: { [ id in RunnerJobType ]: (payload: RunnerJobPayload) =>
   },
   'video-transcription': (_payload: RunnerJobTranscriptionPayload) => {
     return true
-  }
+  },
+  'generate-video-storyboard': (_payload: RunnerJobGenerateStoryboardPayload) => true
 }
 
 export function isJobSupported (job: { type: RunnerJobType, payload: RunnerJobPayload }, enabledJobs?: Set<RunnerJobType>) {

apps/peertube-runner/src/shared/config-manager.ts

@@ -108,6 +108,10 @@ export class ConfigManager {
 
   // ---------------------------------------------------------------------------
 
+  getStoryboardDirectory () {
+    return join(paths.cache, this.id, 'storyboard')
+  }
+
   getTranscodingDirectory () {
     return join(paths.cache, this.id, 'transcoding')
   }

client/src/app/+admin/config/pages/admin-config-general.component.html

@@ -395,6 +395,20 @@
             <ng-container ngProjectAs="description">
               <span i18n>Generate storyboards of local videos using ffmpeg so users can see the video preview in the player while scrubbing the video</span>
             </ng-container>
+
+            <ng-container ngProjectAs="extra">
+              <div class="form-group" formGroupName="remoteRunners" [ngClass]="getStoryboardRunnerDisabledClass()">
+                <my-peertube-checkbox
+                  inputName="storyboardsRemoteRunnersEnabled" formControlName="enabled"
+                  i18n-labelText labelText="Enable remote runners to generate storyboards"
+                >
+                  <ng-container ngProjectAs="description">
+                    <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to generate storyboards.</div>
+                    <div i18n>Remote runners have to register on your instance first.</div>
+                  </ng-container>
+                </my-peertube-checkbox>
+              </div>
+            </ng-container>
           </my-peertube-checkbox>
         </div>
       </ng-container>
@@ -416,10 +430,8 @@
                   i18n-labelText labelText="Enable remote runners for transcription"
                 >
                   <ng-container ngProjectAs="description">
-                    <span i18n>
+                    <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process transcription tasks.</div>
-                      Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process transcription tasks.
+                    <div i18n>Remote runners have to register on your instance first.</div>
-                      Remote runners has to register on your instance first.
-                    </span>
                   </ng-container>
                 </my-peertube-checkbox>
               </div>

client/src/app/+admin/config/pages/admin-config-general.component.ts

@@ -182,6 +182,10 @@ type Form = {
 
   storyboards: FormGroup<{
     enabled: FormControl<boolean>
+
+    remoteRunners: FormGroup<{
+      enabled: FormControl<boolean>
+    }>
   }>
 
   defaults: FormGroup<{
@@ -423,7 +427,10 @@ export class AdminConfigGeneralComponent implements OnInit, OnDestroy, CanCompon
         }
       },
       storyboards: {
-        enabled: null
+        enabled: null,
+        remoteRunners: {
+          enabled: null
+        }
       },
       defaults: {
         publish: {
@@ -536,6 +543,16 @@ export class AdminConfigGeneralComponent implements OnInit, OnDestroy, CanCompon
 
   // ---------------------------------------------------------------------------
 
+  isStoryboardEnabled () {
+    return this.form.value.storyboards.enabled === true
+  }
+
+  getStoryboardRunnerDisabledClass () {
+    return { 'disabled-checkbox-extra': !this.isStoryboardEnabled() }
+  }
+
+  // ---------------------------------------------------------------------------
+
   isAutoFollowIndexEnabled () {
     return this.form.value.followings.instance.autoFollowIndex.enabled === true
   }

client/src/app/+admin/config/pages/admin-config-live.component.html

@@ -167,10 +167,8 @@
               i18n-labelText labelText="Enable remote runners for lives"
             >
               <ng-container ngProjectAs="description">
-                <span i18n>
+                <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process live transcoding.</div>
-                  Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process live transcoding.
+                <div i18n>Remote runners have to register on your instance first.</div>
-                  Remote runners has to register on your instance first.
-                </span>
               </ng-container>
             </my-peertube-checkbox>
           </div>

client/src/app/+admin/config/pages/admin-config-vod.component.html

@@ -185,10 +185,8 @@
             i18n-labelText labelText="Enable remote runners for VOD"
           >
             <ng-container ngProjectAs="description">
-              <span i18n>
+              <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process VOD transcoding.</div>
-                Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process VOD transcoding.
+              <div i18n>Remote runners have to register on your instance first.</div>
-                Remote runners has to register on your instance first.
-              </span>
             </ng-container>
           </my-peertube-checkbox>
         </div>
@@ -262,19 +260,19 @@
             <ng-container ngProjectAs="description" *ngIf="!isTranscodingEnabled()">
               <span i18n>⚠️ You need to enable transcoding first to enable video studio</span>
             </ng-container>
-          </my-peertube-checkbox>
-        </div>
 
-        <div class="form-group" formGroupName="remoteRunners" [ngClass]="getStudioDisabledClass()">
+            <ng-container ngProjectAs="extra">
-          <my-peertube-checkbox
+              <div class="form-group" formGroupName="remoteRunners" [ngClass]="getStudioRunnerDisabledClass()">
-            inputName="videoStudioRemoteRunnersEnabled" formControlName="enabled"
+                <my-peertube-checkbox
-            i18n-labelText labelText="Enable remote runners for studio"
+                  inputName="videoStudioRemoteRunnersEnabled" formControlName="enabled"
-          >
+                  i18n-labelText labelText="Enable remote runners for studio"
-            <ng-container ngProjectAs="description">
+                >
-              <span i18n>
+                  <ng-container ngProjectAs="description">
-                Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process studio transcoding tasks.
+                    <div i18n>Use <a routerLink="/admin/settings/system/runners/runners-list">remote runners</a> to process studio transcoding tasks.</div>
-                Remote runners has to register on your instance first.
+                    <div i18n>Remote runners have to register on your instance first.</div>
-              </span>
+                  </ng-container>
+                </my-peertube-checkbox>
+              </div>
             </ng-container>
           </my-peertube-checkbox>
         </div>

client/src/app/+admin/config/pages/admin-config-vod.component.ts

@@ -226,7 +226,7 @@ export class AdminConfigVODComponent implements OnInit, OnDestroy, CanComponentD
     return { 'disabled-checkbox-extra': !this.isTranscodingEnabled() || this.isRemoteRunnerVODEnabled() }
   }
 
-  getStudioDisabledClass () {
+  getStudioRunnerDisabledClass () {
     return { 'disabled-checkbox-extra': !this.isStudioEnabled() }
   }
 

config/default.yaml

@@ -1137,6 +1137,10 @@ storyboards:
   # Generate storyboards of local videos using ffmpeg so users can see the video preview in the player while scrubbing the video
   enabled: true
 
+  remote_runners:
+    # Use remote runners to generate storyboards instead of processing them locally
+    enabled: false
+
 # Update default PeerTube values
 # Set by API when the field is not provided and put as default value in client
 defaults:

config/production.yaml.example

@@ -340,6 +340,9 @@ security:
   powered_by_header:
     enabled: true
 
+http_server:
+  request_timeout: '5 minutes'
+
 tracker:
   # If you disable the tracker, you disable the P2P on your PeerTube instance
   enabled: true
@@ -1144,6 +1147,10 @@ storyboards:
   # Generate storyboards of local videos using ffmpeg so users can see the video preview in the player while scrubbing the video
   enabled: true
 
+  remote_runners:
+    # Use remote runners to generate storyboards instead of processing them locally
+    enabled: false
+
 # Update default PeerTube values
 # Set by API when the field is not provided and put as default value in client
 defaults:

packages/models/src/runners/runner-jobs/runner-job-payload.model.ts

@@ -14,7 +14,8 @@ export type RunnerJobPayload =
   RunnerJobVODPayload |
   RunnerJobLiveRTMPHLSTranscodingPayload |
   RunnerJobStudioTranscodingPayload |
-  RunnerJobTranscriptionPayload
+  RunnerJobTranscriptionPayload |
+  RunnerJobGenerateStoryboardPayload
 
 // ---------------------------------------------------------------------------
 
@@ -90,6 +91,24 @@ export interface RunnerJobTranscriptionPayload {
   }
 }
 
+export interface RunnerJobGenerateStoryboardPayload {
+  input: {
+    videoFileUrl: string
+  }
+
+  // Computed server-side for consistency across runners
+  sprites: {
+    size: { width: number, height: number }
+    count: { width: number, height: number }
+    duration: number
+  }
+
+  output: {
+    // To upload on an external URL
+    storyboardFileCustomUpload?: RunnerJobCustomUpload
+  }
+}
+
 // ---------------------------------------------------------------------------
 
 export function isAudioMergeTranscodingPayload (payload: RunnerJobPayload): payload is RunnerJobVODAudioMergeTranscodingPayload {

packages/models/src/runners/runner-jobs/runner-job-private-payload.model.ts

@@ -1,15 +1,16 @@
 import { VideoStudioTaskPayload } from '../../server/index.js'
 
 export type RunnerJobVODPrivatePayload =
-  RunnerJobVODWebVideoTranscodingPrivatePayload |
+  | RunnerJobVODWebVideoTranscodingPrivatePayload
-  RunnerJobVODAudioMergeTranscodingPrivatePayload |
+  | RunnerJobVODAudioMergeTranscodingPrivatePayload
-  RunnerJobVODHLSTranscodingPrivatePayload
+  | RunnerJobVODHLSTranscodingPrivatePayload
 
 export type RunnerJobPrivatePayload =
-  RunnerJobVODPrivatePayload |
+  | RunnerJobVODPrivatePayload
-  RunnerJobLiveRTMPHLSTranscodingPrivatePayload |
+  | RunnerJobLiveRTMPHLSTranscodingPrivatePayload
-  RunnerJobVideoStudioTranscodingPrivatePayload |
+  | RunnerJobVideoStudioTranscodingPrivatePayload
-  RunnerJobTranscriptionPrivatePayload
+  | RunnerJobTranscriptionPrivatePayload
+  | RunnerJobGenerateStoryboardPrivatePayload
 
 // ---------------------------------------------------------------------------
 
@@ -52,3 +53,7 @@ export interface RunnerJobVideoStudioTranscodingPrivatePayload {
 export interface RunnerJobTranscriptionPrivatePayload {
   videoUUID: string
 }
+
+export interface RunnerJobGenerateStoryboardPrivatePayload {
+  videoUUID: string
+}

packages/models/src/runners/runner-jobs/runner-job-success-body.model.ts

@@ -13,7 +13,8 @@ export type RunnerJobSuccessPayload =
   VODAudioMergeTranscodingSuccess |
   LiveRTMPHLSTranscodingSuccess |
   VideoStudioTranscodingSuccess |
-  TranscriptionSuccess
+  TranscriptionSuccess |
+  GenerateStoryboardSuccess
 
 export interface VODWebVideoTranscodingSuccess {
   videoFile: Blob | string
@@ -42,6 +43,10 @@ export interface TranscriptionSuccess {
   vttFile: Blob | string
 }
 
+export interface GenerateStoryboardSuccess {
+  storyboardFile: Blob | string
+}
+
 export function isWebVideoOrAudioMergeTranscodingPayloadSuccess (
   payload: RunnerJobSuccessPayload
 ): payload is VODHLSTranscodingSuccess | VODAudioMergeTranscodingSuccess {
@@ -55,3 +60,7 @@ export function isHLSTranscodingPayloadSuccess (payload: RunnerJobSuccessPayload
 export function isTranscriptionPayloadSuccess (payload: RunnerJobSuccessPayload): payload is TranscriptionSuccess {
   return !!(payload as TranscriptionSuccess)?.vttFile
 }
+
+export function isGenerateStoryboardSuccess (payload: RunnerJobSuccessPayload): payload is GenerateStoryboardSuccess {
+  return !!(payload as GenerateStoryboardSuccess)?.storyboardFile
+}

packages/models/src/runners/runner-jobs/runner-job-type.type.ts

@@ -4,4 +4,5 @@ export type RunnerJobType =
   'vod-audio-merge-transcoding' |
   'live-rtmp-hls-transcoding' |
   'video-studio-transcoding' |
-  'video-transcription'
+  'video-transcription' |
+  'generate-video-storyboard'

packages/models/src/server/custom-config.model.ts

@@ -345,6 +345,9 @@ export interface CustomConfig {
 
   storyboards: {
     enabled: boolean
+    remoteRunners: {
+      enabled: boolean
+    }
   }
 
   defaults: {

packages/models/src/server/server-config.model.ts

@@ -418,6 +418,9 @@ export interface ServerConfig {
 
   storyboards: {
     enabled: boolean
+    remoteRunners: {
+      enabled: boolean
+    }
   }
 
   videoTranscription: {

packages/server-commands/src/runners/runner-jobs-command.ts

@@ -4,6 +4,7 @@ import {
   AcceptRunnerJobBody,
   AcceptRunnerJobResult,
   ErrorRunnerJobBody,
+  GenerateStoryboardSuccess,
   HttpStatusCode,
   ListRunnerJobsQuery,
   RequestRunnerJobBody,
@@ -11,6 +12,7 @@ import {
   ResultList,
   RunnerJobAdmin,
   RunnerJobCustomUpload,
+  RunnerJobGenerateStoryboardPayload,
   RunnerJobLiveRTMPHLSTranscodingPayload,
   RunnerJobPayload,
   RunnerJobState,
@@ -26,6 +28,7 @@ import {
   TranscriptionSuccess,
   VODHLSTranscodingSuccess,
   VODWebVideoTranscodingSuccess,
+  isGenerateStoryboardSuccess,
   isHLSTranscodingPayloadSuccess,
   isLiveRTMPHLSTranscodingUpdatePayload,
   isTranscriptionPayloadSuccess,
@@ -265,6 +268,22 @@ export class RunnerJobsCommand extends AbstractCommand {
       payloadWithoutFiles = omit(payloadWithoutFiles as TranscriptionSuccess, [ 'vttFile' ])
     }
 
+    // Generate storyboard success payload contains a storyboard image file
+    if (isGenerateStoryboardSuccess(payload) && payload.storyboardFile) {
+      const reqPayload = options.reqPayload as RunnerJobGenerateStoryboardPayload
+
+      this.updateUploadPayloads({
+        attachesStore: attaches,
+        customUploadsStore: customUploads,
+
+        file: payload.storyboardFile,
+        attachName: 'storyboardFile',
+        customUpload: reqPayload?.output?.storyboardFileCustomUpload
+      })
+
+      payloadWithoutFiles = omit(payloadWithoutFiles as GenerateStoryboardSuccess, [ 'storyboardFile' ])
+    }
+
     return this.uploadRunnerJobRequest({
       ...options,
 
@@ -367,12 +386,17 @@ export class RunnerJobsCommand extends AbstractCommand {
 
     if (!jobUUID) {
       const { availableJobs } = await this.request({ runnerToken })
-      jobUUID = availableJobs[0].uuid
+      // Find a web video transcoding job specifically
+      const webVideoJob = availableJobs.find(j => j.type === 'vod-web-video-transcoding')
+      if (!webVideoJob) throw new Error('No web video transcoding jobs available')
+
+      jobUUID = webVideoJob.uuid
     }
 
     const { job } = await this.accept({ runnerToken, jobUUID })
     const jobToken = job.jobToken
 
+    // Use a proper fixture file path for testing
     const payload: RunnerJobSuccessPayload = { videoFile: 'video_short.mp4' }
     await this.success({ runnerToken, jobUUID, jobToken, payload, reqPayload: undefined })
 

packages/tests/src/api/runners/index.ts

@@ -1,4 +1,5 @@
 export * from './runner-common.js'
+export * from './runner-generate-storyboard.js'
 export * from './runner-live-transcoding.js'
 export * from './runner-socket.js'
 export * from './runner-studio-transcoding.js'

packages/tests/src/api/runners/runner-generate-storyboard.ts

@@ -0,0 +1,92 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { GenerateStoryboardSuccess, RunnerJobGenerateStoryboardPayload } from '@peertube/peertube-models'
+import {
+  PeerTubeServer,
+  cleanupTests,
+  createMultipleServers,
+  doubleFollow,
+  setAccessTokensToServers,
+  setDefaultVideoChannel,
+  waitJobs
+} from '@peertube/peertube-server-commands'
+import { testImage } from '@tests/shared/checks.js'
+import { checkPersistentTmpIsEmpty } from '@tests/shared/directories.js'
+import { expect } from 'chai'
+
+describe('Test runner generate storyboard', function () {
+  let servers: PeerTubeServer[] = []
+  let runnerToken: string
+
+  before(async function () {
+    this.timeout(120_000)
+
+    servers = await createMultipleServers(2)
+
+    await setAccessTokensToServers(servers)
+    await setDefaultVideoChannel(servers)
+
+    await doubleFollow(servers[0], servers[1])
+
+    await servers[0].config.updateExistingConfig({
+      newConfig: {
+        storyboards: {
+          enabled: true,
+          remoteRunners: {
+            enabled: true
+          }
+        }
+      }
+    })
+
+    runnerToken = await servers[0].runners.autoRegisterRunner()
+  })
+
+  async function upload () {
+    const { uuid } = await servers[0].videos.upload({ attributes: { name: 'video', language: undefined } })
+    await waitJobs(servers)
+
+    const { availableJobs } = await servers[0].runnerJobs.request({ runnerToken })
+    expect(availableJobs).to.have.lengthOf(1)
+
+    const jobUUID = availableJobs[0].uuid
+
+    const { job } = await servers[0].runnerJobs.accept<RunnerJobGenerateStoryboardPayload>({ runnerToken, jobUUID })
+    return { uuid, job }
+  }
+
+  it('Should execute a remote generate storyboard job', async function () {
+    this.timeout(240_000)
+
+    const { uuid, job } = await upload()
+
+    expect(job.type === 'generate-video-storyboard')
+    expect(job.payload.input.videoFileUrl).to.exist
+
+    // Check video input file
+    {
+      await servers[0].runnerJobs.getJobFile({ url: job.payload.input.videoFileUrl, jobToken: job.jobToken, runnerToken })
+    }
+
+    const payload: GenerateStoryboardSuccess = {
+      storyboardFile: 'banner.jpg'
+    }
+
+    await servers[0].runnerJobs.success({ runnerToken, jobUUID: job.uuid, jobToken: job.jobToken, payload })
+
+    await waitJobs(servers)
+
+    for (const server of servers) {
+      const { storyboards } = await server.storyboard.list({ id: uuid })
+
+      expect(storyboards).to.have.lengthOf(1)
+
+      await testImage({ name: 'banner.jpg', url: storyboards[0].fileUrl })
+    }
+    await checkPersistentTmpIsEmpty(servers[0])
+  })
+
+  after(async function () {
+    await cleanupTests(servers)
+  })
+})

packages/tests/src/api/server/config.ts

@@ -443,7 +443,10 @@ function buildNewCustomConfig (server: PeerTubeServer): CustomConfig {
       }
     },
     storyboards: {
-      enabled: false
+      enabled: false,
+      remoteRunners: {
+        enabled: true
+      }
     },
     export: {
       users: {

packages/tests/src/api/videos/index.ts

@@ -20,6 +20,7 @@ import './video-schedule-update.js'
 import './video-source.js'
 import './video-static-file-privacy.js'
 import './video-storyboard.js'
+import './video-storyboard-remote-runner.js'
 import './video-transcription.js'
 import './videos-common-filters.js'
 import './videos-history.js'

packages/tests/src/api/videos/video-storyboard.ts

@@ -1,17 +1,11 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
 
-import { expect } from 'chai'
+import { VideoPrivacy } from '@peertube/peertube-models'
-import { readdir } from 'fs/promises'
-import { basename } from 'path'
-import { FIXTURE_URLS } from '@tests/shared/fixture-urls.js'
 import { areHttpImportTestsDisabled } from '@peertube/peertube-node-utils'
-import { HttpStatusCode, VideoPrivacy } from '@peertube/peertube-models'
 import {
   cleanupTests,
   createMultipleServers,
   doubleFollow,
-  makeGetRequest,
-  makeRawRequest,
   PeerTubeServer,
   sendRTMPStream,
   setAccessTokensToServers,
@@ -19,44 +13,11 @@ import {
   stopFfmpeg,
   waitJobs
 } from '@peertube/peertube-server-commands'
-
+import { FIXTURE_URLS } from '@tests/shared/fixture-urls.js'
-async function checkStoryboard (options: {
+import { checkStoryboard } from '@tests/shared/storyboard.js'
-  server: PeerTubeServer
+import { expect } from 'chai'
-  uuid: string
+import { readdir } from 'fs/promises'
-  spriteHeight?: number
+import { basename } from 'path'
-  spriteWidth?: number
-  tilesCount?: number
-  minSize?: number
-  spriteDuration?: number
-}) {
-  const { server, uuid, tilesCount, spriteDuration = 1, spriteHeight = 108, spriteWidth = 192, minSize = 1000 } = options
-
-  const { storyboards } = await server.storyboard.list({ id: uuid })
-
-  expect(storyboards).to.have.lengthOf(1)
-
-  const storyboard = storyboards[0]
-
-  expect(storyboard.spriteDuration).to.equal(spriteDuration)
-  expect(storyboard.spriteHeight).to.equal(spriteHeight)
-  expect(storyboard.spriteWidth).to.equal(spriteWidth)
-  expect(storyboard.storyboardPath).to.exist
-
-  if (tilesCount) {
-    expect(storyboard.totalWidth).to.equal(spriteWidth * Math.min(tilesCount, 11))
-    expect(storyboard.totalHeight).to.equal(spriteHeight * Math.max((tilesCount / 11), 1))
-  }
-
-  {
-    const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })
-    expect(body.length).to.be.above(minSize)
-  }
-
-  {
-    const { body } = await makeRawRequest({ url: storyboard.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
-    expect(body.length).to.be.above(minSize)
-  }
-}
 
 describe('Test video storyboard', function () {
   let servers: PeerTubeServer[]
@@ -205,16 +166,16 @@ describe('Test video storyboard', function () {
     }
 
     {
-      const storyboads = await listFiles()
+      const storyboards = await listFiles()
-      expect(storyboads).to.include(storyboardName)
+      expect(storyboards).to.include(storyboardName)
     }
 
     await servers[0].videos.remove({ id: baseUUID })
     await waitJobs(servers)
 
     {
-      const storyboads = await listFiles()
+      const storyboards = await listFiles()
-      expect(storyboads).to.not.include(storyboardName)
+      expect(storyboards).to.not.include(storyboardName)
     }
   })
 

packages/tests/src/peertube-runner/generate-storyboard.ts

@@ -0,0 +1,189 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { wait } from '@peertube/peertube-core-utils'
+import { RunnerJobState, VideoPrivacy } from '@peertube/peertube-models'
+import { areHttpImportTestsDisabled } from '@peertube/peertube-node-utils'
+import {
+  PeerTubeServer,
+  cleanupTests,
+  createMultipleServers,
+  doubleFollow,
+  sendRTMPStream,
+  setAccessTokensToServers,
+  setDefaultVideoChannel,
+  stopFfmpeg,
+  waitJobs
+} from '@peertube/peertube-server-commands'
+import { checkPeerTubeRunnerCacheIsEmpty } from '@tests/shared/directories.js'
+import { FIXTURE_URLS } from '@tests/shared/fixture-urls.js'
+import { PeerTubeRunnerProcess } from '@tests/shared/peertube-runner-process.js'
+import { checkNoStoryboard, checkStoryboard } from '@tests/shared/storyboard.js'
+import { expect } from 'chai'
+
+describe('Test generate storyboard in peertube-runner program', function () {
+  let servers: PeerTubeServer[] = []
+  let peertubeRunner: PeerTubeRunnerProcess
+
+  before(async function () {
+    this.timeout(120_000)
+
+    servers = await createMultipleServers(2)
+
+    await setAccessTokensToServers(servers)
+    await setDefaultVideoChannel(servers)
+
+    await doubleFollow(servers[0], servers[1])
+
+    await servers[0].config.updateExistingConfig({
+      newConfig: {
+        storyboards: {
+          enabled: true,
+          remoteRunners: {
+            enabled: true
+          }
+        }
+      }
+    })
+
+    const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken()
+
+    peertubeRunner = new PeerTubeRunnerProcess(servers[0])
+    await peertubeRunner.runServer({ jobType: 'generate-video-storyboard' })
+    await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' })
+  })
+
+  describe('Running storyboard generation', function () {
+    describe('Common on filesystem', function () {
+      it('Should run generate storyboard on classic file without transcoding', async function () {
+        this.timeout(360000)
+
+        const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
+        await waitJobs(servers, { runnerJobs: true })
+
+        for (const server of servers) {
+          await checkStoryboard({ server, uuid, tilesCount: 5 })
+        }
+      })
+
+      it('Should run generate storyboard on classic file with transcoding', async function () {
+        this.timeout(360000)
+
+        await servers[0].config.enableMinimumTranscoding()
+
+        const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
+        await waitJobs(servers, { runnerJobs: true })
+
+        for (const server of servers) {
+          await checkStoryboard({ server, uuid, tilesCount: 5 })
+        }
+
+        await servers[0].config.disableTranscoding()
+      })
+
+      it('Should generate a storyboard after HTTP import', async function () {
+        this.timeout(120000)
+
+        if (areHttpImportTestsDisabled()) return
+
+        // 3s video
+        const { video } = await servers[0].videoImports.importVideo({
+          attributes: {
+            targetUrl: FIXTURE_URLS.goodVideo,
+            channelId: servers[0].store.channel.id,
+            privacy: VideoPrivacy.PUBLIC
+          }
+        })
+        await waitJobs(servers, { runnerJobs: true })
+
+        for (const server of servers) {
+          await checkStoryboard({ server, uuid: video.uuid, spriteHeight: 144, tilesCount: 3 })
+        }
+      })
+
+      it('Should generate a storyboard after torrent import', async function () {
+        this.timeout(240000)
+
+        if (areHttpImportTestsDisabled()) return
+
+        // 10s video
+        const { video } = await servers[0].videoImports.importVideo({
+          attributes: {
+            magnetUri: FIXTURE_URLS.magnet,
+            channelId: servers[0].store.channel.id,
+            privacy: VideoPrivacy.PUBLIC
+          }
+        })
+        await waitJobs(servers, { runnerJobs: true })
+
+        for (const server of servers) {
+          await checkStoryboard({ server, uuid: video.uuid, tilesCount: 10 })
+        }
+      })
+
+      it('Should generate a storyboard after a live', async function () {
+        this.timeout(240000)
+
+        await servers[0].config.enableMinimumTranscoding()
+        await servers[0].config.enableLive({ allowReplay: true, transcoding: true, resolutions: 'min' })
+
+        const { live, video } = await servers[0].live.quickCreate({
+          saveReplay: true,
+          permanentLive: false,
+          privacy: VideoPrivacy.PUBLIC
+        })
+
+        const ffmpegCommand = sendRTMPStream({ rtmpBaseUrl: live.rtmpUrl, streamKey: live.streamKey })
+        await servers[0].live.waitUntilPublished({ videoId: video.id })
+
+        await stopFfmpeg(ffmpegCommand)
+
+        await servers[0].live.waitUntilReplacedByReplay({ videoId: video.id })
+        await waitJobs(servers, { runnerJobs: true })
+
+        for (const server of servers) {
+          await checkStoryboard({ server, uuid: video.uuid })
+        }
+      })
+    })
+
+    describe('When generate storyboard is not enabled in runner', function () {
+      before(async function () {
+        await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' })
+        peertubeRunner.kill()
+        await wait(500)
+
+        const registrationToken = await servers[0].runnerRegistrationTokens.getFirstRegistrationToken()
+        await peertubeRunner.runServer({ jobType: 'live-rtmp-hls-transcoding' })
+        await peertubeRunner.registerPeerTubeInstance({ registrationToken, runnerName: 'runner' })
+      })
+
+      it('Should not run generate storyboard', async function () {
+        this.timeout(60000)
+
+        const { uuid } = await servers[0].videos.quickUpload({ name: 'video' })
+        await waitJobs(servers)
+        await wait(2000)
+
+        const { data } = await servers[0].runnerJobs.list({ stateOneOf: [ RunnerJobState.PENDING ] })
+        expect(data.some(j => j.type === 'generate-video-storyboard')).to.be.true
+
+        await checkNoStoryboard({ server: servers[0], uuid })
+      })
+    })
+
+    describe('Check cleanup', function () {
+      it('Should have an empty cache directory', async function () {
+        await checkPeerTubeRunnerCacheIsEmpty(peertubeRunner, 'storyboard')
+      })
+    })
+  })
+
+  after(async function () {
+    if (peertubeRunner) {
+      await peertubeRunner.unregisterPeerTubeInstance({ runnerName: 'runner' })
+      peertubeRunner.kill()
+    }
+
+    await cleanupTests(servers)
+  })
+})

packages/tests/src/peertube-runner/index.ts

@@ -1,5 +1,6 @@
 export * from './client-cli.js'
 export * from './custom-upload.js'
+export * from './generate-storyboard.js'
 export * from './live-transcoding.js'
 export * from './replace-file.js'
 export * from './shutdown.js'

packages/tests/src/shared/directories.ts

@@ -32,7 +32,10 @@ export async function checkDirectoryIsEmpty (server: PeerTubeServer, directory:
   expect(filtered).to.have.lengthOf(0)
 }
 
-export async function checkPeerTubeRunnerCacheIsEmpty (runner: PeerTubeRunnerProcess, subDir: 'transcoding' | 'transcription') {
+export async function checkPeerTubeRunnerCacheIsEmpty (
+  runner: PeerTubeRunnerProcess,
+  subDir: 'transcoding' | 'transcription' | 'storyboard'
+) {
   const directoryPath = join(homedir(), '.cache', 'peertube-runner-nodejs', runner.getId(), subDir)
 
   const directoryExists = await pathExists(directoryPath)

packages/tests/src/shared/storyboard.ts

@@ -0,0 +1,53 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { HttpStatusCode } from '@peertube/peertube-models'
+import { makeGetRequest, makeRawRequest, PeerTubeServer } from '@peertube/peertube-server-commands'
+import { expect } from 'chai'
+
+export async function checkStoryboard (options: {
+  server: PeerTubeServer
+  uuid: string
+  spriteHeight?: number
+  spriteWidth?: number
+  tilesCount?: number
+  minSize?: number
+  spriteDuration?: number
+}) {
+  const { server, uuid, tilesCount, spriteDuration = 1, spriteHeight = 108, spriteWidth = 192, minSize = 1000 } = options
+
+  const { storyboards } = await server.storyboard.list({ id: uuid })
+  expect(storyboards).to.have.lengthOf(1)
+
+  const storyboard = storyboards[0]
+
+  expect(storyboard.spriteDuration).to.equal(spriteDuration)
+  expect(storyboard.spriteHeight).to.equal(spriteHeight)
+  expect(storyboard.spriteWidth).to.equal(spriteWidth)
+  expect(storyboard.storyboardPath).to.exist
+
+  if (tilesCount) {
+    expect(storyboard.totalWidth).to.equal(spriteWidth * Math.min(tilesCount, 11))
+    expect(storyboard.totalHeight).to.equal(spriteHeight * Math.max(tilesCount / 11, 1))
+  }
+
+  {
+    const { body } = await makeGetRequest({ url: server.url, path: storyboard.storyboardPath, expectedStatus: HttpStatusCode.OK_200 })
+    expect(body.length).to.be.above(minSize)
+  }
+
+  {
+    const { body } = await makeRawRequest({ url: storyboard.fileUrl, expectedStatus: HttpStatusCode.OK_200 })
+    expect(body.length).to.be.above(minSize)
+  }
+}
+
+export async function checkNoStoryboard (options: {
+  server: PeerTubeServer
+  uuid: string
+}) {
+  const { server, uuid } = options
+
+  const { storyboards } = await server.storyboard.list({ id: uuid })
+
+  expect(storyboards).to.have.lengthOf(0)
+}

server/core/controllers/api/config.ts

@@ -582,7 +582,10 @@ function customConfig (): CustomConfig {
       }
     },
     storyboards: {
-      enabled: CONFIG.STORYBOARDS.ENABLED
+      enabled: CONFIG.STORYBOARDS.ENABLED,
+      remoteRunners: {
+        enabled: CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED
+      }
     },
     defaults: {
       publish: {

server/core/controllers/api/runners/jobs.ts

@@ -15,6 +15,7 @@ import {
   RunnerJobUpdatePayload,
   ServerErrorCode,
   TranscriptionSuccess,
+  GenerateStoryboardSuccess,
   UserRight,
   VODAudioMergeTranscodingSuccess,
   VODHLSTranscodingSuccess,
@@ -56,8 +57,13 @@ import { RunnerModel } from '@server/models/runner/runner.js'
 import express, { UploadFiles } from 'express'
 
 const postRunnerJobSuccessVideoFiles = createReqFiles(
-  [ 'payload[videoFile]', 'payload[resolutionPlaylistFile]', 'payload[vttFile]' ],
+  [ 'payload[videoFile]', 'payload[resolutionPlaylistFile]', 'payload[vttFile]', 'payload[storyboardFile]' ],
-  { ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT, ...MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT }
+  {
+    ...MIMETYPES.VIDEO.MIMETYPE_EXT,
+    ...MIMETYPES.M3U8.MIMETYPE_EXT,
+    ...MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT,
+    ...MIMETYPES.IMAGE.MIMETYPE_EXT
+  }
 )
 
 const runnerJobUpdateVideoFiles = createReqFiles(
@@ -384,6 +390,14 @@ const jobSuccessPayloadBuilders: {
 
       vttFile: files['payload[vttFile]'][0].path
     }
+  },
+
+  'generate-video-storyboard': (payload: GenerateStoryboardSuccess, files) => {
+    return {
+      ...payload,
+
+      storyboardFile: files['payload[storyboardFile]'][0].path
+    }
   }
 }
 

server/core/controllers/api/videos/source.ts

@@ -8,7 +8,7 @@ import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
 import { regenerateTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
 import { buildNewFile, createVideoSource } from '@server/lib/video-file.js'
-import { buildMoveVideoJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
+import { addRemoteStoryboardJobIfNeeded, buildLocalStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { buildNextVideoState } from '@server/lib/video-state.js'
 import { openapiOperationDoc } from '@server/middlewares/doc.js'
@@ -172,7 +172,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
       }
     },
 
-    buildStoryboardJobIfNeeded({ video, federate: false }),
+    buildLocalStoryboardJobIfNeeded({ video, federate: false }),
 
     {
       type: 'federate-video' as const,
@@ -201,6 +201,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
 
   await JobQueue.Instance.createSequentialJobFlow(...jobs)
 
+  await addRemoteStoryboardJobIfNeeded(video)
   await regenerateTranscriptionTaskIfNeeded(video)
 }
 

server/core/helpers/custom-validators/runners/jobs.ts

@@ -7,7 +7,8 @@ import {
   VODAudioMergeTranscodingSuccess,
   VODHLSTranscodingSuccess,
   VODWebVideoTranscodingSuccess,
-  VideoStudioTranscodingSuccess
+  VideoStudioTranscodingSuccess,
+  GenerateStoryboardSuccess
 } from '@peertube/peertube-models'
 import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants.js'
 import { UploadFilesForCheck } from 'express'
@@ -16,7 +17,15 @@ import { exists, isArray, isFileValid, isSafeFilename } from '../misc.js'
 
 const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS
 
-const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ])
+const runnerJobTypes = new Set([
+  'vod-hls-transcoding',
+  'vod-web-video-transcoding',
+  'vod-audio-merge-transcoding',
+  'live-rtmp-hls-transcoding',
+  'video-studio-transcoding',
+  'video-transcription',
+  'generate-video-storyboard'
+])
 export function isRunnerJobTypeValid (value: RunnerJobType) {
   return runnerJobTypes.has(value)
 }
@@ -27,7 +36,8 @@ export function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload,
     isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
     isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) ||
     isRunnerJobVideoStudioResultPayloadValid(value as VideoStudioTranscodingSuccess, type, files) ||
-    isRunnerJobTranscriptionResultPayloadValid(value as TranscriptionSuccess, type, files)
+    isRunnerJobTranscriptionResultPayloadValid(value as TranscriptionSuccess, type, files) ||
+    isRunnerJobGenerateStoryboardResultPayloadValid(value as GenerateStoryboardSuccess, type, files)
 }
 
 // ---------------------------------------------------------------------------
@@ -42,7 +52,8 @@ export function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, ty
     isRunnerJobVideoStudioUpdatePayloadValid(value, type, files) ||
     isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) ||
     isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) ||
-    isRunnerJobTranscriptionUpdatePayloadValid(value, type, files)
+    isRunnerJobTranscriptionUpdatePayloadValid(value, type, files) ||
+    isRunnerJobGenerateStoryboardUpdatePayloadValid(value, type, files)
 }
 
 // ---------------------------------------------------------------------------
@@ -103,7 +114,7 @@ function isRunnerJobLiveRTMPHLSResultPayloadValid (
   value: LiveRTMPHLSTranscodingSuccess,
   type: RunnerJobType
 ) {
-  return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+  return type === 'live-rtmp-hls-transcoding' && isEmptyUpdatePayload(value)
 }
 
 function isRunnerJobVideoStudioResultPayloadValid (
@@ -124,6 +135,15 @@ function isRunnerJobTranscriptionResultPayloadValid (
     isFileValid({ files, field: 'payload[vttFile]', mimeTypeRegex: null, maxSize: null })
 }
 
+function isRunnerJobGenerateStoryboardResultPayloadValid (
+  value: GenerateStoryboardSuccess,
+  type: RunnerJobType,
+  files: UploadFilesForCheck
+) {
+  return type === 'generate-video-storyboard' &&
+    isFileValid({ files, field: 'payload[storyboardFile]', mimeTypeRegex: null, maxSize: null })
+}
+
 // ---------------------------------------------------------------------------
 
 function isRunnerJobVODWebVideoUpdatePayloadValid (
@@ -131,8 +151,7 @@ function isRunnerJobVODWebVideoUpdatePayloadValid (
   type: RunnerJobType,
   _files: UploadFilesForCheck
 ) {
-  return type === 'vod-web-video-transcoding' &&
+  return type === 'vod-web-video-transcoding' && isEmptyUpdatePayload(value)
-    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
 }
 
 function isRunnerJobVODHLSUpdatePayloadValid (
@@ -140,8 +159,7 @@ function isRunnerJobVODHLSUpdatePayloadValid (
   type: RunnerJobType,
   _files: UploadFilesForCheck
 ) {
-  return type === 'vod-hls-transcoding' &&
+  return type === 'vod-hls-transcoding' && isEmptyUpdatePayload(value)
-    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
 }
 
 function isRunnerJobVODAudioMergeUpdatePayloadValid (
@@ -149,8 +167,7 @@ function isRunnerJobVODAudioMergeUpdatePayloadValid (
   type: RunnerJobType,
   _files: UploadFilesForCheck
 ) {
-  return type === 'vod-audio-merge-transcoding' &&
+  return type === 'vod-audio-merge-transcoding' && isEmptyUpdatePayload(value)
-    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
 }
 
 function isRunnerJobTranscriptionUpdatePayloadValid (
@@ -158,8 +175,7 @@ function isRunnerJobTranscriptionUpdatePayloadValid (
   type: RunnerJobType,
   _files: UploadFilesForCheck
 ) {
-  return type === 'video-transcription' &&
+  return type === 'video-transcription' && isEmptyUpdatePayload(value)
-    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
 }
 
 function isRunnerJobLiveRTMPHLSUpdatePayloadValid (
@@ -201,6 +217,17 @@ function isRunnerJobVideoStudioUpdatePayloadValid (
   type: RunnerJobType,
   _files: UploadFilesForCheck
 ) {
-  return type === 'video-studio-transcoding' &&
+  return type === 'video-studio-transcoding' && isEmptyUpdatePayload(value)
-    (!value || (typeof value === 'object' && Object.keys(value).length === 0))
+}
+
+function isRunnerJobGenerateStoryboardUpdatePayloadValid (
+  value: RunnerJobUpdatePayload,
+  type: RunnerJobType,
+  _files: UploadFilesForCheck
+) {
+  return type === 'generate-video-storyboard' && isEmptyUpdatePayload(value)
+}
+
+function isEmptyUpdatePayload (value: any): boolean {
+  return !value || (typeof value === 'object' && Object.keys(value).length === 0)
 }

server/core/initializers/config.ts

@@ -1099,6 +1099,11 @@ const CONFIG = {
   STORYBOARDS: {
     get ENABLED () {
       return config.get<boolean>('storyboards.enabled')
+    },
+    REMOTE_RUNNERS: {
+      get ENABLED () {
+        return config.get<boolean>('storyboards.remote_runners.enabled')
+      }
     }
   },
   EMAIL: {

server/core/initializers/constants.ts

@@ -291,6 +291,7 @@ export const REPEAT_JOBS: { [id in JobType]?: RepeatOptions } = {
   }
 }
 export const JOB_PRIORITY = {
+  STORYBOARD: 95,
   TRANSCODING: 100,
   VIDEO_STUDIO: 150,
   TRANSCRIPTION: 200

server/core/lib/job-queue/handlers/generate-storyboard.ts

@@ -1,25 +1,20 @@
-import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
+import { FFmpegImage } from '@peertube/peertube-ffmpeg'
 import { GenerateStoryboardPayload, VideoFileStream } from '@peertube/peertube-models'
-import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
 import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
 import { generateImageFilename } from '@server/helpers/image-utils.js'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
-import { deleteFileAndCatch } from '@server/helpers/utils.js'
 import { CONFIG } from '@server/initializers/config.js'
 import { STORYBOARD } from '@server/initializers/constants.js'
-import { sequelizeTypescript } from '@server/initializers/database.js'
-import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
-import { StoryboardModel } from '@server/models/video/storyboard.js'
 import { VideoModel } from '@server/models/video/video.js'
-import { MVideo } from '@server/types/models/index.js'
 import { Job } from 'bullmq'
 import { join } from 'path'
+import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'
 
 const lTagsBase = loggerTagsFactory('storyboard')
 
-async function processGenerateStoryboard (job: Job): Promise<void> {
+export async function processGenerateStoryboard (job: Job): Promise<void> {
   const payload = job.data as GenerateStoryboardPayload
   const lTags = lTagsBase(payload.videoUUID)
 
@@ -46,26 +41,9 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
     }
 
     await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
-      const probe = await ffprobePromise(videoPath)
+      const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)
 
-      const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe)
+      const { totalSprites, spriteDuration } = buildTotalSprites(video)
-      let spriteHeight: number
-      let spriteWidth: number
-
-      if (videoStreamInfo.isPortraitMode) {
-        spriteHeight = STORYBOARD.SPRITE_MAX_SIZE
-        spriteWidth = Math.round(spriteHeight * videoStreamInfo.ratio)
-      } else {
-        spriteWidth = STORYBOARD.SPRITE_MAX_SIZE
-        spriteHeight = Math.round(spriteWidth / videoStreamInfo.ratio)
-      }
-
-      const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
-
-      const filename = generateImageFilename()
-      const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
-
-      const { totalSprites, spriteDuration } = buildSpritesMetadata({ video })
       if (totalSprites === 0) {
         logger.info(`Do not generate a storyboard of ${payload.videoUUID} because the video is not long enough`, lTags)
         return
@@ -76,11 +54,16 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
         maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
       })
 
+      const filename = generateImageFilename()
+      const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
+
       logger.debug(
         `Generating storyboard from video of ${video.uuid} to ${destination}`,
         { ...lTags, totalSprites, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }
       )
 
+      const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
+
       await ffmpeg.generateStoryboardFromVideo({
         destination,
         path: videoPath,
@@ -95,78 +78,23 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
         }
       })
 
-      const imageSize = await getImageSizeFromWorker(destination)
+      await insertStoryboardInDatabase({
+        videoUUID: video.uuid,
+        lTags,
 
-      await retryTransactionWrapper(() => {
+        filename,
-        return sequelizeTypescript.transaction(async transaction => {
+        destination,
-          const videoStillExists = await VideoModel.load(video.id, transaction)
-          if (!videoStillExists) {
-            logger.info(`Video ${payload.videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)
-            deleteFileAndCatch(destination)
-            return
-          }
 
-          const existing = await StoryboardModel.loadByVideo(video.id, transaction)
+        imageSize: await getImageSizeFromWorker(destination),
-          if (existing) await existing.destroy({ transaction })
 
-          await StoryboardModel.create({
+        spriteHeight,
-            filename,
+        spriteWidth,
-            totalHeight: imageSize.height,
+        spriteDuration,
-            totalWidth: imageSize.width,
-            spriteHeight,
-            spriteWidth,
-            spriteDuration,
-            videoId: video.id
-          }, { transaction })
 
-          logger.info(`Storyboard generation ${destination} ended for video ${video.uuid}.`, lTags)
+        federate: payload.federate
-
-          if (payload.federate) {
-            await federateVideoIfNeeded(video, false, transaction)
-          }
-        })
       })
     })
   } finally {
     inputFileMutexReleaser()
   }
 }
-
-// ---------------------------------------------------------------------------
-
-export {
-  processGenerateStoryboard
-}
-
-function buildSpritesMetadata (options: {
-  video: MVideo
-}) {
-  const { video } = options
-
-  if (video.duration < 3) return { spriteDuration: undefined, totalSprites: 0 }
-
-  const maxSprites = Math.min(Math.ceil(video.duration), STORYBOARD.SPRITES_MAX_EDGE_COUNT * STORYBOARD.SPRITES_MAX_EDGE_COUNT)
-
-  const spriteDuration = Math.ceil(video.duration / maxSprites)
-  const totalSprites = Math.ceil(video.duration / spriteDuration)
-
-  // We can generate a single line so we don't need a prime number
-  if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return { spriteDuration, totalSprites }
-
-  return { spriteDuration, totalSprites }
-}
-
-function findGridSize (options: {
-  toFind: number
-  maxEdgeCount: number
-}) {
-  const { toFind, maxEdgeCount } = options
-
-  for (let i = 1; i <= maxEdgeCount; i++) {
-    for (let j = i; j <= maxEdgeCount; j++) {
-      if (toFind <= i * j) return { width: j, height: i }
-    }
-  }
-
-  throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
-}

server/core/lib/job-queue/handlers/video-import.ts

@@ -1,8 +1,5 @@
 import { buildAspectRatio } from '@peertube/peertube-core-utils'
-import {
+import { ffprobePromise, getChaptersFromContainer, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
-  ffprobePromise,
-  getChaptersFromContainer, getVideoStreamDuration
-} from '@peertube/peertube-ffmpeg'
 import {
   ThumbnailType,
   ThumbnailType_Type,
@@ -12,7 +9,8 @@ import {
   VideoImportTorrentPayload,
   VideoImportTorrentPayloadType,
   VideoImportYoutubeDLPayload,
-  VideoImportYoutubeDLPayloadType, VideoState
+  VideoImportYoutubeDLPayloadType,
+  VideoState
 } from '@peertube/peertube-models'
 import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
 import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
@@ -27,7 +25,7 @@ import { isUserQuotaValid } from '@server/lib/user.js'
 import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
 import { replaceChaptersIfNotExist } from '@server/lib/video-chapters.js'
 import { buildNewFile } from '@server/lib/video-file.js'
-import { buildMoveVideoJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
+import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { buildNextVideoState } from '@server/lib/video-state.js'
 import { VideoCaptionModel } from '@server/models/video/video-caption.js'
@@ -305,7 +303,7 @@ async function afterImportSuccess (options: {
   }
 
   // Generate the storyboard in the job queue, and don't forget to federate an update after
-  await JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: true }))
+  await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })
 
   if (await VideoCaptionModel.hasVideoCaption(video.id) !== true && generateTranscription === true) {
     await createTranscriptionTaskIfNeeded(video)

server/core/lib/job-queue/handlers/video-live-ending.ts

@@ -14,7 +14,7 @@ import {
 import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded, updateLocalVideoMiniatureFromExisting } from '@server/lib/thumbnail.js'
 import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding.js'
 import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
-import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
+import { addLocalOrRemoteStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js'
 import { moveToNextState } from '@server/lib/video-state.js'
@@ -41,7 +41,6 @@ import { pathExists, remove } from 'fs-extra/esm'
 import { readdir } from 'fs/promises'
 import { isAbsolute, join } from 'path'
 import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
-import { JobQueue } from '../job-queue.js'
 
 const lTags = loggerTagsFactory('live', 'job')
 
@@ -362,7 +361,7 @@ async function cleanupLiveAndFederate (options: {
 }
 
 function createStoryboardJob (video: MVideo) {
-  return JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: true }))
+  return addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })
 }
 
 async function hasReplayFiles (replayDirectory: string) {

server/core/lib/runners/job-handlers/abstract-job-handler.ts

@@ -1,5 +1,7 @@
 import { pick } from '@peertube/peertube-core-utils'
 import {
+  RunnerJobGenerateStoryboardPayload,
+  RunnerJobGenerateStoryboardPrivatePayload,
   RunnerJobLiveRTMPHLSTranscodingPayload,
   RunnerJobLiveRTMPHLSTranscodingPrivatePayload,
   RunnerJobState,
@@ -28,50 +30,56 @@ import { setAsUpdated } from '@server/models/shared/update.js'
 import { MRunnerJob } from '@server/types/models/runners/index.js'
 
 type CreateRunnerJobArg =
-  {
+  | {
     type: Extract<RunnerJobType, 'vod-web-video-transcoding'>
     payload: RunnerJobVODWebVideoTranscodingPayload
     privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload
-  } |
+  }
-  {
+  | {
     type: Extract<RunnerJobType, 'vod-hls-transcoding'>
     payload: RunnerJobVODHLSTranscodingPayload
     privatePayload: RunnerJobVODHLSTranscodingPrivatePayload
-  } |
+  }
-  {
+  | {
     type: Extract<RunnerJobType, 'vod-audio-merge-transcoding'>
     payload: RunnerJobVODAudioMergeTranscodingPayload
     privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload
-  } |
+  }
-  {
+  | {
     type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'>
     payload: RunnerJobLiveRTMPHLSTranscodingPayload
     privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload
-  } |
+  }
-  {
+  | {
     type: Extract<RunnerJobType, 'video-studio-transcoding'>
     payload: RunnerJobStudioTranscodingPayload
     privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload
-  } |
+  }
-  {
+  | {
+    type: Extract<RunnerJobType, 'generate-video-storyboard'>
+    payload: RunnerJobGenerateStoryboardPayload
+    privatePayload: RunnerJobGenerateStoryboardPrivatePayload
+  }
+  | {
     type: Extract<RunnerJobType, 'video-transcription'>
     payload: RunnerJobTranscriptionPayload
     privatePayload: RunnerJobTranscriptionPrivatePayload
   }
 
-export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> {
+export abstract class AbstractJobHandler<C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> {
-
   protected readonly lTags = loggerTagsFactory('runner')
 
   // ---------------------------------------------------------------------------
 
   abstract create (options: C): Promise<MRunnerJob>
 
-  protected async createRunnerJob (options: CreateRunnerJobArg & {
+  protected async createRunnerJob (
-    jobUUID: string
+    options: CreateRunnerJobArg & {
-    priority: number
+      jobUUID: string
-    dependsOnRunnerJob?: MRunnerJob
+      priority: number
-  }): Promise<MRunnerJob> {
+      dependsOnRunnerJob?: MRunnerJob
+    }
+  ): Promise<MRunnerJob> {
     const { priority, dependsOnRunnerJob } = options
 
     logger.debug('Creating runner job', { options, dependsOnRunnerJob, ...this.lTags(options.type) })

server/core/lib/runners/job-handlers/index.ts

@@ -6,3 +6,4 @@ export * from './video-studio-transcoding-job-handler.js'
 export * from './vod-audio-merge-transcoding-job-handler.js'
 export * from './vod-hls-transcoding-job-handler.js'
 export * from './vod-web-video-transcoding-job-handler.js'
+export * from './video-storyboard-job-handler.js'

server/core/lib/runners/job-handlers/runner-job-handlers.ts

@@ -7,6 +7,7 @@ import { VideoStudioTranscodingJobHandler } from './video-studio-transcoding-job
 import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler.js'
 import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler.js'
 import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler.js'
+import { VideoStoryboardJobHandler } from './video-storyboard-job-handler.js'
 
 const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, RunnerJobUpdatePayload, RunnerJobSuccessPayload>> = {
   'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler,
@@ -14,7 +15,8 @@ const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, Run
   'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler,
   'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler,
   'video-studio-transcoding': VideoStudioTranscodingJobHandler,
-  'video-transcription': TranscriptionJobHandler
+  'video-transcription': TranscriptionJobHandler,
+  'generate-video-storyboard': VideoStoryboardJobHandler
 }
 
 export function getRunnerJobHandlerClass (job: MRunnerJob) {

server/core/lib/runners/job-handlers/transcription-job-handler.ts

@@ -72,7 +72,7 @@ export class TranscriptionJobHandler extends AbstractJobHandler<CreateOptions, R
       jobUUID,
       payload,
       privatePayload,
-      priority: JOB_PRIORITY.TRANSCODING
+      priority: JOB_PRIORITY.TRANSCRIPTION
     })
 
     return job

server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts

@@ -0,0 +1,118 @@
+import {
+  GenerateStoryboardSuccess,
+  RunnerJobGenerateStoryboardPayload,
+  RunnerJobGenerateStoryboardPrivatePayload,
+  RunnerJobUpdatePayload,
+  VideoFileStream
+} from '@peertube/peertube-models'
+import { buildUUID } from '@peertube/peertube-node-utils'
+import { generateImageFilename } from '@server/helpers/image-utils.js'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { JOB_PRIORITY, STORYBOARD } from '@server/initializers/constants.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MRunnerJob } from '@server/types/models/runners/index.js'
+import { move } from 'fs-extra/esm'
+import { join } from 'path'
+import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'
+import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls.js'
+import { AbstractJobHandler } from './abstract-job-handler.js'
+
+const lTagsBase = loggerTagsFactory('storyboard', 'runners')
+
+type CreateOptions = {
+  videoUUID: string
+}
+
+export class VideoStoryboardJobHandler extends AbstractJobHandler<CreateOptions, RunnerJobUpdatePayload, GenerateStoryboardSuccess> {
+  async create (options: CreateOptions) {
+    const { videoUUID } = options
+    const lTags = lTagsBase(videoUUID)
+
+    const jobUUID = buildUUID()
+
+    const video = await VideoModel.loadFull(videoUUID)
+    const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)
+
+    return VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
+      const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)
+
+      const { spriteDuration, totalSprites } = buildTotalSprites(video)
+      if (totalSprites === 0) {
+        logger.info(`Do not generate remote storyboard job of ${videoUUID} because the video is not long enough`, lTags)
+        return
+      }
+
+      const spritesCount = findGridSize({ toFind: totalSprites, maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT })
+
+      const payload: RunnerJobGenerateStoryboardPayload = {
+        input: {
+          videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, videoUUID)
+        },
+        sprites: {
+          size: { height: spriteHeight, width: spriteWidth },
+          count: spritesCount,
+          duration: spriteDuration
+        },
+        output: {}
+      }
+
+      const privatePayload: RunnerJobGenerateStoryboardPrivatePayload = { videoUUID }
+
+      const job = await this.createRunnerJob({
+        type: 'generate-video-storyboard',
+        jobUUID,
+        payload,
+        privatePayload,
+        priority: JOB_PRIORITY.STORYBOARD
+      })
+
+      return job
+    })
+  }
+
+  protected isAbortSupported () {
+    return true
+  }
+
+  protected specificUpdate (_options: { runnerJob: MRunnerJob }) {}
+  protected specificAbort (_options: { runnerJob: MRunnerJob }) {}
+
+  // When runner returns the storyboard image, finish the server-side creation like local job would
+  protected async specificComplete (options: { runnerJob: MRunnerJob, resultPayload: GenerateStoryboardSuccess }) {
+    const { runnerJob, resultPayload } = options
+
+    const video = await VideoModel.loadFull(runnerJob.privatePayload.videoUUID)
+    if (!video) return
+
+    const destinationFilename = generateImageFilename()
+    const destinationPath = join(CONFIG.STORAGE.STORYBOARDS_DIR, destinationFilename)
+
+    await move(resultPayload.storyboardFile as string, destinationPath)
+
+    const { sprites } = runnerJob.payload as RunnerJobGenerateStoryboardPayload
+
+    await insertStoryboardInDatabase({
+      videoUUID: video.uuid,
+      lTags: this.lTags(video.uuid, runnerJob.uuid),
+
+      filename: destinationFilename,
+      destination: destinationPath,
+
+      imageSize: await getImageSizeFromWorker(destinationPath),
+
+      spriteHeight: sprites.size.height,
+      spriteWidth: sprites.size.width,
+      spriteDuration: sprites.duration,
+
+      federate: true
+    })
+
+    logger.info('Runner storyboard job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid))
+  }
+
+  protected specificError (_options: { runnerJob: MRunnerJob }) {}
+  protected specificCancel (_options: { runnerJob: MRunnerJob }) {}
+}

server/core/lib/server-config-manager.ts

@@ -384,7 +384,10 @@ class ServerConfigManager {
       },
 
       storyboards: {
-        enabled: CONFIG.STORYBOARDS.ENABLED
+        enabled: CONFIG.STORYBOARDS.ENABLED,
+        remoteRunners: {
+          enabled: CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED
+        }
       },
 
       webrtc: {

server/core/lib/storyboard.ts

@@ -0,0 +1,98 @@
+import { ffprobePromise, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
+import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
+import { LoggerTags, logger } from '@server/helpers/logger.js'
+import { deleteFileAndCatch } from '@server/helpers/utils.js'
+import { STORYBOARD } from '@server/initializers/constants.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { StoryboardModel } from '@server/models/video/storyboard.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MVideo } from '@server/types/models/index.js'
+import { federateVideoIfNeeded } from './activitypub/videos/federate.js'
+
+export async function buildSpriteSize (videoPath: string) {
+  const probe = await ffprobePromise(videoPath)
+  const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe)
+
+  if (videoStreamInfo.isPortraitMode) {
+    return {
+      spriteHeight: STORYBOARD.SPRITE_MAX_SIZE,
+      spriteWidth: Math.round(STORYBOARD.SPRITE_MAX_SIZE * videoStreamInfo.ratio)
+    }
+  }
+
+  return {
+    spriteWidth: STORYBOARD.SPRITE_MAX_SIZE,
+    spriteHeight: Math.round(STORYBOARD.SPRITE_MAX_SIZE / videoStreamInfo.ratio)
+  }
+}
+
+export function buildTotalSprites (video: MVideo) {
+  if (video.duration < 3) return { spriteDuration: undefined, totalSprites: 0 }
+
+  const maxSprites = Math.min(Math.ceil(video.duration), STORYBOARD.SPRITES_MAX_EDGE_COUNT * STORYBOARD.SPRITES_MAX_EDGE_COUNT)
+
+  const spriteDuration = Math.ceil(video.duration / maxSprites)
+  const totalSprites = Math.ceil(video.duration / spriteDuration)
+
+  // We can generate a single line so we don't need a prime number
+  if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return { spriteDuration, totalSprites }
+
+  return { spriteDuration, totalSprites }
+}
+
+export function findGridSize (options: {
+  toFind: number
+  maxEdgeCount: number
+}) {
+  const { toFind, maxEdgeCount } = options
+
+  for (let i = 1; i <= maxEdgeCount; i++) {
+    for (let j = i; j <= maxEdgeCount; j++) {
+      if (toFind <= i * j) return { width: j, height: i }
+    }
+  }
+
+  throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
+}
+
+export async function insertStoryboardInDatabase (options: {
+  videoUUID: string
+  lTags: LoggerTags
+  filename: string
+  destination: string
+  imageSize: { width: number, height: number }
+  spriteHeight: number
+  spriteWidth: number
+  spriteDuration: number
+  federate: boolean
+}) {
+  const { videoUUID, lTags, imageSize, spriteHeight, spriteWidth, spriteDuration, destination, filename, federate } = options
+
+  await retryTransactionWrapper(() => {
+    return sequelizeTypescript.transaction(async transaction => {
+      const video = await VideoModel.loadFull(videoUUID, transaction)
+      if (!video) {
+        logger.info(`Video ${videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)
+        deleteFileAndCatch(destination)
+        return
+      }
+
+      const existing = await StoryboardModel.loadByVideo(video.id, transaction)
+      if (existing) await existing.destroy({ transaction })
+
+      await StoryboardModel.create({
+        filename,
+        totalHeight: imageSize.height,
+        totalWidth: imageSize.width,
+        spriteHeight,
+        spriteWidth,
+        spriteDuration,
+        videoId: video.id
+      }, { transaction })
+
+      if (federate) {
+        await federateVideoIfNeeded(video, false, transaction)
+      }
+    })
+  })
+}

server/core/lib/transcoding/web-transcoding.ts

@@ -16,10 +16,9 @@ import { copyFile } from 'fs/promises'
 import { basename, join } from 'path'
 import { CONFIG } from '../../initializers/config.js'
 import { VideoFileModel } from '../../models/video/video-file.js'
-import { JobQueue } from '../job-queue/index.js'
 import { generateWebVideoFilename } from '../paths.js'
 import { buildNewFile, saveNewOriginalFileIfNeeded } from '../video-file.js'
-import { buildStoryboardJobIfNeeded } from '../video-jobs.js'
+import { addLocalOrRemoteStoryboardJobIfNeeded } from '../video-jobs.js'
 import { VideoPathManager } from '../video-path-manager.js'
 import { buildFFmpegVOD } from './shared/index.js'
 import { buildOriginalFileResolution } from './transcoding-resolutions.js'
@@ -229,7 +228,7 @@ export async function onWebVideoFileTranscoding (options: {
     video.VideoFiles = await video.$get('VideoFiles')
 
     if (wasAudioFile) {
-      await JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: false }))
+      await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: false })
     }
 
     return { video, videoFile }

server/core/lib/video-jobs.ts

@@ -3,6 +3,7 @@ import { CONFIG } from '@server/initializers/config.js'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
 import { MVideo, MVideoFile, MVideoFullLight, MVideoUUID } from '@server/types/models/index.js'
 import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue.js'
+import { VideoStoryboardJobHandler } from './runners/index.js'
 import { createTranscriptionTaskIfNeeded } from './video-captions.js'
 import { moveFilesIfPrivacyChanged } from './video-privacy.js'
 
@@ -26,13 +27,13 @@ export async function buildMoveVideoJob (options: {
   }
 }
 
-export function buildStoryboardJobIfNeeded (options: {
+export function buildLocalStoryboardJobIfNeeded (options: {
   video: MVideo
   federate: boolean
 }) {
   const { video, federate } = options
 
-  if (CONFIG.STORYBOARDS.ENABLED) {
+  if (CONFIG.STORYBOARDS.ENABLED && !CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED) {
     return {
       type: 'generate-video-storyboard' as 'generate-video-storyboard',
       payload: {
@@ -55,6 +56,28 @@ export function buildStoryboardJobIfNeeded (options: {
   return undefined
 }
 
+export function addRemoteStoryboardJobIfNeeded (video: MVideo) {
+  if (CONFIG.STORYBOARDS.ENABLED !== true) return
+  if (CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED !== true) return
+
+  return new VideoStoryboardJobHandler().create({ videoUUID: video.uuid })
+}
+
+export async function addLocalOrRemoteStoryboardJobIfNeeded (options: {
+  video: MVideo
+  federate: boolean
+}) {
+  const { video, federate } = options
+
+  if (CONFIG.STORYBOARDS.ENABLED !== true) return
+
+  if (CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED === true) {
+    await addRemoteStoryboardJobIfNeeded(video)
+  } else {
+    await JobQueue.Instance.createJob(buildLocalStoryboardJobIfNeeded({ video, federate }))
+  }
+}
+
 export async function addVideoJobsAfterCreation (options: {
   video: MVideo
   videoFile: MVideoFile
@@ -72,7 +95,7 @@ export async function addVideoJobsAfterCreation (options: {
       }
     },
 
-    buildStoryboardJobIfNeeded({ video, federate: false }),
+    buildLocalStoryboardJobIfNeeded({ video, federate: false }),
 
     {
       type: 'notify',
@@ -109,6 +132,8 @@ export async function addVideoJobsAfterCreation (options: {
 
   await JobQueue.Instance.createSequentialJobFlow(...jobs)
 
+  await addRemoteStoryboardJobIfNeeded(video)
+
   if (generateTranscription === true) {
     await createTranscriptionTaskIfNeeded(video)
   }

server/core/lib/video-studio.ts

@@ -12,7 +12,7 @@ import { VideoStudioTranscodingJobHandler } from './runners/index.js'
 import { getTranscodingJobPriority } from './transcoding/transcoding-priority.js'
 import { regenerateTranscriptionTaskIfNeeded } from './video-captions.js'
 import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file.js'
-import { buildStoryboardJobIfNeeded } from './video-jobs.js'
+import { addRemoteStoryboardJobIfNeeded, buildLocalStoryboardJobIfNeeded } from './video-jobs.js'
 import { VideoPathManager } from './video-path-manager.js'
 
 const lTags = loggerTagsFactory('video-studio')
@@ -110,7 +110,7 @@ export async function onVideoStudioEnded (options: {
   await video.save()
 
   await JobQueue.Instance.createSequentialJobFlow(
-    buildStoryboardJobIfNeeded({ video, federate: false }),
+    buildLocalStoryboardJobIfNeeded({ video, federate: false }),
     {
       type: 'federate-video' as 'federate-video',
       payload: {
@@ -129,6 +129,7 @@ export async function onVideoStudioEnded (options: {
     }
   )
 
+  await addRemoteStoryboardJobIfNeeded(video)
   await regenerateTranscriptionTaskIfNeeded(video)
 }
 

server/scripts/create-generate-storyboard-job.ts

@@ -1,10 +1,10 @@
-import { program } from 'commander'
 import { toCompleteUUID } from '@server/helpers/custom-validators/misc.js'
 import { initDatabaseModels } from '@server/initializers/database.js'
 import { JobQueue } from '@server/lib/job-queue/index.js'
+import { addLocalOrRemoteStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
 import { StoryboardModel } from '@server/models/video/storyboard.js'
 import { VideoModel } from '@server/models/video/video.js'
-import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
+import { program } from 'commander'
 
 program
   .description('Generate videos storyboard')
@@ -61,7 +61,7 @@ async function run () {
 
     if (videoFull.isLive) continue
 
-    await JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video: videoFull, federate: true }))
+    await addLocalOrRemoteStoryboardJobIfNeeded({ video: videoFull, federate: true })
 
     console.log(`Created generate-storyboard job for ${videoFull.name}.`)
   }