yetangitu/Peertube
1
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2025-10-03 09:49:20 +02:00
Code Issues Releases Wiki Activity

Feature for runners - handle storyboard-generation-job (#7191)

Browse source 
* Implement processing storyboards by runners

* Fixed storyboard generation by runners

* use common code patterns

* fix import

* improve debug logging for storyboard generation

* config option for storyboard processing with remote-runners

* refactor repetitive pattern

* refactor storyboard related code to share common utlities

* Fix test

* Fix storyboard generation config logic

* Improve logging

* Added tests for storyboard generation with runners

* Refactor PR

---------

Co-authored-by: ilfarpro <ilfarpro@ya.ru>
Co-authored-by: Chocobozzz <me@florianbigard.com>
This commit is contained in:
ilfarpro 2025-09-10 12:50:06 +03:00 • committed by GitHub
parent e74bf8ae2a
commit dd52e8b89e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
50 changed files with 973 additions and 248 deletions
Download patch file Download diff file Expand all files Collapse all files

5
server/core/controllers/api/config.ts
View file

@ -582,7 +582,10 @@ function customConfig (): CustomConfig {
}
},
storyboards: {
enabled: CONFIG.STORYBOARDS.ENABLED
enabled: CONFIG.STORYBOARDS.ENABLED,
remoteRunners: {
enabled: CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED
}
},
defaults: {
publish: {

18
server/core/controllers/api/runners/jobs.ts
View file

@ -15,6 +15,7 @@ import {
RunnerJobUpdatePayload,
ServerErrorCode,
TranscriptionSuccess,
GenerateStoryboardSuccess,
UserRight,
VODAudioMergeTranscodingSuccess,
VODHLSTranscodingSuccess,
@ -56,8 +57,13 @@ import { RunnerModel } from '@server/models/runner/runner.js'
import express, { UploadFiles } from 'express'
const postRunnerJobSuccessVideoFiles = createReqFiles(
[ 'payload[videoFile]', 'payload[resolutionPlaylistFile]', 'payload[vttFile]' ],
{ ...MIMETYPES.VIDEO.MIMETYPE_EXT, ...MIMETYPES.M3U8.MIMETYPE_EXT, ...MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT }
[ 'payload[videoFile]', 'payload[resolutionPlaylistFile]', 'payload[vttFile]', 'payload[storyboardFile]' ],
{
...MIMETYPES.VIDEO.MIMETYPE_EXT,
...MIMETYPES.M3U8.MIMETYPE_EXT,
...MIMETYPES.VIDEO_CAPTIONS.MIMETYPE_EXT,
...MIMETYPES.IMAGE.MIMETYPE_EXT
}
)
const runnerJobUpdateVideoFiles = createReqFiles(
@ -384,6 +390,14 @@ const jobSuccessPayloadBuilders: {
vttFile: files['payload[vttFile]'][0].path
}
},
'generate-video-storyboard': (payload: GenerateStoryboardSuccess, files) => {
return {
...payload,
storyboardFile: files['payload[storyboardFile]'][0].path
}
}
}

5
server/core/controllers/api/videos/source.ts
View file

@ -8,7 +8,7 @@ import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
import { regenerateTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
import { buildNewFile, createVideoSource } from '@server/lib/video-file.js'
import { buildMoveVideoJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { addRemoteStoryboardJobIfNeeded, buildLocalStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
@ -172,7 +172,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
}
},
buildStoryboardJobIfNeeded({ video, federate: false }),
buildLocalStoryboardJobIfNeeded({ video, federate: false }),
{
type: 'federate-video' as const,
@ -201,6 +201,7 @@ async function addVideoJobsAfterUpload (video: MVideoFullLight, videoFile: MVide
await JobQueue.Instance.createSequentialJobFlow(...jobs)
await addRemoteStoryboardJobIfNeeded(video)
await regenerateTranscriptionTaskIfNeeded(video)
}

57
server/core/helpers/custom-validators/runners/jobs.ts
View file

@ -7,7 +7,8 @@ import {
VODAudioMergeTranscodingSuccess,
VODHLSTranscodingSuccess,
VODWebVideoTranscodingSuccess,
VideoStudioTranscodingSuccess
VideoStudioTranscodingSuccess,
GenerateStoryboardSuccess
} from '@peertube/peertube-models'
import { CONSTRAINTS_FIELDS, RUNNER_JOB_STATES } from '@server/initializers/constants.js'
import { UploadFilesForCheck } from 'express'
@ -16,7 +17,15 @@ import { exists, isArray, isFileValid, isSafeFilename } from '../misc.js'
const RUNNER_JOBS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.RUNNER_JOBS
const runnerJobTypes = new Set([ 'vod-hls-transcoding', 'vod-web-video-transcoding', 'vod-audio-merge-transcoding' ])
const runnerJobTypes = new Set([
'vod-hls-transcoding',
'vod-web-video-transcoding',
'vod-audio-merge-transcoding',
'live-rtmp-hls-transcoding',
'video-studio-transcoding',
'video-transcription',
'generate-video-storyboard'
])
export function isRunnerJobTypeValid (value: RunnerJobType) {
return runnerJobTypes.has(value)
}
@ -27,7 +36,8 @@ export function isRunnerJobSuccessPayloadValid (value: RunnerJobSuccessPayload,
isRunnerJobVODAudioMergeResultPayloadValid(value as VODHLSTranscodingSuccess, type, files) ||
isRunnerJobLiveRTMPHLSResultPayloadValid(value as LiveRTMPHLSTranscodingSuccess, type) ||
isRunnerJobVideoStudioResultPayloadValid(value as VideoStudioTranscodingSuccess, type, files) ||
isRunnerJobTranscriptionResultPayloadValid(value as TranscriptionSuccess, type, files)
isRunnerJobTranscriptionResultPayloadValid(value as TranscriptionSuccess, type, files) ||
isRunnerJobGenerateStoryboardResultPayloadValid(value as GenerateStoryboardSuccess, type, files)
}
// ---------------------------------------------------------------------------
@ -42,7 +52,8 @@ export function isRunnerJobUpdatePayloadValid (value: RunnerJobUpdatePayload, ty
isRunnerJobVideoStudioUpdatePayloadValid(value, type, files) ||
isRunnerJobVODAudioMergeUpdatePayloadValid(value, type, files) ||
isRunnerJobLiveRTMPHLSUpdatePayloadValid(value, type, files) ||
isRunnerJobTranscriptionUpdatePayloadValid(value, type, files)
isRunnerJobTranscriptionUpdatePayloadValid(value, type, files) ||
isRunnerJobGenerateStoryboardUpdatePayloadValid(value, type, files)
}
// ---------------------------------------------------------------------------
@ -103,7 +114,7 @@ function isRunnerJobLiveRTMPHLSResultPayloadValid (
value: LiveRTMPHLSTranscodingSuccess,
type: RunnerJobType
) {
return type === 'live-rtmp-hls-transcoding' && (!value || (typeof value === 'object' && Object.keys(value).length === 0))
return type === 'live-rtmp-hls-transcoding' && isEmptyUpdatePayload(value)
}
function isRunnerJobVideoStudioResultPayloadValid (
@ -124,6 +135,15 @@ function isRunnerJobTranscriptionResultPayloadValid (
isFileValid({ files, field: 'payload[vttFile]', mimeTypeRegex: null, maxSize: null })
}
function isRunnerJobGenerateStoryboardResultPayloadValid (
value: GenerateStoryboardSuccess,
type: RunnerJobType,
files: UploadFilesForCheck
) {
return type === 'generate-video-storyboard' &&
isFileValid({ files, field: 'payload[storyboardFile]', mimeTypeRegex: null, maxSize: null })
}
// ---------------------------------------------------------------------------
function isRunnerJobVODWebVideoUpdatePayloadValid (
@ -131,8 +151,7 @@ function isRunnerJobVODWebVideoUpdatePayloadValid (
type: RunnerJobType,
_files: UploadFilesForCheck
) {
return type === 'vod-web-video-transcoding' &&
(!value || (typeof value === 'object' && Object.keys(value).length === 0))
return type === 'vod-web-video-transcoding' && isEmptyUpdatePayload(value)
}
function isRunnerJobVODHLSUpdatePayloadValid (
@ -140,8 +159,7 @@ function isRunnerJobVODHLSUpdatePayloadValid (
type: RunnerJobType,
_files: UploadFilesForCheck
) {
return type === 'vod-hls-transcoding' &&
(!value || (typeof value === 'object' && Object.keys(value).length === 0))
return type === 'vod-hls-transcoding' && isEmptyUpdatePayload(value)
}
function isRunnerJobVODAudioMergeUpdatePayloadValid (
@ -149,8 +167,7 @@ function isRunnerJobVODAudioMergeUpdatePayloadValid (
type: RunnerJobType,
_files: UploadFilesForCheck
) {
return type === 'vod-audio-merge-transcoding' &&
(!value || (typeof value === 'object' && Object.keys(value).length === 0))
return type === 'vod-audio-merge-transcoding' && isEmptyUpdatePayload(value)
}
function isRunnerJobTranscriptionUpdatePayloadValid (
@ -158,8 +175,7 @@ function isRunnerJobTranscriptionUpdatePayloadValid (
type: RunnerJobType,
_files: UploadFilesForCheck
) {
return type === 'video-transcription' &&
(!value || (typeof value === 'object' && Object.keys(value).length === 0))
return type === 'video-transcription' && isEmptyUpdatePayload(value)
}
function isRunnerJobLiveRTMPHLSUpdatePayloadValid (
@ -201,6 +217,17 @@ function isRunnerJobVideoStudioUpdatePayloadValid (
type: RunnerJobType,
_files: UploadFilesForCheck
) {
return type === 'video-studio-transcoding' &&
(!value || (typeof value === 'object' && Object.keys(value).length === 0))
return type === 'video-studio-transcoding' && isEmptyUpdatePayload(value)
}
function isRunnerJobGenerateStoryboardUpdatePayloadValid (
value: RunnerJobUpdatePayload,
type: RunnerJobType,
_files: UploadFilesForCheck
) {
return type === 'generate-video-storyboard' && isEmptyUpdatePayload(value)
}
function isEmptyUpdatePayload (value: any): boolean {
return !value || (typeof value === 'object' && Object.keys(value).length === 0)
}

5
server/core/initializers/config.ts
View file

@ -1099,6 +1099,11 @@ const CONFIG = {
STORYBOARDS: {
get ENABLED () {
return config.get<boolean>('storyboards.enabled')
},
REMOTE_RUNNERS: {
get ENABLED () {
return config.get<boolean>('storyboards.remote_runners.enabled')
}
}
},
EMAIL: {

1
server/core/initializers/constants.ts
View file

@ -291,6 +291,7 @@ export const REPEAT_JOBS: { [id in JobType]?: RepeatOptions } = {
}
}
export const JOB_PRIORITY = {
STORYBOARD: 95,
TRANSCODING: 100,
VIDEO_STUDIO: 150,
TRANSCRIPTION: 200

112
server/core/lib/job-queue/handlers/generate-storyboard.ts
View file

@ -1,25 +1,20 @@
import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
import { FFmpegImage } from '@peertube/peertube-ffmpeg'
import { GenerateStoryboardPayload, VideoFileStream } from '@peertube/peertube-models'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
import { generateImageFilename } from '@server/helpers/image-utils.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { deleteFileAndCatch } from '@server/helpers/utils.js'
import { CONFIG } from '@server/initializers/config.js'
import { STORYBOARD } from '@server/initializers/constants.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { federateVideoIfNeeded } from '@server/lib/activitypub/videos/index.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
import { StoryboardModel } from '@server/models/video/storyboard.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideo } from '@server/types/models/index.js'
import { Job } from 'bullmq'
import { join } from 'path'
import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'
const lTagsBase = loggerTagsFactory('storyboard')
async function processGenerateStoryboard (job: Job): Promise<void> {
export async function processGenerateStoryboard (job: Job): Promise<void> {
const payload = job.data as GenerateStoryboardPayload
const lTags = lTagsBase(payload.videoUUID)
@ -46,26 +41,9 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
}
await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
const probe = await ffprobePromise(videoPath)
const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)
const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe)
let spriteHeight: number
let spriteWidth: number
if (videoStreamInfo.isPortraitMode) {
spriteHeight = STORYBOARD.SPRITE_MAX_SIZE
spriteWidth = Math.round(spriteHeight * videoStreamInfo.ratio)
} else {
spriteWidth = STORYBOARD.SPRITE_MAX_SIZE
spriteHeight = Math.round(spriteWidth / videoStreamInfo.ratio)
}
const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
const filename = generateImageFilename()
const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
const { totalSprites, spriteDuration } = buildSpritesMetadata({ video })
const { totalSprites, spriteDuration } = buildTotalSprites(video)
if (totalSprites === 0) {
logger.info(`Do not generate a storyboard of ${payload.videoUUID} because the video is not long enough`, lTags)
return
@ -76,11 +54,16 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT
})
const filename = generateImageFilename()
const destination = join(CONFIG.STORAGE.STORYBOARDS_DIR, filename)
logger.debug(
`Generating storyboard from video of ${video.uuid} to ${destination}`,
{ ...lTags, totalSprites, spritesCount, spriteDuration, videoDuration: video.duration, spriteHeight, spriteWidth }
)
const ffmpeg = new FFmpegImage(getFFmpegCommandWrapperOptions('thumbnail'))
await ffmpeg.generateStoryboardFromVideo({
destination,
path: videoPath,
@ -95,78 +78,23 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
}
})
const imageSize = await getImageSizeFromWorker(destination)
await insertStoryboardInDatabase({
videoUUID: video.uuid,
lTags,
await retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async transaction => {
const videoStillExists = await VideoModel.load(video.id, transaction)
if (!videoStillExists) {
logger.info(`Video ${payload.videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)
deleteFileAndCatch(destination)
return
}
filename,
destination,
const existing = await StoryboardModel.loadByVideo(video.id, transaction)
if (existing) await existing.destroy({ transaction })
imageSize: await getImageSizeFromWorker(destination),
await StoryboardModel.create({
filename,
totalHeight: imageSize.height,
totalWidth: imageSize.width,
spriteHeight,
spriteWidth,
spriteDuration,
videoId: video.id
}, { transaction })
spriteHeight,
spriteWidth,
spriteDuration,
logger.info(`Storyboard generation ${destination} ended for video ${video.uuid}.`, lTags)
if (payload.federate) {
await federateVideoIfNeeded(video, false, transaction)
}
})
federate: payload.federate
})
})
} finally {
inputFileMutexReleaser()
}
}
// ---------------------------------------------------------------------------
export {
processGenerateStoryboard
}
function buildSpritesMetadata (options: {
video: MVideo
}) {
const { video } = options
if (video.duration < 3) return { spriteDuration: undefined, totalSprites: 0 }
const maxSprites = Math.min(Math.ceil(video.duration), STORYBOARD.SPRITES_MAX_EDGE_COUNT * STORYBOARD.SPRITES_MAX_EDGE_COUNT)
const spriteDuration = Math.ceil(video.duration / maxSprites)
const totalSprites = Math.ceil(video.duration / spriteDuration)
// We can generate a single line so we don't need a prime number
if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return { spriteDuration, totalSprites }
return { spriteDuration, totalSprites }
}
function findGridSize (options: {
toFind: number
maxEdgeCount: number
}) {
const { toFind, maxEdgeCount } = options
for (let i = 1; i <= maxEdgeCount; i++) {
for (let j = i; j <= maxEdgeCount; j++) {
if (toFind <= i * j) return { width: j, height: i }
}
}
throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
}

12
server/core/lib/job-queue/handlers/video-import.ts
View file

@ -1,8 +1,5 @@
import { buildAspectRatio } from '@peertube/peertube-core-utils'
import {
ffprobePromise,
getChaptersFromContainer, getVideoStreamDuration
} from '@peertube/peertube-ffmpeg'
import { ffprobePromise, getChaptersFromContainer, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
import {
ThumbnailType,
ThumbnailType_Type,
@ -12,7 +9,8 @@ import {
VideoImportTorrentPayload,
VideoImportTorrentPayloadType,
VideoImportYoutubeDLPayload,
VideoImportYoutubeDLPayloadType, VideoState
VideoImportYoutubeDLPayloadType,
VideoState
} from '@peertube/peertube-models'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
@ -27,7 +25,7 @@ import { isUserQuotaValid } from '@server/lib/user.js'
import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
import { replaceChaptersIfNotExist } from '@server/lib/video-chapters.js'
import { buildNewFile } from '@server/lib/video-file.js'
import { buildMoveVideoJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { addLocalOrRemoteStoryboardJobIfNeeded, buildMoveVideoJob } from '@server/lib/video-jobs.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
@ -305,7 +303,7 @@ async function afterImportSuccess (options: {
}
// Generate the storyboard in the job queue, and don't forget to federate an update after
await JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: true }))
await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })
if (await VideoCaptionModel.hasVideoCaption(video.id) !== true && generateTranscription === true) {
await createTranscriptionTaskIfNeeded(video)

5
server/core/lib/job-queue/handlers/video-live-ending.ts
View file

@ -14,7 +14,7 @@ import {
import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded, updateLocalVideoMiniatureFromExisting } from '@server/lib/thumbnail.js'
import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding.js'
import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { addLocalOrRemoteStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { isVideoInPublicDirectory } from '@server/lib/video-privacy.js'
import { moveToNextState } from '@server/lib/video-state.js'
@ -41,7 +41,6 @@ import { pathExists, remove } from 'fs-extra/esm'
import { readdir } from 'fs/promises'
import { isAbsolute, join } from 'path'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { JobQueue } from '../job-queue.js'
const lTags = loggerTagsFactory('live', 'job')
@ -362,7 +361,7 @@ async function cleanupLiveAndFederate (options: {
}
function createStoryboardJob (video: MVideo) {
return JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: true }))
return addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: true })
}
async function hasReplayFiles (replayDirectory: string) {

44
server/core/lib/runners/job-handlers/abstract-job-handler.ts
View file

@ -1,5 +1,7 @@
import { pick } from '@peertube/peertube-core-utils'
import {
RunnerJobGenerateStoryboardPayload,
RunnerJobGenerateStoryboardPrivatePayload,
RunnerJobLiveRTMPHLSTranscodingPayload,
RunnerJobLiveRTMPHLSTranscodingPrivatePayload,
RunnerJobState,
@ -28,50 +30,56 @@ import { setAsUpdated } from '@server/models/shared/update.js'
import { MRunnerJob } from '@server/types/models/runners/index.js'
type CreateRunnerJobArg =
{
| {
type: Extract<RunnerJobType, 'vod-web-video-transcoding'>
payload: RunnerJobVODWebVideoTranscodingPayload
privatePayload: RunnerJobVODWebVideoTranscodingPrivatePayload
} |
{
}
| {
type: Extract<RunnerJobType, 'vod-hls-transcoding'>
payload: RunnerJobVODHLSTranscodingPayload
privatePayload: RunnerJobVODHLSTranscodingPrivatePayload
} |
{
}
| {
type: Extract<RunnerJobType, 'vod-audio-merge-transcoding'>
payload: RunnerJobVODAudioMergeTranscodingPayload
privatePayload: RunnerJobVODAudioMergeTranscodingPrivatePayload
} |
{
}
| {
type: Extract<RunnerJobType, 'live-rtmp-hls-transcoding'>
payload: RunnerJobLiveRTMPHLSTranscodingPayload
privatePayload: RunnerJobLiveRTMPHLSTranscodingPrivatePayload
} |
{
}
| {
type: Extract<RunnerJobType, 'video-studio-transcoding'>
payload: RunnerJobStudioTranscodingPayload
privatePayload: RunnerJobVideoStudioTranscodingPrivatePayload
} |
{
}
| {
type: Extract<RunnerJobType, 'generate-video-storyboard'>
payload: RunnerJobGenerateStoryboardPayload
privatePayload: RunnerJobGenerateStoryboardPrivatePayload
}
| {
type: Extract<RunnerJobType, 'video-transcription'>
payload: RunnerJobTranscriptionPayload
privatePayload: RunnerJobTranscriptionPrivatePayload
}
export abstract class AbstractJobHandler <C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> {
export abstract class AbstractJobHandler<C, U extends RunnerJobUpdatePayload, S extends RunnerJobSuccessPayload> {
protected readonly lTags = loggerTagsFactory('runner')
// ---------------------------------------------------------------------------
abstract create (options: C): Promise<MRunnerJob>
protected async createRunnerJob (options: CreateRunnerJobArg & {
jobUUID: string
priority: number
dependsOnRunnerJob?: MRunnerJob
}): Promise<MRunnerJob> {
protected async createRunnerJob (
options: CreateRunnerJobArg & {
jobUUID: string
priority: number
dependsOnRunnerJob?: MRunnerJob
}
): Promise<MRunnerJob> {
const { priority, dependsOnRunnerJob } = options
logger.debug('Creating runner job', { options, dependsOnRunnerJob, ...this.lTags(options.type) })

1
server/core/lib/runners/job-handlers/index.ts
View file

@ -6,3 +6,4 @@ export * from './video-studio-transcoding-job-handler.js'
export * from './vod-audio-merge-transcoding-job-handler.js'
export * from './vod-hls-transcoding-job-handler.js'
export * from './vod-web-video-transcoding-job-handler.js'
export * from './video-storyboard-job-handler.js'

4
server/core/lib/runners/job-handlers/runner-job-handlers.ts
View file

@ -7,6 +7,7 @@ import { VideoStudioTranscodingJobHandler } from './video-studio-transcoding-job
import { VODAudioMergeTranscodingJobHandler } from './vod-audio-merge-transcoding-job-handler.js'
import { VODHLSTranscodingJobHandler } from './vod-hls-transcoding-job-handler.js'
import { VODWebVideoTranscodingJobHandler } from './vod-web-video-transcoding-job-handler.js'
import { VideoStoryboardJobHandler } from './video-storyboard-job-handler.js'
const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, RunnerJobUpdatePayload, RunnerJobSuccessPayload>> = {
'vod-web-video-transcoding': VODWebVideoTranscodingJobHandler,
@ -14,7 +15,8 @@ const processors: Record<RunnerJobType, new() => AbstractJobHandler<unknown, Run
'vod-audio-merge-transcoding': VODAudioMergeTranscodingJobHandler,
'live-rtmp-hls-transcoding': LiveRTMPHLSTranscodingJobHandler,
'video-studio-transcoding': VideoStudioTranscodingJobHandler,
'video-transcription': TranscriptionJobHandler
'video-transcription': TranscriptionJobHandler,
'generate-video-storyboard': VideoStoryboardJobHandler
}
export function getRunnerJobHandlerClass (job: MRunnerJob) {

2
server/core/lib/runners/job-handlers/transcription-job-handler.ts
View file

@ -72,7 +72,7 @@ export class TranscriptionJobHandler extends AbstractJobHandler<CreateOptions, R
jobUUID,
payload,
privatePayload,
priority: JOB_PRIORITY.TRANSCODING
priority: JOB_PRIORITY.TRANSCRIPTION
})
return job

118
server/core/lib/runners/job-handlers/video-storyboard-job-handler.ts Normal file
View file

@ -0,0 +1,118 @@
import {
GenerateStoryboardSuccess,
RunnerJobGenerateStoryboardPayload,
RunnerJobGenerateStoryboardPrivatePayload,
RunnerJobUpdatePayload,
VideoFileStream
} from '@peertube/peertube-models'
import { buildUUID } from '@peertube/peertube-node-utils'
import { generateImageFilename } from '@server/helpers/image-utils.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { JOB_PRIORITY, STORYBOARD } from '@server/initializers/constants.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { getImageSizeFromWorker } from '@server/lib/worker/parent-process.js'
import { VideoModel } from '@server/models/video/video.js'
import { MRunnerJob } from '@server/types/models/runners/index.js'
import { move } from 'fs-extra/esm'
import { join } from 'path'
import { buildSpriteSize, buildTotalSprites, findGridSize, insertStoryboardInDatabase } from '../../storyboard.js'
import { generateRunnerTranscodingVideoInputFileUrl } from '../runner-urls.js'
import { AbstractJobHandler } from './abstract-job-handler.js'
const lTagsBase = loggerTagsFactory('storyboard', 'runners')
type CreateOptions = {
videoUUID: string
}
export class VideoStoryboardJobHandler extends AbstractJobHandler<CreateOptions, RunnerJobUpdatePayload, GenerateStoryboardSuccess> {
async create (options: CreateOptions) {
const { videoUUID } = options
const lTags = lTagsBase(videoUUID)
const jobUUID = buildUUID()
const video = await VideoModel.loadFull(videoUUID)
const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)
return VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
const { spriteHeight, spriteWidth } = await buildSpriteSize(videoPath)
const { spriteDuration, totalSprites } = buildTotalSprites(video)
if (totalSprites === 0) {
logger.info(`Do not generate remote storyboard job of ${videoUUID} because the video is not long enough`, lTags)
return
}
const spritesCount = findGridSize({ toFind: totalSprites, maxEdgeCount: STORYBOARD.SPRITES_MAX_EDGE_COUNT })
const payload: RunnerJobGenerateStoryboardPayload = {
input: {
videoFileUrl: generateRunnerTranscodingVideoInputFileUrl(jobUUID, videoUUID)
},
sprites: {
size: { height: spriteHeight, width: spriteWidth },
count: spritesCount,
duration: spriteDuration
},
output: {}
}
const privatePayload: RunnerJobGenerateStoryboardPrivatePayload = { videoUUID }
const job = await this.createRunnerJob({
type: 'generate-video-storyboard',
jobUUID,
payload,
privatePayload,
priority: JOB_PRIORITY.STORYBOARD
})
return job
})
}
protected isAbortSupported () {
return true
}
protected specificUpdate (_options: { runnerJob: MRunnerJob }) {}
protected specificAbort (_options: { runnerJob: MRunnerJob }) {}
// When runner returns the storyboard image, finish the server-side creation like local job would
protected async specificComplete (options: { runnerJob: MRunnerJob, resultPayload: GenerateStoryboardSuccess }) {
const { runnerJob, resultPayload } = options
const video = await VideoModel.loadFull(runnerJob.privatePayload.videoUUID)
if (!video) return
const destinationFilename = generateImageFilename()
const destinationPath = join(CONFIG.STORAGE.STORYBOARDS_DIR, destinationFilename)
await move(resultPayload.storyboardFile as string, destinationPath)
const { sprites } = runnerJob.payload as RunnerJobGenerateStoryboardPayload
await insertStoryboardInDatabase({
videoUUID: video.uuid,
lTags: this.lTags(video.uuid, runnerJob.uuid),
filename: destinationFilename,
destination: destinationPath,
imageSize: await getImageSizeFromWorker(destinationPath),
spriteHeight: sprites.size.height,
spriteWidth: sprites.size.width,
spriteDuration: sprites.duration,
federate: true
})
logger.info('Runner storyboard job %s for %s ended.', runnerJob.uuid, video.uuid, this.lTags(video.uuid, runnerJob.uuid))
}
protected specificError (_options: { runnerJob: MRunnerJob }) {}
protected specificCancel (_options: { runnerJob: MRunnerJob }) {}
}

5
server/core/lib/server-config-manager.ts
View file

@ -384,7 +384,10 @@ class ServerConfigManager {
},
storyboards: {
enabled: CONFIG.STORYBOARDS.ENABLED
enabled: CONFIG.STORYBOARDS.ENABLED,
remoteRunners: {
enabled: CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED
}
},
webrtc: {

98
server/core/lib/storyboard.ts Normal file
View file

@ -0,0 +1,98 @@
import { ffprobePromise, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
import { LoggerTags, logger } from '@server/helpers/logger.js'
import { deleteFileAndCatch } from '@server/helpers/utils.js'
import { STORYBOARD } from '@server/initializers/constants.js'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { StoryboardModel } from '@server/models/video/storyboard.js'
import { VideoModel } from '@server/models/video/video.js'
import { MVideo } from '@server/types/models/index.js'
import { federateVideoIfNeeded } from './activitypub/videos/federate.js'
export async function buildSpriteSize (videoPath: string) {
const probe = await ffprobePromise(videoPath)
const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe)
if (videoStreamInfo.isPortraitMode) {
return {
spriteHeight: STORYBOARD.SPRITE_MAX_SIZE,
spriteWidth: Math.round(STORYBOARD.SPRITE_MAX_SIZE * videoStreamInfo.ratio)
}
}
return {
spriteWidth: STORYBOARD.SPRITE_MAX_SIZE,
spriteHeight: Math.round(STORYBOARD.SPRITE_MAX_SIZE / videoStreamInfo.ratio)
}
}
export function buildTotalSprites (video: MVideo) {
if (video.duration < 3) return { spriteDuration: undefined, totalSprites: 0 }
const maxSprites = Math.min(Math.ceil(video.duration), STORYBOARD.SPRITES_MAX_EDGE_COUNT * STORYBOARD.SPRITES_MAX_EDGE_COUNT)
const spriteDuration = Math.ceil(video.duration / maxSprites)
const totalSprites = Math.ceil(video.duration / spriteDuration)
// We can generate a single line so we don't need a prime number
if (totalSprites <= STORYBOARD.SPRITES_MAX_EDGE_COUNT) return { spriteDuration, totalSprites }
return { spriteDuration, totalSprites }
}
export function findGridSize (options: {
toFind: number
maxEdgeCount: number
}) {
const { toFind, maxEdgeCount } = options
for (let i = 1; i <= maxEdgeCount; i++) {
for (let j = i; j <= maxEdgeCount; j++) {
if (toFind <= i * j) return { width: j, height: i }
}
}
throw new Error(`Could not find grid size (to find: ${toFind}, max edge count: ${maxEdgeCount}`)
}
export async function insertStoryboardInDatabase (options: {
videoUUID: string
lTags: LoggerTags
filename: string
destination: string
imageSize: { width: number, height: number }
spriteHeight: number
spriteWidth: number
spriteDuration: number
federate: boolean
}) {
const { videoUUID, lTags, imageSize, spriteHeight, spriteWidth, spriteDuration, destination, filename, federate } = options
await retryTransactionWrapper(() => {
return sequelizeTypescript.transaction(async transaction => {
const video = await VideoModel.loadFull(videoUUID, transaction)
if (!video) {
logger.info(`Video ${videoUUID} does not exist anymore, skipping storyboard generation.`, lTags)
deleteFileAndCatch(destination)
return
}
const existing = await StoryboardModel.loadByVideo(video.id, transaction)
if (existing) await existing.destroy({ transaction })
await StoryboardModel.create({
filename,
totalHeight: imageSize.height,
totalWidth: imageSize.width,
spriteHeight,
spriteWidth,
spriteDuration,
videoId: video.id
}, { transaction })
if (federate) {
await federateVideoIfNeeded(video, false, transaction)
}
})
})
}

5
server/core/lib/transcoding/web-transcoding.ts
View file

@ -16,10 +16,9 @@ import { copyFile } from 'fs/promises'
import { basename, join } from 'path'
import { CONFIG } from '../../initializers/config.js'
import { VideoFileModel } from '../../models/video/video-file.js'
import { JobQueue } from '../job-queue/index.js'
import { generateWebVideoFilename } from '../paths.js'
import { buildNewFile, saveNewOriginalFileIfNeeded } from '../video-file.js'
import { buildStoryboardJobIfNeeded } from '../video-jobs.js'
import { addLocalOrRemoteStoryboardJobIfNeeded } from '../video-jobs.js'
import { VideoPathManager } from '../video-path-manager.js'
import { buildFFmpegVOD } from './shared/index.js'
import { buildOriginalFileResolution } from './transcoding-resolutions.js'
@ -229,7 +228,7 @@ export async function onWebVideoFileTranscoding (options: {
video.VideoFiles = await video.$get('VideoFiles')
if (wasAudioFile) {
await JobQueue.Instance.createJob(buildStoryboardJobIfNeeded({ video, federate: false }))
await addLocalOrRemoteStoryboardJobIfNeeded({ video, federate: false })
}
return { video, videoFile }

31
server/core/lib/video-jobs.ts
View file

@ -3,6 +3,7 @@ import { CONFIG } from '@server/initializers/config.js'
import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
import { MVideo, MVideoFile, MVideoFullLight, MVideoUUID } from '@server/types/models/index.js'
import { CreateJobArgument, CreateJobOptions, JobQueue } from './job-queue/job-queue.js'
import { VideoStoryboardJobHandler } from './runners/index.js'
import { createTranscriptionTaskIfNeeded } from './video-captions.js'
import { moveFilesIfPrivacyChanged } from './video-privacy.js'
@ -26,13 +27,13 @@ export async function buildMoveVideoJob (options: {
}
}
export function buildStoryboardJobIfNeeded (options: {
export function buildLocalStoryboardJobIfNeeded (options: {
video: MVideo
federate: boolean
}) {
const { video, federate } = options
if (CONFIG.STORYBOARDS.ENABLED) {
if (CONFIG.STORYBOARDS.ENABLED && !CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED) {
return {
type: 'generate-video-storyboard' as 'generate-video-storyboard',
payload: {
@ -55,6 +56,28 @@ export function buildStoryboardJobIfNeeded (options: {
return undefined
}
export function addRemoteStoryboardJobIfNeeded (video: MVideo) {
if (CONFIG.STORYBOARDS.ENABLED !== true) return
if (CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED !== true) return
return new VideoStoryboardJobHandler().create({ videoUUID: video.uuid })
}
export async function addLocalOrRemoteStoryboardJobIfNeeded (options: {
video: MVideo
federate: boolean
}) {
const { video, federate } = options
if (CONFIG.STORYBOARDS.ENABLED !== true) return
if (CONFIG.STORYBOARDS.REMOTE_RUNNERS.ENABLED === true) {
await addRemoteStoryboardJobIfNeeded(video)
} else {
await JobQueue.Instance.createJob(buildLocalStoryboardJobIfNeeded({ video, federate }))
}
}
export async function addVideoJobsAfterCreation (options: {
video: MVideo
videoFile: MVideoFile
@ -72,7 +95,7 @@ export async function addVideoJobsAfterCreation (options: {
}
},
buildStoryboardJobIfNeeded({ video, federate: false }),
buildLocalStoryboardJobIfNeeded({ video, federate: false }),
{
type: 'notify',
@ -109,6 +132,8 @@ export async function addVideoJobsAfterCreation (options: {
await JobQueue.Instance.createSequentialJobFlow(...jobs)
await addRemoteStoryboardJobIfNeeded(video)
if (generateTranscription === true) {
await createTranscriptionTaskIfNeeded(video)
}

5
server/core/lib/video-studio.ts
View file

@ -12,7 +12,7 @@ import { VideoStudioTranscodingJobHandler } from './runners/index.js'
import { getTranscodingJobPriority } from './transcoding/transcoding-priority.js'
import { regenerateTranscriptionTaskIfNeeded } from './video-captions.js'
import { buildNewFile, removeHLSPlaylist, removeWebVideoFile } from './video-file.js'
import { buildStoryboardJobIfNeeded } from './video-jobs.js'
import { addRemoteStoryboardJobIfNeeded, buildLocalStoryboardJobIfNeeded } from './video-jobs.js'
import { VideoPathManager } from './video-path-manager.js'
const lTags = loggerTagsFactory('video-studio')
@ -110,7 +110,7 @@ export async function onVideoStudioEnded (options: {
await video.save()
await JobQueue.Instance.createSequentialJobFlow(
buildStoryboardJobIfNeeded({ video, federate: false }),
buildLocalStoryboardJobIfNeeded({ video, federate: false }),
{
type: 'federate-video' as 'federate-video',
payload: {
@ -129,6 +129,7 @@ export async function onVideoStudioEnded (options: {
}
)
await addRemoteStoryboardJobIfNeeded(video)
await regenerateTranscriptionTaskIfNeeded(video)
}