Separate HLS audio and video streams

Allows:
  * The HLS player to propose an "Audio only" resolution
  * The live to output an "Audio only" resolution
  * The live to ingest and output an "Audio only" stream

 This feature is under a config for VOD videos and is enabled by default for lives

 In the future we can imagine:
  * To propose multiple audio streams for a specific video
  * To ingest an audio only VOD and just output an audio only "video"
    (the player would play the audio file and PeerTube would not
    generate additional resolutions)

This commit introduce a new way to download videos:
 * Add "/download/videos/generate/:videoId" endpoint where PeerTube can
   mux an audio only and a video only file to a mp4 container
 * The download client modal introduces a new default panel where the
   user can choose resolutions it wants to download

server/core/lib/job-queue/handlers/generate-storyboard.ts

@@ -1,5 +1,5 @@
-import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo, isAudioFile } from '@peertube/peertube-ffmpeg'
-import { GenerateStoryboardPayload } from '@peertube/peertube-models'
+import { FFmpegImage, ffprobePromise, getVideoStreamDimensionsInfo } from '@peertube/peertube-ffmpeg'
+import { GenerateStoryboardPayload, VideoFileStream } from '@peertube/peertube-models'
 import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
 import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
 import { generateImageFilename } from '@server/helpers/image-utils.js'
@@ -34,16 +34,14 @@ async function processGenerateStoryboard (job: Job): Promise<void> {
       return
     }
 
-    const inputFile = video.getMaxQualityFile()
+    const inputFile = video.getMaxQualityFile(VideoFileStream.VIDEO)
+    if (!inputFile) {
+      logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
+      return
+    }
 
     await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async videoPath => {
       const probe = await ffprobePromise(videoPath)
-      const isAudio = await isAudioFile(videoPath, probe)
-
-      if (isAudio) {
-        logger.info('Do not generate a storyboard of %s since the video does not have a video stream', payload.videoUUID, lTags)
-        return
-      }
 
       const videoStreamInfo = await getVideoStreamDimensionsInfo(videoPath, probe)
       let spriteHeight: number

server/core/lib/job-queue/handlers/shared/move-video.ts

@@ -44,7 +44,7 @@ export async function moveToJob (options: {
 
   try {
     const source = await VideoSourceModel.loadLatest(video.id)
-    if (source) {
+    if (source?.keptOriginalFilename) {
       logger.debug(`Moving video source ${source.keptOriginalFilename} file of video ${video.uuid}`, lTags)
 
       await moveVideoSourceFile(source)

server/core/lib/job-queue/handlers/transcoding-job-builder.ts

@@ -1,10 +1,10 @@
-import { Job } from 'bullmq'
+import { pick } from '@peertube/peertube-core-utils'
+import { TranscodingJobBuilderPayload, VideoFileStream } from '@peertube/peertube-models'
 import { createOptimizeOrMergeAudioJobs } from '@server/lib/transcoding/create-transcoding-job.js'
 import { UserModel } from '@server/models/user/user.js'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
 import { VideoModel } from '@server/models/video/video.js'
-import { pick } from '@peertube/peertube-core-utils'
-import { TranscodingJobBuilderPayload } from '@peertube/peertube-models'
+import { Job } from 'bullmq'
 import { logger } from '../../../helpers/logger.js'
 import { JobQueue } from '../job-queue.js'
 
@@ -16,7 +16,7 @@ async function processTranscodingJobBuilder (job: Job) {
   if (payload.optimizeJob) {
     const video = await VideoModel.loadFull(payload.videoUUID)
     const user = await UserModel.loadByVideoId(video.id)
-    const videoFile = video.getMaxQualityFile()
+    const videoFile = video.getMaxQualityFile(VideoFileStream.VIDEO) || video.getMaxQualityFile(VideoFileStream.AUDIO)
 
     await createOptimizeOrMergeAudioJobs({
       ...pick(payload.optimizeJob, [ 'isNewVideo' ]),

server/core/lib/job-queue/handlers/video-import.ts

@@ -129,7 +129,7 @@ type ProcessFileOptions = {
 }
 async function processFile (downloader: () => Promise<string>, videoImport: MVideoImportDefault, options: ProcessFileOptions) {
   let tmpVideoPath: string
-  let videoFile: VideoFileModel
+  let videoFile: MVideoFile
 
   try {
     // Download video from youtubeDL
@@ -163,7 +163,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
       videoImport,
       video: videoImport.Video,
       videoFilePath: tmpVideoPath,
-      videoFile,
+      videoFile: videoFile as VideoFileModel,
       user: videoImport.User
     }
     const acceptedResult = await Hooks.wrapFun(isPostImportVideoAccepted, acceptParameters, hookName)

server/core/lib/job-queue/handlers/video-live-ending.ts

@@ -1,5 +1,5 @@
 import { ffprobePromise, getAudioStream, getVideoStreamDimensionsInfo, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
-import { ThumbnailType, VideoLiveEndingPayload, VideoState } from '@peertube/peertube-models'
+import { ThumbnailType, VideoFileStream, VideoLiveEndingPayload, VideoState } from '@peertube/peertube-models'
 import { peertubeTruncate } from '@server/helpers/core-utils.js'
 import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js'
 import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url.js'
@@ -11,7 +11,7 @@ import {
   getHLSDirectory,
   getLiveReplayBaseDirectory
 } from '@server/lib/paths.js'
-import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
+import { generateLocalVideoMiniature, regenerateMiniaturesIfNeeded, updateLocalVideoMiniatureFromExisting } from '@server/lib/thumbnail.js'
 import { generateHlsPlaylistResolutionFromTS } from '@server/lib/transcoding/hls-transcoding.js'
 import { createTranscriptionTaskIfNeeded } from '@server/lib/video-captions.js'
 import { buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
@@ -25,7 +25,15 @@ import { VideoLiveSessionModel } from '@server/models/video/video-live-session.j
 import { VideoLiveModel } from '@server/models/video/video-live.js'
 import { VideoStreamingPlaylistModel } from '@server/models/video/video-streaming-playlist.js'
 import { VideoModel } from '@server/models/video/video.js'
-import { MVideo, MVideoLive, MVideoLiveSession, MVideoWithAllFiles } from '@server/types/models/index.js'
+import {
+  MThumbnail,
+  MVideo,
+  MVideoLive,
+  MVideoLiveSession,
+  MVideoThumbnail,
+  MVideoWithAllFiles,
+  MVideoWithFileThumbnail
+} from '@server/types/models/index.js'
 import { Job } from 'bullmq'
 import { remove } from 'fs-extra/esm'
 import { readdir } from 'fs/promises'
@@ -97,7 +105,7 @@ export {
 // ---------------------------------------------------------------------------
 
 async function saveReplayToExternalVideo (options: {
-  liveVideo: MVideo
+  liveVideo: MVideoThumbnail
   liveSession: MVideoLiveSession
   publishedAt: string
   replayDirectory: string
@@ -159,21 +167,13 @@ async function saveReplayToExternalVideo (options: {
   try {
     await assignReplayFilesToVideo({ video: replayVideo, replayDirectory })
 
+    logger.info(`Removing replay directory ${replayDirectory}`, lTags(liveVideo.uuid))
     await remove(replayDirectory)
   } finally {
     inputFileMutexReleaser()
   }
 
-  const thumbnails = await generateLocalVideoMiniature({
-    video: replayVideo,
-    videoFile: replayVideo.getMaxQualityFile(),
-    types: [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ],
-    ffprobe: undefined
-  })
-
-  for (const thumbnail of thumbnails) {
-    await replayVideo.addAndSaveThumbnail(thumbnail)
-  }
+  await copyOrRegenerateThumbnails({ liveVideo, replayVideo })
 
   await createStoryboardJob(replayVideo)
   await createTranscriptionTaskIfNeeded(replayVideo)
@@ -181,6 +181,40 @@ async function saveReplayToExternalVideo (options: {
   await moveToNextState({ video: replayVideo, isNewVideo: true })
 }
 
+async function copyOrRegenerateThumbnails (options: {
+  liveVideo: MVideoThumbnail
+  replayVideo: MVideoWithFileThumbnail
+}) {
+  const { liveVideo, replayVideo } = options
+
+  let thumbnails: MThumbnail[] = []
+  const preview = liveVideo.getPreview()
+
+  if (preview?.automaticallyGenerated === false) {
+    thumbnails = await Promise.all(
+      [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ].map(type => {
+        return updateLocalVideoMiniatureFromExisting({
+          inputPath: preview.getPath(),
+          video: replayVideo,
+          type,
+          automaticallyGenerated: false
+        })
+      })
+    )
+  } else {
+    thumbnails = await generateLocalVideoMiniature({
+      video: replayVideo,
+      videoFile: replayVideo.getMaxQualityFile(VideoFileStream.VIDEO) || replayVideo.getMaxQualityFile(VideoFileStream.AUDIO),
+      types: [ ThumbnailType.MINIATURE, ThumbnailType.PREVIEW ],
+      ffprobe: undefined
+    })
+  }
+
+  for (const thumbnail of thumbnails) {
+    await replayVideo.addAndSaveThumbnail(thumbnail)
+  }
+}
+
 async function replaceLiveByReplay (options: {
   video: MVideo
   liveSession: MVideoLiveSession

server/core/lib/job-queue/handlers/video-studio-edition.ts

@@ -1,17 +1,4 @@
-import { Job } from 'bullmq'
-import { remove } from 'fs-extra/esm'
-import { join } from 'path'
-import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
-import { CONFIG } from '@server/initializers/config.js'
-import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js'
-import { isUserQuotaValid } from '@server/lib/user.js'
-import { VideoPathManager } from '@server/lib/video-path-manager.js'
-import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js'
-import { UserModel } from '@server/models/user/user.js'
-import { VideoModel } from '@server/models/video/video.js'
-import { MVideo, MVideoFullLight } from '@server/types/models/index.js'
 import { pick } from '@peertube/peertube-core-utils'
-import { buildUUID } from '@peertube/peertube-node-utils'
 import { FFmpegEdition } from '@peertube/peertube-ffmpeg'
 import {
   VideoStudioEditionPayload,
@@ -22,6 +9,20 @@ import {
   VideoStudioTaskPayload,
   VideoStudioTaskWatermarkPayload
 } from '@peertube/peertube-models'
+import { buildUUID } from '@peertube/peertube-node-utils'
+import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/index.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { VideoTranscodingProfilesManager } from '@server/lib/transcoding/default-transcoding-profiles.js'
+import { isUserQuotaValid } from '@server/lib/user.js'
+import { VideoPathManager } from '@server/lib/video-path-manager.js'
+import { approximateIntroOutroAdditionalSize, onVideoStudioEnded, safeCleanupStudioTMPFiles } from '@server/lib/video-studio.js'
+import { UserModel } from '@server/models/user/user.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MVideo, MVideoFullLight } from '@server/types/models/index.js'
+import { MutexInterface } from 'async-mutex'
+import { Job } from 'bullmq'
+import { remove } from 'fs-extra/esm'
+import { extname, join } from 'path'
 import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
 
 const lTagsBase = loggerTagsFactory('video-studio')
@@ -32,6 +33,8 @@ async function processVideoStudioEdition (job: Job) {
 
   logger.info('Process video studio edition of %s in job %s.', payload.videoUUID, job.id, lTags)
 
+  let inputFileMutexReleaser = await VideoPathManager.Instance.lockFiles(payload.videoUUID)
+
   try {
     const video = await VideoModel.loadFull(payload.videoUUID)
 
@@ -45,18 +48,28 @@ async function processVideoStudioEdition (job: Job) {
 
     await checkUserQuotaOrThrow(video, payload)
 
-    const inputFile = video.getMaxQualityFile()
+    await video.reload()
 
-    const editionResultPath = await VideoPathManager.Instance.makeAvailableVideoFile(inputFile, async originalFilePath => {
+    const editionResultPath = await VideoPathManager.Instance.makeAvailableMaxQualityFiles(video, async ({
+      videoPath: originalVideoFilePath,
+      separatedAudioPath
+    }) => {
       let tmpInputFilePath: string
       let outputPath: string
 
       for (const task of payload.tasks) {
-        const outputFilename = buildUUID() + inputFile.extname
+        const outputFilename = buildUUID() + extname(originalVideoFilePath)
         outputPath = join(CONFIG.STORAGE.TMP_DIR, outputFilename)
 
         await processTask({
-          inputPath: tmpInputFilePath ?? originalFilePath,
+          videoInputPath: tmpInputFilePath ?? originalVideoFilePath,
+
+          separatedAudioInputPath: tmpInputFilePath
+            ? undefined
+            : separatedAudioPath,
+
+          inputFileMutexReleaser,
+
           video,
           outputPath,
           task,
@@ -67,6 +80,7 @@ async function processVideoStudioEdition (job: Job) {
 
         // For the next iteration
         tmpInputFilePath = outputPath
+        inputFileMutexReleaser = undefined
       }
 
       return outputPath
@@ -79,6 +93,8 @@ async function processVideoStudioEdition (job: Job) {
     await safeCleanupStudioTMPFiles(payload.tasks)
 
     throw err
+  } finally {
+    if (inputFileMutexReleaser) inputFileMutexReleaser()
   }
 }
 
@@ -91,7 +107,11 @@ export {
 // ---------------------------------------------------------------------------
 
 type TaskProcessorOptions <T extends VideoStudioTaskPayload = VideoStudioTaskPayload> = {
-  inputPath: string
+  videoInputPath: string
+  separatedAudioInputPath?: string
+
+  inputFileMutexReleaser: MutexInterface.Releaser
+
   outputPath: string
   video: MVideo
   task: T
@@ -122,7 +142,7 @@ function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntr
   logger.debug('Will add intro/outro to the video.', { options, ...lTags })
 
   return buildFFmpegEdition().addIntroOutro({
-    ...pick(options, [ 'inputPath', 'outputPath' ]),
+    ...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
 
     introOutroPath: task.options.file,
     type: task.name === 'add-intro'
@@ -137,7 +157,7 @@ function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
   logger.debug('Will cut the video.', { options, ...lTags })
 
   return buildFFmpegEdition().cutVideo({
-    ...pick(options, [ 'inputPath', 'outputPath' ]),
+    ...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
 
     start: task.options.start,
     end: task.options.end
@@ -150,7 +170,7 @@ function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWater
   logger.debug('Will add watermark to the video.', { options, ...lTags })
 
   return buildFFmpegEdition().addWatermark({
-    ...pick(options, [ 'inputPath', 'outputPath' ]),
+    ...pick(options, [ 'inputFileMutexReleaser', 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
 
     watermarkPath: task.options.file,
 

server/core/lib/job-queue/handlers/video-transcoding.ts

@@ -1,4 +1,10 @@
-import { Job } from 'bullmq'
+import {
+  HLSTranscodingPayload,
+  MergeAudioTranscodingPayload,
+  NewWebVideoResolutionTranscodingPayload,
+  OptimizeTranscodingPayload,
+  VideoTranscodingPayload
+} from '@peertube/peertube-models'
 import { onTranscodingEnded } from '@server/lib/transcoding/ended-transcoding.js'
 import { generateHlsPlaylistResolution } from '@server/lib/transcoding/hls-transcoding.js'
 import { mergeAudioVideofile, optimizeOriginalVideofile, transcodeNewWebVideoResolution } from '@server/lib/transcoding/web-transcoding.js'
@@ -8,13 +14,7 @@ import { moveToFailedTranscodingState } from '@server/lib/video-state.js'
 import { UserModel } from '@server/models/user/user.js'
 import { VideoJobInfoModel } from '@server/models/video/video-job-info.js'
 import { MUser, MUserId, MVideoFullLight } from '@server/types/models/index.js'
-import {
-  HLSTranscodingPayload,
-  MergeAudioTranscodingPayload,
-  NewWebVideoResolutionTranscodingPayload,
-  OptimizeTranscodingPayload,
-  VideoTranscodingPayload
-} from '@peertube/peertube-models'
+import { Job } from 'bullmq'
 import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
 import { VideoModel } from '../../../models/video/video.js'
 
@@ -87,7 +87,7 @@ async function handleWebVideoMergeAudioJob (job: Job, payload: MergeAudioTransco
 async function handleWebVideoOptimizeJob (job: Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
   logger.info('Handling optimize transcoding job for %s.', video.uuid, lTags(video.uuid), { payload })
 
-  await optimizeOriginalVideofile({ video, inputVideoFile: video.getMaxQualityFile(), quickTranscode: payload.quickTranscode, job })
+  await optimizeOriginalVideofile({ video, quickTranscode: payload.quickTranscode, job })
 
   logger.info('Optimize transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
 
@@ -103,7 +103,7 @@ async function handleNewWebVideoResolutionJob (job: Job, payload: NewWebVideoRes
 
   logger.info('Web Video transcoding job for %s ended.', video.uuid, lTags(video.uuid), { payload })
 
-  await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })
+  await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video })
 }
 
 // ---------------------------------------------------------------------------
@@ -117,20 +117,24 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg:
   try {
     video = await VideoModel.loadFull(videoArg.uuid)
 
-    const videoFileInput = payload.copyCodecs
-      ? video.getWebVideoFile(payload.resolution)
-      : video.getMaxQualityFile()
+    const { videoFile, separatedAudioFile } = video.getMaxQualityAudioAndVideoFiles()
 
-    const videoOrStreamingPlaylist = videoFileInput.getVideoOrStreamingPlaylist()
+    const videoFileInputs = payload.copyCodecs
+      ? [ video.getWebVideoFileMinResolution(payload.resolution) ]
+      : [ videoFile, separatedAudioFile ].filter(v => !!v)
 
-    await VideoPathManager.Instance.makeAvailableVideoFile(videoFileInput.withVideoOrPlaylist(videoOrStreamingPlaylist), videoInputPath => {
+    await VideoPathManager.Instance.makeAvailableVideoFiles(videoFileInputs, ([ videoPath, separatedAudioPath ]) => {
       return generateHlsPlaylistResolution({
         video,
-        videoInputPath,
+
+        videoInputPath: videoPath,
+        separatedAudioInputPath: separatedAudioPath,
+
         inputFileMutexReleaser,
         resolution: payload.resolution,
         fps: payload.fps,
         copyCodecs: payload.copyCodecs,
+        separatedAudio: payload.separatedAudio,
         job
       })
     })
@@ -146,5 +150,5 @@ async function handleHLSJob (job: Job, payload: HLSTranscodingPayload, videoArg:
     await removeAllWebVideoFiles(video)
   }
 
-  await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: true, video })
+  await onTranscodingEnded({ isNewVideo: payload.isNewVideo, moveVideoToNextState: !payload.hasChildren, video })
 }