Separate HLS audio and video streams

Allows:
  * The HLS player to propose an "Audio only" resolution
  * The live to output an "Audio only" resolution
  * The live to ingest and output an "Audio only" stream

 This feature is under a config for VOD videos and is enabled by default for lives

 In the future we can imagine:
  * To propose multiple audio streams for a specific video
  * To ingest an audio only VOD and just output an audio only "video"
    (the player would play the audio file and PeerTube would not
    generate additional resolutions)

This commit introduce a new way to download videos:
 * Add "/download/videos/generate/:videoId" endpoint where PeerTube can
   mux an audio only and a video only file to a mp4 container
 * The download client modal introduces a new default panel where the
   user can choose resolutions it wants to download

apps/peertube-runner/src/server/process/shared/common.ts

@@ -1,9 +1,10 @@
-import { remove } from 'fs-extra/esm'
-import { join } from 'path'
+import { pick } from '@peertube/peertube-core-utils'
 import { FFmpegEdition, FFmpegLive, FFmpegVOD, getDefaultAvailableEncoders, getDefaultEncodersToTry } from '@peertube/peertube-ffmpeg'
 import { RunnerJob, RunnerJobPayload } from '@peertube/peertube-models'
 import { buildUUID } from '@peertube/peertube-node-utils'
 import { PeerTubeServer } from '@peertube/peertube-server-commands'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
 import { ConfigManager, downloadFile, logger } from '../../../shared/index.js'
 import { getWinstonLogger } from './winston-logger.js'
 
@@ -35,6 +36,18 @@ export async function downloadInputFile (options: {
   return destination
 }
 
+export async function downloadSeparatedAudioFileIfNeeded (options: {
+  urls: string[]
+  job: JobWithToken
+  runnerToken: string
+}) {
+  const { urls } = options
+
+  if (!urls || urls.length === 0) return undefined
+
+  return downloadInputFile({ url: urls[0], ...pick(options, [ 'job', 'runnerToken' ]) })
+}
+
 export function scheduleTranscodingProgress (options: {
   server: PeerTubeServer
   runnerToken: string

apps/peertube-runner/src/server/process/shared/process-live.ts

@@ -1,9 +1,11 @@
-import { FSWatcher, watch } from 'chokidar'
-import { FfmpegCommand } from 'fluent-ffmpeg'
-import { ensureDir, remove } from 'fs-extra/esm'
-import { basename, join } from 'path'
 import { wait } from '@peertube/peertube-core-utils'
-import { ffprobePromise, getVideoStreamBitrate, getVideoStreamDimensionsInfo, hasAudioStream } from '@peertube/peertube-ffmpeg'
+import {
+  ffprobePromise,
+  getVideoStreamBitrate,
+  getVideoStreamDimensionsInfo,
+  hasAudioStream,
+  hasVideoStream
+} from '@peertube/peertube-ffmpeg'
 import {
   LiveRTMPHLSTranscodingSuccess,
   LiveRTMPHLSTranscodingUpdatePayload,
@@ -12,6 +14,10 @@ import {
   ServerErrorCode
 } from '@peertube/peertube-models'
 import { buildUUID } from '@peertube/peertube-node-utils'
+import { FSWatcher, watch } from 'chokidar'
+import { FfmpegCommand } from 'fluent-ffmpeg'
+import { ensureDir, remove } from 'fs-extra/esm'
+import { basename, join } from 'path'
 import { ConfigManager } from '../../../shared/config-manager.js'
 import { logger } from '../../../shared/index.js'
 import { buildFFmpegLive, ProcessOptions } from './common.js'
@@ -51,6 +57,7 @@ export class ProcessLiveRTMPHLSTranscoding {
         logger.info({ probe }, `Probed ${payload.input.rtmpUrl}`)
 
         const hasAudio = await hasAudioStream(payload.input.rtmpUrl, probe)
+        const hasVideo = await hasVideoStream(payload.input.rtmpUrl, probe)
         const bitrate = await getVideoStreamBitrate(payload.input.rtmpUrl, probe)
         const { ratio } = await getVideoStreamDimensionsInfo(payload.input.rtmpUrl, probe)
 
@@ -103,11 +110,13 @@ export class ProcessLiveRTMPHLSTranscoding {
           segmentDuration: payload.output.segmentDuration,
 
           toTranscode: payload.output.toTranscode,
+          splitAudioAndVideo: true,
 
           bitrate,
           ratio,
 
           hasAudio,
+          hasVideo,
           probe
         })
 

apps/peertube-runner/src/server/process/shared/process-studio.ts

@@ -1,5 +1,3 @@
-import { remove } from 'fs-extra/esm'
-import { join } from 'path'
 import { pick } from '@peertube/peertube-core-utils'
 import {
   RunnerJobStudioTranscodingPayload,
@@ -12,17 +10,30 @@ import {
   VideoStudioTranscodingSuccess
 } from '@peertube/peertube-models'
 import { buildUUID } from '@peertube/peertube-node-utils'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
 import { ConfigManager } from '../../../shared/config-manager.js'
 import { logger } from '../../../shared/index.js'
-import { buildFFmpegEdition, downloadInputFile, JobWithToken, ProcessOptions, scheduleTranscodingProgress } from './common.js'
+import {
+  buildFFmpegEdition,
+  downloadInputFile,
+  downloadSeparatedAudioFileIfNeeded,
+  JobWithToken,
+  ProcessOptions,
+  scheduleTranscodingProgress
+} from './common.js'
 
 export async function processStudioTranscoding (options: ProcessOptions<RunnerJobStudioTranscodingPayload>) {
   const { server, job, runnerToken } = options
   const payload = job.payload
 
-  let inputPath: string
+  let videoInputPath: string
+  let separatedAudioInputPath: string
+
+  let tmpVideoInputFilePath: string
+  let tmpSeparatedAudioInputFilePath: string
+
   let outputPath: string
-  let tmpInputFilePath: string
 
   let tasksProgress = 0
 
@@ -36,8 +47,11 @@ export async function processStudioTranscoding (options: ProcessOptions<RunnerJo
   try {
     logger.info(`Downloading input file ${payload.input.videoFileUrl} for job ${job.jobToken}`)
 
-    inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
-    tmpInputFilePath = inputPath
+    videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+    separatedAudioInputPath = await downloadSeparatedAudioFileIfNeeded({ urls: payload.input.separatedAudioFileUrl, runnerToken, job })
+
+    tmpVideoInputFilePath = videoInputPath
+    tmpSeparatedAudioInputFilePath = separatedAudioInputPath
 
     logger.info(`Input file ${payload.input.videoFileUrl} downloaded for job ${job.jobToken}. Running studio transcoding tasks.`)
 
@@ -46,17 +60,20 @@ export async function processStudioTranscoding (options: ProcessOptions<RunnerJo
       outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), outputFilename)
 
       await processTask({
-        inputPath: tmpInputFilePath,
+        videoInputPath: tmpVideoInputFilePath,
+        separatedAudioInputPath: tmpSeparatedAudioInputFilePath,
         outputPath,
         task,
         job,
         runnerToken
       })
 
-      if (tmpInputFilePath) await remove(tmpInputFilePath)
+      if (tmpVideoInputFilePath) await remove(tmpVideoInputFilePath)
+      if (tmpSeparatedAudioInputFilePath) await remove(tmpSeparatedAudioInputFilePath)
 
       // For the next iteration
-      tmpInputFilePath = outputPath
+      tmpVideoInputFilePath = outputPath
+      tmpSeparatedAudioInputFilePath = undefined
 
       tasksProgress += Math.floor(100 / payload.tasks.length)
     }
@@ -72,7 +89,8 @@ export async function processStudioTranscoding (options: ProcessOptions<RunnerJo
       payload: successBody
     })
   } finally {
-    if (tmpInputFilePath) await remove(tmpInputFilePath)
+    if (tmpVideoInputFilePath) await remove(tmpVideoInputFilePath)
+    if (tmpSeparatedAudioInputFilePath) await remove(tmpSeparatedAudioInputFilePath)
     if (outputPath) await remove(outputPath)
     if (updateProgressInterval) clearInterval(updateProgressInterval)
   }
@@ -83,8 +101,11 @@ export async function processStudioTranscoding (options: ProcessOptions<RunnerJo
 // ---------------------------------------------------------------------------
 
 type TaskProcessorOptions <T extends VideoStudioTaskPayload = VideoStudioTaskPayload> = {
-  inputPath: string
+  videoInputPath: string
+  separatedAudioInputPath: string
+
   outputPath: string
+
   task: T
   runnerToken: string
   job: JobWithToken
@@ -107,15 +128,15 @@ async function processTask (options: TaskProcessorOptions) {
 }
 
 async function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTaskIntroPayload | VideoStudioTaskOutroPayload>) {
-  const { inputPath, task, runnerToken, job } = options
+  const { videoInputPath, task, runnerToken, job } = options
 
-  logger.debug('Adding intro/outro to ' + inputPath)
+  logger.debug(`Adding intro/outro to ${videoInputPath}`)
 
   const introOutroPath = await downloadInputFile({ url: task.options.file, runnerToken, job })
 
   try {
     await buildFFmpegEdition().addIntroOutro({
-      ...pick(options, [ 'inputPath', 'outputPath' ]),
+      ...pick(options, [ 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
 
       introOutroPath,
       type: task.name === 'add-intro'
@@ -128,12 +149,12 @@ async function processAddIntroOutro (options: TaskProcessorOptions<VideoStudioTa
 }
 
 function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
-  const { inputPath, task } = options
+  const { videoInputPath, task } = options
 
-  logger.debug(`Cutting ${inputPath}`)
+  logger.debug(`Cutting ${videoInputPath}`)
 
   return buildFFmpegEdition().cutVideo({
-    ...pick(options, [ 'inputPath', 'outputPath' ]),
+    ...pick(options, [ 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
 
     start: task.options.start,
     end: task.options.end
@@ -141,15 +162,15 @@ function processCut (options: TaskProcessorOptions<VideoStudioTaskCutPayload>) {
 }
 
 async function processAddWatermark (options: TaskProcessorOptions<VideoStudioTaskWatermarkPayload>) {
-  const { inputPath, task, runnerToken, job } = options
+  const { videoInputPath, task, runnerToken, job } = options
 
-  logger.debug('Adding watermark to ' + inputPath)
+  logger.debug(`Adding watermark to ${videoInputPath}`)
 
   const watermarkPath = await downloadInputFile({ url: task.options.file, runnerToken, job })
 
   try {
     await buildFFmpegEdition().addWatermark({
-      ...pick(options, [ 'inputPath', 'outputPath' ]),
+      ...pick(options, [ 'videoInputPath', 'separatedAudioInputPath', 'outputPath' ]),
 
       watermarkPath,
 

apps/peertube-runner/src/server/process/shared/process-vod.ts

@@ -1,5 +1,3 @@
-import { remove } from 'fs-extra/esm'
-import { join } from 'path'
 import {
   RunnerJobVODAudioMergeTranscodingPayload,
   RunnerJobVODHLSTranscodingPayload,
@@ -9,9 +7,17 @@ import {
   VODWebVideoTranscodingSuccess
 } from '@peertube/peertube-models'
 import { buildUUID } from '@peertube/peertube-node-utils'
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
 import { ConfigManager } from '../../../shared/config-manager.js'
 import { logger } from '../../../shared/index.js'
-import { buildFFmpegVOD, downloadInputFile, ProcessOptions, scheduleTranscodingProgress } from './common.js'
+import {
+  buildFFmpegVOD,
+  downloadInputFile,
+  downloadSeparatedAudioFileIfNeeded,
+  ProcessOptions,
+  scheduleTranscodingProgress
+} from './common.js'
 
 export async function processWebVideoTranscoding (options: ProcessOptions<RunnerJobVODWebVideoTranscodingPayload>) {
   const { server, job, runnerToken } = options
@@ -19,7 +25,8 @@ export async function processWebVideoTranscoding (options: ProcessOptions<Runner
   const payload = job.payload
 
   let ffmpegProgress: number
-  let inputPath: string
+  let videoInputPath: string
+  let separatedAudioInputPath: string
 
   const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`)
 
@@ -33,7 +40,8 @@ export async function processWebVideoTranscoding (options: ProcessOptions<Runner
   try {
     logger.info(`Downloading input file ${payload.input.videoFileUrl} for web video transcoding job ${job.jobToken}`)
 
-    inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+    videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+    separatedAudioInputPath = await downloadSeparatedAudioFileIfNeeded({ urls: payload.input.separatedAudioFileUrl, runnerToken, job })
 
     logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running web video transcoding.`)
 
@@ -44,7 +52,8 @@ export async function processWebVideoTranscoding (options: ProcessOptions<Runner
     await ffmpegVod.transcode({
       type: 'video',
 
-      inputPath,
+      videoInputPath,
+      separatedAudioInputPath,
 
       outputPath,
 
@@ -65,7 +74,8 @@ export async function processWebVideoTranscoding (options: ProcessOptions<Runner
       payload: successBody
     })
   } finally {
-    if (inputPath) await remove(inputPath)
+    if (videoInputPath) await remove(videoInputPath)
+    if (separatedAudioInputPath) await remove(separatedAudioInputPath)
     if (outputPath) await remove(outputPath)
     if (updateProgressInterval) clearInterval(updateProgressInterval)
   }
@@ -76,7 +86,8 @@ export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVO
   const payload = job.payload
 
   let ffmpegProgress: number
-  let inputPath: string
+  let videoInputPath: string
+  let separatedAudioInputPath: string
 
   const uuid = buildUUID()
   const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `${uuid}-${payload.output.resolution}.m3u8`)
@@ -93,7 +104,8 @@ export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVO
   try {
     logger.info(`Downloading input file ${payload.input.videoFileUrl} for HLS transcoding job ${job.jobToken}`)
 
-    inputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+    videoInputPath = await downloadInputFile({ url: payload.input.videoFileUrl, runnerToken, job })
+    separatedAudioInputPath = await downloadSeparatedAudioFileIfNeeded({ urls: payload.input.separatedAudioFileUrl, runnerToken, job })
 
     logger.info(`Downloaded input file ${payload.input.videoFileUrl} for job ${job.jobToken}. Running HLS transcoding.`)
 
@@ -104,14 +116,18 @@ export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVO
     await ffmpegVod.transcode({
       type: 'hls',
       copyCodecs: false,
-      inputPath,
+
+      videoInputPath,
+      separatedAudioInputPath,
+
       hlsPlaylist: { videoFilename },
       outputPath,
 
       inputFileMutexReleaser: () => {},
 
       resolution: payload.output.resolution,
-      fps: payload.output.fps
+      fps: payload.output.fps,
+      separatedAudio: payload.output.separatedAudio
     })
 
     const successBody: VODHLSTranscodingSuccess = {
@@ -126,7 +142,8 @@ export async function processHLSTranscoding (options: ProcessOptions<RunnerJobVO
       payload: successBody
     })
   } finally {
-    if (inputPath) await remove(inputPath)
+    if (videoInputPath) await remove(videoInputPath)
+    if (separatedAudioInputPath) await remove(separatedAudioInputPath)
     if (outputPath) await remove(outputPath)
     if (videoPath) await remove(videoPath)
     if (updateProgressInterval) clearInterval(updateProgressInterval)
@@ -139,7 +156,7 @@ export async function processAudioMergeTranscoding (options: ProcessOptions<Runn
 
   let ffmpegProgress: number
   let audioPath: string
-  let inputPath: string
+  let previewPath: string
 
   const outputPath = join(ConfigManager.Instance.getTranscodingDirectory(), `output-${buildUUID()}.mp4`)
 
@@ -157,7 +174,7 @@ export async function processAudioMergeTranscoding (options: ProcessOptions<Runn
     )
 
     audioPath = await downloadInputFile({ url: payload.input.audioFileUrl, runnerToken, job })
-    inputPath = await downloadInputFile({ url: payload.input.previewFileUrl, runnerToken, job })
+    previewPath = await downloadInputFile({ url: payload.input.previewFileUrl, runnerToken, job })
 
     logger.info(
       `Downloaded input files ${payload.input.audioFileUrl} and ${payload.input.previewFileUrl} ` +
@@ -172,7 +189,7 @@ export async function processAudioMergeTranscoding (options: ProcessOptions<Runn
       type: 'merge-audio',
 
       audioPath,
-      inputPath,
+      videoInputPath: previewPath,
 
       outputPath,
 
@@ -194,7 +211,7 @@ export async function processAudioMergeTranscoding (options: ProcessOptions<Runn
     })
   } finally {
     if (audioPath) await remove(audioPath)
-    if (inputPath) await remove(inputPath)
+    if (previewPath) await remove(previewPath)
     if (outputPath) await remove(outputPath)
     if (updateProgressInterval) clearInterval(updateProgressInterval)
   }