Separate HLS audio and video streams

Allows:
  * The HLS player to propose an "Audio only" resolution
  * The live to output an "Audio only" resolution
  * The live to ingest and output an "Audio only" stream

 This feature is under a config for VOD videos and is enabled by default for lives

 In the future we can imagine:
  * To propose multiple audio streams for a specific video
  * To ingest an audio only VOD and just output an audio only "video"
    (the player would play the audio file and PeerTube would not
    generate additional resolutions)

This commit introduce a new way to download videos:
 * Add "/download/videos/generate/:videoId" endpoint where PeerTube can
   mux an audio only and a video only file to a mp4 container
 * The download client modal introduces a new default panel where the
   user can choose resolutions it wants to download

server/core/lib/user-import-export/exporters/abstract-user-exporter.ts

@@ -8,7 +8,7 @@ export type ExportResult <T> = {
 
   staticFiles: {
     archivePath: string
-    createrReadStream: () => Promise<Readable>
+    readStreamFactory: () => Promise<Readable>
   }[]
 
   activityPub?: ActivityPubActor | ActivityPubOrderedCollection<string>

server/core/lib/user-import-export/exporters/actor-exporter.ts

@@ -59,7 +59,7 @@ export abstract class ActorExporter <T> extends AbstractUserExporter<T> {
 
       staticFiles.push({
         archivePath: archivePathBuilder(image.filename),
-        createrReadStream: () => Promise.resolve(createReadStream(image.getPath()))
+        readStreamFactory: () => Promise.resolve(createReadStream(image.getPath()))
       })
 
       const relativePath = join(this.relativeStaticDirPath, archivePathBuilder(image.filename))

server/core/lib/user-import-export/exporters/video-playlists-exporter.ts

@@ -26,7 +26,7 @@ export class VideoPlaylistsExporter extends AbstractUserExporter <VideoPlaylists
 
         staticFiles.push({
           archivePath: this.getArchiveThumbnailPath(playlist, thumbnail),
-          createrReadStream: () => Promise.resolve(createReadStream(thumbnail.getPath()))
+          readStreamFactory: () => Promise.resolve(createReadStream(thumbnail.getPath()))
         })
 
         archiveFiles.thumbnail = join(this.relativeStaticDirPath, this.getArchiveThumbnailPath(playlist, thumbnail))

server/core/lib/user-import-export/exporters/videos-exporter.ts

@@ -6,6 +6,7 @@ import { audiencify, getAudience } from '@server/lib/activitypub/audience.js'
 import { buildCreateActivity } from '@server/lib/activitypub/send/send-create.js'
 import { buildChaptersAPHasPart } from '@server/lib/activitypub/video-chapters.js'
 import { getHLSFileReadStream, getOriginalFileReadStream, getWebVideoFileReadStream } from '@server/lib/object-storage/videos.js'
+import { muxToMergeVideoFiles } from '@server/lib/video-file.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { VideoCaptionModel } from '@server/models/video/video-caption.js'
 import { VideoChannelModel } from '@server/models/video/video-channel.js'
@@ -16,7 +17,8 @@ import { VideoSourceModel } from '@server/models/video/video-source.js'
 import { VideoModel } from '@server/models/video/video.js'
 import {
   MStreamingPlaylistFiles,
-  MThumbnail, MVideo, MVideoAP, MVideoCaption,
+  MThumbnail,
+  MVideo, MVideoAP, MVideoCaption,
   MVideoCaptionLanguageUrl,
   MVideoChapter,
   MVideoFile,
@@ -27,7 +29,7 @@ import { MVideoSource } from '@server/types/models/video/video-source.js'
 import Bluebird from 'bluebird'
 import { createReadStream } from 'fs'
 import { extname, join } from 'path'
-import { Readable } from 'stream'
+import { PassThrough, Readable } from 'stream'
 import { AbstractUserExporter, ExportResult } from './abstract-user-exporter.js'
 
 export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
@@ -89,13 +91,13 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
     // Then fetch more attributes for AP serialization
     const videoAP = await video.lightAPToFullAP(undefined)
 
-    const { relativePathsFromJSON, staticFiles } = await this.exportVideoFiles({ video, captions })
+    const { relativePathsFromJSON, staticFiles, exportedVideoFileOrSource } = await this.exportVideoFiles({ video, captions })
 
     return {
       json: this.exportVideoJSON({ video, captions, live, passwords, source, chapters, archiveFiles: relativePathsFromJSON }),
       staticFiles,
       relativePathsFromJSON,
-      activityPubOutbox: await this.exportVideoAP(videoAP, chapters)
+      activityPubOutbox: await this.exportVideoAP(videoAP, chapters, exportedVideoFileOrSource)
     }
   }
 
@@ -250,8 +252,11 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
 
   // ---------------------------------------------------------------------------
 
-  private async exportVideoAP (video: MVideoAP, chapters: MVideoChapter[]): Promise<ActivityCreate<VideoObject>> {
-    const videoFile = video.getMaxQualityFile()
+  private async exportVideoAP (
+    video: MVideoAP,
+    chapters: MVideoChapter[],
+    exportedVideoFileOrSource: MVideoFile | MVideoSource
+  ): Promise<ActivityCreate<VideoObject>> {
     const icon = video.getPreview()
 
     const audience = getAudience(video.VideoChannel.Account.Actor, video.privacy === VideoPrivacy.PUBLIC)
@@ -274,13 +279,19 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
 
       hasParts: buildChaptersAPHasPart(video, chapters),
 
-      attachment: this.options.withVideoFiles && videoFile
+      attachment: this.options.withVideoFiles && exportedVideoFileOrSource
         ? [
           {
             type: 'Video' as 'Video',
-            url: join(this.options.relativeStaticDirPath, this.getArchiveVideoFilePath(video, videoFile)),
+            url: join(this.options.relativeStaticDirPath, this.getArchiveVideoFilePath(video, exportedVideoFileOrSource)),
 
-            ...pick(videoFile.toActivityPubObject(video), [ 'mediaType', 'height', 'size', 'fps' ])
+            // FIXME: typings
+            ...pick((exportedVideoFileOrSource as MVideoFile & MVideoSource).toActivityPubObject(video), [
+              'mediaType',
+              'height',
+              'size',
+              'fps'
+            ])
           }
         ]
         : undefined
@@ -298,6 +309,9 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
     const { video, captions } = options
 
     const staticFiles: ExportResult<VideoExportJSON>['staticFiles'] = []
+
+    let exportedVideoFileOrSource: MVideoFile | MVideoSource
+
     const relativePathsFromJSON = {
       videoFile: null as string,
       thumbnail: null as string,
@@ -305,32 +319,32 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
     }
 
     if (this.options.withVideoFiles) {
-      const source = await VideoSourceModel.loadLatest(video.id)
-      const maxQualityFile = video.getMaxQualityFile()
+      const { source, videoFile, separatedAudioFile } = await this.getArchiveVideo(video)
 
-      // Prefer using original file if possible
-      const file = source?.keptOriginalFilename
-        ? source
-        : maxQualityFile
-
-      if (file) {
-        const videoPath = this.getArchiveVideoFilePath(video, file)
+      if (source || videoFile || separatedAudioFile) {
+        const videoPath = this.getArchiveVideoFilePath(video, source || videoFile || separatedAudioFile)
 
         staticFiles.push({
           archivePath: videoPath,
-          createrReadStream: () => file === source
+
+          // Prefer using original file if possible
+          readStreamFactory: () => source?.keptOriginalFilename
             ? this.generateVideoSourceReadStream(source)
-            : this.generateVideoFileReadStream(video, maxQualityFile)
+            : this.generateVideoFileReadStream({ video, videoFile, separatedAudioFile })
         })
 
         relativePathsFromJSON.videoFile = join(this.relativeStaticDirPath, videoPath)
+
+        exportedVideoFileOrSource = source?.keptOriginalFilename
+          ? source
+          : videoFile || separatedAudioFile
       }
     }
 
     for (const caption of captions) {
       staticFiles.push({
         archivePath: this.getArchiveCaptionFilePath(video, caption),
-        createrReadStream: () => Promise.resolve(createReadStream(caption.getFSPath()))
+        readStreamFactory: () => Promise.resolve(createReadStream(caption.getFSPath()))
       })
 
       relativePathsFromJSON.captions[caption.language] = join(this.relativeStaticDirPath, this.getArchiveCaptionFilePath(video, caption))
@@ -340,13 +354,13 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
     if (thumbnail) {
       staticFiles.push({
         archivePath: this.getArchiveThumbnailFilePath(video, thumbnail),
-        createrReadStream: () => Promise.resolve(createReadStream(thumbnail.getPath()))
+        readStreamFactory: () => Promise.resolve(createReadStream(thumbnail.getPath()))
       })
 
       relativePathsFromJSON.thumbnail = join(this.relativeStaticDirPath, this.getArchiveThumbnailFilePath(video, thumbnail))
     }
 
-    return { staticFiles, relativePathsFromJSON }
+    return { staticFiles, relativePathsFromJSON, exportedVideoFileOrSource }
   }
 
   private async generateVideoSourceReadStream (source: MVideoSource): Promise<Readable> {
@@ -359,7 +373,22 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
     return stream
   }
 
-  private async generateVideoFileReadStream (video: MVideoFullLight, videoFile: MVideoFile): Promise<Readable> {
+  private async generateVideoFileReadStream (options: {
+    videoFile: MVideoFile
+    separatedAudioFile: MVideoFile
+    video: MVideoFullLight
+  }): Promise<Readable> {
+    const { video, videoFile, separatedAudioFile } = options
+
+    if (separatedAudioFile) {
+      const stream = new PassThrough()
+
+      muxToMergeVideoFiles({ video, videoFiles: [ videoFile, separatedAudioFile ], output: stream })
+        .catch(err => logger.error('Cannot mux video files', { err }))
+
+      return Promise.resolve(stream)
+    }
+
     if (videoFile.storage === FileStorage.FILE_SYSTEM) {
       return createReadStream(VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile))
     }
@@ -371,8 +400,18 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
     return stream
   }
 
-  private getArchiveVideoFilePath (video: MVideo, file: { filename?: string, keptOriginalFilename?: string }) {
-    return join('video-files', video.uuid + extname(file.filename || file.keptOriginalFilename))
+  private async getArchiveVideo (video: MVideoFullLight) {
+    const source = await VideoSourceModel.loadLatest(video.id)
+
+    const { videoFile, separatedAudioFile } = video.getMaxQualityAudioAndVideoFiles()
+
+    if (source?.keptOriginalFilename) return { source }
+
+    return { videoFile, separatedAudioFile }
+  }
+
+  private getArchiveVideoFilePath (video: MVideo, file: { keptOriginalFilename?: string, filename?: string }) {
+    return join('video-files', video.uuid + extname(file.keptOriginalFilename || file.filename))
   }
 
   private getArchiveCaptionFilePath (video: MVideo, caption: MVideoCaptionLanguageUrl) {

server/core/lib/user-import-export/user-exporter.ts

@@ -114,7 +114,7 @@ export class UserExporter {
 
     return new Promise<void>(async (res, rej) => {
       this.archive.on('warning', err => {
-        logger.warn('Warning to archive a file in ' + exportModel.filename, { err })
+        logger.warn('Warning to archive a file in ' + exportModel.filename, { ...lTags(), err })
       })
 
       this.archive.on('error', err => {
@@ -127,7 +127,7 @@ export class UserExporter {
         for (const { exporter, jsonFilename } of this.buildExporters(exportModel, user)) {
           const { json, staticFiles, activityPub, activityPubOutbox } = await exporter.export()
 
-          logger.debug('Adding JSON file ' + jsonFilename + ' in archive ' + exportModel.filename)
+          logger.debug(`Adding JSON file ${jsonFilename} in archive ${exportModel.filename}`, lTags())
           this.appendJSON(json, join('peertube', jsonFilename))
 
           if (activityPub) {
@@ -144,12 +144,12 @@ export class UserExporter {
           for (const file of staticFiles) {
             const archivePath = join('files', parse(jsonFilename).name, file.archivePath)
 
-            logger.debug(`Adding static file ${archivePath} in archive`)
+            logger.debug(`Adding static file ${archivePath} in archive`, lTags())
 
             try {
-              await this.addToArchiveAndWait(await file.createrReadStream(), archivePath)
+              await this.addToArchiveAndWait(await file.readStreamFactory(), archivePath)
             } catch (err) {
-              logger.error(`Cannot add ${archivePath} in archive`, { err })
+              logger.error(`Cannot add ${archivePath} in archive`, { err, ...lTags() })
             }
           }
         }
@@ -287,10 +287,14 @@ export class UserExporter {
 
       this.archive.on('entry', entryListener)
 
+      logger.error('Adding stream ' + archivePath)
+
       // Prevent sending a stream that has an error on open resulting in a stucked archiving process
       stream.once('readable', () => {
         if (errored) return
 
+        logger.error('Readable stream ' + archivePath)
+
         this.archive.append(stream, { name: archivePath })
       })
     })

server/core/lib/user-import-export/user-importer.ts

@@ -73,7 +73,7 @@ export class UserImporter {
       importModel.resultSummary = resultSummary
       await saveInTransactionWithRetries(importModel)
     } catch (err) {
-      logger.error('Cannot import user archive', { toto: 'coucou', err, ...lTags() })
+      logger.error('Cannot import user archive', { err, ...lTags() })
 
       try {
         importModel.state = UserImportState.ERRORED