Add config option to keep original video file (basic first version) (#6157)

* testing not removing old file and adding columb to db

* implement feature

* remove unnecessary config changes

* use only keptOriginalFileName, change keptOriginalFileName to keptOriginalFilename for consistency with with videoFile table, slight refactor with basename()

* save original video files to dedicated directory original-video-files

* begin implementing object storage (bucket) support

---------

Co-authored-by: chagai.friedlander <chagai.friedlander@fairkom.eu>
Co-authored-by: Ian <ian.kraft@hotmail.com>
Co-authored-by: Chocobozzz <me@florianbigard.com>

packages/server-commands/src/server/config-command.ts

@@ -106,6 +106,19 @@ export class ConfigCommand extends AbstractCommand {
 
   // ---------------------------------------------------------------------------
 
+  keepSourceFile () {
+    return this.updateExistingSubConfig({
+      newConfig: {
+        transcoding: {
+          originalFile: {
+            keep: true
+          }
+        }
+      }
+    })
+  }
+  // ---------------------------------------------------------------------------
+
   enableChannelSync () {
     return this.setChannelSyncEnabled(true)
   }
@@ -234,13 +247,17 @@ export class ConfigCommand extends AbstractCommand {
     webVideo?: boolean // default true
     hls?: boolean // default true
     with0p?: boolean // default false
+    keepOriginal?: boolean // default false
   } = {}) {
-    const { webVideo = true, hls = true, with0p = false } = options
+    const { webVideo = true, hls = true, with0p = false, keepOriginal = false } = options
 
     return this.updateExistingSubConfig({
       newConfig: {
         transcoding: {
           enabled: true,
+          originalFile: {
+            keep: keepOriginal
+          },
 
           allowAudioFiles: true,
           allowAdditionalExtensions: true,
@@ -261,13 +278,17 @@ export class ConfigCommand extends AbstractCommand {
   enableMinimumTranscoding (options: {
     webVideo?: boolean // default true
     hls?: boolean // default true
+    keepOriginal?: boolean // default false
   } = {}) {
-    const { webVideo = true, hls = true } = options
+    const { webVideo = true, hls = true, keepOriginal = false } = options
 
     return this.updateExistingSubConfig({
       newConfig: {
         transcoding: {
           enabled: true,
+          originalFile: {
+            keep: keepOriginal
+          },
 
           allowAudioFiles: true,
           allowAdditionalExtensions: true,
@@ -560,6 +581,9 @@ export class ConfigCommand extends AbstractCommand {
       },
       transcoding: {
         enabled: true,
+        originalFile: {
+          keep: false
+        },
         remoteRunners: {
           enabled: false
         },

packages/server-commands/src/server/object-storage-command.ts

@@ -1,5 +1,5 @@
-import { randomInt } from 'crypto'
 import { HttpStatusCode } from '@peertube/peertube-models'
+import { randomInt } from 'crypto'
 import { makePostBodyRequest } from '../requests/index.js'
 
 export class ObjectStorageCommand {
@@ -50,6 +50,14 @@ export class ObjectStorageCommand {
 
         web_videos: {
           bucket_name: this.getMockWebVideosBucketName()
+        },
+
+        user_exports: {
+          bucket_name: this.getMockUserExportBucketName()
+        },
+
+        original_video_files: {
+          bucket_name: this.getMockOriginalFileBucketName()
         }
       }
     }
@@ -63,6 +71,14 @@ export class ObjectStorageCommand {
     return `http://${this.getMockStreamingPlaylistsBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
   }
 
+  getMockUserExportBaseUrl () {
+    return `http://${this.getMockUserExportBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
+  }
+
+  getMockOriginalFileBaseUrl () {
+    return `http://${this.getMockOriginalFileBucketName()}.${ObjectStorageCommand.getMockEndpointHost()}/`
+  }
+
   async prepareDefaultMockBuckets () {
     await this.createMockBucket(this.getMockStreamingPlaylistsBucketName())
     await this.createMockBucket(this.getMockWebVideosBucketName())
@@ -100,6 +116,14 @@ export class ObjectStorageCommand {
     return this.getMockBucketName(name)
   }
 
+  getMockUserExportBucketName (name = 'user-exports') {
+    return this.getMockBucketName(name)
+  }
+
+  getMockOriginalFileBucketName (name = 'original-video-files') {
+    return this.getMockBucketName(name)
+  }
+
   getMockBucketName (name: string) {
     return `${this.seed}-${name}`
   }

packages/server-commands/src/server/server.ts

@@ -379,6 +379,7 @@ export class PeerTubeServer {
         avatars: this.getDirectoryPath('avatars') + '/',
         web_videos: this.getDirectoryPath('web-videos') + '/',
         streaming_playlists: this.getDirectoryPath('streaming-playlists') + '/',
+        original_video_files: this.getDirectoryPath('original-video-files') + '/',
         redundancy: this.getDirectoryPath('redundancy') + '/',
         logs: this.getDirectoryPath('logs') + '/',
         previews: this.getDirectoryPath('previews') + '/',

packages/server-commands/src/server/servers-command.ts

@@ -1,10 +1,10 @@
-import { exec } from 'child_process'
-import { copy, ensureDir, remove } from 'fs-extra/esm'
-import { readdir, readFile } from 'fs/promises'
-import { basename, join } from 'path'
 import { wait } from '@peertube/peertube-core-utils'
 import { HttpStatusCode } from '@peertube/peertube-models'
-import { getFileSize, isGithubCI, root } from '@peertube/peertube-node-utils'
+import { isGithubCI, root } from '@peertube/peertube-node-utils'
+import { exec } from 'child_process'
+import { copy, ensureDir, remove } from 'fs-extra/esm'
+import { readFile, readdir } from 'fs/promises'
+import { basename, join } from 'path'
 import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
 
 export class ServersCommand extends AbstractCommand {
@@ -84,6 +84,8 @@ export class ServersCommand extends AbstractCommand {
     return files.length
   }
 
+  // ---------------------------------------------------------------------------
+
   buildWebVideoFilePath (fileUrl: string) {
     return this.buildDirectory(join('web-videos', basename(fileUrl)))
   }
@@ -92,13 +94,9 @@ export class ServersCommand extends AbstractCommand {
     return this.buildDirectory(join('streaming-playlists', 'hls', videoUUID, basename(fileUrl)))
   }
 
+  // ---------------------------------------------------------------------------
+
   getLogContent () {
     return readFile(this.buildDirectory('logs/peertube.log'))
   }
-
-  async getServerFileSize (subPath: string) {
-    const path = this.server.servers.buildDirectory(subPath)
-
-    return getFileSize(path)
-  }
 }

packages/server-commands/src/users/user-exports-command.ts

@@ -1,7 +1,8 @@
-import { HttpStatusCode, ResultList, UserExport, UserExportRequestResult, UserExportState } from '@peertube/peertube-models'
-import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
 import { wait } from '@peertube/peertube-core-utils'
-import { unwrapBody } from '../requests/requests.js'
+import { HttpStatusCode, ResultList, UserExport, UserExportRequestResult, UserExportState } from '@peertube/peertube-models'
+import { writeFile } from 'fs/promises'
+import { makeRawRequest, unwrapBody } from '../requests/requests.js'
+import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
 
 export class UserExportsCommand extends AbstractCommand {
 
@@ -49,6 +50,22 @@ export class UserExportsCommand extends AbstractCommand {
     })
   }
 
+  async downloadLatestArchive (options: OverrideCommandOptions & {
+    userId: number
+    destination: string
+  }) {
+    const { data } = await this.list(options)
+
+    const res = await makeRawRequest({
+      url: data[0].privateDownloadUrl,
+      responseType: 'arraybuffer',
+      redirects: 1,
+      expectedStatus: HttpStatusCode.OK_200
+    })
+
+    await writeFile(options.destination, res.body)
+  }
+
   async deleteAllArchives (options: OverrideCommandOptions & {
     userId: number
   }) {