Add config option to keep original video file (basic first version) (#6157)

* testing not removing old file and adding columb to db

* implement feature

* remove unnecessary config changes

* use only keptOriginalFileName, change keptOriginalFileName to keptOriginalFilename for consistency with with videoFile table, slight refactor with basename()

* save original video files to dedicated directory original-video-files

* begin implementing object storage (bucket) support

---------

Co-authored-by: chagai.friedlander <chagai.friedlander@fairkom.eu>
Co-authored-by: Ian <ian.kraft@hotmail.com>
Co-authored-by: Chocobozzz <me@florianbigard.com>

server/core/controllers/api/config.ts

@@ -320,6 +320,9 @@ function customConfig (): CustomConfig {
     },
     transcoding: {
       enabled: CONFIG.TRANSCODING.ENABLED,
+      originalFile: {
+        keep: CONFIG.TRANSCODING.ORIGINAL_FILE.KEEP
+      },
       remoteRunners: {
         enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED
       },

server/core/controllers/api/videos/live.ts

@@ -184,9 +184,9 @@ async function addLiveVideo (req: express.Request, res: express.Response) {
       duration: 0,
       state: VideoState.WAITING_FOR_LIVE,
       isLive: true,
-      filename: null
+      inputFilename: null
     },
-    videoFilePath: undefined,
+    videoFile: undefined,
     user: res.locals.oauth.token.User,
     thumbnails
   })

server/core/controllers/api/videos/source.ts

@@ -1,20 +1,20 @@
-import express from 'express'
-import { move } from 'fs-extra/esm'
+import { buildAspectRatio } from '@peertube/peertube-core-utils'
+import { VideoState } from '@peertube/peertube-models'
 import { sequelizeTypescript } from '@server/initializers/database.js'
 import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
 import { Hooks } from '@server/lib/plugins/hooks.js'
 import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
 import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
-import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
-import { buildNewFile } from '@server/lib/video-file.js'
+import { buildNewFile, createVideoSource } from '@server/lib/video-file.js'
+import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { buildNextVideoState } from '@server/lib/video-state.js'
 import { openapiOperationDoc } from '@server/middlewares/doc.js'
 import { VideoModel } from '@server/models/video/video.js'
-import { VideoSourceModel } from '@server/models/video/video-source.js'
 import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
-import { VideoState } from '@peertube/peertube-models'
+import express from 'express'
+import { move } from 'fs-extra/esm'
 import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
 import {
   asyncMiddleware,
@@ -23,7 +23,6 @@ import {
   replaceVideoSourceResumableValidator,
   videoSourceGetLatestValidator
 } from '../../../middlewares/index.js'
-import { buildAspectRatio } from '@peertube/peertube-core-utils'
 
 const lTags = loggerTagsFactory('api', 'video')
 
@@ -61,7 +60,7 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R
   const videoPhysicalFile = res.locals.updateVideoFileResumable
   const user = res.locals.oauth.token.User
 
-  const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' })
+  const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video', ffprobe: res.locals.ffprobe })
   const originalFilename = videoPhysicalFile.originalname
 
   const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoAll.uuid)
@@ -114,13 +113,15 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R
 
     await removeOldFiles({ video, files: oldWebVideoFiles, playlists: oldStreamingPlaylists })
 
-    const source = await VideoSourceModel.create({
-      filename: originalFilename,
-      videoId: video.id,
+    const source = await createVideoSource({
+      inputFilename: originalFilename,
+      inputProbe: res.locals.ffprobe,
+      inputPath: destination,
+      video,
       createdAt: inputFileUpdatedAt
     })
 
-    await regenerateMiniaturesIfNeeded(video)
+    await regenerateMiniaturesIfNeeded(video, res.locals.ffprobe)
     await video.VideoChannel.setAsUpdated()
     await addVideoJobsAfterUpload(video, video.getMaxQualityFile())
 

server/core/controllers/api/videos/upload.ts

@@ -1,12 +1,14 @@
-import express from 'express'
+import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
+import { HttpStatusCode, ThumbnailType, VideoCreate } from '@peertube/peertube-models'
+import { uuidToShort } from '@peertube/peertube-node-utils'
 import { getResumableUploadPath } from '@server/helpers/upload.js'
+import { LocalVideoCreator } from '@server/lib/local-video-creator.js'
 import { Redis } from '@server/lib/redis.js'
 import { setupUploadResumableRoutes, uploadx } from '@server/lib/uploadx.js'
 import { buildNextVideoState } from '@server/lib/video-state.js'
 import { openapiOperationDoc } from '@server/middlewares/doc.js'
-import { uuidToShort } from '@peertube/peertube-node-utils'
-import { HttpStatusCode, ThumbnailType, VideoCreate } from '@peertube/peertube-models'
-import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js'
+import express from 'express'
+import { VideoAuditView, auditLoggerFactory, getAuditIdFromRes } from '../../../helpers/audit-logger.js'
 import { createReqFiles } from '../../../helpers/express-utils.js'
 import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
 import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js'
@@ -19,8 +21,6 @@ import {
   videosAddResumableInitValidator,
   videosAddResumableValidator
 } from '../../../middlewares/index.js'
-import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
-import { LocalVideoCreator } from '@server/lib/local-video-creator.js'
 
 const lTags = loggerTagsFactory('api', 'video')
 const auditLogger = auditLoggerFactory('videos')
@@ -134,7 +134,12 @@ async function addVideo (options: {
 
   const localVideoCreator = new LocalVideoCreator({
     lTags,
-    videoFilePath: videoPhysicalFile.path,
+
+    videoFile: {
+      path: videoPhysicalFile.path,
+      probe: res.locals.ffprobe
+    },
+
     user: res.locals.oauth.token.User,
     channel: res.locals.videoChannel,
 
@@ -148,7 +153,7 @@ async function addVideo (options: {
       ...videoInfo,
 
       duration: videoPhysicalFile.duration,
-      filename: videoPhysicalFile.originalname,
+      inputFilename: videoPhysicalFile.originalname,
       state: buildNextVideoState(),
       isLive: false
     },