yetangitu/Peertube
1
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2025-10-05 02:39:33 +02:00
Code Issues Releases Wiki Activity

Add config option to keep original video file (basic first version) (#6157)

Browse source 
* testing not removing old file and adding columb to db

* implement feature

* remove unnecessary config changes

* use only keptOriginalFileName, change keptOriginalFileName to keptOriginalFilename for consistency with with videoFile table, slight refactor with basename()

* save original video files to dedicated directory original-video-files

* begin implementing object storage (bucket) support

---------

Co-authored-by: chagai.friedlander <chagai.friedlander@fairkom.eu>
Co-authored-by: Ian <ian.kraft@hotmail.com>
Co-authored-by: Chocobozzz <me@florianbigard.com>
This commit is contained in:
chagai95 2024-03-15 15:47:18 +01:00 • committed by GitHub
parent ae31e90c30
commit e57c3024f4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
75 changed files with 1653 additions and 801 deletions
Download patch file Download diff file Expand all files Collapse all files

3
server/core/controllers/api/config.ts
View file

@ -320,6 +320,9 @@ function customConfig (): CustomConfig {
},
transcoding: {
enabled: CONFIG.TRANSCODING.ENABLED,
originalFile: {
keep: CONFIG.TRANSCODING.ORIGINAL_FILE.KEEP
},
remoteRunners: {
enabled: CONFIG.TRANSCODING.REMOTE_RUNNERS.ENABLED
},

4
server/core/controllers/api/videos/live.ts
View file

@ -184,9 +184,9 @@ async function addLiveVideo (req: express.Request, res: express.Response) {
duration: 0,
state: VideoState.WAITING_FOR_LIVE,
isLive: true,
filename: null
inputFilename: null
},
videoFilePath: undefined,
videoFile: undefined,
user: res.locals.oauth.token.User,
thumbnails
})

25
server/core/controllers/api/videos/source.ts
View file

@ -1,20 +1,20 @@
import express from 'express'
import { move } from 'fs-extra/esm'
import { buildAspectRatio } from '@peertube/peertube-core-utils'
import { VideoState } from '@peertube/peertube-models'
import { sequelizeTypescript } from '@server/initializers/database.js'
import { CreateJobArgument, CreateJobOptions, JobQueue } from '@server/lib/job-queue/index.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { regenerateMiniaturesIfNeeded } from '@server/lib/thumbnail.js'
import { setupUploadResumableRoutes } from '@server/lib/uploadx.js'
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
import { buildNewFile } from '@server/lib/video-file.js'
import { buildNewFile, createVideoSource } from '@server/lib/video-file.js'
import { buildMoveJob, buildStoryboardJobIfNeeded } from '@server/lib/video-jobs.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { VideoModel } from '@server/models/video/video.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { MStreamingPlaylistFiles, MVideo, MVideoFile, MVideoFullLight } from '@server/types/models/index.js'
import { VideoState } from '@peertube/peertube-models'
import express from 'express'
import { move } from 'fs-extra/esm'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import {
asyncMiddleware,
@ -23,7 +23,6 @@ import {
replaceVideoSourceResumableValidator,
videoSourceGetLatestValidator
} from '../../../middlewares/index.js'
import { buildAspectRatio } from '@peertube/peertube-core-utils'
const lTags = loggerTagsFactory('api', 'video')
@ -61,7 +60,7 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R
const videoPhysicalFile = res.locals.updateVideoFileResumable
const user = res.locals.oauth.token.User
const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video' })
const videoFile = await buildNewFile({ path: videoPhysicalFile.path, mode: 'web-video', ffprobe: res.locals.ffprobe })
const originalFilename = videoPhysicalFile.originalname
const videoFileMutexReleaser = await VideoPathManager.Instance.lockFiles(res.locals.videoAll.uuid)
@ -114,13 +113,15 @@ async function replaceVideoSourceResumable (req: express.Request, res: express.R
await removeOldFiles({ video, files: oldWebVideoFiles, playlists: oldStreamingPlaylists })
const source = await VideoSourceModel.create({
filename: originalFilename,
videoId: video.id,
const source = await createVideoSource({
inputFilename: originalFilename,
inputProbe: res.locals.ffprobe,
inputPath: destination,
video,
createdAt: inputFileUpdatedAt
})
await regenerateMiniaturesIfNeeded(video)
await regenerateMiniaturesIfNeeded(video, res.locals.ffprobe)
await video.VideoChannel.setAsUpdated()
await addVideoJobsAfterUpload(video, video.getMaxQualityFile())

21
server/core/controllers/api/videos/upload.ts
View file

@ -1,12 +1,14 @@
import express from 'express'
import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
import { HttpStatusCode, ThumbnailType, VideoCreate } from '@peertube/peertube-models'
import { uuidToShort } from '@peertube/peertube-node-utils'
import { getResumableUploadPath } from '@server/helpers/upload.js'
import { LocalVideoCreator } from '@server/lib/local-video-creator.js'
import { Redis } from '@server/lib/redis.js'
import { setupUploadResumableRoutes, uploadx } from '@server/lib/uploadx.js'
import { buildNextVideoState } from '@server/lib/video-state.js'
import { openapiOperationDoc } from '@server/middlewares/doc.js'
import { uuidToShort } from '@peertube/peertube-node-utils'
import { HttpStatusCode, ThumbnailType, VideoCreate } from '@peertube/peertube-models'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger.js'
import express from 'express'
import { VideoAuditView, auditLoggerFactory, getAuditIdFromRes } from '../../../helpers/audit-logger.js'
import { createReqFiles } from '../../../helpers/express-utils.js'
import { logger, loggerTagsFactory } from '../../../helpers/logger.js'
import { CONSTRAINTS_FIELDS, MIMETYPES } from '../../../initializers/constants.js'
@ -19,8 +21,6 @@ import {
videosAddResumableInitValidator,
videosAddResumableValidator
} from '../../../middlewares/index.js'
import { ffprobePromise, getChaptersFromContainer } from '@peertube/peertube-ffmpeg'
import { LocalVideoCreator } from '@server/lib/local-video-creator.js'
const lTags = loggerTagsFactory('api', 'video')
const auditLogger = auditLoggerFactory('videos')
@ -134,7 +134,12 @@ async function addVideo (options: {
const localVideoCreator = new LocalVideoCreator({
lTags,
videoFilePath: videoPhysicalFile.path,
videoFile: {
path: videoPhysicalFile.path,
probe: res.locals.ffprobe
},
user: res.locals.oauth.token.User,
channel: res.locals.videoChannel,
@ -148,7 +153,7 @@ async function addVideo (options: {
...videoInfo,
duration: videoPhysicalFile.duration,
filename: videoPhysicalFile.originalname,
inputFilename: videoPhysicalFile.originalname,
state: buildNextVideoState(),
isLive: false
},

53
server/core/controllers/download.ts
View file

@ -1,12 +1,14 @@
import cors from 'cors'
import express from 'express'
import { forceNumber } from '@peertube/peertube-core-utils'
import { FileStorage, HttpStatusCode, VideoStreamingPlaylistType } from '@peertube/peertube-models'
import { logger } from '@server/helpers/logger.js'
import { VideoTorrentsSimpleFileCache } from '@server/lib/files-cache/index.js'
import {
generateHLSFilePresignedUrl,
generateOriginalFilePresignedUrl,
generateUserExportPresignedUrl,
generateWebVideoPresignedUrl
} from '@server/lib/object-storage/index.js'
import { getFSUserExportFilePath } from '@server/lib/paths.js'
import { Hooks } from '@server/lib/plugins/hooks.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import {
@ -17,15 +19,16 @@ import {
MVideoFile,
MVideoFullLight
} from '@server/types/models/index.js'
import { forceNumber } from '@peertube/peertube-core-utils'
import { HttpStatusCode, FileStorage, VideoStreamingPlaylistType } from '@peertube/peertube-models'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import cors from 'cors'
import express from 'express'
import { STATIC_DOWNLOAD_PATHS } from '../initializers/constants.js'
import {
asyncMiddleware, optionalAuthenticate,
originalVideoFileDownloadValidator,
userExportDownloadValidator,
videosDownloadValidator
} from '../middlewares/index.js'
import { getFSUserExportFilePath } from '@server/lib/paths.js'
const downloadRouter = express.Router()
@ -40,7 +43,7 @@ downloadRouter.use(
STATIC_DOWNLOAD_PATHS.VIDEOS + ':id-:resolution([0-9]+).:extension',
optionalAuthenticate,
asyncMiddleware(videosDownloadValidator),
asyncMiddleware(downloadVideoFile)
asyncMiddleware(downloadWebVideoFile)
)
downloadRouter.use(
@ -51,11 +54,18 @@ downloadRouter.use(
)
downloadRouter.use(
STATIC_DOWNLOAD_PATHS.USER_EXPORT + ':filename',
STATIC_DOWNLOAD_PATHS.USER_EXPORTS + ':filename',
asyncMiddleware(userExportDownloadValidator), // Include JWT token authentication
asyncMiddleware(downloadUserExport)
)
downloadRouter.use(
STATIC_DOWNLOAD_PATHS.ORIGINAL_VIDEO_FILE + ':filename',
optionalAuthenticate,
asyncMiddleware(originalVideoFileDownloadValidator),
asyncMiddleware(downloadOriginalFile)
)
// ---------------------------------------------------------------------------
export {
@ -91,7 +101,7 @@ async function downloadTorrent (req: express.Request, res: express.Response) {
return res.download(result.path, result.downloadName)
}
async function downloadVideoFile (req: express.Request, res: express.Response) {
async function downloadWebVideoFile (req: express.Request, res: express.Response) {
const video = res.locals.videoAll
const videoFile = getVideoFile(req, video.VideoFiles)
@ -184,6 +194,19 @@ function downloadUserExport (req: express.Request, res: express.Response) {
return Promise.resolve()
}
function downloadOriginalFile (req: express.Request, res: express.Response) {
const videoSource = res.locals.videoSource
const downloadFilename = videoSource.inputFilename
if (videoSource.storage === FileStorage.OBJECT_STORAGE) {
return redirectOriginalFileToObjectStorage({ res, videoSource, downloadFilename })
}
res.download(VideoPathManager.Instance.getFSOriginalVideoFilePath(videoSource.keptOriginalFilename), downloadFilename)
return Promise.resolve()
}
// ---------------------------------------------------------------------------
function getVideoFile (req: express.Request, files: MVideoFile[]) {
@ -262,3 +285,17 @@ async function redirectUserExportToObjectStorage (options: {
return res.redirect(url)
}
async function redirectOriginalFileToObjectStorage (options: {
res: express.Response
downloadFilename: string
videoSource: MVideoSource
}) {
const { res, downloadFilename, videoSource } = options
const url = await generateOriginalFilePresignedUrl({ videoSource, downloadFilename })
logger.debug('Generating pre-signed URL %s for original video file %s', url, videoSource.keptOriginalFilename)
return res.redirect(url)
}

4
server/core/controllers/static.ts
View file

@ -31,12 +31,12 @@ const privateWebVideoStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUI
staticRouter.use(
[ STATIC_PATHS.PRIVATE_WEB_VIDEOS, STATIC_PATHS.LEGACY_PRIVATE_WEB_VIDEOS ],
...privateWebVideoStaticMiddlewares,
express.static(DIRECTORIES.VIDEOS.PRIVATE, { fallthrough: false }),
express.static(DIRECTORIES.WEB_VIDEOS.PRIVATE, { fallthrough: false }),
handleStaticError
)
staticRouter.use(
[ STATIC_PATHS.WEB_VIDEOS, STATIC_PATHS.LEGACY_WEB_VIDEOS ],
express.static(DIRECTORIES.VIDEOS.PUBLIC, { fallthrough: false }),
express.static(DIRECTORIES.WEB_VIDEOS.PUBLIC, { fallthrough: false }),
handleStaticError
)