mirror of
https://github.com/Chocobozzz/PeerTube.git
synced 2025-10-05 19:42:24 +02:00
Resumable video uploads (#3933)
* WIP: resumable video uploads relates to #324 * fix review comments * video upload: error handling * fix audio upload * fixes after self review * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/middlewares/validators/videos/videos.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * Update server/controllers/api/videos/index.ts Co-authored-by: Rigel Kent <par@rigelk.eu> * update after code review * refactor upload route - restore multipart upload route - move resumable to dedicated upload-resumable route - move checks to middleware - do not leak internal fs structure in response * fix yarn.lock upon rebase * factorize addVideo for reuse in both endpoints * add resumable upload API to openapi spec * add initial test and test helper for resumable upload * typings for videoAddResumable middleware * avoid including aws and google packages via node-uploadx, by only including uploadx/core * rename ex-isAudioBg to more explicit name mentioning it is a preview file for audio * add video-upload-tmp-folder-cleaner job * stronger typing of video upload middleware * reduce dependency to @uploadx/core * add audio upload test * refactor resumable uploads cleanup from job to scheduler * refactor resumable uploads scheduler to compare to last execution time * make resumable upload validator to always cleanup on failure * move legacy upload request building outside of uploadVideo test helper * filter upload-resumable middlewares down to POST, PUT, DELETE also begin to type metadata * merge add duration functions * stronger typings and documentation for uploadx behaviour, move init validator up * refactor(client/video-edit): options > uploadxOptions * refactor(client/video-edit): remove obsolete else * scheduler/remove-dangling-resum: rename tag * refactor(server/video): add UploadVideoFiles type * refactor(mw/validators): restructure eslint disable * refactor(mw/validators/videos): rename import * refactor(client/vid-upload): rename html elem id * refactor(sched/remove-dangl): move fn to method * refactor(mw/async): add method typing * refactor(mw/vali/video): double quote > single * refactor(server/upload-resum): express use > all * proper http methud enum server/middlewares/async.ts * properly type http methods * factorize common video upload validation steps * add check for maximum partially uploaded file size * fix audioBg use * fix extname(filename) in addVideo * document parameters for uploadx's resumable protocol * clear META files in scheduler * last audio refactor before cramming preview in the initial POST form data * refactor as mulitpart/form-data initial post request this allows preview/thumbnail uploads alongside the initial request, and cleans up the upload form * Add more tests for resumable uploads * Refactor remove dangling resumable uploads * Prepare changelog * Add more resumable upload tests * Remove user quota check for resumable uploads * Fix upload error handler * Update nginx template for upload-resumable * Cleanup comment * Remove unused express methods * Prefer to use got instead of raw http * Don't retry on error 500 Co-authored-by: Rigel Kent <par@rigelk.eu> Co-authored-by: Rigel Kent <sendmemail@rigelk.eu> Co-authored-by: Chocobozzz <me@florianbigard.com>
This commit is contained in:
kontrollanten
GitHub
parent
d29ced1a85
commit
f6d6e7f861
46 changed files with 2454 additions and 1293 deletions
|
|
@ -1,4 +1,6 @@
|
|||
import { InboxManager } from '@server/lib/activitypub/inbox-manager'
|
||||
import { RemoveDanglingResumableUploadsScheduler } from '@server/lib/schedulers/remove-dangling-resumable-uploads-scheduler'
|
||||
import { SendDebugCommand } from '@shared/models'
|
||||
import * as express from 'express'
|
||||
import { UserRight } from '../../../../shared/models/users'
|
||||
import { authenticate, ensureUserHasRight } from '../../../middlewares'
|
||||
|
|
@ -11,6 +13,12 @@ debugRouter.get('/debug',
|
|||
getDebug
|
||||
)
|
||||
|
||||
debugRouter.post('/debug/run-command',
|
||||
authenticate,
|
||||
ensureUserHasRight(UserRight.MANAGE_DEBUG),
|
||||
runCommand
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export {
|
||||
|
|
@ -25,3 +33,13 @@ function getDebug (req: express.Request, res: express.Response) {
|
|||
activityPubMessagesWaiting: InboxManager.Instance.getActivityPubMessagesWaiting()
|
||||
})
|
||||
}
|
||||
|
||||
async function runCommand (req: express.Request, res: express.Response) {
|
||||
const body: SendDebugCommand = req.body
|
||||
|
||||
if (body.command === 'remove-dandling-resumable-uploads') {
|
||||
await RemoveDanglingResumableUploadsScheduler.Instance.execute()
|
||||
}
|
||||
|
||||
return res.sendStatus(204)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@ import * as express from 'express'
|
|||
import { move } from 'fs-extra'
|
||||
import { extname } from 'path'
|
||||
import toInt from 'validator/lib/toInt'
|
||||
import { deleteResumableUploadMetaFile, getResumableUploadPath } from '@server/helpers/upload'
|
||||
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
|
||||
import { changeVideoChannelShare } from '@server/lib/activitypub/share'
|
||||
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
|
||||
|
|
@ -10,8 +11,9 @@ import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnail
|
|||
import { generateVideoFilename, getVideoFilePath } from '@server/lib/video-paths'
|
||||
import { getServerActor } from '@server/models/application/application'
|
||||
import { MVideo, MVideoFile, MVideoFullLight } from '@server/types/models'
|
||||
import { uploadx } from '@uploadx/core'
|
||||
import { VideoCreate, VideosCommonQuery, VideoState, VideoUpdate } from '../../../../shared'
|
||||
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
|
||||
import { HttpStatusCode } from '../../../../shared/core-utils/miscs'
|
||||
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
|
||||
import { resetSequelizeInstance, retryTransactionWrapper } from '../../../helpers/database-utils'
|
||||
import { buildNSFWFilter, createReqFiles, getCountVideos } from '../../../helpers/express-utils'
|
||||
|
|
@ -47,7 +49,9 @@ import {
|
|||
setDefaultPagination,
|
||||
setDefaultVideosSort,
|
||||
videoFileMetadataGetValidator,
|
||||
videosAddValidator,
|
||||
videosAddLegacyValidator,
|
||||
videosAddResumableInitValidator,
|
||||
videosAddResumableValidator,
|
||||
videosCustomGetValidator,
|
||||
videosGetValidator,
|
||||
videosRemoveValidator,
|
||||
|
|
@ -69,6 +73,7 @@ import { watchingRouter } from './watching'
|
|||
const lTags = loggerTagsFactory('api', 'video')
|
||||
const auditLogger = auditLoggerFactory('videos')
|
||||
const videosRouter = express.Router()
|
||||
const uploadxMiddleware = uploadx.upload({ directory: getResumableUploadPath() })
|
||||
|
||||
const reqVideoFileAdd = createReqFiles(
|
||||
[ 'videofile', 'thumbnailfile', 'previewfile' ],
|
||||
|
|
@ -79,6 +84,16 @@ const reqVideoFileAdd = createReqFiles(
|
|||
previewfile: CONFIG.STORAGE.TMP_DIR
|
||||
}
|
||||
)
|
||||
|
||||
const reqVideoFileAddResumable = createReqFiles(
|
||||
[ 'thumbnailfile', 'previewfile' ],
|
||||
MIMETYPES.IMAGE.MIMETYPE_EXT,
|
||||
{
|
||||
thumbnailfile: getResumableUploadPath(),
|
||||
previewfile: getResumableUploadPath()
|
||||
}
|
||||
)
|
||||
|
||||
const reqVideoFileUpdate = createReqFiles(
|
||||
[ 'thumbnailfile', 'previewfile' ],
|
||||
MIMETYPES.IMAGE.MIMETYPE_EXT,
|
||||
|
|
@ -111,18 +126,39 @@ videosRouter.get('/',
|
|||
commonVideosFiltersValidator,
|
||||
asyncMiddleware(listVideos)
|
||||
)
|
||||
|
||||
videosRouter.post('/upload',
|
||||
authenticate,
|
||||
reqVideoFileAdd,
|
||||
asyncMiddleware(videosAddLegacyValidator),
|
||||
asyncRetryTransactionMiddleware(addVideoLegacy)
|
||||
)
|
||||
|
||||
videosRouter.post('/upload-resumable',
|
||||
authenticate,
|
||||
reqVideoFileAddResumable,
|
||||
asyncMiddleware(videosAddResumableInitValidator),
|
||||
uploadxMiddleware
|
||||
)
|
||||
|
||||
videosRouter.delete('/upload-resumable',
|
||||
authenticate,
|
||||
uploadxMiddleware
|
||||
)
|
||||
|
||||
videosRouter.put('/upload-resumable',
|
||||
authenticate,
|
||||
uploadxMiddleware, // uploadx doesn't use call next() before the file upload completes
|
||||
asyncMiddleware(videosAddResumableValidator),
|
||||
asyncMiddleware(addVideoResumable)
|
||||
)
|
||||
|
||||
videosRouter.put('/:id',
|
||||
authenticate,
|
||||
reqVideoFileUpdate,
|
||||
asyncMiddleware(videosUpdateValidator),
|
||||
asyncRetryTransactionMiddleware(updateVideo)
|
||||
)
|
||||
videosRouter.post('/upload',
|
||||
authenticate,
|
||||
reqVideoFileAdd,
|
||||
asyncMiddleware(videosAddValidator),
|
||||
asyncRetryTransactionMiddleware(addVideo)
|
||||
)
|
||||
|
||||
videosRouter.get('/:id/description',
|
||||
asyncMiddleware(videosGetValidator),
|
||||
|
|
@ -157,23 +193,23 @@ export {
|
|||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function listVideoCategories (req: express.Request, res: express.Response) {
|
||||
function listVideoCategories (_req: express.Request, res: express.Response) {
|
||||
res.json(VIDEO_CATEGORIES)
|
||||
}
|
||||
|
||||
function listVideoLicences (req: express.Request, res: express.Response) {
|
||||
function listVideoLicences (_req: express.Request, res: express.Response) {
|
||||
res.json(VIDEO_LICENCES)
|
||||
}
|
||||
|
||||
function listVideoLanguages (req: express.Request, res: express.Response) {
|
||||
function listVideoLanguages (_req: express.Request, res: express.Response) {
|
||||
res.json(VIDEO_LANGUAGES)
|
||||
}
|
||||
|
||||
function listVideoPrivacies (req: express.Request, res: express.Response) {
|
||||
function listVideoPrivacies (_req: express.Request, res: express.Response) {
|
||||
res.json(VIDEO_PRIVACIES)
|
||||
}
|
||||
|
||||
async function addVideo (req: express.Request, res: express.Response) {
|
||||
async function addVideoLegacy (req: express.Request, res: express.Response) {
|
||||
// Uploading the video could be long
|
||||
// Set timeout to 10 minutes, as Express's default is 2 minutes
|
||||
req.setTimeout(1000 * 60 * 10, () => {
|
||||
|
|
@ -183,13 +219,42 @@ async function addVideo (req: express.Request, res: express.Response) {
|
|||
|
||||
const videoPhysicalFile = req.files['videofile'][0]
|
||||
const videoInfo: VideoCreate = req.body
|
||||
const files = req.files
|
||||
|
||||
const videoData = buildLocalVideoFromReq(videoInfo, res.locals.videoChannel.id)
|
||||
videoData.state = CONFIG.TRANSCODING.ENABLED ? VideoState.TO_TRANSCODE : VideoState.PUBLISHED
|
||||
videoData.duration = videoPhysicalFile['duration'] // duration was added by a previous middleware
|
||||
return addVideo({ res, videoPhysicalFile, videoInfo, files })
|
||||
}
|
||||
|
||||
async function addVideoResumable (_req: express.Request, res: express.Response) {
|
||||
const videoPhysicalFile = res.locals.videoFileResumable
|
||||
const videoInfo = videoPhysicalFile.metadata
|
||||
const files = { previewfile: videoInfo.previewfile }
|
||||
|
||||
// Don't need the meta file anymore
|
||||
await deleteResumableUploadMetaFile(videoPhysicalFile.path)
|
||||
|
||||
return addVideo({ res, videoPhysicalFile, videoInfo, files })
|
||||
}
|
||||
|
||||
async function addVideo (options: {
|
||||
res: express.Response
|
||||
videoPhysicalFile: express.VideoUploadFile
|
||||
videoInfo: VideoCreate
|
||||
files: express.UploadFiles
|
||||
}) {
|
||||
const { res, videoPhysicalFile, videoInfo, files } = options
|
||||
const videoChannel = res.locals.videoChannel
|
||||
const user = res.locals.oauth.token.User
|
||||
|
||||
const videoData = buildLocalVideoFromReq(videoInfo, videoChannel.id)
|
||||
|
||||
videoData.state = CONFIG.TRANSCODING.ENABLED
|
||||
? VideoState.TO_TRANSCODE
|
||||
: VideoState.PUBLISHED
|
||||
|
||||
videoData.duration = videoPhysicalFile.duration // duration was added by a previous middleware
|
||||
|
||||
const video = new VideoModel(videoData) as MVideoFullLight
|
||||
video.VideoChannel = res.locals.videoChannel
|
||||
video.VideoChannel = videoChannel
|
||||
video.url = getLocalVideoActivityPubUrl(video) // We use the UUID, so set the URL after building the object
|
||||
|
||||
const videoFile = new VideoFileModel({
|
||||
|
|
@ -217,7 +282,7 @@ async function addVideo (req: express.Request, res: express.Response) {
|
|||
|
||||
const [ thumbnailModel, previewModel ] = await buildVideoThumbnailsFromReq({
|
||||
video,
|
||||
files: req.files,
|
||||
files,
|
||||
fallback: type => generateVideoMiniature({ video, videoFile, type })
|
||||
})
|
||||
|
||||
|
|
@ -253,7 +318,7 @@ async function addVideo (req: express.Request, res: express.Response) {
|
|||
|
||||
await autoBlacklistVideoIfNeeded({
|
||||
video,
|
||||
user: res.locals.oauth.token.User,
|
||||
user,
|
||||
isRemote: false,
|
||||
isNew: true,
|
||||
transaction: t
|
||||
|
|
@ -282,7 +347,7 @@ async function addVideo (req: express.Request, res: express.Response) {
|
|||
.catch(err => logger.error('Cannot federate or notify video creation %s', video.url, { err, ...lTags(video.uuid) }))
|
||||
|
||||
if (video.state === VideoState.TO_TRANSCODE) {
|
||||
await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User)
|
||||
await addOptimizeOrMergeAudioJob(videoCreated, videoFile, user)
|
||||
}
|
||||
|
||||
Hooks.runAction('action:api.video.uploaded', { video: videoCreated })
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue