mirror of
https://github.com/Chocobozzz/PeerTube.git
synced 2025-10-03 17:59:37 +02:00
Add global rate limit to video download
This commit is contained in:
parent
49a6211f25
commit
25c5507a03
10 changed files with 354 additions and 235 deletions
|
@ -538,6 +538,12 @@ nsfw_flags_settings:
|
||||||
# using NSFW flags (violent content, etc.) set by video authors
|
# using NSFW flags (violent content, etc.) set by video authors
|
||||||
enabled: true
|
enabled: true
|
||||||
|
|
||||||
|
download_generate_video:
|
||||||
|
# Max parallel downloads on your instance
|
||||||
|
# Each download spawns an ffmpeg process
|
||||||
|
# The ffmpeg process ends when users have downloaded the entire file or cancelled the download
|
||||||
|
max_parallel_downloads: 100
|
||||||
|
|
||||||
cache:
|
cache:
|
||||||
previews:
|
previews:
|
||||||
size: 500 # Max number of previews you want to cache
|
size: 500 # Max number of previews you want to cache
|
||||||
|
|
|
@ -15,6 +15,9 @@ rates_limit:
|
||||||
signup:
|
signup:
|
||||||
window: 5 minutes
|
window: 5 minutes
|
||||||
max: 200
|
max: 200
|
||||||
|
download_generate_video:
|
||||||
|
window: 5 seconds
|
||||||
|
max: 500000
|
||||||
|
|
||||||
database:
|
database:
|
||||||
hostname: '127.0.0.1'
|
hostname: '127.0.0.1'
|
||||||
|
|
|
@ -534,7 +534,13 @@ webrtc:
|
||||||
nsfw_flags_settings:
|
nsfw_flags_settings:
|
||||||
# Allow logged-in/anonymous users to have a more granular control over their NSFW policy
|
# Allow logged-in/anonymous users to have a more granular control over their NSFW policy
|
||||||
# using NSFW flags (violent content, etc.) set by video authors
|
# using NSFW flags (violent content, etc.) set by video authors
|
||||||
enabled: false
|
enabled: true
|
||||||
|
|
||||||
|
download_generate_video:
|
||||||
|
# Max parallel downloads on your instance
|
||||||
|
# Each download spawns an ffmpeg process
|
||||||
|
# The ffmpeg process ends when users have downloaded the entire file or cancelled the download
|
||||||
|
max_parallel_downloads: 100
|
||||||
|
|
||||||
###############################################################################
|
###############################################################################
|
||||||
#
|
#
|
||||||
|
@ -1092,11 +1098,6 @@ client:
|
||||||
# You can automatically redirect your users on this external platform when they click on the login button
|
# You can automatically redirect your users on this external platform when they click on the login button
|
||||||
redirect_on_single_external_auth: false
|
redirect_on_single_external_auth: false
|
||||||
|
|
||||||
# Allow logged-in/anonymous users to have a more granular control over their NSFW policy
|
|
||||||
# using NSFW flags (violent content, etc.) set by video authors
|
|
||||||
nsfw_flags_settings:
|
|
||||||
enabled: true
|
|
||||||
|
|
||||||
open_in_app:
|
open_in_app:
|
||||||
android:
|
android:
|
||||||
# Use an intent URL: https://developer.chrome.com/docs/android/intents
|
# Use an intent URL: https://developer.chrome.com/docs/android/intents
|
||||||
|
|
|
@ -123,13 +123,64 @@ describe('Test generate download API validator', function () {
|
||||||
await server.videos.generateDownload({ videoId, videoFileIds, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
|
await server.videos.generateDownload({ videoId, videoFileIds, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
|
||||||
})
|
})
|
||||||
|
|
||||||
it('Should suceed with the correct params', async function () {
|
it('Should succeed with the correct params', async function () {
|
||||||
const videoFileIds = [ audioStreamId, videoStreamIds[0] ]
|
const videoFileIds = [ audioStreamId, videoStreamIds[0] ]
|
||||||
|
|
||||||
await server.videos.generateDownload({ videoId, videoFileIds })
|
await server.videos.generateDownload({ videoId, videoFileIds })
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe('Download rate limit', function () {
|
||||||
|
let videoId: string
|
||||||
|
let fileId: number
|
||||||
|
|
||||||
|
before(async function () {
|
||||||
|
this.timeout(60000)
|
||||||
|
|
||||||
|
await server.kill()
|
||||||
|
|
||||||
|
await server.run({
|
||||||
|
download_generate_video: {
|
||||||
|
max_parallel_downloads: 2
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
const { uuid } = await server.videos.quickUpload({ name: 'video' })
|
||||||
|
videoId = uuid
|
||||||
|
await waitJobs([ server ])
|
||||||
|
|
||||||
|
const video = await server.videos.get({ id: uuid })
|
||||||
|
fileId = video.files[0].id
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should succeed with a single download', async function () {
|
||||||
|
const videoFileIds = [ fileId ]
|
||||||
|
|
||||||
|
for (let i = 0; i < 3; i++) {
|
||||||
|
await server.videos.generateDownload({ videoId, videoFileIds })
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
it('Should fail with too many parallel downloads', async function () {
|
||||||
|
const videoFileIds = [ fileId ]
|
||||||
|
|
||||||
|
const promises: Promise<any>[] = []
|
||||||
|
|
||||||
|
for (let i = 0; i < 10; i++) {
|
||||||
|
promises.push(
|
||||||
|
server.videos.generateDownload({ videoId, videoFileIds })
|
||||||
|
.catch(err => {
|
||||||
|
if (err.message.includes('429')) return
|
||||||
|
|
||||||
|
throw err
|
||||||
|
})
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
await Promise.all(promises)
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
after(async function () {
|
after(async function () {
|
||||||
await cleanupTests([ server ])
|
await cleanupTests([ server ])
|
||||||
})
|
})
|
||||||
|
|
|
@ -12,7 +12,7 @@ import {
|
||||||
} from '@server/lib/object-storage/index.js'
|
} from '@server/lib/object-storage/index.js'
|
||||||
import { getFSUserExportFilePath } from '@server/lib/paths.js'
|
import { getFSUserExportFilePath } from '@server/lib/paths.js'
|
||||||
import { Hooks } from '@server/lib/plugins/hooks.js'
|
import { Hooks } from '@server/lib/plugins/hooks.js'
|
||||||
import { muxToMergeVideoFiles } from '@server/lib/video-file.js'
|
import { VideoDownload } from '@server/lib/video-download.js'
|
||||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||||
import {
|
import {
|
||||||
MStreamingPlaylist,
|
MStreamingPlaylist,
|
||||||
|
@ -27,7 +27,9 @@ import cors from 'cors'
|
||||||
import express from 'express'
|
import express from 'express'
|
||||||
import { DOWNLOAD_PATHS, WEBSERVER } from '../initializers/constants.js'
|
import { DOWNLOAD_PATHS, WEBSERVER } from '../initializers/constants.js'
|
||||||
import {
|
import {
|
||||||
asyncMiddleware, buildRateLimiter, optionalAuthenticate,
|
asyncMiddleware,
|
||||||
|
buildRateLimiter,
|
||||||
|
optionalAuthenticate,
|
||||||
originalVideoFileDownloadValidator,
|
originalVideoFileDownloadValidator,
|
||||||
userExportDownloadValidator,
|
userExportDownloadValidator,
|
||||||
videosDownloadValidator,
|
videosDownloadValidator,
|
||||||
|
@ -244,6 +246,13 @@ async function downloadGeneratedVideoFile (req: express.Request, res: express.Re
|
||||||
|
|
||||||
if (!checkAllowResult(res, allowParameters, allowedResult)) return
|
if (!checkAllowResult(res, allowParameters, allowedResult)) return
|
||||||
|
|
||||||
|
if (VideoDownload.totalDownloads > CONFIG.DOWNLOAD_GENERATE_VIDEO.MAX_PARALLEL_DOWNLOADS) {
|
||||||
|
return res.fail({
|
||||||
|
status: HttpStatusCode.TOO_MANY_REQUESTS_429,
|
||||||
|
message: `Too many parallel downloads on this server. Please try again later.`
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
const maxResolutionFile = maxBy(videoFiles, 'resolution')
|
const maxResolutionFile = maxBy(videoFiles, 'resolution')
|
||||||
|
|
||||||
// Prefer m4a extension for the user if this is a mp4 audio file only
|
// Prefer m4a extension for the user if this is a mp4 audio file only
|
||||||
|
@ -260,7 +269,7 @@ async function downloadGeneratedVideoFile (req: express.Request, res: express.Re
|
||||||
|
|
||||||
res.type(extname)
|
res.type(extname)
|
||||||
|
|
||||||
await muxToMergeVideoFiles({ video, videoFiles, output: res })
|
await new VideoDownload({ video, videoFiles }).muxToMergeVideoFiles(res)
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
|
@ -249,7 +249,8 @@ export function checkMissedConfig () {
|
||||||
'live.transcoding.remote_runners.enabled',
|
'live.transcoding.remote_runners.enabled',
|
||||||
'storyboards.enabled',
|
'storyboards.enabled',
|
||||||
'webrtc.stun_servers',
|
'webrtc.stun_servers',
|
||||||
'nsfw_flags_settings.enabled'
|
'nsfw_flags_settings.enabled',
|
||||||
|
'download_generate_video.max_parallel_downloads'
|
||||||
]
|
]
|
||||||
|
|
||||||
const requiredAlternatives = [
|
const requiredAlternatives = [
|
||||||
|
|
|
@ -78,6 +78,10 @@ const CONFIG = {
|
||||||
ENABLED: config.get<boolean>('nsfw_flags_settings.enabled')
|
ENABLED: config.get<boolean>('nsfw_flags_settings.enabled')
|
||||||
},
|
},
|
||||||
|
|
||||||
|
DOWNLOAD_GENERATE_VIDEO: {
|
||||||
|
MAX_PARALLEL_DOWNLOADS: config.get<number>('download_generate_video.max_parallel_downloads')
|
||||||
|
},
|
||||||
|
|
||||||
CLIENT: {
|
CLIENT: {
|
||||||
VIDEOS: {
|
VIDEOS: {
|
||||||
MINIATURE: {
|
MINIATURE: {
|
||||||
|
|
|
@ -11,7 +11,7 @@ import {
|
||||||
getOriginalFileReadStream,
|
getOriginalFileReadStream,
|
||||||
getWebVideoFileReadStream
|
getWebVideoFileReadStream
|
||||||
} from '@server/lib/object-storage/videos.js'
|
} from '@server/lib/object-storage/videos.js'
|
||||||
import { muxToMergeVideoFiles } from '@server/lib/video-file.js'
|
import { VideoDownload } from '@server/lib/video-download.js'
|
||||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||||
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
||||||
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
||||||
|
@ -391,7 +391,8 @@ export class VideosExporter extends AbstractUserExporter<VideoExportJSON> {
|
||||||
if (separatedAudioFile) {
|
if (separatedAudioFile) {
|
||||||
const stream = new PassThrough()
|
const stream = new PassThrough()
|
||||||
|
|
||||||
muxToMergeVideoFiles({ video, videoFiles: [ videoFile, separatedAudioFile ], output: stream })
|
await new VideoDownload({ video, videoFiles: [ videoFile, separatedAudioFile ] })
|
||||||
|
.muxToMergeVideoFiles(stream)
|
||||||
.catch(err => logger.error('Cannot mux video files', { err }))
|
.catch(err => logger.error('Cannot mux video files', { err }))
|
||||||
|
|
||||||
return Promise.resolve(stream)
|
return Promise.resolve(stream)
|
||||||
|
|
262
server/core/lib/video-download.ts
Normal file
262
server/core/lib/video-download.ts
Normal file
|
@ -0,0 +1,262 @@
|
||||||
|
import { FFmpegContainer } from '@peertube/peertube-ffmpeg'
|
||||||
|
import { FileStorage } from '@peertube/peertube-models'
|
||||||
|
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/ffmpeg-options.js'
|
||||||
|
import { logger } from '@server/helpers/logger.js'
|
||||||
|
import { buildRequestError, doRequestAndSaveToFile, generateRequestStream } from '@server/helpers/requests.js'
|
||||||
|
import { REQUEST_TIMEOUTS } from '@server/initializers/constants.js'
|
||||||
|
import { MVideoFile, MVideoThumbnail } from '@server/types/models/index.js'
|
||||||
|
import { remove } from 'fs-extra/esm'
|
||||||
|
import { Readable, Writable } from 'stream'
|
||||||
|
import { lTags } from './object-storage/shared/index.js'
|
||||||
|
import {
|
||||||
|
getHLSFileReadStream,
|
||||||
|
getWebVideoFileReadStream,
|
||||||
|
makeHLSFileAvailable,
|
||||||
|
makeWebVideoFileAvailable
|
||||||
|
} from './object-storage/videos.js'
|
||||||
|
import { VideoPathManager } from './video-path-manager.js'
|
||||||
|
|
||||||
|
export class VideoDownload {
|
||||||
|
static totalDownloads = 0
|
||||||
|
|
||||||
|
private readonly inputs: (string | Readable)[] = []
|
||||||
|
|
||||||
|
private readonly tmpDestinations: string[] = []
|
||||||
|
private ffmpegContainer: FFmpegContainer
|
||||||
|
|
||||||
|
private readonly video: MVideoThumbnail
|
||||||
|
private readonly videoFiles: MVideoFile[]
|
||||||
|
|
||||||
|
constructor (options: {
|
||||||
|
video: MVideoThumbnail
|
||||||
|
videoFiles: MVideoFile[]
|
||||||
|
}) {
|
||||||
|
this.video = options.video
|
||||||
|
this.videoFiles = options.videoFiles
|
||||||
|
}
|
||||||
|
|
||||||
|
async muxToMergeVideoFiles (output: Writable) {
|
||||||
|
return new Promise<void>(async (res, rej) => {
|
||||||
|
try {
|
||||||
|
VideoDownload.totalDownloads++
|
||||||
|
|
||||||
|
const maxResolution = await this.buildMuxInputs(rej)
|
||||||
|
|
||||||
|
// Include cover to audio file?
|
||||||
|
const { coverPath, isTmpDestination } = maxResolution === 0
|
||||||
|
? await this.buildCoverInput()
|
||||||
|
: { coverPath: undefined, isTmpDestination: false }
|
||||||
|
|
||||||
|
if (coverPath && isTmpDestination) {
|
||||||
|
this.tmpDestinations.push(coverPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Muxing files for video ${this.video.url}`, { inputs: this.inputsToLog(), ...lTags(this.video.uuid) })
|
||||||
|
|
||||||
|
this.ffmpegContainer = new FFmpegContainer(getFFmpegCommandWrapperOptions('vod'))
|
||||||
|
|
||||||
|
try {
|
||||||
|
await this.ffmpegContainer.mergeInputs({
|
||||||
|
inputs: this.inputs,
|
||||||
|
output,
|
||||||
|
logError: false,
|
||||||
|
|
||||||
|
// Include a cover if this is an audio file
|
||||||
|
coverPath
|
||||||
|
})
|
||||||
|
|
||||||
|
logger.info(`Mux ended for video ${this.video.url}`, { inputs: this.inputsToLog(), ...lTags(this.video.uuid) })
|
||||||
|
|
||||||
|
res()
|
||||||
|
} catch (err) {
|
||||||
|
const message = err?.message || ''
|
||||||
|
|
||||||
|
if (message.includes('Output stream closed')) {
|
||||||
|
logger.info(`Client aborted mux for video ${this.video.url}`, lTags(this.video.uuid))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.warn(`Cannot mux files of video ${this.video.url}`, { err, inputs: this.inputsToLog(), ...lTags(this.video.uuid) })
|
||||||
|
|
||||||
|
if (err.inputStreamError) {
|
||||||
|
err.inputStreamError = buildRequestError(err.inputStreamError)
|
||||||
|
}
|
||||||
|
|
||||||
|
throw err
|
||||||
|
} finally {
|
||||||
|
this.ffmpegContainer.forceKill()
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
rej(err)
|
||||||
|
} finally {
|
||||||
|
this.cleanup()
|
||||||
|
.catch(cleanupErr => logger.error('Cannot cleanup after mux error', { err: cleanupErr, ...lTags(this.video.uuid) }))
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Build mux inputs
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
private async buildMuxInputs (rej: (err: Error) => void) {
|
||||||
|
let maxResolution = 0
|
||||||
|
|
||||||
|
for (const videoFile of this.videoFiles) {
|
||||||
|
if (!videoFile) continue
|
||||||
|
|
||||||
|
maxResolution = Math.max(maxResolution, videoFile.resolution)
|
||||||
|
|
||||||
|
const { input, isTmpDestination } = await this.buildMuxInput(
|
||||||
|
videoFile,
|
||||||
|
err => {
|
||||||
|
logger.warn(`Cannot build mux input of video ${this.video.url}`, {
|
||||||
|
err,
|
||||||
|
inputs: this.inputsToLog(),
|
||||||
|
...lTags(this.video.uuid)
|
||||||
|
})
|
||||||
|
|
||||||
|
this.cleanup()
|
||||||
|
.catch(cleanupErr => logger.error('Cannot cleanup after mux error', { err: cleanupErr, ...lTags(this.video.uuid) }))
|
||||||
|
|
||||||
|
rej(buildRequestError(err as any))
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
this.inputs.push(input)
|
||||||
|
|
||||||
|
if (isTmpDestination === true) this.tmpDestinations.push(input)
|
||||||
|
}
|
||||||
|
|
||||||
|
return maxResolution
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildMuxInput (
|
||||||
|
videoFile: MVideoFile,
|
||||||
|
onStreamError: (err: Error) => void
|
||||||
|
): Promise<{ input: Readable, isTmpDestination: false } | { input: string, isTmpDestination: boolean }> {
|
||||||
|
// Remote
|
||||||
|
if (this.video.remote === true) {
|
||||||
|
return this.buildMuxRemoteInput(videoFile, onStreamError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Local on FS
|
||||||
|
if (videoFile.storage === FileStorage.FILE_SYSTEM) {
|
||||||
|
return this.buildMuxLocalFSInput(videoFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Local on object storage
|
||||||
|
return this.buildMuxLocalObjectStorageInput(videoFile)
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildMuxRemoteInput (videoFile: MVideoFile, onStreamError: (err: Error) => void) {
|
||||||
|
const timeout = REQUEST_TIMEOUTS.VIDEO_FILE
|
||||||
|
|
||||||
|
const videoSizeKB = videoFile.size / 1000
|
||||||
|
const bodyKBLimit = videoSizeKB + 0.1 * videoSizeKB
|
||||||
|
|
||||||
|
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
|
||||||
|
if (videoFile.isAudio()) {
|
||||||
|
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
|
||||||
|
|
||||||
|
// > 1GB
|
||||||
|
if (bodyKBLimit > 1000 * 1000) {
|
||||||
|
throw new Error('Cannot download remote video file > 1GB')
|
||||||
|
}
|
||||||
|
|
||||||
|
await doRequestAndSaveToFile(videoFile.fileUrl, destination, { timeout, bodyKBLimit })
|
||||||
|
|
||||||
|
return { input: destination, isTmpDestination: true as const }
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
input: generateRequestStream(videoFile.fileUrl, { timeout, bodyKBLimit }).on('error', onStreamError),
|
||||||
|
isTmpDestination: false as const
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private buildMuxLocalFSInput (videoFile: MVideoFile) {
|
||||||
|
return { input: VideoPathManager.Instance.getFSVideoFileOutputPath(this.video, videoFile), isTmpDestination: false }
|
||||||
|
}
|
||||||
|
|
||||||
|
private async buildMuxLocalObjectStorageInput (videoFile: MVideoFile) {
|
||||||
|
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
|
||||||
|
if (videoFile.hasAudio() && !videoFile.hasVideo()) {
|
||||||
|
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
|
||||||
|
|
||||||
|
if (videoFile.isHLS()) {
|
||||||
|
await makeHLSFileAvailable(this.video.getHLSPlaylist(), videoFile.filename, destination)
|
||||||
|
} else {
|
||||||
|
await makeWebVideoFileAvailable(videoFile.filename, destination)
|
||||||
|
}
|
||||||
|
|
||||||
|
return { input: destination, isTmpDestination: true as const }
|
||||||
|
}
|
||||||
|
|
||||||
|
if (videoFile.isHLS()) {
|
||||||
|
const { stream } = await getHLSFileReadStream({
|
||||||
|
playlist: this.video.getHLSPlaylist().withVideo(this.video),
|
||||||
|
filename: videoFile.filename,
|
||||||
|
rangeHeader: undefined
|
||||||
|
})
|
||||||
|
|
||||||
|
return { input: stream, isTmpDestination: false as const }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Web video
|
||||||
|
const { stream } = await getWebVideoFileReadStream({
|
||||||
|
filename: videoFile.filename,
|
||||||
|
rangeHeader: undefined
|
||||||
|
})
|
||||||
|
|
||||||
|
return { input: stream, isTmpDestination: false as const }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
private async buildCoverInput () {
|
||||||
|
const preview = this.video.getPreview()
|
||||||
|
|
||||||
|
if (this.video.isOwned()) return { coverPath: preview?.getPath() }
|
||||||
|
|
||||||
|
if (preview.fileUrl) {
|
||||||
|
const destination = VideoPathManager.Instance.buildTMPDestination(preview.filename)
|
||||||
|
|
||||||
|
await doRequestAndSaveToFile(preview.fileUrl, destination)
|
||||||
|
|
||||||
|
return { coverPath: destination, isTmpDestination: true }
|
||||||
|
}
|
||||||
|
|
||||||
|
return { coverPath: undefined }
|
||||||
|
}
|
||||||
|
|
||||||
|
private inputsToLog () {
|
||||||
|
return this.inputs.map(i => {
|
||||||
|
if (typeof i === 'string') return i
|
||||||
|
|
||||||
|
return 'ReadableStream'
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
private async cleanup () {
|
||||||
|
VideoDownload.totalDownloads--
|
||||||
|
|
||||||
|
for (const destination of this.tmpDestinations) {
|
||||||
|
await remove(destination)
|
||||||
|
.catch(err => logger.error('Cannot remove tmp destination', { err, destination, ...lTags(this.video.uuid) }))
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const input of this.inputs) {
|
||||||
|
if (input instanceof Readable) {
|
||||||
|
if (!input.destroyed) input.destroy()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (this.ffmpegContainer) {
|
||||||
|
this.ffmpegContainer.forceKill()
|
||||||
|
this.ffmpegContainer = undefined
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.debug(`Cleaned muxing for video ${this.video.url}`, { inputs: this.inputsToLog(), ...lTags(this.video.uuid) })
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,5 +1,4 @@
|
||||||
import {
|
import {
|
||||||
FFmpegContainer,
|
|
||||||
ffprobePromise,
|
ffprobePromise,
|
||||||
getVideoStreamDimensionsInfo,
|
getVideoStreamDimensionsInfo,
|
||||||
getVideoStreamFPS,
|
getVideoStreamFPS,
|
||||||
|
@ -9,25 +8,15 @@ import {
|
||||||
} from '@peertube/peertube-ffmpeg'
|
} from '@peertube/peertube-ffmpeg'
|
||||||
import { FileStorage, VideoFileFormatFlag, VideoFileMetadata, VideoFileStream, VideoResolution } from '@peertube/peertube-models'
|
import { FileStorage, VideoFileFormatFlag, VideoFileMetadata, VideoFileStream, VideoResolution } from '@peertube/peertube-models'
|
||||||
import { getFileSize, getLowercaseExtension } from '@peertube/peertube-node-utils'
|
import { getFileSize, getLowercaseExtension } from '@peertube/peertube-node-utils'
|
||||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/ffmpeg-options.js'
|
|
||||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||||
import { buildRequestError, doRequestAndSaveToFile, generateRequestStream } from '@server/helpers/requests.js'
|
|
||||||
import { CONFIG } from '@server/initializers/config.js'
|
import { CONFIG } from '@server/initializers/config.js'
|
||||||
import { MIMETYPES, REQUEST_TIMEOUTS } from '@server/initializers/constants.js'
|
import { MIMETYPES } from '@server/initializers/constants.js'
|
||||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||||
import { VideoSourceModel } from '@server/models/video/video-source.js'
|
import { VideoSourceModel } from '@server/models/video/video-source.js'
|
||||||
import { MVideo, MVideoFile, MVideoId, MVideoThumbnail, MVideoWithAllFiles } from '@server/types/models/index.js'
|
import { MVideo, MVideoFile, MVideoId, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||||
import { FfprobeData } from 'fluent-ffmpeg'
|
import { FfprobeData } from 'fluent-ffmpeg'
|
||||||
import { move, remove } from 'fs-extra/esm'
|
import { move, remove } from 'fs-extra/esm'
|
||||||
import { Readable, Writable } from 'stream'
|
import { storeOriginalVideoFile } from './object-storage/videos.js'
|
||||||
import { lTags } from './object-storage/shared/index.js'
|
|
||||||
import {
|
|
||||||
getHLSFileReadStream,
|
|
||||||
getWebVideoFileReadStream,
|
|
||||||
makeHLSFileAvailable,
|
|
||||||
makeWebVideoFileAvailable,
|
|
||||||
storeOriginalVideoFile
|
|
||||||
} from './object-storage/videos.js'
|
|
||||||
import { generateHLSVideoFilename, generateWebVideoFilename } from './paths.js'
|
import { generateHLSVideoFilename, generateWebVideoFilename } from './paths.js'
|
||||||
import { VideoPathManager } from './video-path-manager.js'
|
import { VideoPathManager } from './video-path-manager.js'
|
||||||
|
|
||||||
|
@ -260,211 +249,3 @@ export async function saveNewOriginalFileIfNeeded (video: MVideo, videoFile: MVi
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
export async function muxToMergeVideoFiles (options: {
|
|
||||||
video: MVideoThumbnail
|
|
||||||
videoFiles: MVideoFile[]
|
|
||||||
output: Writable
|
|
||||||
}) {
|
|
||||||
const { video, videoFiles, output } = options
|
|
||||||
|
|
||||||
const inputs: (string | Readable)[] = []
|
|
||||||
const tmpDestinations: string[] = []
|
|
||||||
let ffmpegContainer: FFmpegContainer
|
|
||||||
|
|
||||||
return new Promise<void>(async (res, rej) => {
|
|
||||||
const cleanup = async () => {
|
|
||||||
for (const destination of tmpDestinations) {
|
|
||||||
await remove(destination)
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const input of inputs) {
|
|
||||||
if (input instanceof Readable) {
|
|
||||||
if (!input.destroyed) input.destroy()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ffmpegContainer) {
|
|
||||||
ffmpegContainer.forceKill()
|
|
||||||
ffmpegContainer = undefined
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
let maxResolution = 0
|
|
||||||
|
|
||||||
for (const videoFile of videoFiles) {
|
|
||||||
if (!videoFile) continue
|
|
||||||
|
|
||||||
maxResolution = Math.max(maxResolution, videoFile.resolution)
|
|
||||||
|
|
||||||
const { input, isTmpDestination } = await buildMuxInput(
|
|
||||||
video,
|
|
||||||
videoFile,
|
|
||||||
err => {
|
|
||||||
logger.warn(`Cannot build mux input of video ${video.url}`, { err, inputs: inputsToLog, ...lTags(video.uuid) })
|
|
||||||
|
|
||||||
cleanup()
|
|
||||||
.catch(cleanupErr => logger.error('Cannot cleanup after mux error', { err: cleanupErr, ...lTags(video.uuid) }))
|
|
||||||
|
|
||||||
rej(buildRequestError(err as any))
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
inputs.push(input)
|
|
||||||
|
|
||||||
if (isTmpDestination === true) tmpDestinations.push(input)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Include cover to audio file?
|
|
||||||
const { coverPath, isTmpDestination } = maxResolution === 0
|
|
||||||
? await buildCoverInput(video)
|
|
||||||
: { coverPath: undefined, isTmpDestination: false }
|
|
||||||
|
|
||||||
if (coverPath && isTmpDestination) tmpDestinations.push(coverPath)
|
|
||||||
|
|
||||||
const inputsToLog = inputs.map(i => {
|
|
||||||
if (typeof i === 'string') return i
|
|
||||||
|
|
||||||
return 'ReadableStream'
|
|
||||||
})
|
|
||||||
|
|
||||||
logger.info(`Muxing files for video ${video.url}`, { inputs: inputsToLog, ...lTags(video.uuid) })
|
|
||||||
|
|
||||||
ffmpegContainer = new FFmpegContainer(getFFmpegCommandWrapperOptions('vod'))
|
|
||||||
|
|
||||||
try {
|
|
||||||
await ffmpegContainer.mergeInputs({
|
|
||||||
inputs,
|
|
||||||
output,
|
|
||||||
logError: false,
|
|
||||||
|
|
||||||
// Include a cover if this is an audio file
|
|
||||||
coverPath
|
|
||||||
})
|
|
||||||
|
|
||||||
logger.info(`Mux ended for video ${video.url}`, { inputs: inputsToLog, ...lTags(video.uuid) })
|
|
||||||
|
|
||||||
res()
|
|
||||||
} catch (err) {
|
|
||||||
const message = err?.message || ''
|
|
||||||
|
|
||||||
if (message.includes('Output stream closed')) {
|
|
||||||
logger.info(`Client aborted mux for video ${video.url}`, lTags(video.uuid))
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
logger.warn(`Cannot mux files of video ${video.url}`, { err, inputs: inputsToLog, ...lTags(video.uuid) })
|
|
||||||
|
|
||||||
if (err.inputStreamError) {
|
|
||||||
err.inputStreamError = buildRequestError(err.inputStreamError)
|
|
||||||
}
|
|
||||||
|
|
||||||
throw err
|
|
||||||
} finally {
|
|
||||||
ffmpegContainer.forceKill()
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
rej(err)
|
|
||||||
} finally {
|
|
||||||
await cleanup()
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async function buildMuxInput (
|
|
||||||
video: MVideo,
|
|
||||||
videoFile: MVideoFile,
|
|
||||||
onStreamError: (err: Error) => void
|
|
||||||
): Promise<{ input: Readable, isTmpDestination: false } | { input: string, isTmpDestination: boolean }> {
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Remote
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
if (video.remote === true) {
|
|
||||||
const timeout = REQUEST_TIMEOUTS.VIDEO_FILE
|
|
||||||
|
|
||||||
const videoSizeKB = videoFile.size / 1000
|
|
||||||
const bodyKBLimit = videoSizeKB + 0.1 * videoSizeKB
|
|
||||||
|
|
||||||
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
|
|
||||||
if (videoFile.isAudio()) {
|
|
||||||
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
|
|
||||||
|
|
||||||
// > 1GB
|
|
||||||
if (bodyKBLimit > 1000 * 1000) {
|
|
||||||
throw new Error('Cannot download remote video file > 1GB')
|
|
||||||
}
|
|
||||||
|
|
||||||
await doRequestAndSaveToFile(videoFile.fileUrl, destination, { timeout, bodyKBLimit })
|
|
||||||
|
|
||||||
return { input: destination, isTmpDestination: true }
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
input: generateRequestStream(videoFile.fileUrl, { timeout, bodyKBLimit }).on('error', onStreamError),
|
|
||||||
isTmpDestination: false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Local on FS
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
if (videoFile.storage === FileStorage.FILE_SYSTEM) {
|
|
||||||
return { input: VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile), isTmpDestination: false }
|
|
||||||
}
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Local on object storage
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
|
|
||||||
if (videoFile.hasAudio() && !videoFile.hasVideo()) {
|
|
||||||
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
|
|
||||||
|
|
||||||
if (videoFile.isHLS()) {
|
|
||||||
await makeHLSFileAvailable(video.getHLSPlaylist(), videoFile.filename, destination)
|
|
||||||
} else {
|
|
||||||
await makeWebVideoFileAvailable(videoFile.filename, destination)
|
|
||||||
}
|
|
||||||
|
|
||||||
return { input: destination, isTmpDestination: true }
|
|
||||||
}
|
|
||||||
|
|
||||||
if (videoFile.isHLS()) {
|
|
||||||
const { stream } = await getHLSFileReadStream({
|
|
||||||
playlist: video.getHLSPlaylist().withVideo(video),
|
|
||||||
filename: videoFile.filename,
|
|
||||||
rangeHeader: undefined
|
|
||||||
})
|
|
||||||
|
|
||||||
return { input: stream, isTmpDestination: false }
|
|
||||||
}
|
|
||||||
|
|
||||||
// Web video
|
|
||||||
const { stream } = await getWebVideoFileReadStream({
|
|
||||||
filename: videoFile.filename,
|
|
||||||
rangeHeader: undefined
|
|
||||||
})
|
|
||||||
|
|
||||||
return { input: stream, isTmpDestination: false }
|
|
||||||
}
|
|
||||||
|
|
||||||
async function buildCoverInput (video: MVideoThumbnail) {
|
|
||||||
const preview = video.getPreview()
|
|
||||||
|
|
||||||
if (video.isOwned()) return { coverPath: preview?.getPath() }
|
|
||||||
|
|
||||||
if (preview.fileUrl) {
|
|
||||||
const destination = VideoPathManager.Instance.buildTMPDestination(preview.filename)
|
|
||||||
|
|
||||||
await doRequestAndSaveToFile(preview.fileUrl, destination)
|
|
||||||
|
|
||||||
return { coverPath: destination, isTmpDestination: true }
|
|
||||||
}
|
|
||||||
|
|
||||||
return { coverPath: undefined }
|
|
||||||
}
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue