mirror of
https://github.com/Chocobozzz/PeerTube.git
synced 2025-10-03 09:49:20 +02:00
Add global rate limit to video download
This commit is contained in:
parent
49a6211f25
commit
25c5507a03
10 changed files with 354 additions and 235 deletions
|
@ -538,6 +538,12 @@ nsfw_flags_settings:
|
|||
# using NSFW flags (violent content, etc.) set by video authors
|
||||
enabled: true
|
||||
|
||||
download_generate_video:
|
||||
# Max parallel downloads on your instance
|
||||
# Each download spawns an ffmpeg process
|
||||
# The ffmpeg process ends when users have downloaded the entire file or cancelled the download
|
||||
max_parallel_downloads: 100
|
||||
|
||||
cache:
|
||||
previews:
|
||||
size: 500 # Max number of previews you want to cache
|
||||
|
|
|
@ -15,6 +15,9 @@ rates_limit:
|
|||
signup:
|
||||
window: 5 minutes
|
||||
max: 200
|
||||
download_generate_video:
|
||||
window: 5 seconds
|
||||
max: 500000
|
||||
|
||||
database:
|
||||
hostname: '127.0.0.1'
|
||||
|
|
|
@ -534,7 +534,13 @@ webrtc:
|
|||
nsfw_flags_settings:
|
||||
# Allow logged-in/anonymous users to have a more granular control over their NSFW policy
|
||||
# using NSFW flags (violent content, etc.) set by video authors
|
||||
enabled: false
|
||||
enabled: true
|
||||
|
||||
download_generate_video:
|
||||
# Max parallel downloads on your instance
|
||||
# Each download spawns an ffmpeg process
|
||||
# The ffmpeg process ends when users have downloaded the entire file or cancelled the download
|
||||
max_parallel_downloads: 100
|
||||
|
||||
###############################################################################
|
||||
#
|
||||
|
@ -1092,11 +1098,6 @@ client:
|
|||
# You can automatically redirect your users on this external platform when they click on the login button
|
||||
redirect_on_single_external_auth: false
|
||||
|
||||
# Allow logged-in/anonymous users to have a more granular control over their NSFW policy
|
||||
# using NSFW flags (violent content, etc.) set by video authors
|
||||
nsfw_flags_settings:
|
||||
enabled: true
|
||||
|
||||
open_in_app:
|
||||
android:
|
||||
# Use an intent URL: https://developer.chrome.com/docs/android/intents
|
||||
|
|
|
@ -123,13 +123,64 @@ describe('Test generate download API validator', function () {
|
|||
await server.videos.generateDownload({ videoId, videoFileIds, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
|
||||
})
|
||||
|
||||
it('Should suceed with the correct params', async function () {
|
||||
it('Should succeed with the correct params', async function () {
|
||||
const videoFileIds = [ audioStreamId, videoStreamIds[0] ]
|
||||
|
||||
await server.videos.generateDownload({ videoId, videoFileIds })
|
||||
})
|
||||
})
|
||||
|
||||
describe('Download rate limit', function () {
|
||||
let videoId: string
|
||||
let fileId: number
|
||||
|
||||
before(async function () {
|
||||
this.timeout(60000)
|
||||
|
||||
await server.kill()
|
||||
|
||||
await server.run({
|
||||
download_generate_video: {
|
||||
max_parallel_downloads: 2
|
||||
}
|
||||
})
|
||||
|
||||
const { uuid } = await server.videos.quickUpload({ name: 'video' })
|
||||
videoId = uuid
|
||||
await waitJobs([ server ])
|
||||
|
||||
const video = await server.videos.get({ id: uuid })
|
||||
fileId = video.files[0].id
|
||||
})
|
||||
|
||||
it('Should succeed with a single download', async function () {
|
||||
const videoFileIds = [ fileId ]
|
||||
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await server.videos.generateDownload({ videoId, videoFileIds })
|
||||
}
|
||||
})
|
||||
|
||||
it('Should fail with too many parallel downloads', async function () {
|
||||
const videoFileIds = [ fileId ]
|
||||
|
||||
const promises: Promise<any>[] = []
|
||||
|
||||
for (let i = 0; i < 10; i++) {
|
||||
promises.push(
|
||||
server.videos.generateDownload({ videoId, videoFileIds })
|
||||
.catch(err => {
|
||||
if (err.message.includes('429')) return
|
||||
|
||||
throw err
|
||||
})
|
||||
)
|
||||
}
|
||||
|
||||
await Promise.all(promises)
|
||||
})
|
||||
})
|
||||
|
||||
after(async function () {
|
||||
await cleanupTests([ server ])
|
||||
})
|
||||
|
|
|
@ -12,7 +12,7 @@ import {
|
|||
} from '@server/lib/object-storage/index.js'
|
||||
import { getFSUserExportFilePath } from '@server/lib/paths.js'
|
||||
import { Hooks } from '@server/lib/plugins/hooks.js'
|
||||
import { muxToMergeVideoFiles } from '@server/lib/video-file.js'
|
||||
import { VideoDownload } from '@server/lib/video-download.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import {
|
||||
MStreamingPlaylist,
|
||||
|
@ -27,7 +27,9 @@ import cors from 'cors'
|
|||
import express from 'express'
|
||||
import { DOWNLOAD_PATHS, WEBSERVER } from '../initializers/constants.js'
|
||||
import {
|
||||
asyncMiddleware, buildRateLimiter, optionalAuthenticate,
|
||||
asyncMiddleware,
|
||||
buildRateLimiter,
|
||||
optionalAuthenticate,
|
||||
originalVideoFileDownloadValidator,
|
||||
userExportDownloadValidator,
|
||||
videosDownloadValidator,
|
||||
|
@ -244,6 +246,13 @@ async function downloadGeneratedVideoFile (req: express.Request, res: express.Re
|
|||
|
||||
if (!checkAllowResult(res, allowParameters, allowedResult)) return
|
||||
|
||||
if (VideoDownload.totalDownloads > CONFIG.DOWNLOAD_GENERATE_VIDEO.MAX_PARALLEL_DOWNLOADS) {
|
||||
return res.fail({
|
||||
status: HttpStatusCode.TOO_MANY_REQUESTS_429,
|
||||
message: `Too many parallel downloads on this server. Please try again later.`
|
||||
})
|
||||
}
|
||||
|
||||
const maxResolutionFile = maxBy(videoFiles, 'resolution')
|
||||
|
||||
// Prefer m4a extension for the user if this is a mp4 audio file only
|
||||
|
@ -260,7 +269,7 @@ async function downloadGeneratedVideoFile (req: express.Request, res: express.Re
|
|||
|
||||
res.type(extname)
|
||||
|
||||
await muxToMergeVideoFiles({ video, videoFiles, output: res })
|
||||
await new VideoDownload({ video, videoFiles }).muxToMergeVideoFiles(res)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
|
@ -249,7 +249,8 @@ export function checkMissedConfig () {
|
|||
'live.transcoding.remote_runners.enabled',
|
||||
'storyboards.enabled',
|
||||
'webrtc.stun_servers',
|
||||
'nsfw_flags_settings.enabled'
|
||||
'nsfw_flags_settings.enabled',
|
||||
'download_generate_video.max_parallel_downloads'
|
||||
]
|
||||
|
||||
const requiredAlternatives = [
|
||||
|
|
|
@ -78,6 +78,10 @@ const CONFIG = {
|
|||
ENABLED: config.get<boolean>('nsfw_flags_settings.enabled')
|
||||
},
|
||||
|
||||
DOWNLOAD_GENERATE_VIDEO: {
|
||||
MAX_PARALLEL_DOWNLOADS: config.get<number>('download_generate_video.max_parallel_downloads')
|
||||
},
|
||||
|
||||
CLIENT: {
|
||||
VIDEOS: {
|
||||
MINIATURE: {
|
||||
|
|
|
@ -11,7 +11,7 @@ import {
|
|||
getOriginalFileReadStream,
|
||||
getWebVideoFileReadStream
|
||||
} from '@server/lib/object-storage/videos.js'
|
||||
import { muxToMergeVideoFiles } from '@server/lib/video-file.js'
|
||||
import { VideoDownload } from '@server/lib/video-download.js'
|
||||
import { VideoPathManager } from '@server/lib/video-path-manager.js'
|
||||
import { VideoCaptionModel } from '@server/models/video/video-caption.js'
|
||||
import { VideoChannelModel } from '@server/models/video/video-channel.js'
|
||||
|
@ -391,7 +391,8 @@ export class VideosExporter extends AbstractUserExporter<VideoExportJSON> {
|
|||
if (separatedAudioFile) {
|
||||
const stream = new PassThrough()
|
||||
|
||||
muxToMergeVideoFiles({ video, videoFiles: [ videoFile, separatedAudioFile ], output: stream })
|
||||
await new VideoDownload({ video, videoFiles: [ videoFile, separatedAudioFile ] })
|
||||
.muxToMergeVideoFiles(stream)
|
||||
.catch(err => logger.error('Cannot mux video files', { err }))
|
||||
|
||||
return Promise.resolve(stream)
|
||||
|
|
262
server/core/lib/video-download.ts
Normal file
262
server/core/lib/video-download.ts
Normal file
|
@ -0,0 +1,262 @@
|
|||
import { FFmpegContainer } from '@peertube/peertube-ffmpeg'
|
||||
import { FileStorage } from '@peertube/peertube-models'
|
||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/ffmpeg-options.js'
|
||||
import { logger } from '@server/helpers/logger.js'
|
||||
import { buildRequestError, doRequestAndSaveToFile, generateRequestStream } from '@server/helpers/requests.js'
|
||||
import { REQUEST_TIMEOUTS } from '@server/initializers/constants.js'
|
||||
import { MVideoFile, MVideoThumbnail } from '@server/types/models/index.js'
|
||||
import { remove } from 'fs-extra/esm'
|
||||
import { Readable, Writable } from 'stream'
|
||||
import { lTags } from './object-storage/shared/index.js'
|
||||
import {
|
||||
getHLSFileReadStream,
|
||||
getWebVideoFileReadStream,
|
||||
makeHLSFileAvailable,
|
||||
makeWebVideoFileAvailable
|
||||
} from './object-storage/videos.js'
|
||||
import { VideoPathManager } from './video-path-manager.js'
|
||||
|
||||
export class VideoDownload {
|
||||
static totalDownloads = 0
|
||||
|
||||
private readonly inputs: (string | Readable)[] = []
|
||||
|
||||
private readonly tmpDestinations: string[] = []
|
||||
private ffmpegContainer: FFmpegContainer
|
||||
|
||||
private readonly video: MVideoThumbnail
|
||||
private readonly videoFiles: MVideoFile[]
|
||||
|
||||
constructor (options: {
|
||||
video: MVideoThumbnail
|
||||
videoFiles: MVideoFile[]
|
||||
}) {
|
||||
this.video = options.video
|
||||
this.videoFiles = options.videoFiles
|
||||
}
|
||||
|
||||
async muxToMergeVideoFiles (output: Writable) {
|
||||
return new Promise<void>(async (res, rej) => {
|
||||
try {
|
||||
VideoDownload.totalDownloads++
|
||||
|
||||
const maxResolution = await this.buildMuxInputs(rej)
|
||||
|
||||
// Include cover to audio file?
|
||||
const { coverPath, isTmpDestination } = maxResolution === 0
|
||||
? await this.buildCoverInput()
|
||||
: { coverPath: undefined, isTmpDestination: false }
|
||||
|
||||
if (coverPath && isTmpDestination) {
|
||||
this.tmpDestinations.push(coverPath)
|
||||
}
|
||||
|
||||
logger.info(`Muxing files for video ${this.video.url}`, { inputs: this.inputsToLog(), ...lTags(this.video.uuid) })
|
||||
|
||||
this.ffmpegContainer = new FFmpegContainer(getFFmpegCommandWrapperOptions('vod'))
|
||||
|
||||
try {
|
||||
await this.ffmpegContainer.mergeInputs({
|
||||
inputs: this.inputs,
|
||||
output,
|
||||
logError: false,
|
||||
|
||||
// Include a cover if this is an audio file
|
||||
coverPath
|
||||
})
|
||||
|
||||
logger.info(`Mux ended for video ${this.video.url}`, { inputs: this.inputsToLog(), ...lTags(this.video.uuid) })
|
||||
|
||||
res()
|
||||
} catch (err) {
|
||||
const message = err?.message || ''
|
||||
|
||||
if (message.includes('Output stream closed')) {
|
||||
logger.info(`Client aborted mux for video ${this.video.url}`, lTags(this.video.uuid))
|
||||
return
|
||||
}
|
||||
|
||||
logger.warn(`Cannot mux files of video ${this.video.url}`, { err, inputs: this.inputsToLog(), ...lTags(this.video.uuid) })
|
||||
|
||||
if (err.inputStreamError) {
|
||||
err.inputStreamError = buildRequestError(err.inputStreamError)
|
||||
}
|
||||
|
||||
throw err
|
||||
} finally {
|
||||
this.ffmpegContainer.forceKill()
|
||||
}
|
||||
} catch (err) {
|
||||
rej(err)
|
||||
} finally {
|
||||
this.cleanup()
|
||||
.catch(cleanupErr => logger.error('Cannot cleanup after mux error', { err: cleanupErr, ...lTags(this.video.uuid) }))
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Build mux inputs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
private async buildMuxInputs (rej: (err: Error) => void) {
|
||||
let maxResolution = 0
|
||||
|
||||
for (const videoFile of this.videoFiles) {
|
||||
if (!videoFile) continue
|
||||
|
||||
maxResolution = Math.max(maxResolution, videoFile.resolution)
|
||||
|
||||
const { input, isTmpDestination } = await this.buildMuxInput(
|
||||
videoFile,
|
||||
err => {
|
||||
logger.warn(`Cannot build mux input of video ${this.video.url}`, {
|
||||
err,
|
||||
inputs: this.inputsToLog(),
|
||||
...lTags(this.video.uuid)
|
||||
})
|
||||
|
||||
this.cleanup()
|
||||
.catch(cleanupErr => logger.error('Cannot cleanup after mux error', { err: cleanupErr, ...lTags(this.video.uuid) }))
|
||||
|
||||
rej(buildRequestError(err as any))
|
||||
}
|
||||
)
|
||||
|
||||
this.inputs.push(input)
|
||||
|
||||
if (isTmpDestination === true) this.tmpDestinations.push(input)
|
||||
}
|
||||
|
||||
return maxResolution
|
||||
}
|
||||
|
||||
private async buildMuxInput (
|
||||
videoFile: MVideoFile,
|
||||
onStreamError: (err: Error) => void
|
||||
): Promise<{ input: Readable, isTmpDestination: false } | { input: string, isTmpDestination: boolean }> {
|
||||
// Remote
|
||||
if (this.video.remote === true) {
|
||||
return this.buildMuxRemoteInput(videoFile, onStreamError)
|
||||
}
|
||||
|
||||
// Local on FS
|
||||
if (videoFile.storage === FileStorage.FILE_SYSTEM) {
|
||||
return this.buildMuxLocalFSInput(videoFile)
|
||||
}
|
||||
|
||||
// Local on object storage
|
||||
return this.buildMuxLocalObjectStorageInput(videoFile)
|
||||
}
|
||||
|
||||
private async buildMuxRemoteInput (videoFile: MVideoFile, onStreamError: (err: Error) => void) {
|
||||
const timeout = REQUEST_TIMEOUTS.VIDEO_FILE
|
||||
|
||||
const videoSizeKB = videoFile.size / 1000
|
||||
const bodyKBLimit = videoSizeKB + 0.1 * videoSizeKB
|
||||
|
||||
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
|
||||
if (videoFile.isAudio()) {
|
||||
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
|
||||
|
||||
// > 1GB
|
||||
if (bodyKBLimit > 1000 * 1000) {
|
||||
throw new Error('Cannot download remote video file > 1GB')
|
||||
}
|
||||
|
||||
await doRequestAndSaveToFile(videoFile.fileUrl, destination, { timeout, bodyKBLimit })
|
||||
|
||||
return { input: destination, isTmpDestination: true as const }
|
||||
}
|
||||
|
||||
return {
|
||||
input: generateRequestStream(videoFile.fileUrl, { timeout, bodyKBLimit }).on('error', onStreamError),
|
||||
isTmpDestination: false as const
|
||||
}
|
||||
}
|
||||
|
||||
private buildMuxLocalFSInput (videoFile: MVideoFile) {
|
||||
return { input: VideoPathManager.Instance.getFSVideoFileOutputPath(this.video, videoFile), isTmpDestination: false }
|
||||
}
|
||||
|
||||
private async buildMuxLocalObjectStorageInput (videoFile: MVideoFile) {
|
||||
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
|
||||
if (videoFile.hasAudio() && !videoFile.hasVideo()) {
|
||||
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
|
||||
|
||||
if (videoFile.isHLS()) {
|
||||
await makeHLSFileAvailable(this.video.getHLSPlaylist(), videoFile.filename, destination)
|
||||
} else {
|
||||
await makeWebVideoFileAvailable(videoFile.filename, destination)
|
||||
}
|
||||
|
||||
return { input: destination, isTmpDestination: true as const }
|
||||
}
|
||||
|
||||
if (videoFile.isHLS()) {
|
||||
const { stream } = await getHLSFileReadStream({
|
||||
playlist: this.video.getHLSPlaylist().withVideo(this.video),
|
||||
filename: videoFile.filename,
|
||||
rangeHeader: undefined
|
||||
})
|
||||
|
||||
return { input: stream, isTmpDestination: false as const }
|
||||
}
|
||||
|
||||
// Web video
|
||||
const { stream } = await getWebVideoFileReadStream({
|
||||
filename: videoFile.filename,
|
||||
rangeHeader: undefined
|
||||
})
|
||||
|
||||
return { input: stream, isTmpDestination: false as const }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
private async buildCoverInput () {
|
||||
const preview = this.video.getPreview()
|
||||
|
||||
if (this.video.isOwned()) return { coverPath: preview?.getPath() }
|
||||
|
||||
if (preview.fileUrl) {
|
||||
const destination = VideoPathManager.Instance.buildTMPDestination(preview.filename)
|
||||
|
||||
await doRequestAndSaveToFile(preview.fileUrl, destination)
|
||||
|
||||
return { coverPath: destination, isTmpDestination: true }
|
||||
}
|
||||
|
||||
return { coverPath: undefined }
|
||||
}
|
||||
|
||||
private inputsToLog () {
|
||||
return this.inputs.map(i => {
|
||||
if (typeof i === 'string') return i
|
||||
|
||||
return 'ReadableStream'
|
||||
})
|
||||
}
|
||||
|
||||
private async cleanup () {
|
||||
VideoDownload.totalDownloads--
|
||||
|
||||
for (const destination of this.tmpDestinations) {
|
||||
await remove(destination)
|
||||
.catch(err => logger.error('Cannot remove tmp destination', { err, destination, ...lTags(this.video.uuid) }))
|
||||
}
|
||||
|
||||
for (const input of this.inputs) {
|
||||
if (input instanceof Readable) {
|
||||
if (!input.destroyed) input.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
if (this.ffmpegContainer) {
|
||||
this.ffmpegContainer.forceKill()
|
||||
this.ffmpegContainer = undefined
|
||||
}
|
||||
|
||||
logger.debug(`Cleaned muxing for video ${this.video.url}`, { inputs: this.inputsToLog(), ...lTags(this.video.uuid) })
|
||||
}
|
||||
}
|
|
@ -1,5 +1,4 @@
|
|||
import {
|
||||
FFmpegContainer,
|
||||
ffprobePromise,
|
||||
getVideoStreamDimensionsInfo,
|
||||
getVideoStreamFPS,
|
||||
|
@ -9,25 +8,15 @@ import {
|
|||
} from '@peertube/peertube-ffmpeg'
|
||||
import { FileStorage, VideoFileFormatFlag, VideoFileMetadata, VideoFileStream, VideoResolution } from '@peertube/peertube-models'
|
||||
import { getFileSize, getLowercaseExtension } from '@peertube/peertube-node-utils'
|
||||
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/ffmpeg-options.js'
|
||||
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
|
||||
import { buildRequestError, doRequestAndSaveToFile, generateRequestStream } from '@server/helpers/requests.js'
|
||||
import { CONFIG } from '@server/initializers/config.js'
|
||||
import { MIMETYPES, REQUEST_TIMEOUTS } from '@server/initializers/constants.js'
|
||||
import { MIMETYPES } from '@server/initializers/constants.js'
|
||||
import { VideoFileModel } from '@server/models/video/video-file.js'
|
||||
import { VideoSourceModel } from '@server/models/video/video-source.js'
|
||||
import { MVideo, MVideoFile, MVideoId, MVideoThumbnail, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { MVideo, MVideoFile, MVideoId, MVideoWithAllFiles } from '@server/types/models/index.js'
|
||||
import { FfprobeData } from 'fluent-ffmpeg'
|
||||
import { move, remove } from 'fs-extra/esm'
|
||||
import { Readable, Writable } from 'stream'
|
||||
import { lTags } from './object-storage/shared/index.js'
|
||||
import {
|
||||
getHLSFileReadStream,
|
||||
getWebVideoFileReadStream,
|
||||
makeHLSFileAvailable,
|
||||
makeWebVideoFileAvailable,
|
||||
storeOriginalVideoFile
|
||||
} from './object-storage/videos.js'
|
||||
import { storeOriginalVideoFile } from './object-storage/videos.js'
|
||||
import { generateHLSVideoFilename, generateWebVideoFilename } from './paths.js'
|
||||
import { VideoPathManager } from './video-path-manager.js'
|
||||
|
||||
|
@ -260,211 +249,3 @@ export async function saveNewOriginalFileIfNeeded (video: MVideo, videoFile: MVi
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export async function muxToMergeVideoFiles (options: {
|
||||
video: MVideoThumbnail
|
||||
videoFiles: MVideoFile[]
|
||||
output: Writable
|
||||
}) {
|
||||
const { video, videoFiles, output } = options
|
||||
|
||||
const inputs: (string | Readable)[] = []
|
||||
const tmpDestinations: string[] = []
|
||||
let ffmpegContainer: FFmpegContainer
|
||||
|
||||
return new Promise<void>(async (res, rej) => {
|
||||
const cleanup = async () => {
|
||||
for (const destination of tmpDestinations) {
|
||||
await remove(destination)
|
||||
}
|
||||
|
||||
for (const input of inputs) {
|
||||
if (input instanceof Readable) {
|
||||
if (!input.destroyed) input.destroy()
|
||||
}
|
||||
}
|
||||
|
||||
if (ffmpegContainer) {
|
||||
ffmpegContainer.forceKill()
|
||||
ffmpegContainer = undefined
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
let maxResolution = 0
|
||||
|
||||
for (const videoFile of videoFiles) {
|
||||
if (!videoFile) continue
|
||||
|
||||
maxResolution = Math.max(maxResolution, videoFile.resolution)
|
||||
|
||||
const { input, isTmpDestination } = await buildMuxInput(
|
||||
video,
|
||||
videoFile,
|
||||
err => {
|
||||
logger.warn(`Cannot build mux input of video ${video.url}`, { err, inputs: inputsToLog, ...lTags(video.uuid) })
|
||||
|
||||
cleanup()
|
||||
.catch(cleanupErr => logger.error('Cannot cleanup after mux error', { err: cleanupErr, ...lTags(video.uuid) }))
|
||||
|
||||
rej(buildRequestError(err as any))
|
||||
}
|
||||
)
|
||||
|
||||
inputs.push(input)
|
||||
|
||||
if (isTmpDestination === true) tmpDestinations.push(input)
|
||||
}
|
||||
|
||||
// Include cover to audio file?
|
||||
const { coverPath, isTmpDestination } = maxResolution === 0
|
||||
? await buildCoverInput(video)
|
||||
: { coverPath: undefined, isTmpDestination: false }
|
||||
|
||||
if (coverPath && isTmpDestination) tmpDestinations.push(coverPath)
|
||||
|
||||
const inputsToLog = inputs.map(i => {
|
||||
if (typeof i === 'string') return i
|
||||
|
||||
return 'ReadableStream'
|
||||
})
|
||||
|
||||
logger.info(`Muxing files for video ${video.url}`, { inputs: inputsToLog, ...lTags(video.uuid) })
|
||||
|
||||
ffmpegContainer = new FFmpegContainer(getFFmpegCommandWrapperOptions('vod'))
|
||||
|
||||
try {
|
||||
await ffmpegContainer.mergeInputs({
|
||||
inputs,
|
||||
output,
|
||||
logError: false,
|
||||
|
||||
// Include a cover if this is an audio file
|
||||
coverPath
|
||||
})
|
||||
|
||||
logger.info(`Mux ended for video ${video.url}`, { inputs: inputsToLog, ...lTags(video.uuid) })
|
||||
|
||||
res()
|
||||
} catch (err) {
|
||||
const message = err?.message || ''
|
||||
|
||||
if (message.includes('Output stream closed')) {
|
||||
logger.info(`Client aborted mux for video ${video.url}`, lTags(video.uuid))
|
||||
return
|
||||
}
|
||||
|
||||
logger.warn(`Cannot mux files of video ${video.url}`, { err, inputs: inputsToLog, ...lTags(video.uuid) })
|
||||
|
||||
if (err.inputStreamError) {
|
||||
err.inputStreamError = buildRequestError(err.inputStreamError)
|
||||
}
|
||||
|
||||
throw err
|
||||
} finally {
|
||||
ffmpegContainer.forceKill()
|
||||
}
|
||||
} catch (err) {
|
||||
rej(err)
|
||||
} finally {
|
||||
await cleanup()
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
async function buildMuxInput (
|
||||
video: MVideo,
|
||||
videoFile: MVideoFile,
|
||||
onStreamError: (err: Error) => void
|
||||
): Promise<{ input: Readable, isTmpDestination: false } | { input: string, isTmpDestination: boolean }> {
|
||||
// ---------------------------------------------------------------------------
|
||||
// Remote
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
if (video.remote === true) {
|
||||
const timeout = REQUEST_TIMEOUTS.VIDEO_FILE
|
||||
|
||||
const videoSizeKB = videoFile.size / 1000
|
||||
const bodyKBLimit = videoSizeKB + 0.1 * videoSizeKB
|
||||
|
||||
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
|
||||
if (videoFile.isAudio()) {
|
||||
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
|
||||
|
||||
// > 1GB
|
||||
if (bodyKBLimit > 1000 * 1000) {
|
||||
throw new Error('Cannot download remote video file > 1GB')
|
||||
}
|
||||
|
||||
await doRequestAndSaveToFile(videoFile.fileUrl, destination, { timeout, bodyKBLimit })
|
||||
|
||||
return { input: destination, isTmpDestination: true }
|
||||
}
|
||||
|
||||
return {
|
||||
input: generateRequestStream(videoFile.fileUrl, { timeout, bodyKBLimit }).on('error', onStreamError),
|
||||
isTmpDestination: false
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Local on FS
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
if (videoFile.storage === FileStorage.FILE_SYSTEM) {
|
||||
return { input: VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile), isTmpDestination: false }
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Local on object storage
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
|
||||
if (videoFile.hasAudio() && !videoFile.hasVideo()) {
|
||||
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
|
||||
|
||||
if (videoFile.isHLS()) {
|
||||
await makeHLSFileAvailable(video.getHLSPlaylist(), videoFile.filename, destination)
|
||||
} else {
|
||||
await makeWebVideoFileAvailable(videoFile.filename, destination)
|
||||
}
|
||||
|
||||
return { input: destination, isTmpDestination: true }
|
||||
}
|
||||
|
||||
if (videoFile.isHLS()) {
|
||||
const { stream } = await getHLSFileReadStream({
|
||||
playlist: video.getHLSPlaylist().withVideo(video),
|
||||
filename: videoFile.filename,
|
||||
rangeHeader: undefined
|
||||
})
|
||||
|
||||
return { input: stream, isTmpDestination: false }
|
||||
}
|
||||
|
||||
// Web video
|
||||
const { stream } = await getWebVideoFileReadStream({
|
||||
filename: videoFile.filename,
|
||||
rangeHeader: undefined
|
||||
})
|
||||
|
||||
return { input: stream, isTmpDestination: false }
|
||||
}
|
||||
|
||||
async function buildCoverInput (video: MVideoThumbnail) {
|
||||
const preview = video.getPreview()
|
||||
|
||||
if (video.isOwned()) return { coverPath: preview?.getPath() }
|
||||
|
||||
if (preview.fileUrl) {
|
||||
const destination = VideoPathManager.Instance.buildTMPDestination(preview.filename)
|
||||
|
||||
await doRequestAndSaveToFile(preview.fileUrl, destination)
|
||||
|
||||
return { coverPath: destination, isTmpDestination: true }
|
||||
}
|
||||
|
||||
return { coverPath: undefined }
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue