yetangitu/Peertube
1
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2025-10-03 09:49:20 +02:00
Code Issues Releases Wiki Activity

Add global rate limit to video download

Browse source
This commit is contained in:
Chocobozzz 2025-05-14 15:16:08 +02:00
parent 49a6211f25
commit 25c5507a03
No known key found for this signature in database
GPG key ID: 583A612D890159BE
10 changed files with 354 additions and 235 deletions
Download patch file Download diff file Expand all files Collapse all files

225
server/core/lib/video-file.ts
View file

@ -1,5 +1,4 @@
import {
FFmpegContainer,
ffprobePromise,
getVideoStreamDimensionsInfo,
getVideoStreamFPS,
@ -9,25 +8,15 @@ import {
} from '@peertube/peertube-ffmpeg'
import { FileStorage, VideoFileFormatFlag, VideoFileMetadata, VideoFileStream, VideoResolution } from '@peertube/peertube-models'
import { getFileSize, getLowercaseExtension } from '@peertube/peertube-node-utils'
import { getFFmpegCommandWrapperOptions } from '@server/helpers/ffmpeg/ffmpeg-options.js'
import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
import { buildRequestError, doRequestAndSaveToFile, generateRequestStream } from '@server/helpers/requests.js'
import { CONFIG } from '@server/initializers/config.js'
import { MIMETYPES, REQUEST_TIMEOUTS } from '@server/initializers/constants.js'
import { MIMETYPES } from '@server/initializers/constants.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoSourceModel } from '@server/models/video/video-source.js'
import { MVideo, MVideoFile, MVideoId, MVideoThumbnail, MVideoWithAllFiles } from '@server/types/models/index.js'
import { MVideo, MVideoFile, MVideoId, MVideoWithAllFiles } from '@server/types/models/index.js'
import { FfprobeData } from 'fluent-ffmpeg'
import { move, remove } from 'fs-extra/esm'
import { Readable, Writable } from 'stream'
import { lTags } from './object-storage/shared/index.js'
import {
getHLSFileReadStream,
getWebVideoFileReadStream,
makeHLSFileAvailable,
makeWebVideoFileAvailable,
storeOriginalVideoFile
} from './object-storage/videos.js'
import { storeOriginalVideoFile } from './object-storage/videos.js'
import { generateHLSVideoFilename, generateWebVideoFilename } from './paths.js'
import { VideoPathManager } from './video-path-manager.js'
@ -260,211 +249,3 @@ export async function saveNewOriginalFileIfNeeded (video: MVideo, videoFile: MVi
}
}
}
// ---------------------------------------------------------------------------
export async function muxToMergeVideoFiles (options: {
video: MVideoThumbnail
videoFiles: MVideoFile[]
output: Writable
}) {
const { video, videoFiles, output } = options
const inputs: (string | Readable)[] = []
const tmpDestinations: string[] = []
let ffmpegContainer: FFmpegContainer
return new Promise<void>(async (res, rej) => {
const cleanup = async () => {
for (const destination of tmpDestinations) {
await remove(destination)
}
for (const input of inputs) {
if (input instanceof Readable) {
if (!input.destroyed) input.destroy()
}
}
if (ffmpegContainer) {
ffmpegContainer.forceKill()
ffmpegContainer = undefined
}
}
try {
let maxResolution = 0
for (const videoFile of videoFiles) {
if (!videoFile) continue
maxResolution = Math.max(maxResolution, videoFile.resolution)
const { input, isTmpDestination } = await buildMuxInput(
video,
videoFile,
err => {
logger.warn(`Cannot build mux input of video ${video.url}`, { err, inputs: inputsToLog, ...lTags(video.uuid) })
cleanup()
.catch(cleanupErr => logger.error('Cannot cleanup after mux error', { err: cleanupErr, ...lTags(video.uuid) }))
rej(buildRequestError(err as any))
}
)
inputs.push(input)
if (isTmpDestination === true) tmpDestinations.push(input)
}
// Include cover to audio file?
const { coverPath, isTmpDestination } = maxResolution === 0
? await buildCoverInput(video)
: { coverPath: undefined, isTmpDestination: false }
if (coverPath && isTmpDestination) tmpDestinations.push(coverPath)
const inputsToLog = inputs.map(i => {
if (typeof i === 'string') return i
return 'ReadableStream'
})
logger.info(`Muxing files for video ${video.url}`, { inputs: inputsToLog, ...lTags(video.uuid) })
ffmpegContainer = new FFmpegContainer(getFFmpegCommandWrapperOptions('vod'))
try {
await ffmpegContainer.mergeInputs({
inputs,
output,
logError: false,
// Include a cover if this is an audio file
coverPath
})
logger.info(`Mux ended for video ${video.url}`, { inputs: inputsToLog, ...lTags(video.uuid) })
res()
} catch (err) {
const message = err?.message || ''
if (message.includes('Output stream closed')) {
logger.info(`Client aborted mux for video ${video.url}`, lTags(video.uuid))
return
}
logger.warn(`Cannot mux files of video ${video.url}`, { err, inputs: inputsToLog, ...lTags(video.uuid) })
if (err.inputStreamError) {
err.inputStreamError = buildRequestError(err.inputStreamError)
}
throw err
} finally {
ffmpegContainer.forceKill()
}
} catch (err) {
rej(err)
} finally {
await cleanup()
}
})
}
async function buildMuxInput (
video: MVideo,
videoFile: MVideoFile,
onStreamError: (err: Error) => void
): Promise<{ input: Readable, isTmpDestination: false } | { input: string, isTmpDestination: boolean }> {
// ---------------------------------------------------------------------------
// Remote
// ---------------------------------------------------------------------------
if (video.remote === true) {
const timeout = REQUEST_TIMEOUTS.VIDEO_FILE
const videoSizeKB = videoFile.size / 1000
const bodyKBLimit = videoSizeKB + 0.1 * videoSizeKB
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
if (videoFile.isAudio()) {
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
// > 1GB
if (bodyKBLimit > 1000 * 1000) {
throw new Error('Cannot download remote video file > 1GB')
}
await doRequestAndSaveToFile(videoFile.fileUrl, destination, { timeout, bodyKBLimit })
return { input: destination, isTmpDestination: true }
}
return {
input: generateRequestStream(videoFile.fileUrl, { timeout, bodyKBLimit }).on('error', onStreamError),
isTmpDestination: false
}
}
// ---------------------------------------------------------------------------
// Local on FS
// ---------------------------------------------------------------------------
if (videoFile.storage === FileStorage.FILE_SYSTEM) {
return { input: VideoPathManager.Instance.getFSVideoFileOutputPath(video, videoFile), isTmpDestination: false }
}
// ---------------------------------------------------------------------------
// Local on object storage
// ---------------------------------------------------------------------------
// FFmpeg doesn't support multiple input streams, so download the audio file on disk directly
if (videoFile.hasAudio() && !videoFile.hasVideo()) {
const destination = VideoPathManager.Instance.buildTMPDestination(videoFile.filename)
if (videoFile.isHLS()) {
await makeHLSFileAvailable(video.getHLSPlaylist(), videoFile.filename, destination)
} else {
await makeWebVideoFileAvailable(videoFile.filename, destination)
}
return { input: destination, isTmpDestination: true }
}
if (videoFile.isHLS()) {
const { stream } = await getHLSFileReadStream({
playlist: video.getHLSPlaylist().withVideo(video),
filename: videoFile.filename,
rangeHeader: undefined
})
return { input: stream, isTmpDestination: false }
}
// Web video
const { stream } = await getWebVideoFileReadStream({
filename: videoFile.filename,
rangeHeader: undefined
})
return { input: stream, isTmpDestination: false }
}
async function buildCoverInput (video: MVideoThumbnail) {
const preview = video.getPreview()
if (video.isOwned()) return { coverPath: preview?.getPath() }
if (preview.fileUrl) {
const destination = VideoPathManager.Instance.buildTMPDestination(preview.filename)
await doRequestAndSaveToFile(preview.fileUrl, destination)
return { coverPath: destination, isTmpDestination: true }
}
return { coverPath: undefined }
}