yetangitu/Peertube
1
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2025-10-04 02:09:37 +02:00
Code Issues Releases Wiki Activity

Add config option to keep original video file (basic first version) (#6157)

Browse source 
* testing not removing old file and adding columb to db

* implement feature

* remove unnecessary config changes

* use only keptOriginalFileName, change keptOriginalFileName to keptOriginalFilename for consistency with with videoFile table, slight refactor with basename()

* save original video files to dedicated directory original-video-files

* begin implementing object storage (bucket) support

---------

Co-authored-by: chagai.friedlander <chagai.friedlander@fairkom.eu>
Co-authored-by: Ian <ian.kraft@hotmail.com>
Co-authored-by: Chocobozzz <me@florianbigard.com>
This commit is contained in:
chagai95 2024-03-15 15:47:18 +01:00 • committed by GitHub
parent ae31e90c30
commit e57c3024f4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
75 changed files with 1653 additions and 801 deletions
Download patch file Download diff file Expand all files Collapse all files

15
server/core/lib/object-storage/keys.ts
View file

@ -1,25 +1,22 @@
import { join } from 'path'
import { MStreamingPlaylistVideo } from '@server/types/models/index.js'
function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) {
export function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) {
return join(generateHLSObjectBaseStorageKey(playlist), filename)
}
function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) {
export function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) {
return join(playlist.getStringType(), playlist.Video.uuid)
}
function generateWebVideoObjectStorageKey (filename: string) {
export function generateWebVideoObjectStorageKey (filename: string) {
return filename
}
function generateUserExportObjectStorageKey (filename: string) {
export function generateOriginalVideoObjectStorageKey (filename: string) {
return filename
}
export {
generateHLSObjectStorageKey,
generateHLSObjectBaseStorageKey,
generateWebVideoObjectStorageKey,
generateUserExportObjectStorageKey
export function generateUserExportObjectStorageKey (filename: string) {
return filename
}

31
server/core/lib/object-storage/pre-signed-urls.ts
View file

@ -1,8 +1,14 @@
import { CONFIG } from '@server/initializers/config.js'
import { MStreamingPlaylistVideo, MUserExport, MVideoFile } from '@server/types/models/index.js'
import { generateHLSObjectStorageKey, generateUserExportObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import {
generateHLSObjectStorageKey,
generateOriginalVideoObjectStorageKey,
generateUserExportObjectStorageKey,
generateWebVideoObjectStorageKey
} from './keys.js'
import { buildKey, getClient } from './shared/index.js'
import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls.js'
import { getObjectStoragePublicFileUrl } from './urls.js'
export async function generateWebVideoPresignedUrl (options: {
file: MVideoFile
@ -16,7 +22,7 @@ export async function generateWebVideoPresignedUrl (options: {
downloadFilename
})
return getWebVideoPublicFileUrl(url)
return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
}
export async function generateHLSFilePresignedUrl (options: {
@ -32,7 +38,7 @@ export async function generateHLSFilePresignedUrl (options: {
downloadFilename
})
return getHLSPublicFileUrl(url)
return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
export async function generateUserExportPresignedUrl (options: {
@ -47,7 +53,22 @@ export async function generateUserExportPresignedUrl (options: {
downloadFilename
})
return getHLSPublicFileUrl(url)
return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.USER_EXPORTS)
}
export async function generateOriginalFilePresignedUrl (options: {
videoSource: MVideoSource
downloadFilename: string
}) {
const { videoSource, downloadFilename } = options
const url = await generatePresignedUrl({
bucket: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES.BUCKET_NAME,
key: buildKey(generateOriginalVideoObjectStorageKey(videoSource.keptOriginalFilename), CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES),
downloadFilename
})
return getObjectStoragePublicFileUrl(url, CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
}
// ---------------------------------------------------------------------------

33
server/core/lib/object-storage/urls.ts
View file

@ -1,23 +1,15 @@
import { CONFIG } from '@server/initializers/config.js'
import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants.js'
import { MVideoUUID } from '@server/types/models/index.js'
import { BucketInfo, buildKey, getEndpointParsed } from './shared/index.js'
function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
export function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
}
// ---------------------------------------------------------------------------
function getWebVideoPublicFileUrl (fileUrl: string) {
const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL
if (!baseUrl) return fileUrl
return replaceByBaseUrl(fileUrl, baseUrl)
}
function getHLSPublicFileUrl (fileUrl: string) {
const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL
export function getObjectStoragePublicFileUrl (fileUrl: string, objectStorageConfig: { BASE_URL: string }) {
const baseUrl = objectStorageConfig.BASE_URL
if (!baseUrl) return fileUrl
return replaceByBaseUrl(fileUrl, baseUrl)
@ -25,28 +17,13 @@ function getHLSPublicFileUrl (fileUrl: string) {
// ---------------------------------------------------------------------------
function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
export function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}`
}
function getWebVideoPrivateFileUrl (filename: string) {
export function getWebVideoPrivateFileUrl (filename: string) {
return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename
}
// ---------------------------------------------------------------------------
export {
getInternalUrl,
getWebVideoPublicFileUrl,
getHLSPublicFileUrl,
getHLSPrivateFileUrl,
getWebVideoPrivateFileUrl,
replaceByBaseUrl
}
// ---------------------------------------------------------------------------
function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) {

95
server/core/lib/object-storage/videos.ts
View file

@ -1,14 +1,20 @@
import { basename, join } from 'path'
import { logger } from '@server/helpers/logger.js'
import { CONFIG } from '@server/initializers/config.js'
import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models/index.js'
import { MVideoSource } from '@server/types/models/video/video-source.js'
import { basename, join } from 'path'
import { getHLSDirectory } from '../paths.js'
import { VideoPathManager } from '../video-path-manager.js'
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js'
import {
generateHLSObjectBaseStorageKey,
generateHLSObjectStorageKey,
generateOriginalVideoObjectStorageKey,
generateWebVideoObjectStorageKey
} from './keys.js'
import {
createObjectReadStream,
listKeysOfPrefix,
lTags,
listKeysOfPrefix,
makeAvailable,
removeObject,
removeObjectByFullKey,
@ -19,13 +25,13 @@ import {
updatePrefixACL
} from './shared/index.js'
function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
export function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
// ---------------------------------------------------------------------------
function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
export function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
return storeObject({
inputPath: join(getHLSDirectory(playlist.Video), filename),
objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
@ -34,7 +40,7 @@ function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename:
})
}
function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
export function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
return storeObject({
inputPath: path,
objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
@ -43,7 +49,7 @@ function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string)
})
}
function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) {
export function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) {
return storeContent({
content,
inputPath: path,
@ -55,7 +61,7 @@ function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: strin
// ---------------------------------------------------------------------------
function storeWebVideoFile (video: MVideo, file: MVideoFile) {
export function storeWebVideoFile (video: MVideo, file: MVideoFile) {
return storeObject({
inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
objectStorageKey: generateWebVideoObjectStorageKey(file.filename),
@ -66,7 +72,18 @@ function storeWebVideoFile (video: MVideo, file: MVideoFile) {
// ---------------------------------------------------------------------------
async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) {
export function storeOriginalVideoFile (inputPath: string, filename: string) {
return storeObject({
inputPath,
objectStorageKey: generateOriginalVideoObjectStorageKey(filename),
bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES,
isPrivate: true
})
}
// ---------------------------------------------------------------------------
export async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) {
await updateObjectACL({
objectStorageKey: generateWebVideoObjectStorageKey(file.filename),
bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS,
@ -74,7 +91,7 @@ async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) {
})
}
async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
export async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
await updatePrefixACL({
prefix: generateHLSObjectBaseStorageKey(playlist),
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
@ -84,31 +101,37 @@ async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
// ---------------------------------------------------------------------------
function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
export function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) {
export function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) {
return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) {
export function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) {
return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
function removeHLSFileObjectStorageByFullKey (key: string) {
export function removeHLSFileObjectStorageByFullKey (key: string) {
return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
// ---------------------------------------------------------------------------
function removeWebVideoObjectStorage (videoFile: MVideoFile) {
export function removeWebVideoObjectStorage (videoFile: MVideoFile) {
return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
}
// ---------------------------------------------------------------------------
async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
export function removeOriginalFileObjectStorage (videoSource: MVideoSource) {
return removeObject(generateOriginalVideoObjectStorageKey(videoSource.keptOriginalFilename), CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES)
}
// ---------------------------------------------------------------------------
export async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
const key = generateHLSObjectStorageKey(playlist, filename)
logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())
@ -122,7 +145,7 @@ async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename
return destination
}
async function makeWebVideoFileAvailable (filename: string, destination: string) {
export async function makeWebVideoFileAvailable (filename: string, destination: string) {
const key = generateWebVideoObjectStorageKey(filename)
logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags())
@ -138,7 +161,7 @@ async function makeWebVideoFileAvailable (filename: string, destination: string)
// ---------------------------------------------------------------------------
function getWebVideoFileReadStream (options: {
export function getWebVideoFileReadStream (options: {
filename: string
rangeHeader: string
}) {
@ -153,7 +176,7 @@ function getWebVideoFileReadStream (options: {
})
}
function getHLSFileReadStream (options: {
export function getHLSFileReadStream (options: {
playlist: MStreamingPlaylistVideo
filename: string
rangeHeader: string
@ -169,29 +192,17 @@ function getHLSFileReadStream (options: {
})
}
// ---------------------------------------------------------------------------
export function getOriginalFileReadStream (options: {
keptOriginalFilename: string
rangeHeader: string
}) {
const { keptOriginalFilename, rangeHeader } = options
export {
listHLSFileKeysOf,
const key = generateOriginalVideoObjectStorageKey(keptOriginalFilename)
storeWebVideoFile,
storeHLSFileFromFilename,
storeHLSFileFromPath,
storeHLSFileFromContent,
updateWebVideoFileACL,
updateHLSFilesACL,
removeHLSObjectStorage,
removeHLSFileObjectStorageByFilename,
removeHLSFileObjectStorageByPath,
removeHLSFileObjectStorageByFullKey,
removeWebVideoObjectStorage,
makeWebVideoFileAvailable,
makeHLSFileAvailable,
getWebVideoFileReadStream,
getHLSFileReadStream
return createObjectReadStream({
key,
bucketInfo: CONFIG.OBJECT_STORAGE.ORIGINAL_VIDEO_FILES,
rangeHeader
})
}