yetangitu/Peertube
1
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2025-10-05 10:49:28 +02:00
Code Issues Releases Wiki Activity

Live supports object storage

Browse source 
* Sync live files (segments, master playlist, resolution playlist,
   segment sha file) into object storage
 * Automatically delete them when the live ends
 * Segment sha file is now a file on disk, and not stored in memory
   anymore
This commit is contained in:
Chocobozzz 2022-10-04 10:03:17 +02:00
parent 9c0cdc5047
commit cfd57d2ca0
No known key found for this signature in database
GPG key ID: 583A612D890159BE
21 changed files with 615 additions and 307 deletions
Download patch file Download diff file Expand all files Collapse all files

25
server/lib/object-storage/shared/object-storage-helpers.ts
View file

@ -22,6 +22,24 @@ type BucketInfo = {
PREFIX?: string
}
async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
const s3Client = getClient()
const commandPrefix = bucketInfo.PREFIX + prefix
const listCommand = new ListObjectsV2Command({
Bucket: bucketInfo.BUCKET_NAME,
Prefix: commandPrefix
})
const listedObjects = await s3Client.send(listCommand)
if (isArray(listedObjects.Contents) !== true) return []
return listedObjects.Contents.map(c => c.Key)
}
// ---------------------------------------------------------------------------
async function storeObject (options: {
inputPath: string
objectStorageKey: string
@ -36,6 +54,8 @@ async function storeObject (options: {
return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo })
}
// ---------------------------------------------------------------------------
async function removeObject (filename: string, bucketInfo: BucketInfo) {
const command = new DeleteObjectCommand({
Bucket: bucketInfo.BUCKET_NAME,
@ -89,6 +109,8 @@ async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
if (listedObjects.IsTruncated) await removePrefix(prefix, bucketInfo)
}
// ---------------------------------------------------------------------------
async function makeAvailable (options: {
key: string
destination: string
@ -122,7 +144,8 @@ export {
storeObject,
removeObject,
removePrefix,
makeAvailable
makeAvailable,
listKeysOfPrefix
}
// ---------------------------------------------------------------------------

37
server/lib/object-storage/videos.ts
View file

@ -1,19 +1,35 @@
import { join } from 'path'
import { basename, join } from 'path'
import { logger } from '@server/helpers/logger'
import { CONFIG } from '@server/initializers/config'
import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models'
import { getHLSDirectory } from '../paths'
import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebTorrentObjectStorageKey } from './keys'
import { lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
import { listKeysOfPrefix, lTags, makeAvailable, removeObject, removePrefix, storeObject } from './shared'
function storeHLSFile (playlist: MStreamingPlaylistVideo, filename: string, path?: string) {
function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
// ---------------------------------------------------------------------------
function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
return storeObject({
inputPath: path ?? join(getHLSDirectory(playlist.Video), filename),
inputPath: join(getHLSDirectory(playlist.Video), filename),
objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
})
}
function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
return storeObject({
inputPath: path,
objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
})
}
// ---------------------------------------------------------------------------
function storeWebTorrentFile (filename: string) {
return storeObject({
inputPath: join(CONFIG.STORAGE.VIDEOS_DIR, filename),
@ -22,6 +38,8 @@ function storeWebTorrentFile (filename: string) {
})
}
// ---------------------------------------------------------------------------
function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
@ -30,10 +48,14 @@ function removeHLSFileObjectStorage (playlist: MStreamingPlaylistVideo, filename
return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
}
// ---------------------------------------------------------------------------
function removeWebTorrentObjectStorage (videoFile: MVideoFile) {
return removeObject(generateWebTorrentObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.VIDEOS)
}
// ---------------------------------------------------------------------------
async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
const key = generateHLSObjectStorageKey(playlist, filename)
@ -62,9 +84,14 @@ async function makeWebTorrentFileAvailable (filename: string, destination: strin
return destination
}
// ---------------------------------------------------------------------------
export {
listHLSFileKeysOf,
storeWebTorrentFile,
storeHLSFile,
storeHLSFileFromFilename,
storeHLSFileFromPath,
removeHLSObjectStorage,
removeHLSFileObjectStorage,