server/server -> server/core

server/core/lib/object-storage/index.ts

@@ -0,0 +1,5 @@
+export * from './keys.js'
+export * from './proxy.js'
+export * from './pre-signed-urls.js'
+export * from './urls.js'
+export * from './videos.js'

server/core/lib/object-storage/keys.ts

@@ -0,0 +1,20 @@
+import { join } from 'path'
+import { MStreamingPlaylistVideo } from '@server/types/models/index.js'
+
+function generateHLSObjectStorageKey (playlist: MStreamingPlaylistVideo, filename: string) {
+  return join(generateHLSObjectBaseStorageKey(playlist), filename)
+}
+
+function generateHLSObjectBaseStorageKey (playlist: MStreamingPlaylistVideo) {
+  return join(playlist.getStringType(), playlist.Video.uuid)
+}
+
+function generateWebVideoObjectStorageKey (filename: string) {
+  return filename
+}
+
+export {
+  generateHLSObjectStorageKey,
+  generateHLSObjectBaseStorageKey,
+  generateWebVideoObjectStorageKey
+}

server/core/lib/object-storage/pre-signed-urls.ts

@@ -0,0 +1,50 @@
+import { CONFIG } from '@server/initializers/config.js'
+import { MStreamingPlaylistVideo, MVideoFile } from '@server/types/models/index.js'
+import { generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js'
+import { buildKey, getClient } from './shared/index.js'
+import { getHLSPublicFileUrl, getWebVideoPublicFileUrl } from './urls.js'
+
+export async function generateWebVideoPresignedUrl (options: {
+  file: MVideoFile
+  downloadFilename: string
+}) {
+  const { file, downloadFilename } = options
+
+  const key = generateWebVideoObjectStorageKey(file.filename)
+
+  const { GetObjectCommand } = await import('@aws-sdk/client-s3')
+  const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner')
+
+  const command = new GetObjectCommand({
+    Bucket: CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BUCKET_NAME,
+    Key: buildKey(key, CONFIG.OBJECT_STORAGE.WEB_VIDEOS),
+    ResponseContentDisposition: `attachment; filename=${downloadFilename}`
+  })
+
+  const url = await getSignedUrl(await getClient(), command, { expiresIn: 3600 * 24 })
+
+  return getWebVideoPublicFileUrl(url)
+}
+
+export async function generateHLSFilePresignedUrl (options: {
+  streamingPlaylist: MStreamingPlaylistVideo
+  file: MVideoFile
+  downloadFilename: string
+}) {
+  const { streamingPlaylist, file, downloadFilename } = options
+
+  const key = generateHLSObjectStorageKey(streamingPlaylist, file.filename)
+
+  const { GetObjectCommand } = await import('@aws-sdk/client-s3')
+  const { getSignedUrl } = await import('@aws-sdk/s3-request-presigner')
+
+  const command = new GetObjectCommand({
+    Bucket: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BUCKET_NAME,
+    Key: buildKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS),
+    ResponseContentDisposition: `attachment; filename=${downloadFilename}`
+  })
+
+  const url = await getSignedUrl(await getClient(), command, { expiresIn: 3600 * 24 })
+
+  return getHLSPublicFileUrl(url)
+}

server/core/lib/object-storage/proxy.ts

@@ -0,0 +1,100 @@
+import express from 'express'
+import { PassThrough, pipeline } from 'stream'
+import { HttpStatusCode } from '@peertube/peertube-models'
+import { buildReinjectVideoFileTokenQuery } from '@server/controllers/shared/m3u8-playlist.js'
+import { logger } from '@server/helpers/logger.js'
+import { StreamReplacer } from '@server/helpers/stream-replacer.js'
+import { MStreamingPlaylist, MVideo } from '@server/types/models/index.js'
+import { injectQueryToPlaylistUrls } from '../hls.js'
+import { getHLSFileReadStream, getWebVideoFileReadStream } from './videos.js'
+
+import type { GetObjectCommandOutput } from '@aws-sdk/client-s3'
+
+export async function proxifyWebVideoFile (options: {
+  req: express.Request
+  res: express.Response
+  filename: string
+}) {
+  const { req, res, filename } = options
+
+  logger.debug('Proxifying Web Video file %s from object storage.', filename)
+
+  try {
+    const { response: s3Response, stream } = await getWebVideoFileReadStream({
+      filename,
+      rangeHeader: req.header('range')
+    })
+
+    setS3Headers(res, s3Response)
+
+    return stream.pipe(res)
+  } catch (err) {
+    return handleObjectStorageFailure(res, err)
+  }
+}
+
+export async function proxifyHLS (options: {
+  req: express.Request
+  res: express.Response
+  playlist: MStreamingPlaylist
+  video: MVideo
+  filename: string
+  reinjectVideoFileToken: boolean
+}) {
+  const { req, res, playlist, video, filename, reinjectVideoFileToken } = options
+
+  logger.debug('Proxifying HLS file %s from object storage.', filename)
+
+  try {
+    const { response: s3Response, stream } = await getHLSFileReadStream({
+      playlist: playlist.withVideo(video),
+      filename,
+      rangeHeader: req.header('range')
+    })
+
+    setS3Headers(res, s3Response)
+
+    const streamReplacer = reinjectVideoFileToken
+      ? new StreamReplacer(line => injectQueryToPlaylistUrls(line, buildReinjectVideoFileTokenQuery(req, filename.endsWith('master.m3u8'))))
+      : new PassThrough()
+
+    return pipeline(
+      stream,
+      streamReplacer,
+      res,
+      err => {
+        if (!err) return
+
+        handleObjectStorageFailure(res, err)
+      }
+    )
+  } catch (err) {
+    return handleObjectStorageFailure(res, err)
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+function handleObjectStorageFailure (res: express.Response, err: Error) {
+  if (err.name === 'NoSuchKey') {
+    logger.debug('Could not find key in object storage to proxify private HLS video file.', { err })
+    return res.sendStatus(HttpStatusCode.NOT_FOUND_404)
+  }
+
+  logger.error('Object storage failure', { err })
+
+  return res.fail({
+    status: HttpStatusCode.INTERNAL_SERVER_ERROR_500,
+    message: err.message,
+    type: err.name
+  })
+}
+
+function setS3Headers (res: express.Response, s3Response: GetObjectCommandOutput) {
+  if (s3Response.$metadata.httpStatusCode === HttpStatusCode.PARTIAL_CONTENT_206) {
+    res.setHeader('Content-Range', s3Response.ContentRange)
+    res.status(HttpStatusCode.PARTIAL_CONTENT_206)
+  }
+}

server/core/lib/object-storage/shared/client.ts

@@ -0,0 +1,78 @@
+import type { S3Client } from '@aws-sdk/client-s3'
+import { logger } from '@server/helpers/logger.js'
+import { isProxyEnabled } from '@server/helpers/proxy.js'
+import { getAgent } from '@server/helpers/requests.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { lTags } from './logger.js'
+
+async function getProxyRequestHandler () {
+  if (!isProxyEnabled()) return null
+
+  const { agent } = getAgent()
+
+  const { NodeHttpHandler } = await import('@aws-sdk/node-http-handler')
+
+  return new NodeHttpHandler({
+    httpAgent: agent.http,
+    httpsAgent: agent.https
+  })
+}
+
+let endpointParsed: URL
+function getEndpointParsed () {
+  if (endpointParsed) return endpointParsed
+
+  endpointParsed = new URL(getEndpoint())
+
+  return endpointParsed
+}
+
+let s3ClientPromise: Promise<S3Client>
+function getClient () {
+  if (s3ClientPromise) return s3ClientPromise
+
+  s3ClientPromise = (async () => {
+    const OBJECT_STORAGE = CONFIG.OBJECT_STORAGE
+
+    const { S3Client } = await import('@aws-sdk/client-s3')
+
+    const s3Client = new S3Client({
+      endpoint: getEndpoint(),
+      region: OBJECT_STORAGE.REGION,
+      credentials: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID
+        ? {
+          accessKeyId: OBJECT_STORAGE.CREDENTIALS.ACCESS_KEY_ID,
+          secretAccessKey: OBJECT_STORAGE.CREDENTIALS.SECRET_ACCESS_KEY
+        }
+        : undefined,
+      requestHandler: await getProxyRequestHandler()
+    })
+
+    logger.info('Initialized S3 client %s with region %s.', getEndpoint(), OBJECT_STORAGE.REGION, lTags())
+
+    return s3Client
+  })()
+
+  return s3ClientPromise
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getEndpointParsed,
+  getClient
+}
+
+// ---------------------------------------------------------------------------
+
+let endpoint: string
+function getEndpoint () {
+  if (endpoint) return endpoint
+
+  const endpointConfig = CONFIG.OBJECT_STORAGE.ENDPOINT
+  endpoint = endpointConfig.startsWith('http://') || endpointConfig.startsWith('https://')
+    ? CONFIG.OBJECT_STORAGE.ENDPOINT
+    : 'https://' + CONFIG.OBJECT_STORAGE.ENDPOINT
+
+  return endpoint
+}

server/core/lib/object-storage/shared/index.ts

@@ -0,0 +1,3 @@
+export * from './client.js'
+export * from './logger.js'
+export * from './object-storage-helpers.js'

server/core/lib/object-storage/shared/logger.ts

@@ -0,0 +1,7 @@
+import { loggerTagsFactory } from '@server/helpers/logger.js'
+
+const lTags = loggerTagsFactory('object-storage')
+
+export {
+  lTags
+}

server/core/lib/object-storage/shared/object-storage-helpers.ts

@@ -0,0 +1,345 @@
+import { pipelinePromise } from '@server/helpers/core-utils.js'
+import { isArray } from '@server/helpers/custom-validators/misc.js'
+import { logger } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+import Bluebird from 'bluebird'
+import { createReadStream, createWriteStream, ReadStream } from 'fs'
+import { ensureDir } from 'fs-extra/esm'
+import { dirname } from 'path'
+import { Readable } from 'stream'
+import { getInternalUrl } from '../urls.js'
+import { getClient } from './client.js'
+import { lTags } from './logger.js'
+
+import type { _Object, CompleteMultipartUploadCommandOutput, PutObjectCommandInput, S3Client } from '@aws-sdk/client-s3'
+
+type BucketInfo = {
+  BUCKET_NAME: string
+  PREFIX?: string
+}
+
+async function listKeysOfPrefix (prefix: string, bucketInfo: BucketInfo) {
+  const s3Client = await getClient()
+
+  const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
+
+  const commandPrefix = bucketInfo.PREFIX + prefix
+  const listCommand = new ListObjectsV2Command({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Prefix: commandPrefix
+  })
+
+  const listedObjects = await s3Client.send(listCommand)
+
+  if (isArray(listedObjects.Contents) !== true) return []
+
+  return listedObjects.Contents.map(c => c.Key)
+}
+
+// ---------------------------------------------------------------------------
+
+async function storeObject (options: {
+  inputPath: string
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}): Promise<string> {
+  const { inputPath, objectStorageKey, bucketInfo, isPrivate } = options
+
+  logger.debug('Uploading file %s to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
+
+  const fileStream = createReadStream(inputPath)
+
+  return uploadToStorage({ objectStorageKey, content: fileStream, bucketInfo, isPrivate })
+}
+
+async function storeContent (options: {
+  content: string
+  inputPath: string
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}): Promise<string> {
+  const { content, objectStorageKey, bucketInfo, inputPath, isPrivate } = options
+
+  logger.debug('Uploading %s content to %s%s in bucket %s', inputPath, bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, lTags())
+
+  return uploadToStorage({ objectStorageKey, content, bucketInfo, isPrivate })
+}
+
+// ---------------------------------------------------------------------------
+
+async function updateObjectACL (options: {
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}) {
+  const { objectStorageKey, bucketInfo, isPrivate } = options
+
+  const acl = getACL(isPrivate)
+  if (!acl) return
+
+  const key = buildKey(objectStorageKey, bucketInfo)
+
+  logger.debug('Updating ACL file %s in bucket %s', key, bucketInfo.BUCKET_NAME, lTags())
+
+  const { PutObjectAclCommand } = await import('@aws-sdk/client-s3')
+
+  const command = new PutObjectAclCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: key,
+    ACL: acl
+  })
+
+  const client = await getClient()
+  await client.send(command)
+}
+
+async function updatePrefixACL (options: {
+  prefix: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}) {
+  const { prefix, bucketInfo, isPrivate } = options
+
+  const acl = getACL(isPrivate)
+  if (!acl) return
+
+  const { PutObjectAclCommand } = await import('@aws-sdk/client-s3')
+
+  logger.debug('Updating ACL of files in prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
+
+  return applyOnPrefix({
+    prefix,
+    bucketInfo,
+    commandBuilder: obj => {
+      logger.debug('Updating ACL of %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
+
+      return new PutObjectAclCommand({
+        Bucket: bucketInfo.BUCKET_NAME,
+        Key: obj.Key,
+        ACL: acl
+      })
+    }
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function removeObject (objectStorageKey: string, bucketInfo: BucketInfo) {
+  const key = buildKey(objectStorageKey, bucketInfo)
+
+  return removeObjectByFullKey(key, bucketInfo)
+}
+
+async function removeObjectByFullKey (fullKey: string, bucketInfo: BucketInfo) {
+  logger.debug('Removing file %s in bucket %s', fullKey, bucketInfo.BUCKET_NAME, lTags())
+
+  const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
+
+  const command = new DeleteObjectCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: fullKey
+  })
+
+  const client = await getClient()
+
+  return client.send(command)
+}
+
+async function removePrefix (prefix: string, bucketInfo: BucketInfo) {
+  logger.debug('Removing prefix %s in bucket %s', prefix, bucketInfo.BUCKET_NAME, lTags())
+
+  const { DeleteObjectCommand } = await import('@aws-sdk/client-s3')
+
+  return applyOnPrefix({
+    prefix,
+    bucketInfo,
+    commandBuilder: obj => {
+      logger.debug('Removing %s inside prefix %s in bucket %s', obj.Key, prefix, bucketInfo.BUCKET_NAME, lTags())
+
+      return new DeleteObjectCommand({
+        Bucket: bucketInfo.BUCKET_NAME,
+        Key: obj.Key
+      })
+    }
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+async function makeAvailable (options: {
+  key: string
+  destination: string
+  bucketInfo: BucketInfo
+}) {
+  const { key, destination, bucketInfo } = options
+
+  await ensureDir(dirname(options.destination))
+
+  const { GetObjectCommand } = await import('@aws-sdk/client-s3')
+
+  const command = new GetObjectCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: buildKey(key, bucketInfo)
+  })
+
+  const client = await getClient()
+  const response = await client.send(command)
+
+  const file = createWriteStream(destination)
+  await pipelinePromise(response.Body as Readable, file)
+
+  file.close()
+}
+
+function buildKey (key: string, bucketInfo: BucketInfo) {
+  return bucketInfo.PREFIX + key
+}
+
+// ---------------------------------------------------------------------------
+
+async function createObjectReadStream (options: {
+  key: string
+  bucketInfo: BucketInfo
+  rangeHeader: string
+}) {
+  const { key, bucketInfo, rangeHeader } = options
+
+  const { GetObjectCommand } = await import('@aws-sdk/client-s3')
+
+  const command = new GetObjectCommand({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: buildKey(key, bucketInfo),
+    Range: rangeHeader
+  })
+
+  const client = await getClient()
+  const response = await client.send(command)
+
+  return {
+    response,
+    stream: response.Body as Readable
+  }
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  type BucketInfo,
+
+  buildKey,
+
+  storeObject,
+  storeContent,
+
+  removeObject,
+  removeObjectByFullKey,
+  removePrefix,
+
+  makeAvailable,
+
+  updateObjectACL,
+  updatePrefixACL,
+
+  listKeysOfPrefix,
+  createObjectReadStream
+}
+
+// ---------------------------------------------------------------------------
+
+async function uploadToStorage (options: {
+  content: ReadStream | string
+  objectStorageKey: string
+  bucketInfo: BucketInfo
+  isPrivate: boolean
+}) {
+  const { content, objectStorageKey, bucketInfo, isPrivate } = options
+
+  const input: PutObjectCommandInput = {
+    Body: content,
+    Bucket: bucketInfo.BUCKET_NAME,
+    Key: buildKey(objectStorageKey, bucketInfo)
+  }
+
+  const acl = getACL(isPrivate)
+  if (acl) input.ACL = acl
+
+  const { Upload } = await import('@aws-sdk/lib-storage')
+
+  const parallelUploads3 = new Upload({
+    client: await getClient(),
+    queueSize: 4,
+    partSize: CONFIG.OBJECT_STORAGE.MAX_UPLOAD_PART,
+
+    // `leavePartsOnError` must be set to `true` to avoid silently dropping failed parts
+    // More detailed explanation:
+    // https://github.com/aws/aws-sdk-js-v3/blob/v3.164.0/lib/lib-storage/src/Upload.ts#L274
+    // https://github.com/aws/aws-sdk-js-v3/issues/2311#issuecomment-939413928
+    leavePartsOnError: true,
+    params: input
+  })
+
+  const response = (await parallelUploads3.done()) as CompleteMultipartUploadCommandOutput
+  // Check is needed even if the HTTP status code is 200 OK
+  // For more information, see https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
+  if (!response.Bucket) {
+    const message = `Error uploading ${objectStorageKey} to bucket ${bucketInfo.BUCKET_NAME}`
+    logger.error(message, { response, ...lTags() })
+    throw new Error(message)
+  }
+
+  logger.debug(
+    'Completed %s%s in bucket %s',
+    bucketInfo.PREFIX, objectStorageKey, bucketInfo.BUCKET_NAME, { ...lTags(), reseponseMetadata: response.$metadata }
+  )
+
+  return getInternalUrl(bucketInfo, objectStorageKey)
+}
+
+async function applyOnPrefix (options: {
+  prefix: string
+  bucketInfo: BucketInfo
+  commandBuilder: (obj: _Object) => Parameters<S3Client['send']>[0]
+
+  continuationToken?: string
+}) {
+  const { prefix, bucketInfo, commandBuilder, continuationToken } = options
+
+  const s3Client = await getClient()
+
+  const { ListObjectsV2Command } = await import('@aws-sdk/client-s3')
+
+  const commandPrefix = buildKey(prefix, bucketInfo)
+  const listCommand = new ListObjectsV2Command({
+    Bucket: bucketInfo.BUCKET_NAME,
+    Prefix: commandPrefix,
+    ContinuationToken: continuationToken
+  })
+
+  const listedObjects = await s3Client.send(listCommand)
+
+  if (isArray(listedObjects.Contents) !== true) {
+    const message = `Cannot apply function on ${commandPrefix} prefix in bucket ${bucketInfo.BUCKET_NAME}: no files listed.`
+
+    logger.error(message, { response: listedObjects, ...lTags() })
+    throw new Error(message)
+  }
+
+  await Bluebird.map(listedObjects.Contents, object => {
+    const command = commandBuilder(object)
+
+    return s3Client.send(command)
+  }, { concurrency: 10 })
+
+  // Repeat if not all objects could be listed at once (limit of 1000?)
+  if (listedObjects.IsTruncated) {
+    await applyOnPrefix({ ...options, continuationToken: listedObjects.ContinuationToken })
+  }
+}
+
+function getACL (isPrivate: boolean) {
+  return isPrivate
+    ? CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PRIVATE
+    : CONFIG.OBJECT_STORAGE.UPLOAD_ACL.PUBLIC
+}

server/core/lib/object-storage/urls.ts

@@ -0,0 +1,63 @@
+import { CONFIG } from '@server/initializers/config.js'
+import { OBJECT_STORAGE_PROXY_PATHS, WEBSERVER } from '@server/initializers/constants.js'
+import { MVideoUUID } from '@server/types/models/index.js'
+import { BucketInfo, buildKey, getEndpointParsed } from './shared/index.js'
+
+function getInternalUrl (config: BucketInfo, keyWithoutPrefix: string) {
+  return getBaseUrl(config) + buildKey(keyWithoutPrefix, config)
+}
+
+// ---------------------------------------------------------------------------
+
+function getWebVideoPublicFileUrl (fileUrl: string) {
+  const baseUrl = CONFIG.OBJECT_STORAGE.WEB_VIDEOS.BASE_URL
+  if (!baseUrl) return fileUrl
+
+  return replaceByBaseUrl(fileUrl, baseUrl)
+}
+
+function getHLSPublicFileUrl (fileUrl: string) {
+  const baseUrl = CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS.BASE_URL
+  if (!baseUrl) return fileUrl
+
+  return replaceByBaseUrl(fileUrl, baseUrl)
+}
+
+// ---------------------------------------------------------------------------
+
+function getHLSPrivateFileUrl (video: MVideoUUID, filename: string) {
+  return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + video.uuid + `/${filename}`
+}
+
+function getWebVideoPrivateFileUrl (filename: string) {
+  return WEBSERVER.URL + OBJECT_STORAGE_PROXY_PATHS.PRIVATE_WEB_VIDEOS + filename
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  getInternalUrl,
+
+  getWebVideoPublicFileUrl,
+  getHLSPublicFileUrl,
+
+  getHLSPrivateFileUrl,
+  getWebVideoPrivateFileUrl,
+
+  replaceByBaseUrl
+}
+
+// ---------------------------------------------------------------------------
+
+function getBaseUrl (bucketInfo: BucketInfo, baseUrl?: string) {
+  if (baseUrl) return baseUrl
+
+  return `${getEndpointParsed().protocol}//${bucketInfo.BUCKET_NAME}.${getEndpointParsed().host}/`
+}
+
+const regex = new RegExp('https?://[^/]+')
+function replaceByBaseUrl (fileUrl: string, baseUrl: string) {
+  if (!fileUrl) return fileUrl
+
+  return fileUrl.replace(regex, baseUrl)
+}

server/core/lib/object-storage/videos.ts

@@ -0,0 +1,197 @@
+import { basename, join } from 'path'
+import { logger } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { MStreamingPlaylistVideo, MVideo, MVideoFile } from '@server/types/models/index.js'
+import { getHLSDirectory } from '../paths.js'
+import { VideoPathManager } from '../video-path-manager.js'
+import { generateHLSObjectBaseStorageKey, generateHLSObjectStorageKey, generateWebVideoObjectStorageKey } from './keys.js'
+import {
+  createObjectReadStream,
+  listKeysOfPrefix,
+  lTags,
+  makeAvailable,
+  removeObject,
+  removeObjectByFullKey,
+  removePrefix,
+  storeContent,
+  storeObject,
+  updateObjectACL,
+  updatePrefixACL
+} from './shared/index.js'
+
+function listHLSFileKeysOf (playlist: MStreamingPlaylistVideo) {
+  return listKeysOfPrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+// ---------------------------------------------------------------------------
+
+function storeHLSFileFromFilename (playlist: MStreamingPlaylistVideo, filename: string) {
+  return storeObject({
+    inputPath: join(getHLSDirectory(playlist.Video), filename),
+    objectStorageKey: generateHLSObjectStorageKey(playlist, filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
+  })
+}
+
+function storeHLSFileFromPath (playlist: MStreamingPlaylistVideo, path: string) {
+  return storeObject({
+    inputPath: path,
+    objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
+  })
+}
+
+function storeHLSFileFromContent (playlist: MStreamingPlaylistVideo, path: string, content: string) {
+  return storeContent({
+    content,
+    inputPath: path,
+    objectStorageKey: generateHLSObjectStorageKey(playlist, basename(path)),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function storeWebVideoFile (video: MVideo, file: MVideoFile) {
+  return storeObject({
+    inputPath: VideoPathManager.Instance.getFSVideoFileOutputPath(video, file),
+    objectStorageKey: generateWebVideoObjectStorageKey(file.filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS,
+    isPrivate: video.hasPrivateStaticPath()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+async function updateWebVideoFileACL (video: MVideo, file: MVideoFile) {
+  await updateObjectACL({
+    objectStorageKey: generateWebVideoObjectStorageKey(file.filename),
+    bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS,
+    isPrivate: video.hasPrivateStaticPath()
+  })
+}
+
+async function updateHLSFilesACL (playlist: MStreamingPlaylistVideo) {
+  await updatePrefixACL({
+    prefix: generateHLSObjectBaseStorageKey(playlist),
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    isPrivate: playlist.Video.hasPrivateStaticPath()
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+function removeHLSObjectStorage (playlist: MStreamingPlaylistVideo) {
+  return removePrefix(generateHLSObjectBaseStorageKey(playlist), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+function removeHLSFileObjectStorageByFilename (playlist: MStreamingPlaylistVideo, filename: string) {
+  return removeObject(generateHLSObjectStorageKey(playlist, filename), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+function removeHLSFileObjectStorageByPath (playlist: MStreamingPlaylistVideo, path: string) {
+  return removeObject(generateHLSObjectStorageKey(playlist, basename(path)), CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+function removeHLSFileObjectStorageByFullKey (key: string) {
+  return removeObjectByFullKey(key, CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS)
+}
+
+// ---------------------------------------------------------------------------
+
+function removeWebVideoObjectStorage (videoFile: MVideoFile) {
+  return removeObject(generateWebVideoObjectStorageKey(videoFile.filename), CONFIG.OBJECT_STORAGE.WEB_VIDEOS)
+}
+
+// ---------------------------------------------------------------------------
+
+async function makeHLSFileAvailable (playlist: MStreamingPlaylistVideo, filename: string, destination: string) {
+  const key = generateHLSObjectStorageKey(playlist, filename)
+
+  logger.info('Fetching HLS file %s from object storage to %s.', key, destination, lTags())
+
+  await makeAvailable({
+    key,
+    destination,
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS
+  })
+
+  return destination
+}
+
+async function makeWebVideoFileAvailable (filename: string, destination: string) {
+  const key = generateWebVideoObjectStorageKey(filename)
+
+  logger.info('Fetching Web Video file %s from object storage to %s.', key, destination, lTags())
+
+  await makeAvailable({
+    key,
+    destination,
+    bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS
+  })
+
+  return destination
+}
+
+// ---------------------------------------------------------------------------
+
+function getWebVideoFileReadStream (options: {
+  filename: string
+  rangeHeader: string
+}) {
+  const { filename, rangeHeader } = options
+
+  const key = generateWebVideoObjectStorageKey(filename)
+
+  return createObjectReadStream({
+    key,
+    bucketInfo: CONFIG.OBJECT_STORAGE.WEB_VIDEOS,
+    rangeHeader
+  })
+}
+
+function getHLSFileReadStream (options: {
+  playlist: MStreamingPlaylistVideo
+  filename: string
+  rangeHeader: string
+}) {
+  const { playlist, filename, rangeHeader } = options
+
+  const key = generateHLSObjectStorageKey(playlist, filename)
+
+  return createObjectReadStream({
+    key,
+    bucketInfo: CONFIG.OBJECT_STORAGE.STREAMING_PLAYLISTS,
+    rangeHeader
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export {
+  listHLSFileKeysOf,
+
+  storeWebVideoFile,
+  storeHLSFileFromFilename,
+  storeHLSFileFromPath,
+  storeHLSFileFromContent,
+
+  updateWebVideoFileACL,
+  updateHLSFilesACL,
+
+  removeHLSObjectStorage,
+  removeHLSFileObjectStorageByFilename,
+  removeHLSFileObjectStorageByPath,
+  removeHLSFileObjectStorageByFullKey,
+
+  removeWebVideoObjectStorage,
+
+  makeWebVideoFileAvailable,
+  makeHLSFileAvailable,
+
+  getWebVideoFileReadStream,
+  getHLSFileReadStream
+}