Limit user tokens cache

server/lib/files-cache/abstract-video-static-file-cache.ts

@@ -0,0 +1,52 @@
+import * as AsyncLRU from 'async-lru'
+import { createWriteStream, remove } from 'fs-extra'
+import { logger } from '../../helpers/logger'
+import { VideoModel } from '../../models/video/video'
+import { fetchRemoteVideoStaticFile } from '../activitypub'
+
+export abstract class AbstractVideoStaticFileCache <T> {
+
+  protected lru
+
+  abstract getFilePath (params: T): Promise<string>
+
+  // Load and save the remote file, then return the local path from filesystem
+  protected abstract loadRemoteFile (key: string): Promise<string>
+
+  init (max: number, maxAge: number) {
+    this.lru = new AsyncLRU({
+      max,
+      maxAge,
+      load: (key, cb) => {
+        this.loadRemoteFile(key)
+          .then(res => cb(null, res))
+          .catch(err => cb(err))
+      }
+    })
+
+    this.lru.on('evict', (obj: { key: string, value: string }) => {
+      remove(obj.value)
+        .then(() => logger.debug('%s evicted from %s', obj.value, this.constructor.name))
+    })
+  }
+
+  protected loadFromLRU (key: string) {
+    return new Promise<string>((res, rej) => {
+      this.lru.get(key, (err, value) => {
+        err ? rej(err) : res(value)
+      })
+    })
+  }
+
+  protected saveRemoteVideoFileAndReturnPath (video: VideoModel, remoteStaticPath: string, destPath: string) {
+    return new Promise<string>((res, rej) => {
+      const req = fetchRemoteVideoStaticFile(video, remoteStaticPath, rej)
+
+      const stream = createWriteStream(destPath)
+
+      req.pipe(stream)
+         .on('error', (err) => rej(err))
+         .on('finish', () => res(destPath))
+    })
+  }
+}

server/lib/files-cache/actor-follow-score-cache.ts

@@ -0,0 +1,46 @@
+import { ACTOR_FOLLOW_SCORE } from '../../initializers'
+import { logger } from '../../helpers/logger'
+
+// Cache follows scores, instead of writing them too often in database
+// Keep data in memory, we don't really need Redis here as we don't really care to loose some scores
+class ActorFollowScoreCache {
+
+  private static instance: ActorFollowScoreCache
+  private pendingFollowsScore: { [ url: string ]: number } = {}
+
+  private constructor () {}
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  updateActorFollowsScore (goodInboxes: string[], badInboxes: string[]) {
+    if (goodInboxes.length === 0 && badInboxes.length === 0) return
+
+    logger.info('Updating %d good actor follows and %d bad actor follows scores in cache.', goodInboxes.length, badInboxes.length)
+
+    for (const goodInbox of goodInboxes) {
+      if (this.pendingFollowsScore[goodInbox] === undefined) this.pendingFollowsScore[goodInbox] = 0
+
+      this.pendingFollowsScore[goodInbox] += ACTOR_FOLLOW_SCORE.BONUS
+    }
+
+    for (const badInbox of badInboxes) {
+      if (this.pendingFollowsScore[badInbox] === undefined) this.pendingFollowsScore[badInbox] = 0
+
+      this.pendingFollowsScore[badInbox] += ACTOR_FOLLOW_SCORE.PENALTY
+    }
+  }
+
+  getPendingFollowsScoreCopy () {
+    return this.pendingFollowsScore
+  }
+
+  clearPendingFollowsScore () {
+    this.pendingFollowsScore = {}
+  }
+}
+
+export {
+  ActorFollowScoreCache
+}

server/lib/files-cache/index.ts

@@ -0,0 +1,3 @@
+export * from './actor-follow-score-cache'
+export * from './videos-preview-cache'
+export * from './videos-caption-cache'

server/lib/files-cache/videos-caption-cache.ts

@@ -0,0 +1,53 @@
+import { join } from 'path'
+import { FILES_CACHE, CONFIG } from '../../initializers'
+import { VideoModel } from '../../models/video/video'
+import { VideoCaptionModel } from '../../models/video/video-caption'
+import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+
+type GetPathParam = { videoId: string, language: string }
+
+class VideosCaptionCache extends AbstractVideoStaticFileCache <GetPathParam> {
+
+  private static readonly KEY_DELIMITER = '%'
+  private static instance: VideosCaptionCache
+
+  private constructor () {
+    super()
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  async getFilePath (params: GetPathParam) {
+    const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(params.videoId, params.language)
+    if (!videoCaption) return undefined
+
+    if (videoCaption.isOwned()) return join(CONFIG.STORAGE.CAPTIONS_DIR, videoCaption.getCaptionName())
+
+    const key = params.videoId + VideosCaptionCache.KEY_DELIMITER + params.language
+    return this.loadFromLRU(key)
+  }
+
+  protected async loadRemoteFile (key: string) {
+    const [ videoId, language ] = key.split(VideosCaptionCache.KEY_DELIMITER)
+
+    const videoCaption = await VideoCaptionModel.loadByVideoIdAndLanguage(videoId, language)
+    if (!videoCaption) return undefined
+
+    if (videoCaption.isOwned()) throw new Error('Cannot load remote caption of owned video.')
+
+    // Used to fetch the path
+    const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(videoId)
+    if (!video) return undefined
+
+    const remoteStaticPath = videoCaption.getCaptionStaticPath()
+    const destPath = join(FILES_CACHE.VIDEO_CAPTIONS.DIRECTORY, videoCaption.getCaptionName())
+
+    return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
+  }
+}
+
+export {
+  VideosCaptionCache
+}

server/lib/files-cache/videos-preview-cache.ts

@@ -0,0 +1,42 @@
+import { join } from 'path'
+import { FILES_CACHE, CONFIG, STATIC_PATHS } from '../../initializers'
+import { VideoModel } from '../../models/video/video'
+import { AbstractVideoStaticFileCache } from './abstract-video-static-file-cache'
+
+class VideosPreviewCache extends AbstractVideoStaticFileCache <string> {
+
+  private static instance: VideosPreviewCache
+
+  private constructor () {
+    super()
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+
+  async getFilePath (videoUUID: string) {
+    const video = await VideoModel.loadByUUIDWithFile(videoUUID)
+    if (!video) return undefined
+
+    if (video.isOwned()) return join(CONFIG.STORAGE.PREVIEWS_DIR, video.getPreviewName())
+
+    return this.loadFromLRU(videoUUID)
+  }
+
+  protected async loadRemoteFile (key: string) {
+    const video = await VideoModel.loadAndPopulateAccountAndServerAndTags(key)
+    if (!video) return undefined
+
+    if (video.isOwned()) throw new Error('Cannot load remote preview of owned video.')
+
+    const remoteStaticPath = join(STATIC_PATHS.PREVIEWS, video.getPreviewName())
+    const destPath = join(FILES_CACHE.PREVIEWS.DIRECTORY, video.getPreviewName())
+
+    return this.saveRemoteVideoFileAndReturnPath(video, remoteStaticPath, destPath)
+  }
+}
+
+export {
+  VideosPreviewCache
+}