server/server -> server/core

server/core/lib/worker/parent-process.ts

@@ -0,0 +1,77 @@
+import { join } from 'path'
+import Piscina from 'piscina'
+import { JOB_CONCURRENCY, WORKER_THREADS } from '@server/initializers/constants.js'
+import httpBroadcast from './workers/http-broadcast.js'
+import downloadImage from './workers/image-downloader.js'
+import processImage from './workers/image-processor.js'
+
+let downloadImageWorker: Piscina
+
+function downloadImageFromWorker (options: Parameters<typeof downloadImage>[0]): Promise<ReturnType<typeof downloadImage>> {
+  if (!downloadImageWorker) {
+    downloadImageWorker = new Piscina({
+      filename: new URL(join('workers', 'image-downloader.js'), import.meta.url).href,
+      concurrentTasksPerWorker: WORKER_THREADS.DOWNLOAD_IMAGE.CONCURRENCY,
+      maxThreads: WORKER_THREADS.DOWNLOAD_IMAGE.MAX_THREADS
+    })
+  }
+
+  return downloadImageWorker.run(options)
+}
+
+// ---------------------------------------------------------------------------
+
+let processImageWorker: Piscina
+
+function processImageFromWorker (options: Parameters<typeof processImage>[0]): Promise<ReturnType<typeof processImage>> {
+  if (!processImageWorker) {
+    processImageWorker = new Piscina({
+      filename: new URL(join('workers', 'image-processor.js'), import.meta.url).href,
+      concurrentTasksPerWorker: WORKER_THREADS.PROCESS_IMAGE.CONCURRENCY,
+      maxThreads: WORKER_THREADS.PROCESS_IMAGE.MAX_THREADS
+    })
+  }
+
+  return processImageWorker.run(options)
+}
+
+// ---------------------------------------------------------------------------
+
+let parallelHTTPBroadcastWorker: Piscina
+
+function parallelHTTPBroadcastFromWorker (options: Parameters<typeof httpBroadcast>[0]): Promise<ReturnType<typeof httpBroadcast>> {
+  if (!parallelHTTPBroadcastWorker) {
+    parallelHTTPBroadcastWorker = new Piscina({
+      filename: new URL(join('workers', 'http-broadcast.js'), import.meta.url).href,
+      // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
+      concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast-parallel'],
+      maxThreads: 1
+    })
+  }
+
+  return parallelHTTPBroadcastWorker.run(options)
+}
+
+// ---------------------------------------------------------------------------
+
+let sequentialHTTPBroadcastWorker: Piscina
+
+function sequentialHTTPBroadcastFromWorker (options: Parameters<typeof httpBroadcast>[0]): Promise<ReturnType<typeof httpBroadcast>> {
+  if (!sequentialHTTPBroadcastWorker) {
+    sequentialHTTPBroadcastWorker = new Piscina({
+      filename: new URL(join('workers', 'http-broadcast.js'), import.meta.url).href,
+      // Keep it sync with job concurrency so the worker will accept all the requests sent by the parallelized jobs
+      concurrentTasksPerWorker: JOB_CONCURRENCY['activitypub-http-broadcast'],
+      maxThreads: 1
+    })
+  }
+
+  return sequentialHTTPBroadcastWorker.run(options)
+}
+
+export {
+  downloadImageFromWorker,
+  processImageFromWorker,
+  parallelHTTPBroadcastFromWorker,
+  sequentialHTTPBroadcastFromWorker
+}

server/core/lib/worker/workers/http-broadcast.ts

@@ -0,0 +1,28 @@
+import Bluebird from 'bluebird'
+import { logger } from '@server/helpers/logger.js'
+import { doRequest, PeerTubeRequestOptions } from '@server/helpers/requests.js'
+import { BROADCAST_CONCURRENCY } from '@server/initializers/constants.js'
+
+async function httpBroadcast (payload: {
+  uris: string[]
+  requestOptions: PeerTubeRequestOptions
+}) {
+  const { uris, requestOptions } = payload
+
+  const badUrls: string[] = []
+  const goodUrls: string[] = []
+
+  await Bluebird.map(uris, async uri => {
+    try {
+      await doRequest(uri, requestOptions)
+      goodUrls.push(uri)
+    } catch (err) {
+      logger.debug('HTTP broadcast to %s failed.', uri, { err })
+      badUrls.push(uri)
+    }
+  }, { concurrency: BROADCAST_CONCURRENCY })
+
+  return { goodUrls, badUrls }
+}
+
+export default httpBroadcast

server/core/lib/worker/workers/image-downloader.ts

@@ -0,0 +1,31 @@
+import { remove } from 'fs-extra/esm'
+import { join } from 'path'
+import { processImage } from '@server/helpers/image-utils.js'
+import { doRequestAndSaveToFile } from '@server/helpers/requests.js'
+import { CONFIG } from '@server/initializers/config.js'
+
+async function downloadImage (options: {
+  url: string
+  destDir: string
+  destName: string
+  size: { width: number, height: number }
+}) {
+  const { url, destDir, destName, size } = options
+
+  const tmpPath = join(CONFIG.STORAGE.TMP_DIR, 'pending-' + destName)
+  await doRequestAndSaveToFile(url, tmpPath)
+
+  const destPath = join(destDir, destName)
+
+  try {
+    await processImage({ path: tmpPath, destination: destPath, newSize: size })
+  } catch (err) {
+    await remove(tmpPath)
+
+    throw err
+  }
+
+  return destPath
+}
+
+export default downloadImage

server/core/lib/worker/workers/image-processor.ts

@@ -0,0 +1,3 @@
+import { processImage } from '@server/helpers/image-utils.js'
+
+export default processImage