Merge branch 'release/7.2.0' into develop

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v7.2.3
+
+### SECURITY
+
+ * Upgrade `multer` dependency to prevent Denial of Service with a malformed request
+
+### Bug fixes
+
+ * Fix channel synchronization that duplicates video imports
+
+
 ## v7.2.2
 
 ### SECURITY

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peertube-client",
-  "version": "7.2.2",
+  "version": "7.2.3",
   "private": true,
   "license": "AGPL-3.0",
   "author": {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "peertube",
   "description": "PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.",
-  "version": "7.2.2",
+  "version": "7.2.3",
   "private": true,
   "licence": "AGPL-3.0",
   "engines": {
@@ -163,7 +163,7 @@
     "maxmind": "^4.3.6",
     "memoizee": "^0.4.14",
     "morgan": "^1.5.3",
-    "multer": "^2.0.1",
+    "multer": "^2.0.2",
     "node-html-parser": "^7.0.1",
     "node-media-server": "^2.1.4",
     "nodemailer": "^7.0.3",

packages/tests/src/api/videos/video-channel-syncs.ts

@@ -22,7 +22,6 @@ describe('Test channel synchronizations', function () {
   if (areYoutubeImportTestsDisabled()) return
 
   function runSuite (mode: 'youtube-dl' | 'yt-dlp') {
-
     describe('Sync using ' + mode, function () {
       let servers: PeerTubeServer[]
       let sqlCommands: SQLCommand[] = []
@@ -30,6 +29,8 @@ describe('Test channel synchronizations', function () {
       let startTestDate: Date
 
       let rootChannelSyncId: number
+      let videoToDelete: number
+
       const userInfo = {
         accessToken: '',
         username: 'user1',
@@ -41,8 +42,8 @@ describe('Test channel synchronizations', function () {
       async function changeDateForSync (channelSyncId: number, newDate: string) {
         await sqlCommands[0].updateQuery(
           `UPDATE "videoChannelSync" ` +
-          `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
-          `WHERE id=${channelSyncId}`
+            `SET "createdAt"='${newDate}', "lastSyncAt"='${newDate}' ` +
+            `WHERE id=${channelSyncId}`
         )
       }
 
@@ -288,21 +289,34 @@ describe('Test channel synchronizations', function () {
           }
         })
 
-        const { videoChannelSync: { id: videoChannelSyncId } } = await servers[0].channelSyncs.create({
+        const { videoChannelSync } = await servers[0].channelSyncs.create({
           attributes: {
             externalChannelUrl: FIXTURE_URLS.youtubePlaylist,
             videoChannelId: channelId
           }
         })
+        rootChannelSyncId = videoChannelSync.id
 
-        await forceSyncAll(videoChannelSyncId)
+        await forceSyncAll(rootChannelSyncId)
 
         {
-
           const { total, data } = await listAllVideosOfChannel('channel2')
           expect(total).to.equal(2)
           expect(data[0].name).to.equal('test')
           expect(data[1].name).to.equal('small video - youtube')
+
+          videoToDelete = data[1].id
+        }
+      })
+
+      it('Should not re-import deleted videos', async function () {
+        await servers[0].videos.remove({ id: videoToDelete })
+        await forceSyncAll(rootChannelSyncId)
+
+        {
+          const { total, data } = await listAllVideosOfChannel('channel2')
+          expect(total).to.equal(1)
+          expect(data[0].name).to.equal('test')
         }
       })
 

server/core/lib/sync-channel.ts

@@ -1,11 +1,11 @@
+import { VideoChannelSyncState, VideoPrivacy } from '@peertube/peertube-models'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { YoutubeDLWrapper } from '@server/helpers/youtube-dl/index.js'
 import { CONFIG } from '@server/initializers/config.js'
 import { buildYoutubeDLImport } from '@server/lib/video-pre-import.js'
 import { UserModel } from '@server/models/user/user.js'
 import { VideoImportModel } from '@server/models/video/video-import.js'
-import { MChannel, MChannelAccountDefault, MChannelSync } from '@server/types/models/index.js'
-import { VideoChannelSyncState, VideoPrivacy } from '@peertube/peertube-models'
+import { MChannelAccountDefault, MChannelSync } from '@server/types/models/index.js'
 import { CreateJobArgument, JobQueue } from './job-queue/index.js'
 import { ServerConfigManager } from './server-config-manager.js'
 
@@ -38,7 +38,9 @@ export async function synchronizeChannel (options: {
 
     logger.info(
       'Fetched %d candidate URLs for sync channel %s.',
-      targetUrls.length, channel.Actor.preferredUsername, { targetUrls, ...lTags() }
+      targetUrls.length,
+      channel.Actor.preferredUsername,
+      { targetUrls, ...lTags() }
     )
 
     if (targetUrls.length === 0) {
@@ -56,7 +58,7 @@ export async function synchronizeChannel (options: {
       logger.debug(`Import candidate: ${targetUrl}`, lTags())
 
       try {
-        if (await skipImport(channel, targetUrl, onlyAfter)) continue
+        if (await skipImport({ channel, channelSync, targetUrl, onlyAfter })) continue
 
         const { job } = await buildYoutubeDLImport({
           user,
@@ -92,9 +94,19 @@ export async function synchronizeChannel (options: {
 
 // ---------------------------------------------------------------------------
 
-async function skipImport (channel: MChannel, targetUrl: string, onlyAfter?: Date) {
-  if (await VideoImportModel.urlAlreadyImported(channel.id, targetUrl)) {
-    logger.debug('%s is already imported for channel %s, skipping video channel synchronization.', targetUrl, channel.name, lTags())
+async function skipImport (options: {
+  channel: MChannelAccountDefault
+  channelSync: MChannelSync
+  targetUrl: string
+  onlyAfter?: Date
+}) {
+  const { channel, channelSync, targetUrl, onlyAfter } = options
+
+  if (await VideoImportModel.urlAlreadyImported({ channelId: channel.id, channelSyncId: channelSync?.id, targetUrl })) {
+    logger.debug(
+      `${targetUrl} is already imported for channel ${channel.name}, skipping video channel synchronization.`,
+      { channelSync, ...lTags() }
+    )
     return true
   }
 

server/core/models/video/video-import.ts

@@ -208,18 +208,44 @@ export class VideoImportModel extends SequelizeModel<VideoImportModel> {
     ]).then(([ total, data ]) => ({ total, data }))
   }
 
-  static async urlAlreadyImported (channelId: number, targetUrl: string): Promise<boolean> {
-    const element = await VideoImportModel.unscoped().findOne({
-      where: {
-        targetUrl,
-        state: {
-          [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
-        },
-        videoChannelSyncId: channelId
+  static async urlAlreadyImported (options: {
+    targetUrl: string
+    channelId: number
+    channelSyncId?: number
+  }): Promise<boolean> {
+    const { channelSyncId, channelId, targetUrl } = options
+
+    const baseWhere = {
+      targetUrl,
+      state: {
+        [Op.in]: [ VideoImportState.PENDING, VideoImportState.PROCESSING, VideoImportState.SUCCESS ]
       }
+    }
+
+    const bySyncId = channelSyncId
+      ? VideoImportModel.unscoped().findOne({
+        where: {
+          ...baseWhere,
+
+          videoChannelSyncId: channelSyncId
+        }
+      })
+      : Promise.resolve(undefined)
+
+    const byChannelId = VideoImportModel.unscoped().findOne({
+      where: baseWhere,
+      include: [
+        {
+          model: VideoModel.unscoped(),
+          required: true,
+          where: {
+            channelId
+          }
+        }
+      ]
     })
 
-    return !!element
+    return (await Promise.all([ bySyncId, byChannelId ])).some(e => !!e)
   }
 
   getTargetIdentifier () {

yarn.lock

@@ -8095,10 +8095,10 @@ msgpackr@^1.11.2:
   optionalDependencies:
     msgpackr-extract "^3.0.2"
 
-multer@^2.0.1:
-  version "2.0.1"
-  resolved "https://registry.yarnpkg.com/multer/-/multer-2.0.1.tgz#3ed335ed2b96240e3df9e23780c91cfcf5d29202"
-  integrity sha512-Ug8bXeTIUlxurg8xLTEskKShvcKDZALo1THEX5E41pYCD2sCVub5/kIRIGqWNoqV6szyLyQKV6mD4QUrWE5GCQ==
+multer@^2.0.2:
+  version "2.0.2"
+  resolved "https://registry.yarnpkg.com/multer/-/multer-2.0.2.tgz#08a8aa8255865388c387aaf041426b0c87bf58dd"
+  integrity sha512-u7f2xaZ/UG8oLXHvtF/oWTRvT44p9ecwBBqTwgJVq0+4BW1g8OW01TyMEGWBHbyMOYVHXslaut7qEQ1meATXgw==
   dependencies:
     append-field "^1.0.0"
     busboy "^1.6.0"