Merge branch 'release/7.1.0' into develop

CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## v7.1.1
+
+### SECURITY
+
+  * This version fixes important vulnerabilities, that will be detailed on Tuesday, April 15
+
+### Bug fixes
+
+  * Fix playlist page margins
+  * Fix danger button border
+  * Fix unsubscribe button label for channels
+  * Fix remote subscribe on iOS
+  * Add Podcast feed to subscribe button
+  * Always display technical information tab in *About* page
+  * Fix menu button auto font-size to prevent overflow in some locales
+  * Correctly inject multiple `rel="me"` links with supported markdown fields
+  * Fix adding studio watermark with audio/video split HLS file
+  * Reset video state on studio failure
+  * Fix updating a user in administration
+  * Fix error when getting a S3 object with some S3 providers
+  * Specify charset when uploading caption files in S3
+  * Fix theme color parsing with some web browsers
+  * Improve channel description in custom markup miniature
+  * Ensure ffmpeg process is killed if download is aborted
+  * Correctly reload playlist on playlist change in watch page
+  * Use `indexifembedded` in embeds instead of `noindex`
+  * Fix extra space on links of remote comments
+  * Don't convert webp images to jpeg
+
+
 ## v7.1.0
 
 ### IMPORTANT NOTES

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "peertube-client",
-  "version": "7.1.0",
+  "version": "7.1.1",
   "private": true,
   "license": "AGPL-3.0",
   "author": {

package.json

@@ -1,7 +1,7 @@
 {
   "name": "peertube",
   "description": "PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.",
-  "version": "7.1.0",
+  "version": "7.1.1",
   "private": true,
   "licence": "AGPL-3.0",
   "engines": {

packages/models/src/server/job.model.ts

@@ -72,6 +72,7 @@ export type ActivitypubHttpFetcherPayload = {
   uri: string
   type: FetchType
   videoId?: number
+  accountId?: number
 }
 
 export type ActivitypubHttpUnicastPayload = {
@@ -129,19 +130,18 @@ export type VideoRedundancyPayload = {
   videoId: number
 }
 
-export type ManageVideoTorrentPayload =
+export type ManageVideoTorrentPayload = {
-  {
+  action: 'create'
-    action: 'create'
+  videoId: number
-    videoId: number
+  videoFileId: number
-    videoFileId: number
+} | {
-  } | {
+  action: 'update-metadata'
-    action: 'update-metadata'
 
-    videoId?: number
+  videoId?: number
-    streamingPlaylistId?: number
+  streamingPlaylistId?: number
 
-    videoFileId: number
+  videoFileId: number
-  }
+}
 
 // Video transcoding payloads
 
@@ -182,7 +182,7 @@ export interface OptimizeTranscodingPayload extends BaseTranscodingPayload {
 }
 
 export type VideoTranscodingPayload =
-  HLSTranscodingPayload
+  | HLSTranscodingPayload
   | NewWebVideoResolutionTranscodingPayload
   | OptimizeTranscodingPayload
   | MergeAudioTranscodingPayload
@@ -255,10 +255,10 @@ export type VideoStudioTaskWatermarkPayload = {
 }
 
 export type VideoStudioTaskPayload =
-  VideoStudioTaskCutPayload |
+  | VideoStudioTaskCutPayload
-  VideoStudioTaskIntroPayload |
+  | VideoStudioTaskIntroPayload
-  VideoStudioTaskOutroPayload |
+  | VideoStudioTaskOutroPayload
-  VideoStudioTaskWatermarkPayload
+  | VideoStudioTaskWatermarkPayload
 
 export interface VideoStudioEditionPayload {
   videoUUID: string
@@ -280,11 +280,10 @@ export interface AfterVideoChannelImportPayload {
 
 // ---------------------------------------------------------------------------
 
-export type NotifyPayload =
+export type NotifyPayload = {
-  {
+  action: 'new-video'
-    action: 'new-video'
+  videoUUID: string
-    videoUUID: string
+}
-  }
 
 // ---------------------------------------------------------------------------
 

packages/tests/src/api/check-params/index.ts

@@ -21,6 +21,7 @@ import './registrations.js'
 import './runners.js'
 import './search.js'
 import './services.js'
+import './static.js'
 import './transcoding.js'
 import './two-factor.js'
 import './upload-quota.js'

packages/tests/src/api/check-params/static.ts

@@ -0,0 +1,95 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
+
+import { getHLS } from '@peertube/peertube-core-utils'
+import { HttpStatusCode, VideoDetails, VideoPrivacy } from '@peertube/peertube-models'
+import {
+  cleanupTests,
+  createSingleServer,
+  makeGetRequest,
+  PeerTubeServer,
+  setAccessTokensToServers,
+  waitJobs
+} from '@peertube/peertube-server-commands'
+import { expect } from 'chai'
+import { basename } from 'path'
+
+describe('Test static endpoints validators', function () {
+  let server: PeerTubeServer
+
+  let privateVideo: VideoDetails
+  let privateM3U8: string
+
+  let publicVideo: VideoDetails
+
+  // ---------------------------------------------------------------
+
+  before(async function () {
+    this.timeout(300_000)
+
+    server = await createSingleServer(1)
+    await setAccessTokensToServers([ server ])
+    await server.config.enableMinimumTranscoding({ hls: true })
+
+    {
+      const { uuid } = await server.videos.quickUpload({ name: 'video 1', privacy: VideoPrivacy.PRIVATE })
+      await waitJobs([ server ])
+
+      privateVideo = await server.videos.getWithToken({ id: uuid })
+      privateM3U8 = basename(getHLS(privateVideo).playlistUrl)
+    }
+
+    {
+      const { uuid } = await server.videos.quickUpload({ name: 'video 2', privacy: VideoPrivacy.PUBLIC })
+      await waitJobs([ server ])
+
+      publicVideo = await server.videos.get({ id: uuid })
+    }
+
+    await waitJobs([ server ])
+  })
+
+  describe('Getting m3u8 playlist', function () {
+    it('Should fail with an invalid video UUID', async function () {
+      await makeGetRequest({
+        url: server.url,
+        token: server.accessToken,
+        path: '/static/streaming-playlists/hls/private/toto/' + privateM3U8
+      })
+    })
+
+    it('Should fail with an invalid playlist name', async function () {
+      await makeGetRequest({
+        url: server.url,
+        token: server.accessToken,
+        path: '/static/streaming-playlists/hls/private/' + privateVideo.uuid + '/' + privateM3U8.replace('.m3u8', '.mp4'),
+        expectedStatus: HttpStatusCode.NOT_FOUND_404
+      })
+    })
+
+    it('Should fail with another m3u8 playlist of another video', async function () {
+      await makeGetRequest({
+        url: server.url,
+        headers: {
+          'x-peertube-video-password': 'fake'
+        },
+        path: '/static/streaming-playlists/hls/private/' + publicVideo.uuid + '/..%2f' + privateVideo.uuid + '%2f' + privateM3U8
+      })
+    })
+
+    it('Should succeed with the correct params', async function () {
+      const { text } = await makeGetRequest({
+        url: server.url,
+        token: server.accessToken,
+        path: '/static/streaming-playlists/hls/private/' + privateVideo.uuid + '/' + privateM3U8,
+        expectedStatus: HttpStatusCode.OK_200
+      })
+
+      expect(text).to.contain('#EXTM3U')
+      expect(text).to.contain(basename(getHLS(privateVideo).files[0].playlistUrl))
+    })
+  })
+
+  after(async function () {
+    await cleanupTests([ server ])
+  })
+})

packages/tests/src/api/check-params/user-import.ts

@@ -3,7 +3,8 @@
 import { HttpStatusCode } from '@peertube/peertube-models'
 import {
   cleanupTests,
-  createSingleServer, PeerTubeServer,
+  createSingleServer,
+  PeerTubeServer,
   setAccessTokensToServers,
   waitJobs
 } from '@peertube/peertube-server-commands'
@@ -37,7 +38,6 @@ describe('Test user import API validators', function () {
   })
 
   describe('Request import', function () {
-
     it('Should fail if import is disabled', async function () {
       await server.config.disableUserImport()
 
@@ -121,7 +121,9 @@ describe('Test user import API validators', function () {
         'export-bad-video.zip',
         'export-without-videos.zip',
         'export-bad-structure.zip',
-        'export-bad-structure.zip'
+        'export-bad-structure.zip',
+        'export-crash.zip',
+        'zip-bomb.zip'
       ]
 
       const tokens: string[] = []
@@ -141,7 +143,6 @@ describe('Test user import API validators', function () {
   })
 
   describe('Get latest import status', function () {
-
     it('Should fail without token', async function () {
       await server.userImports.getLatestImport({ userId, token: null, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 })
     })

packages/tests/src/api/check-params/video-files.ts

@@ -12,7 +12,7 @@ import {
   waitJobs
 } from '@peertube/peertube-server-commands'
 
-describe('Test videos files', function () {
+describe('Test videos files API validators', function () {
   let servers: PeerTubeServer[]
 
   let userToken: string

packages/tests/src/api/check-params/video-playlists.ts

@@ -26,6 +26,7 @@ describe('Test video playlists API validator', function () {
   let userAccessToken: string
 
   let playlist: VideoPlaylistCreateResult
+  let userPlaylist: VideoPlaylistCreateResult
   let privatePlaylistUUID: string
 
   let watchLaterPlaylistId: number
@@ -45,6 +46,8 @@ describe('Test video playlists API validator', function () {
     await setDefaultVideoChannel([ server ])
 
     userAccessToken = await server.users.generateUserAndToken('user1')
+    const user = await server.users.getMyInfo({ token: userAccessToken })
+
     videoId = (await server.videos.quickUpload({ name: 'video 1' })).id
 
     command = server.playlists
@@ -70,6 +73,17 @@ describe('Test video playlists API validator', function () {
       })
     }
 
+    {
+      userPlaylist = await command.create({
+        token: userAccessToken,
+        attributes: {
+          displayName: 'user playlist',
+          privacy: VideoPlaylistPrivacy.PUBLIC,
+          videoChannelId: user.videoChannels[0].id
+        }
+      })
+    }
+
     {
       const created = await command.create({
         attributes: {
@@ -246,7 +260,7 @@ describe('Test video playlists API validator', function () {
     })
 
     it('Should fail with an unknown video channel id', async function () {
-      const params = getBase({ videoChannelId: 42 }, { expectedStatus: HttpStatusCode.NOT_FOUND_404 })
+      const params = getBase({ videoChannelId: 42 }, { expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
 
       await command.create(params)
       await command.update(getUpdate(params, playlist.shortUUID))
@@ -292,6 +306,13 @@ describe('Test video playlists API validator', function () {
       ))
     })
 
+    it('Should fail to set a playlist to a channel owned by another user', async function () {
+      const params = getBase({}, { token: userAccessToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 })
+
+      await command.create(params)
+      await command.update(getUpdate(params, userPlaylist.shortUUID))
+    })
+
     it('Should fail to update the watch later playlist', async function () {
       await command.update(getUpdate(
         getBase({}, { expectedStatus: HttpStatusCode.BAD_REQUEST_400 }),

packages/tests/src/api/videos/video-playlists.ts

@@ -117,7 +117,6 @@ describe('Test video playlists', function () {
   })
 
   describe('Check playlists filters and privacies', function () {
-
     it('Should list video playlist privacies', async function () {
       const privacies = await commands[0].getPrivacies()
 
@@ -169,7 +168,6 @@ describe('Test video playlists', function () {
 
         let playlist: VideoPlaylist = null
         for (const body of [ bodyList, bodyChannel ]) {
-
           expect(body.total).to.equal(1)
           expect(body.data).to.have.lengthOf(1)
 
@@ -218,7 +216,6 @@ describe('Test video playlists', function () {
   })
 
   describe('Create and federate playlists', function () {
-
     it('Should create a playlist on server 1 and have the playlist on server 2 and 3', async function () {
       this.timeout(30000)
 
@@ -345,7 +342,6 @@ describe('Test video playlists', function () {
   })
 
   describe('List playlists', function () {
-
     it('Should correctly list the playlists', async function () {
       this.timeout(30000)
 
@@ -495,7 +491,6 @@ describe('Test video playlists', function () {
   })
 
   describe('Update playlists', function () {
-
     it('Should update a playlist', async function () {
       this.timeout(30000)
 
@@ -535,7 +530,6 @@ describe('Test video playlists', function () {
   })
 
   describe('Element timestamps', function () {
-
     it('Should create a playlist containing different startTimestamp/endTimestamp videos', async function () {
       this.timeout(120000)
 
@@ -659,12 +653,14 @@ describe('Test video playlists', function () {
       group1 = [ servers[0] ]
       group2 = [ servers[1], servers[2] ]
 
+      const myInfo = await servers[0].users.getMyInfo({ token: userTokenServer1 })
+
       const playlist = await commands[0].create({
         token: userTokenServer1,
         attributes: {
           displayName: 'playlist 56',
           privacy: VideoPlaylistPrivacy.PUBLIC,
-          videoChannelId: servers[0].store.channel.id
+          videoChannelId: myInfo.videoChannels[0].id
         }
       })
 
@@ -820,7 +816,6 @@ describe('Test video playlists', function () {
   })
 
   describe('Managing playlist elements', function () {
-
     it('Should reorder the playlist', async function () {
       this.timeout(30000)
 
@@ -1094,7 +1089,6 @@ describe('Test video playlists', function () {
   })
 
   describe('Playlist deletion', function () {
-
     it('Should delete the playlist on server 1 and delete on server 2 and 3', async function () {
       this.timeout(30000)
 

server/core/controllers/static.ts

@@ -5,7 +5,9 @@ import {
   ensureCanAccessPrivateVideoHLSFiles,
   ensureCanAccessVideoPrivateWebVideoFiles,
   handleStaticError,
-  optionalAuthenticate
+  optionalAuthenticate,
+  privateHLSFileValidator,
+  privateM3U8PlaylistValidator
 } from '@server/middlewares/index.js'
 import cors from 'cors'
 import express from 'express'
@@ -55,17 +57,20 @@ const privateHLSStaticMiddlewares = CONFIG.STATIC_FILES.PRIVATE_FILES_REQUIRE_AU
   : []
 
 staticRouter.use(
-  STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:playlistName.m3u8',
+  STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:playlistNameWithoutExtension([a-z0-9-]+).m3u8',
+  privateM3U8PlaylistValidator,
   ...privateHLSStaticMiddlewares,
   asyncMiddleware(servePrivateM3U8)
 )
 
 staticRouter.use(
-  STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS,
+  STATIC_PATHS.STREAMING_PLAYLISTS.PRIVATE_HLS + ':videoUUID/:filename',
+  privateHLSFileValidator,
   ...privateHLSStaticMiddlewares,
-  express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, { fallthrough: false }),
+  servePrivateHLSFile
-  handleStaticError
 )
+// ---------------------------------------------------------------------------
+
 staticRouter.use(
   STATIC_PATHS.STREAMING_PLAYLISTS.HLS,
   express.static(DIRECTORIES.HLS_STREAMING_PLAYLIST.PUBLIC, { fallthrough: false }),
@@ -80,9 +85,15 @@ export {
 
 // ---------------------------------------------------------------------------
 
+function servePrivateHLSFile (req: express.Request, res: express.Response) {
+  const path = join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, req.params.videoUUID, req.params.filename)
+
+  return res.sendFile(path)
+}
+
 async function servePrivateM3U8 (req: express.Request, res: express.Response) {
-  const path = join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, req.params.videoUUID, req.params.playlistName + '.m3u8')
+  const path = join(DIRECTORIES.HLS_STREAMING_PLAYLIST.PRIVATE, req.params.videoUUID, req.params.playlistNameWithoutExtension + '.m3u8')
-  const filename = req.params.playlistName + '.m3u8'
+  const filename = req.params.playlistNameWithoutExtension + '.m3u8'
 
   let playlistContent: string
 

server/core/helpers/unzip.ts

@@ -7,7 +7,14 @@ import { logger, loggerTagsFactory } from './logger.js'
 
 const lTags = loggerTagsFactory('unzip')
 
-export async function unzip (source: string, destination: string) {
+export async function unzip (options: {
+  source: string
+  destination: string
+  maxSize: number // in bytes
+  maxFiles: number
+}) {
+  const { source, destination } = options
+
   await ensureDir(destination)
 
   logger.info(`Unzip ${source} to ${destination}`, lTags())
@@ -16,9 +23,27 @@ export async function unzip (source: string, destination: string) {
     yauzl.open(source, { lazyEntries: true }, (err, zipFile) => {
       if (err) return rej(err)
 
+      zipFile.on('error', err => rej(err))
+
+      let decompressedSize = 0
+      let entries = 0
+
       zipFile.readEntry()
 
       zipFile.on('entry', async entry => {
+        decompressedSize += entry.uncompressedSize
+        entries++
+
+        if (decompressedSize > options.maxSize) {
+          zipFile.close()
+          return rej(new Error(`Unzipped size exceeds ${options.maxSize} bytes`))
+        }
+
+        if (entries > options.maxFiles) {
+          zipFile.close()
+          return rej(new Error(`Unzipped files count exceeds ${options.maxFiles}`))
+        }
+
         const entryPath = join(destination, entry.fileName)
 
         try {

server/core/lib/activitypub/actors/check-actor.ts

@@ -0,0 +1,8 @@
+import { MActorHostOnly } from '@server/types/models/index.js'
+
+export function haveActorsSameRemoteHost (base: MActorHostOnly, other: MActorHostOnly) {
+  if (!base.serverId || !other.serverId) return false
+  if (base.serverId !== other.serverId) return false
+
+  return true
+}

server/core/lib/activitypub/actors/get.ts

@@ -143,7 +143,7 @@ async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, ref
 async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) {
   // We created a new account: fetch the playlists
   if (created === true && actor.Account && accountPlaylistsUrl) {
-    const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' }
+    const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists', accountId: actor.Account.id }
     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
   }
 }

server/core/lib/activitypub/actors/index.ts

@@ -1,3 +1,4 @@
+export * from './check-actor.js'
 export * from './get.js'
 export * from './image.js'
 export * from './keys.js'

server/core/lib/activitypub/crawl.ts

@@ -6,10 +6,10 @@ import { logger } from '../../helpers/logger.js'
 import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants.js'
 import { fetchAP } from './activity.js'
 
-type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
+type HandlerFunction<T> = (items: T[]) => Promise<any> | Bluebird<any>
 type CleanerFunction = (startedDate: Date) => Promise<any>
 
-async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
+export async function crawlCollectionPage<T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
   let url = argUrl
 
   logger.info('Crawling ActivityPub data on %s.', url)
@@ -23,6 +23,8 @@ async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction
   let i = 0
   let nextLink = firstBody.first
   while (nextLink && i < limit) {
+    i++
+
     let body: any
 
     if (typeof nextLink === 'string') {
@@ -40,7 +42,6 @@ async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction
     }
 
     nextLink = body.next
-    i++
 
     if (Array.isArray(body.orderedItems)) {
       const items = body.orderedItems
@@ -52,7 +53,3 @@ async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction
 
   if (cleaner) await retryTransactionWrapper(cleaner, startDate)
 }
-
-export {
-  crawlCollectionPage
-}

server/core/lib/activitypub/playlists/create-update.ts

@@ -1,5 +1,4 @@
 import { HttpStatusCode, PlaylistObject } from '@peertube/peertube-models'
-import { AttributesOnly } from '@peertube/peertube-typescript-utils'
 import { isArray } from '@server/helpers/custom-validators/misc.js'
 import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
@@ -10,11 +9,18 @@ import { updateRemotePlaylistMiniatureFromUrl } from '@server/lib/thumbnail.js'
 import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
 import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
 import { FilteredModelAttributes } from '@server/types/index.js'
-import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models/index.js'
+import {
+  MAccountHost,
+  MThumbnail,
+  MVideoPlaylist,
+  MVideoPlaylistFull,
+  MVideoPlaylistVideosLength
+} from '@server/types/models/index.js'
 import Bluebird from 'bluebird'
 import { getAPId } from '../activity.js'
 import { getOrCreateAPActor } from '../actors/index.js'
 import { crawlCollectionPage } from '../crawl.js'
+import { checkUrlsSameHost } from '../url.js'
 import { getOrCreateAPVideo } from '../videos/index.js'
 import {
   fetchRemotePlaylistElement,
@@ -22,11 +28,17 @@ import {
   playlistElementObjectToDBAttributes,
   playlistObjectToDBAttributes
 } from './shared/index.js'
+import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
 
 const lTags = loggerTagsFactory('ap', 'video-playlist')
 
-async function createAccountPlaylists (playlistUrls: string[]) {
+export async function createAccountPlaylists (playlistUrls: string[], account: MAccountHost) {
   await Bluebird.map(playlistUrls, async playlistUrl => {
+    if (!checkUrlsSameHost(playlistUrl, account.Actor.url)) {
+      logger.warn(`Playlist ${playlistUrl} is not on the same host as owner account ${account.Actor.url}`, lTags(playlistUrl))
+      return
+    }
+
     try {
       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
       if (exists === true) return
@@ -37,17 +49,31 @@ async function createAccountPlaylists (playlistUrls: string[]) {
         throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
       }
 
-      return createOrUpdateVideoPlaylist(playlistObject)
+      return createOrUpdateVideoPlaylist({ playlistObject, contextUrl: playlistUrl })
     } catch (err) {
       logger.warn(`Cannot create or update playlist ${playlistUrl}`, { err, ...lTags(playlistUrl) })
     }
   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 }
 
-async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) {
+export async function createOrUpdateVideoPlaylist (options: {
+  playlistObject: PlaylistObject
+  // Which is the context where we retrieved the playlist
+  // Can be the actor that signed the activity URL or the playlist URL we fetched
+  contextUrl: string
+  to?: string[]
+}) {
+  const { playlistObject, contextUrl, to } = options
+
+  if (!checkUrlsSameHost(playlistObject.id, contextUrl)) {
+    throw new Error(`Playlist ${playlistObject.id} is not on the same host as context URL ${contextUrl}`)
+  }
+
   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to)
 
-  await setVideoChannel(playlistObject, playlistAttributes)
+  const channel = await getRemotePlaylistChannel(playlistObject)
+  playlistAttributes.videoChannelId = channel.id
+  playlistAttributes.ownerAccountId = channel.accountId
 
   const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true })
 
@@ -65,28 +91,26 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?:
 }
 
 // ---------------------------------------------------------------------------
-
+// Private
-export {
-  createAccountPlaylists,
-  createOrUpdateVideoPlaylist
-}
-
 // ---------------------------------------------------------------------------
 
-async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
+async function getRemotePlaylistChannel (playlistObject: PlaylistObject) {
   if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) {
     throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject))
   }
 
-  const actor = await getOrCreateAPActor(getAPId(playlistObject.attributedTo[0]), 'all')
+  const channelUrl = getAPId(playlistObject.attributedTo[0])
-
+  if (!checkUrlsSameHost(channelUrl, playlistObject.id)) {
-  if (!actor.VideoChannel) {
+    throw new Error(`Playlist ${playlistObject.id} and "attributedTo" channel ${channelUrl} are not on the same host`)
-    logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
-    return
   }
 
-  playlistAttributes.videoChannelId = actor.VideoChannel.id
+  const actor = await getOrCreateAPActor(channelUrl, 'all')
-  playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id
+
+  if (!actor.VideoChannel) {
+    throw new Error(`Playlist ${playlistObject.id} "attributedTo" is not a video channel.`)
+  }
+
+  return actor.VideoChannel
 }
 
 async function fetchElementUrls (playlistObject: PlaylistObject) {
@@ -97,7 +121,7 @@ async function fetchElementUrls (playlistObject: PlaylistObject) {
     return Promise.resolve()
   })
 
-  return accItems
+  return accItems.filter(i => isActivityPubUrlValid(i))
 }
 
 async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {

server/core/lib/activitypub/playlists/get.ts

@@ -1,14 +1,11 @@
 import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
 import { MVideoPlaylistFullSummary } from '@server/types/models/index.js'
-import { APObjectId } from '@peertube/peertube-models'
 import { getAPId } from '../activity.js'
 import { createOrUpdateVideoPlaylist } from './create-update.js'
 import { scheduleRefreshIfNeeded } from './refresh.js'
 import { fetchRemoteVideoPlaylist } from './shared/index.js'
 
-async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObjectId): Promise<MVideoPlaylistFullSummary> {
+export async function getOrCreateAPVideoPlaylist (playlistUrl: string): Promise<MVideoPlaylistFullSummary> {
-  const playlistUrl = getAPId(playlistObjectArg)
-
   const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl)
 
   if (playlistFromDatabase) {
@@ -21,15 +18,9 @@ async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObjectId): Promi
   if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl)
 
   // playlistUrl is just an alias/redirection, so process object id instead
-  if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject)
+  if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(getAPId(playlistObject))
 
-  const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject)
+  const playlistCreated = await createOrUpdateVideoPlaylist({ playlistObject, contextUrl: playlistUrl })
 
   return playlistCreated
 }
-
-// ---------------------------------------------------------------------------
-
-export {
-  getOrCreateAPVideoPlaylist
-}

server/core/lib/activitypub/playlists/refresh.ts

@@ -1,8 +1,8 @@
+import { HttpStatusCode } from '@peertube/peertube-models'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { PeerTubeRequestError } from '@server/helpers/requests.js'
 import { JobQueue } from '@server/lib/job-queue/index.js'
-import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models/index.js'
+import { MVideoPlaylist, MVideoPlaylistOwnerDefault } from '@server/types/models/index.js'
-import { HttpStatusCode } from '@peertube/peertube-models'
 import { createOrUpdateVideoPlaylist } from './create-update.js'
 import { fetchRemoteVideoPlaylist } from './shared/index.js'
 
@@ -12,7 +12,7 @@ function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) {
   JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } })
 }
 
-async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
+async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwnerDefault): Promise<MVideoPlaylistOwnerDefault> {
   if (!videoPlaylist.isOutdated()) return videoPlaylist
 
   const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url)
@@ -29,7 +29,7 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
       return videoPlaylist
     }
 
-    await createOrUpdateVideoPlaylist(playlistObject)
+    await createOrUpdateVideoPlaylist({ playlistObject, contextUrl: videoPlaylist.url })
 
     return videoPlaylist
   } catch (err) {
@@ -50,6 +50,6 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
 }
 
 export {
-  scheduleRefreshIfNeeded,
+  refreshVideoPlaylistIfNeeded,
-  refreshVideoPlaylistIfNeeded
+  scheduleRefreshIfNeeded
 }

server/core/lib/activitypub/playlists/shared/object-to-model-attributes.ts

@@ -16,8 +16,8 @@ export function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to
     privacy,
     url: playlistObject.id,
     uuid: playlistObject.uuid,
-    ownerAccountId: null,
     videoChannelId: null,
+    ownerAccountId: null,
     createdAt: new Date(playlistObject.published),
     updatedAt: new Date(playlistObject.updated)
   } as AttributesOnly<VideoPlaylistModel>

server/core/lib/activitypub/process/process-create.ts

@@ -184,8 +184,7 @@ async function processCreatePlaylist (
   byActor: MActorSignature
 ) {
   const byAccount = byActor.Account
-
   if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url)
 
-  await createOrUpdateVideoPlaylist(playlistObject, activity.to)
+  await createOrUpdateVideoPlaylist({ playlistObject, contextUrl: byActor.url, to: activity.to })
 }

server/core/lib/activitypub/process/process-update.ts

@@ -127,5 +127,5 @@ async function processUpdatePlaylist (
   const byAccount = byActor.Account
   if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url)
 
-  await createOrUpdateVideoPlaylist(playlistObject, activity.to)
+  await createOrUpdateVideoPlaylist({ playlistObject, contextUrl: byActor.url, to: activity.to })
 }

server/core/lib/activitypub/videos/updater.ts

@@ -1,4 +1,3 @@
-import { Transaction } from 'sequelize'
 import { VideoObject, VideoPrivacy } from '@peertube/peertube-models'
 import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils.js'
 import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js'
@@ -8,12 +7,14 @@ import { Hooks } from '@server/lib/plugins/hooks.js'
 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
 import { VideoLiveModel } from '@server/models/video/video-live.js'
 import {
-  MActor,
+  MActorHost,
   MChannelAccountLight,
   MChannelId,
   MVideoAccountLightBlacklistAllFiles,
   MVideoFullLight
 } from '@server/types/models/index.js'
+import { Transaction } from 'sequelize'
+import { haveActorsSameRemoteHost } from '../actors/check-actor.js'
 import { APVideoAbstractBuilder, getVideoAttributesFromObject, updateVideoRates } from './shared/index.js'
 
 export class APVideoUpdater extends APVideoAbstractBuilder {
@@ -40,7 +41,8 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
 
   async update (overrideTo?: string[]) {
     logger.debug(
-      'Updating remote video "%s".', this.videoObject.uuid,
+      'Updating remote video "%s".',
+      this.videoObject.uuid,
       { videoObject: this.videoObject, ...this.lTags() }
     )
 
@@ -111,13 +113,9 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
   }
 
   // Check we can update the channel: we trust the remote server
-  private checkChannelUpdateOrThrow (newChannelActor: MActor) {
+  private checkChannelUpdateOrThrow (newChannelActor: MActorHost) {
-    if (!this.oldVideoChannel.Actor.serverId || !newChannelActor.serverId) {
+    if (haveActorsSameRemoteHost(this.oldVideoChannel.Actor, newChannelActor) !== true) {
-      throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
+      throw new Error(`Actor ${this.oldVideoChannel.Actor.url} is not on the same host as ${newChannelActor.url}`)
-    }
-
-    if (this.oldVideoChannel.Actor.serverId !== newChannelActor.serverId) {
-      throw new Error(`New channel ${newChannelActor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`)
     }
   }
 

server/core/lib/job-queue/handlers/activitypub-http-fetcher.ts

@@ -4,12 +4,13 @@ import { logger } from '../../../helpers/logger.js'
 import { VideoModel } from '../../../models/video/video.js'
 import { VideoCommentModel } from '../../../models/video/video-comment.js'
 import { VideoShareModel } from '../../../models/video/video-share.js'
-import { MVideoFullLight } from '../../../types/models/index.js'
+import { MAccountDefault, MVideoFullLight } from '../../../types/models/index.js'
 import { crawlCollectionPage } from '../../activitypub/crawl.js'
 import { createAccountPlaylists } from '../../activitypub/playlists/index.js'
 import { processActivities } from '../../activitypub/process/index.js'
 import { addVideoShares } from '../../activitypub/share.js'
 import { addVideoComments } from '../../activitypub/video-comments.js'
+import { AccountModel } from '@server/models/account/account.js'
 
 async function processActivityPubHttpFetcher (job: Job) {
   logger.info('Processing ActivityPub fetcher in job %s.', job.id)
@@ -19,14 +20,17 @@ async function processActivityPubHttpFetcher (job: Job) {
   let video: MVideoFullLight
   if (payload.videoId) video = await VideoModel.loadFull(payload.videoId)
 
-  const fetcherType: { [ id in FetchType ]: (items: any[]) => Promise<any> } = {
+  let account: MAccountDefault
+  if (payload.accountId) account = await AccountModel.load(payload.accountId)
+
+  const fetcherType: { [id in FetchType]: (items: any[]) => Promise<any> } = {
     'activity': items => processActivities(items, { outboxUrl: payload.uri, fromFetch: true }),
     'video-shares': items => addVideoShares(items, video),
     'video-comments': items => addVideoComments(items),
-    'account-playlists': items => createAccountPlaylists(items)
+    'account-playlists': items => createAccountPlaylists(items, account)
   }
 
-  const cleanerType: { [ id in FetchType ]?: (crawlStartDate: Date) => Promise<any> } = {
+  const cleanerType: { [id in FetchType]?: (crawlStartDate: Date) => Promise<any> } = {
     'video-shares': crawlStartDate => VideoShareModel.cleanOldSharesOf(video.id, crawlStartDate),
     'video-comments': crawlStartDate => VideoCommentModel.cleanOldCommentsOf(video.id, crawlStartDate)
   }

server/core/lib/user-import-export/user-importer.ts

@@ -1,5 +1,5 @@
 import { UserImportResultSummary, UserImportState } from '@peertube/peertube-models'
-import { getFilenameWithoutExt } from '@peertube/peertube-node-utils'
+import { getFilenameWithoutExt, getFileSize } from '@peertube/peertube-node-utils'
 import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { unzip } from '@server/helpers/unzip.js'
@@ -20,6 +20,7 @@ import { UserVideoHistoryImporter } from './importers/user-video-history-importe
 import { VideoPlaylistsImporter } from './importers/video-playlists-importer.js'
 import { VideosImporter } from './importers/videos-importer.js'
 import { WatchedWordsListsImporter } from './importers/watched-words-lists-importer.js'
+import { parseBytes } from '@server/helpers/core-utils.js'
 
 const lTags = loggerTagsFactory('user-import')
 
@@ -51,7 +52,14 @@ export class UserImporter {
       const inputZip = getFSUserImportFilePath(importModel)
       this.extractedDirectory = join(dirname(inputZip), getFilenameWithoutExt(inputZip))
 
-      await unzip(inputZip, this.extractedDirectory)
+      await unzip({
+        source: inputZip,
+        destination: this.extractedDirectory,
+        // Videos that take a lot of space don't have a good compression ratio
+        // Keep a minimum of 1GB if the archive doesn't contain video files
+        maxSize: Math.max(await getFileSize(inputZip) * 2, parseBytes('1GB')),
+        maxFiles: 10000
+      })
 
       const user = await UserModel.loadByIdFull(importModel.userId)
 

server/core/middlewares/validators/shared/videos.ts

@@ -83,7 +83,7 @@ export async function doesVideoChannelOfAccountExist (channelId: number, user: M
   const videoChannel = await VideoChannelModel.loadAndPopulateAccount(channelId)
 
   if (videoChannel === null) {
-    res.fail({ message: 'Unknown video "video channel" for this instance.' })
+    res.fail({ message: `Unknown ${channelId} on this instance.` })
     return false
   }
 
@@ -94,9 +94,7 @@ export async function doesVideoChannelOfAccountExist (channelId: number, user: M
   }
 
   if (videoChannel.Account.id !== user.Account.id) {
-    res.fail({
+    res.fail({ message: `Unknown channel ${channelId} for this account.` })
-      message: 'Unknown video "video channel" for this account.'
-    })
     return false
   }
 

server/core/middlewares/validators/static.ts

@@ -1,28 +1,35 @@
 import { HttpStatusCode } from '@peertube/peertube-models'
-import { exists, isSafePeerTubeFilenameWithoutExtension, isUUIDValid, toBooleanOrNull } from '@server/helpers/custom-validators/misc.js'
+import {
+  exists,
+  isSafeFilename,
+  isSafePeerTubeFilenameWithoutExtension,
+  isUUIDValid,
+  toBooleanOrNull
+} from '@server/helpers/custom-validators/misc.js'
 import { logger } from '@server/helpers/logger.js'
 import { LRU_CACHE } from '@server/initializers/constants.js'
 import { VideoFileModel } from '@server/models/video/video-file.js'
 import { VideoModel } from '@server/models/video/video.js'
 import { MStreamingPlaylist, MVideoFile, MVideoThumbnailBlacklist } from '@server/types/models/index.js'
 import express from 'express'
-import { query } from 'express-validator'
+import { param, query } from 'express-validator'
 import { LRUCache } from 'lru-cache'
-import { basename, dirname } from 'path'
+import { basename } from 'path'
 import { areValidationErrors, checkCanAccessVideoStaticFiles, isValidVideoPasswordHeader } from './shared/index.js'
 
 type LRUValue = {
   allowed: boolean
   video?: MVideoThumbnailBlacklist
   file?: MVideoFile
-  playlist?: MStreamingPlaylist }
+  playlist?: MStreamingPlaylist
+}
 
 const staticFileTokenBypass = new LRUCache<string, LRUValue>({
   max: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.MAX_SIZE,
   ttl: LRU_CACHE.STATIC_VIDEO_FILES_RIGHTS_CHECK.TTL
 })
 
-const ensureCanAccessVideoPrivateWebVideoFiles = [
+export const ensureCanAccessVideoPrivateWebVideoFiles = [
   query('videoFileToken').optional().custom(exists),
 
   isValidVideoPasswordHeader(),
@@ -61,32 +68,50 @@ const ensureCanAccessVideoPrivateWebVideoFiles = [
   }
 ]
 
-const ensureCanAccessPrivateVideoHLSFiles = [
+export const privateM3U8PlaylistValidator = [
-  query('videoFileToken')
+  param('videoUUID')
-    .optional()
+    .custom(isUUIDValid),
-    .custom(exists),
+
+  param('playlistNameWithoutExtension')
+    .custom(v => isSafePeerTubeFilenameWithoutExtension(v)),
 
   query('reinjectVideoFileToken')
     .optional()
     .customSanitizer(toBooleanOrNull)
     .isBoolean().withMessage('Should be a valid reinjectVideoFileToken boolean'),
 
-  query('playlistName')
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res)) return
+
+    return next()
+  }
+]
+
+export const privateHLSFileValidator = [
+  param('videoUUID')
+    .custom(isUUIDValid),
+
+  param('filename')
+    .custom(v => isSafeFilename(v)),
+
+  (req: express.Request, res: express.Response, next: express.NextFunction) => {
+    if (areValidationErrors(req, res)) return
+
+    return next()
+  }
+]
+
+export const ensureCanAccessPrivateVideoHLSFiles = [
+  query('videoFileToken')
     .optional()
-    .customSanitizer(isSafePeerTubeFilenameWithoutExtension),
+    .custom(exists),
 
   isValidVideoPasswordHeader(),
 
   async (req: express.Request, res: express.Response, next: express.NextFunction) => {
     if (areValidationErrors(req, res)) return
 
-    const videoUUID = basename(dirname(req.originalUrl))
+    const videoUUID = req.params.videoUUID
-
-    if (!isUUIDValid(videoUUID)) {
-      logger.debug('Path does not contain valid video UUID to serve static file %s', req.originalUrl)
-
-      return res.sendStatus(HttpStatusCode.FORBIDDEN_403)
-    }
 
     const token = extractTokenOrDie(req, res)
     if (!token) return
@@ -121,10 +146,6 @@ const ensureCanAccessPrivateVideoHLSFiles = [
   }
 ]
 
-export {
-  ensureCanAccessPrivateVideoHLSFiles, ensureCanAccessVideoPrivateWebVideoFiles
-}
-
 // ---------------------------------------------------------------------------
 
 async function isWebVideoAllowed (req: express.Request, res: express.Response) {

server/core/middlewares/validators/videos/video-playlists.ts

@@ -1,5 +1,3 @@
-import express from 'express'
-import { body, param, query, ValidationChain } from 'express-validator'
 import { forceNumber } from '@peertube/peertube-core-utils'
 import {
   HttpStatusCode,
@@ -12,6 +10,8 @@ import {
 } from '@peertube/peertube-models'
 import { ExpressPromiseHandler } from '@server/types/express-handler.js'
 import { MUserAccountId } from '@server/types/models/index.js'
+import express from 'express'
+import { body, param, query, ValidationChain } from 'express-validator'
 import {
   isArrayOf,
   isIdOrUUIDValid,
@@ -37,7 +37,7 @@ import { MVideoPlaylist } from '../../../types/models/video/video-playlist.js'
 import { authenticatePromise } from '../../auth.js'
 import {
   areValidationErrors,
-  doesVideoChannelIdExist,
+  doesVideoChannelOfAccountExist,
   doesVideoExist,
   doesVideoPlaylistExist,
   isValidPlaylistIdParam,
@@ -52,7 +52,9 @@ const videoPlaylistsAddValidator = getCommonPlaylistEditAttributes().concat([
     if (areValidationErrors(req, res)) return cleanUpReqFiles(req)
 
     const body: VideoPlaylistCreate = req.body
-    if (body.videoChannelId && !await doesVideoChannelIdExist(body.videoChannelId, res)) return cleanUpReqFiles(req)
+    if (body.videoChannelId && !await doesVideoChannelOfAccountExist(body.videoChannelId, res.locals.oauth.token.User, res)) {
+      return cleanUpReqFiles(req)
+    }
 
     if (
       !body.videoChannelId &&
@@ -88,7 +90,8 @@ const videoPlaylistsUpdateValidator = getCommonPlaylistEditAttributes().concat([
     const body: VideoPlaylistUpdate = req.body
 
     const newPrivacy = body.privacy || videoPlaylist.privacy
-    if (newPrivacy === VideoPlaylistPrivacy.PUBLIC &&
+    if (
+      newPrivacy === VideoPlaylistPrivacy.PUBLIC &&
       (
         (!videoPlaylist.videoChannelId && !body.videoChannelId) ||
         body.videoChannelId === null
@@ -105,7 +108,9 @@ const videoPlaylistsUpdateValidator = getCommonPlaylistEditAttributes().concat([
       return res.fail({ message: 'Cannot update a watch later playlist.' })
     }
 
-    if (body.videoChannelId && !await doesVideoChannelIdExist(body.videoChannelId, res)) return cleanUpReqFiles(req)
+    if (body.videoChannelId && !await doesVideoChannelOfAccountExist(body.videoChannelId, res.locals.oauth.token.User, res)) {
+      return cleanUpReqFiles(req)
+    }
 
     return next()
   }
@@ -350,21 +355,17 @@ const doVideosInPlaylistExistValidator = [
 // ---------------------------------------------------------------------------
 
 export {
+  commonVideoPlaylistFiltersValidator,
+  doVideosInPlaylistExistValidator,
+  videoPlaylistElementAPGetValidator,
   videoPlaylistsAddValidator,
-  videoPlaylistsUpdateValidator,
+  videoPlaylistsAddVideoValidator,
   videoPlaylistsDeleteValidator,
   videoPlaylistsGetValidator,
-  videoPlaylistsSearchValidator,
-
-  videoPlaylistsAddVideoValidator,
-  videoPlaylistsUpdateOrRemoveVideoValidator,
   videoPlaylistsReorderVideosValidator,
-
+  videoPlaylistsSearchValidator,
-  videoPlaylistElementAPGetValidator,
+  videoPlaylistsUpdateOrRemoveVideoValidator,
-
+  videoPlaylistsUpdateValidator
-  commonVideoPlaylistFiltersValidator,
-
-  doVideosInPlaylistExistValidator
 }
 
 // ---------------------------------------------------------------------------
@@ -375,7 +376,7 @@ function getCommonPlaylistEditAttributes () {
       .custom((value, { req }) => isVideoImageValid(req.files, 'thumbnailfile'))
       .withMessage(
         'This thumbnail file is not supported or too large. Please, make sure it is of the following type: ' +
-        CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.IMAGE.EXTNAME.join(', ')
+          CONSTRAINTS_FIELDS.VIDEO_PLAYLISTS.IMAGE.EXTNAME.join(', ')
       ),
 
     body('description')