Merge branch 'release/7.1.0' into develop

server/core/lib/activitypub/actors/check-actor.ts

@@ -0,0 +1,8 @@
+import { MActorHostOnly } from '@server/types/models/index.js'
+
+export function haveActorsSameRemoteHost (base: MActorHostOnly, other: MActorHostOnly) {
+  if (!base.serverId || !other.serverId) return false
+  if (base.serverId !== other.serverId) return false
+
+  return true
+}

server/core/lib/activitypub/actors/get.ts

@@ -143,7 +143,7 @@ async function scheduleOutboxFetchIfNeeded (actor: MActor, created: boolean, ref
 async function schedulePlaylistFetchIfNeeded (actor: MActorAccountId, created: boolean, accountPlaylistsUrl: string) {
   // We created a new account: fetch the playlists
   if (created === true && actor.Account && accountPlaylistsUrl) {
-    const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists' }
+    const payload = { uri: accountPlaylistsUrl, type: 'account-playlists' as 'account-playlists', accountId: actor.Account.id }
     await JobQueue.Instance.createJob({ type: 'activitypub-http-fetcher', payload })
   }
 }

server/core/lib/activitypub/actors/index.ts

@@ -1,3 +1,4 @@
+export * from './check-actor.js'
 export * from './get.js'
 export * from './image.js'
 export * from './keys.js'

server/core/lib/activitypub/crawl.ts

@@ -6,10 +6,10 @@ import { logger } from '../../helpers/logger.js'
 import { ACTIVITY_PUB, WEBSERVER } from '../../initializers/constants.js'
 import { fetchAP } from './activity.js'
 
-type HandlerFunction<T> = (items: T[]) => (Promise<any> | Bluebird<any>)
+type HandlerFunction<T> = (items: T[]) => Promise<any> | Bluebird<any>
 type CleanerFunction = (startedDate: Date) => Promise<any>
 
-async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
+export async function crawlCollectionPage<T> (argUrl: string, handler: HandlerFunction<T>, cleaner?: CleanerFunction) {
   let url = argUrl
 
   logger.info('Crawling ActivityPub data on %s.', url)
@@ -23,6 +23,8 @@ async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction
   let i = 0
   let nextLink = firstBody.first
   while (nextLink && i < limit) {
+    i++
+
     let body: any
 
     if (typeof nextLink === 'string') {
@@ -40,7 +42,6 @@ async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction
     }
 
     nextLink = body.next
-    i++
 
     if (Array.isArray(body.orderedItems)) {
       const items = body.orderedItems
@@ -52,7 +53,3 @@ async function crawlCollectionPage <T> (argUrl: string, handler: HandlerFunction
 
   if (cleaner) await retryTransactionWrapper(cleaner, startDate)
 }
-
-export {
-  crawlCollectionPage
-}

server/core/lib/activitypub/playlists/create-update.ts

@@ -1,5 +1,4 @@
 import { HttpStatusCode, PlaylistObject } from '@peertube/peertube-models'
-import { AttributesOnly } from '@peertube/peertube-typescript-utils'
 import { isArray } from '@server/helpers/custom-validators/misc.js'
 import { retryTransactionWrapper } from '@server/helpers/database-utils.js'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
@@ -10,11 +9,18 @@ import { updateRemotePlaylistMiniatureFromUrl } from '@server/lib/thumbnail.js'
 import { VideoPlaylistElementModel } from '@server/models/video/video-playlist-element.js'
 import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
 import { FilteredModelAttributes } from '@server/types/index.js'
-import { MThumbnail, MVideoPlaylist, MVideoPlaylistFull, MVideoPlaylistVideosLength } from '@server/types/models/index.js'
+import {
+  MAccountHost,
+  MThumbnail,
+  MVideoPlaylist,
+  MVideoPlaylistFull,
+  MVideoPlaylistVideosLength
+} from '@server/types/models/index.js'
 import Bluebird from 'bluebird'
 import { getAPId } from '../activity.js'
 import { getOrCreateAPActor } from '../actors/index.js'
 import { crawlCollectionPage } from '../crawl.js'
+import { checkUrlsSameHost } from '../url.js'
 import { getOrCreateAPVideo } from '../videos/index.js'
 import {
   fetchRemotePlaylistElement,
@@ -22,11 +28,17 @@ import {
   playlistElementObjectToDBAttributes,
   playlistObjectToDBAttributes
 } from './shared/index.js'
+import { isActivityPubUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
 
 const lTags = loggerTagsFactory('ap', 'video-playlist')
 
-async function createAccountPlaylists (playlistUrls: string[]) {
+export async function createAccountPlaylists (playlistUrls: string[], account: MAccountHost) {
   await Bluebird.map(playlistUrls, async playlistUrl => {
+    if (!checkUrlsSameHost(playlistUrl, account.Actor.url)) {
+      logger.warn(`Playlist ${playlistUrl} is not on the same host as owner account ${account.Actor.url}`, lTags(playlistUrl))
+      return
+    }
+
     try {
       const exists = await VideoPlaylistModel.doesPlaylistExist(playlistUrl)
       if (exists === true) return
@@ -37,17 +49,31 @@ async function createAccountPlaylists (playlistUrls: string[]) {
         throw new Error(`Cannot refresh remote playlist ${playlistUrl}: invalid body.`)
       }
 
-      return createOrUpdateVideoPlaylist(playlistObject)
+      return createOrUpdateVideoPlaylist({ playlistObject, contextUrl: playlistUrl })
     } catch (err) {
       logger.warn(`Cannot create or update playlist ${playlistUrl}`, { err, ...lTags(playlistUrl) })
     }
   }, { concurrency: CRAWL_REQUEST_CONCURRENCY })
 }
 
-async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?: string[]) {
+export async function createOrUpdateVideoPlaylist (options: {
+  playlistObject: PlaylistObject
+  // Which is the context where we retrieved the playlist
+  // Can be the actor that signed the activity URL or the playlist URL we fetched
+  contextUrl: string
+  to?: string[]
+}) {
+  const { playlistObject, contextUrl, to } = options
+
+  if (!checkUrlsSameHost(playlistObject.id, contextUrl)) {
+    throw new Error(`Playlist ${playlistObject.id} is not on the same host as context URL ${contextUrl}`)
+  }
+
   const playlistAttributes = playlistObjectToDBAttributes(playlistObject, to || playlistObject.to)
 
-  await setVideoChannel(playlistObject, playlistAttributes)
+  const channel = await getRemotePlaylistChannel(playlistObject)
+  playlistAttributes.videoChannelId = channel.id
+  playlistAttributes.ownerAccountId = channel.accountId
 
   const [ upsertPlaylist ] = await VideoPlaylistModel.upsert<MVideoPlaylistVideosLength>(playlistAttributes, { returning: true })
 
@@ -65,28 +91,26 @@ async function createOrUpdateVideoPlaylist (playlistObject: PlaylistObject, to?:
 }
 
 // ---------------------------------------------------------------------------
-
-export {
-  createAccountPlaylists,
-  createOrUpdateVideoPlaylist
-}
-
+// Private
 // ---------------------------------------------------------------------------
 
-async function setVideoChannel (playlistObject: PlaylistObject, playlistAttributes: AttributesOnly<VideoPlaylistModel>) {
+async function getRemotePlaylistChannel (playlistObject: PlaylistObject) {
   if (!isArray(playlistObject.attributedTo) || playlistObject.attributedTo.length !== 1) {
     throw new Error('Not attributed to for playlist object ' + getAPId(playlistObject))
   }
 
-  const actor = await getOrCreateAPActor(getAPId(playlistObject.attributedTo[0]), 'all')
-
-  if (!actor.VideoChannel) {
-    logger.warn('Playlist "attributedTo" %s is not a video channel.', playlistObject.id, { playlistObject, ...lTags(playlistObject.id) })
-    return
+  const channelUrl = getAPId(playlistObject.attributedTo[0])
+  if (!checkUrlsSameHost(channelUrl, playlistObject.id)) {
+    throw new Error(`Playlist ${playlistObject.id} and "attributedTo" channel ${channelUrl} are not on the same host`)
   }
 
-  playlistAttributes.videoChannelId = actor.VideoChannel.id
-  playlistAttributes.ownerAccountId = actor.VideoChannel.Account.id
+  const actor = await getOrCreateAPActor(channelUrl, 'all')
+
+  if (!actor.VideoChannel) {
+    throw new Error(`Playlist ${playlistObject.id} "attributedTo" is not a video channel.`)
+  }
+
+  return actor.VideoChannel
 }
 
 async function fetchElementUrls (playlistObject: PlaylistObject) {
@@ -97,7 +121,7 @@ async function fetchElementUrls (playlistObject: PlaylistObject) {
     return Promise.resolve()
   })
 
-  return accItems
+  return accItems.filter(i => isActivityPubUrlValid(i))
 }
 
 async function updatePlaylistThumbnail (playlistObject: PlaylistObject, playlist: MVideoPlaylistFull) {

server/core/lib/activitypub/playlists/get.ts

@@ -1,14 +1,11 @@
 import { VideoPlaylistModel } from '@server/models/video/video-playlist.js'
 import { MVideoPlaylistFullSummary } from '@server/types/models/index.js'
-import { APObjectId } from '@peertube/peertube-models'
 import { getAPId } from '../activity.js'
 import { createOrUpdateVideoPlaylist } from './create-update.js'
 import { scheduleRefreshIfNeeded } from './refresh.js'
 import { fetchRemoteVideoPlaylist } from './shared/index.js'
 
-async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObjectId): Promise<MVideoPlaylistFullSummary> {
-  const playlistUrl = getAPId(playlistObjectArg)
-
+export async function getOrCreateAPVideoPlaylist (playlistUrl: string): Promise<MVideoPlaylistFullSummary> {
   const playlistFromDatabase = await VideoPlaylistModel.loadByUrlWithAccountAndChannelSummary(playlistUrl)
 
   if (playlistFromDatabase) {
@@ -21,15 +18,9 @@ async function getOrCreateAPVideoPlaylist (playlistObjectArg: APObjectId): Promi
   if (!playlistObject) throw new Error('Cannot fetch remote playlist with url: ' + playlistUrl)
 
   // playlistUrl is just an alias/redirection, so process object id instead
-  if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(playlistObject)
+  if (playlistObject.id !== playlistUrl) return getOrCreateAPVideoPlaylist(getAPId(playlistObject))
 
-  const playlistCreated = await createOrUpdateVideoPlaylist(playlistObject)
+  const playlistCreated = await createOrUpdateVideoPlaylist({ playlistObject, contextUrl: playlistUrl })
 
   return playlistCreated
 }
-
-// ---------------------------------------------------------------------------
-
-export {
-  getOrCreateAPVideoPlaylist
-}

server/core/lib/activitypub/playlists/refresh.ts

@@ -1,8 +1,8 @@
+import { HttpStatusCode } from '@peertube/peertube-models'
 import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { PeerTubeRequestError } from '@server/helpers/requests.js'
 import { JobQueue } from '@server/lib/job-queue/index.js'
-import { MVideoPlaylist, MVideoPlaylistOwner } from '@server/types/models/index.js'
-import { HttpStatusCode } from '@peertube/peertube-models'
+import { MVideoPlaylist, MVideoPlaylistOwnerDefault } from '@server/types/models/index.js'
 import { createOrUpdateVideoPlaylist } from './create-update.js'
 import { fetchRemoteVideoPlaylist } from './shared/index.js'
 
@@ -12,7 +12,7 @@ function scheduleRefreshIfNeeded (playlist: MVideoPlaylist) {
   JobQueue.Instance.createJobAsync({ type: 'activitypub-refresher', payload: { type: 'video-playlist', url: playlist.url } })
 }
 
-async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner): Promise<MVideoPlaylistOwner> {
+async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwnerDefault): Promise<MVideoPlaylistOwnerDefault> {
   if (!videoPlaylist.isOutdated()) return videoPlaylist
 
   const lTags = loggerTagsFactory('ap', 'video-playlist', 'refresh', videoPlaylist.uuid, videoPlaylist.url)
@@ -29,7 +29,7 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
       return videoPlaylist
     }
 
-    await createOrUpdateVideoPlaylist(playlistObject)
+    await createOrUpdateVideoPlaylist({ playlistObject, contextUrl: videoPlaylist.url })
 
     return videoPlaylist
   } catch (err) {
@@ -50,6 +50,6 @@ async function refreshVideoPlaylistIfNeeded (videoPlaylist: MVideoPlaylistOwner)
 }
 
 export {
-  scheduleRefreshIfNeeded,
-  refreshVideoPlaylistIfNeeded
+  refreshVideoPlaylistIfNeeded,
+  scheduleRefreshIfNeeded
 }

server/core/lib/activitypub/playlists/shared/object-to-model-attributes.ts

@@ -16,8 +16,8 @@ export function playlistObjectToDBAttributes (playlistObject: PlaylistObject, to
     privacy,
     url: playlistObject.id,
     uuid: playlistObject.uuid,
-    ownerAccountId: null,
     videoChannelId: null,
+    ownerAccountId: null,
     createdAt: new Date(playlistObject.published),
     updatedAt: new Date(playlistObject.updated)
   } as AttributesOnly<VideoPlaylistModel>

server/core/lib/activitypub/process/process-create.ts

@@ -184,8 +184,7 @@ async function processCreatePlaylist (
   byActor: MActorSignature
 ) {
   const byAccount = byActor.Account
-
   if (!byAccount) throw new Error('Cannot create video playlist with the non account actor ' + byActor.url)
 
-  await createOrUpdateVideoPlaylist(playlistObject, activity.to)
+  await createOrUpdateVideoPlaylist({ playlistObject, contextUrl: byActor.url, to: activity.to })
 }

server/core/lib/activitypub/process/process-update.ts

@@ -127,5 +127,5 @@ async function processUpdatePlaylist (
   const byAccount = byActor.Account
   if (!byAccount) throw new Error('Cannot update video playlist with the non account actor ' + byActor.url)
 
-  await createOrUpdateVideoPlaylist(playlistObject, activity.to)
+  await createOrUpdateVideoPlaylist({ playlistObject, contextUrl: byActor.url, to: activity.to })
 }

server/core/lib/activitypub/videos/updater.ts

@@ -1,4 +1,3 @@
-import { Transaction } from 'sequelize'
 import { VideoObject, VideoPrivacy } from '@peertube/peertube-models'
 import { resetSequelizeInstance, runInReadCommittedTransaction } from '@server/helpers/database-utils.js'
 import { logger, loggerTagsFactory, LoggerTagsFn } from '@server/helpers/logger.js'
@@ -8,12 +7,14 @@ import { Hooks } from '@server/lib/plugins/hooks.js'
 import { autoBlacklistVideoIfNeeded } from '@server/lib/video-blacklist.js'
 import { VideoLiveModel } from '@server/models/video/video-live.js'
 import {
-  MActor,
+  MActorHost,
   MChannelAccountLight,
   MChannelId,
   MVideoAccountLightBlacklistAllFiles,
   MVideoFullLight
 } from '@server/types/models/index.js'
+import { Transaction } from 'sequelize'
+import { haveActorsSameRemoteHost } from '../actors/check-actor.js'
 import { APVideoAbstractBuilder, getVideoAttributesFromObject, updateVideoRates } from './shared/index.js'
 
 export class APVideoUpdater extends APVideoAbstractBuilder {
@@ -40,7 +41,8 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
 
   async update (overrideTo?: string[]) {
     logger.debug(
-      'Updating remote video "%s".', this.videoObject.uuid,
+      'Updating remote video "%s".',
+      this.videoObject.uuid,
       { videoObject: this.videoObject, ...this.lTags() }
     )
 
@@ -111,13 +113,9 @@ export class APVideoUpdater extends APVideoAbstractBuilder {
   }
 
   // Check we can update the channel: we trust the remote server
-  private checkChannelUpdateOrThrow (newChannelActor: MActor) {
-    if (!this.oldVideoChannel.Actor.serverId || !newChannelActor.serverId) {
-      throw new Error('Cannot check old channel/new channel validity because `serverId` is null')
-    }
-
-    if (this.oldVideoChannel.Actor.serverId !== newChannelActor.serverId) {
-      throw new Error(`New channel ${newChannelActor.url} is not on the same server than new channel ${this.oldVideoChannel.Actor.url}`)
+  private checkChannelUpdateOrThrow (newChannelActor: MActorHost) {
+    if (haveActorsSameRemoteHost(this.oldVideoChannel.Actor, newChannelActor) !== true) {
+      throw new Error(`Actor ${this.oldVideoChannel.Actor.url} is not on the same host as ${newChannelActor.url}`)
     }
   }