diff --git a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts index 2a3ad18b1..2273265ac 100644 --- a/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts +++ b/server/core/lib/schedulers/video-channel-sync-latest-scheduler.ts @@ -29,17 +29,22 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler { logger.info( 'Creating video import jobs for "%s" sync with external channel "%s"', - channel.Actor.preferredUsername, sync.externalChannelUrl + channel.Actor.preferredUsername, + sync.externalChannelUrl ) - const onlyAfter = sync.lastSyncAt || sync.createdAt + // We can't rely on publication date for playlist elements + // For example, an old video may have been added to a playlist since the last sync + const skipPublishedBefore = this.isPlaylistUrl(sync.externalChannelUrl) + ? undefined + : sync.lastSyncAt || sync.createdAt await synchronizeChannel({ channel, externalChannelUrl: sync.externalChannelUrl, videosCountLimit: CONFIG.IMPORT.VIDEO_CHANNEL_SYNCHRONIZATION.VIDEOS_LIMIT_PER_SYNCHRONIZATION, channelSync: sync, - onlyAfter + skipPublishedBefore }) } } @@ -47,4 +52,14 @@ export class VideoChannelSyncLatestScheduler extends AbstractScheduler { static get Instance () { return this.instance || (this.instance = new this()) } + + private isPlaylistUrl (url: string): boolean { + const parsed = new URL(url) + const pathname = parsed.pathname.toLowerCase() + + return pathname.startsWith('/playlist/') || // Dailymotion playlist + pathname.startsWith('/showcase/') || // Vimeo playlist + pathname.startsWith('/playlist?') || // YouTube playlist + pathname.startsWith('/w/p/') // PeerTube playlist + } } diff --git a/server/core/lib/sync-channel.ts b/server/core/lib/sync-channel.ts index d83a26043..ad642e481 100644 --- a/server/core/lib/sync-channel.ts +++ b/server/core/lib/sync-channel.ts @@ -16,9 +16,9 @@ export async function synchronizeChannel (options: { externalChannelUrl: string videosCountLimit: number channelSync?: MChannelSync - onlyAfter?: Date + skipPublishedBefore?: Date }) { - const { channel, externalChannelUrl, videosCountLimit, onlyAfter, channelSync } = options + const { channel, externalChannelUrl, videosCountLimit, skipPublishedBefore, channelSync } = options if (channelSync) { channelSync.state = VideoChannelSyncState.PROCESSING @@ -58,7 +58,7 @@ export async function synchronizeChannel (options: { logger.debug(`Import candidate: ${targetUrl}`, lTags()) try { - if (await skipImport({ channel, channelSync, targetUrl, onlyAfter })) continue + if (await skipImport({ channel, channelSync, targetUrl, skipPublishedBefore })) continue const { job } = await buildYoutubeDLImport({ user, @@ -98,9 +98,9 @@ async function skipImport (options: { channel: MChannelAccountDefault channelSync: MChannelSync targetUrl: string - onlyAfter?: Date + skipPublishedBefore?: Date }) { - const { channel, channelSync, targetUrl, onlyAfter } = options + const { channel, channelSync, targetUrl, skipPublishedBefore } = options if (await VideoImportModel.urlAlreadyImported({ channelId: channel.id, channelSyncId: channelSync?.id, targetUrl })) { logger.debug( @@ -110,7 +110,7 @@ async function skipImport (options: { return true } - if (onlyAfter) { + if (skipPublishedBefore) { const youtubeDL = new YoutubeDLWrapper( targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'), @@ -119,7 +119,7 @@ async function skipImport (options: { const videoInfo = await youtubeDL.getInfoForDownload() - const onlyAfterWithoutTime = new Date(onlyAfter) + const onlyAfterWithoutTime = new Date(skipPublishedBefore) onlyAfterWithoutTime.setHours(0, 0, 0, 0) if (videoInfo.originallyPublishedAtWithoutTime.getTime() < onlyAfterWithoutTime.getTime()) {