server/server -> server/core

server/core/lib/views/shared/index.ts

@@ -0,0 +1,3 @@
+export * from './video-viewer-counters.js'
+export * from './video-viewer-stats.js'
+export * from './video-views.js'

server/core/lib/views/shared/video-viewer-counters.ts

@@ -0,0 +1,197 @@
+import { buildUUID, isTestOrDevInstance, sha256 } from '@peertube/peertube-node-utils'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { VIEW_LIFETIME } from '@server/initializers/constants.js'
+import { sendView } from '@server/lib/activitypub/send/send-view.js'
+import { PeerTubeSocket } from '@server/lib/peertube-socket.js'
+import { getServerActor } from '@server/models/application/application.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { MVideo, MVideoImmutable } from '@server/types/models/index.js'
+
+const lTags = loggerTagsFactory('views')
+
+export type ViewerScope = 'local' | 'remote'
+export type VideoScope = 'local' | 'remote'
+
+type Viewer = {
+  expires: number
+  id: string
+  viewerScope: ViewerScope
+  videoScope: VideoScope
+  lastFederation?: number
+}
+
+export class VideoViewerCounters {
+
+  // expires is new Date().getTime()
+  private readonly viewersPerVideo = new Map<number, Viewer[]>()
+  private readonly idToViewer = new Map<string, Viewer>()
+
+  private readonly salt = buildUUID()
+
+  private processingViewerCounters = false
+
+  constructor () {
+    setInterval(() => this.cleanViewerCounters(), VIEW_LIFETIME.VIEWER_COUNTER)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async addLocalViewer (options: {
+    video: MVideoImmutable
+    ip: string
+  }) {
+    const { video, ip } = options
+
+    logger.debug('Adding local viewer to video viewers counter %s.', video.uuid, { ...lTags(video.uuid) })
+
+    const viewerId = this.generateViewerId(ip, video.uuid)
+    const viewer = this.idToViewer.get(viewerId)
+
+    if (viewer) {
+      viewer.expires = this.buildViewerExpireTime()
+      await this.federateViewerIfNeeded(video, viewer)
+
+      return false
+    }
+
+    const newViewer = await this.addViewerToVideo({ viewerId, video, viewerScope: 'local' })
+    await this.federateViewerIfNeeded(video, newViewer)
+
+    return true
+  }
+
+  async addRemoteViewer (options: {
+    video: MVideo
+    viewerId: string
+    viewerExpires: Date
+  }) {
+    const { video, viewerExpires, viewerId } = options
+
+    logger.debug('Adding remote viewer to video %s.', video.uuid, { ...lTags(video.uuid) })
+
+    await this.addViewerToVideo({ video, viewerExpires, viewerId, viewerScope: 'remote' })
+
+    return true
+  }
+
+  // ---------------------------------------------------------------------------
+
+  getTotalViewers (options: {
+    viewerScope: ViewerScope
+    videoScope: VideoScope
+  }) {
+    let total = 0
+
+    for (const viewers of this.viewersPerVideo.values()) {
+      total += viewers.filter(v => v.viewerScope === options.viewerScope && v.videoScope === options.videoScope).length
+    }
+
+    return total
+  }
+
+  getViewers (video: MVideo) {
+    const viewers = this.viewersPerVideo.get(video.id)
+    if (!viewers) return 0
+
+    return viewers.length
+  }
+
+  buildViewerExpireTime () {
+    return new Date().getTime() + VIEW_LIFETIME.VIEWER_COUNTER
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async addViewerToVideo (options: {
+    video: MVideoImmutable
+    viewerId: string
+    viewerScope: ViewerScope
+    viewerExpires?: Date
+  }) {
+    const { video, viewerExpires, viewerId, viewerScope } = options
+
+    let watchers = this.viewersPerVideo.get(video.id)
+
+    if (!watchers) {
+      watchers = []
+      this.viewersPerVideo.set(video.id, watchers)
+    }
+
+    const expires = viewerExpires
+      ? viewerExpires.getTime()
+      : this.buildViewerExpireTime()
+
+    const videoScope: VideoScope = video.remote
+      ? 'remote'
+      : 'local'
+
+    const viewer = { id: viewerId, expires, videoScope, viewerScope }
+    watchers.push(viewer)
+
+    this.idToViewer.set(viewerId, viewer)
+
+    await this.notifyClients(video.id, watchers.length)
+
+    return viewer
+  }
+
+  private async cleanViewerCounters () {
+    if (this.processingViewerCounters) return
+    this.processingViewerCounters = true
+
+    if (!isTestOrDevInstance()) logger.info('Cleaning video viewers.', lTags())
+
+    try {
+      for (const videoId of this.viewersPerVideo.keys()) {
+        const notBefore = new Date().getTime()
+
+        const viewers = this.viewersPerVideo.get(videoId)
+
+        // Only keep not expired viewers
+        const newViewers: Viewer[] = []
+
+        // Filter new viewers
+        for (const viewer of viewers) {
+          if (viewer.expires > notBefore) {
+            newViewers.push(viewer)
+          } else {
+            this.idToViewer.delete(viewer.id)
+          }
+        }
+
+        if (newViewers.length === 0) this.viewersPerVideo.delete(videoId)
+        else this.viewersPerVideo.set(videoId, newViewers)
+
+        await this.notifyClients(videoId, newViewers.length)
+      }
+    } catch (err) {
+      logger.error('Error in video clean viewers scheduler.', { err, ...lTags() })
+    }
+
+    this.processingViewerCounters = false
+  }
+
+  private async notifyClients (videoId: string | number, viewersLength: number) {
+    const video = await VideoModel.loadImmutableAttributes(videoId)
+    if (!video) return
+
+    PeerTubeSocket.Instance.sendVideoViewsUpdate(video, viewersLength)
+
+    logger.debug('Video viewers update for %s is %d.', video.url, viewersLength, lTags())
+  }
+
+  private generateViewerId (ip: string, videoUUID: string) {
+    return sha256(this.salt + '-' + ip + '-' + videoUUID)
+  }
+
+  private async federateViewerIfNeeded (video: MVideoImmutable, viewer: Viewer) {
+    // Federate the viewer if it's been a "long" time we did not
+    const now = new Date().getTime()
+    const federationLimit = now - (VIEW_LIFETIME.VIEWER_COUNTER * 0.75)
+
+    if (viewer.lastFederation && viewer.lastFederation > federationLimit) return
+
+    await sendView({ byActor: await getServerActor(), video, type: 'viewer', viewerIdentifier: viewer.id })
+    viewer.lastFederation = now
+  }
+}

server/core/lib/views/shared/video-viewer-stats.ts

@@ -0,0 +1,196 @@
+import { Transaction } from 'sequelize'
+import { VideoViewEvent } from '@peertube/peertube-models'
+import { isTestOrDevInstance } from '@peertube/peertube-node-utils'
+import { GeoIP } from '@server/helpers/geo-ip.js'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { MAX_LOCAL_VIEWER_WATCH_SECTIONS, VIEW_LIFETIME } from '@server/initializers/constants.js'
+import { sequelizeTypescript } from '@server/initializers/database.js'
+import { sendCreateWatchAction } from '@server/lib/activitypub/send/index.js'
+import { getLocalVideoViewerActivityPubUrl } from '@server/lib/activitypub/url.js'
+import { Redis } from '@server/lib/redis.js'
+import { VideoModel } from '@server/models/video/video.js'
+import { LocalVideoViewerWatchSectionModel } from '@server/models/view/local-video-viewer-watch-section.js'
+import { LocalVideoViewerModel } from '@server/models/view/local-video-viewer.js'
+import { MVideo, MVideoImmutable } from '@server/types/models/index.js'
+
+const lTags = loggerTagsFactory('views')
+
+type LocalViewerStats = {
+  firstUpdated: number // Date.getTime()
+  lastUpdated: number // Date.getTime()
+
+  watchSections: {
+    start: number
+    end: number
+  }[]
+
+  watchTime: number
+
+  country: string
+
+  videoId: number
+}
+
+export class VideoViewerStats {
+  private processingViewersStats = false
+
+  constructor () {
+    setInterval(() => this.processViewerStats(), VIEW_LIFETIME.VIEWER_STATS)
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async addLocalViewer (options: {
+    video: MVideoImmutable
+    currentTime: number
+    ip: string
+    viewEvent?: VideoViewEvent
+  }) {
+    const { video, ip, viewEvent, currentTime } = options
+
+    logger.debug('Adding local viewer to video stats %s.', video.uuid, { currentTime, viewEvent, ...lTags(video.uuid) })
+
+    return this.updateLocalViewerStats({ video, viewEvent, currentTime, ip })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  async getWatchTime (videoId: number, ip: string) {
+    const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId })
+
+    return stats?.watchTime || 0
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async updateLocalViewerStats (options: {
+    video: MVideoImmutable
+    ip: string
+    currentTime: number
+    viewEvent?: VideoViewEvent
+  }) {
+    const { video, ip, viewEvent, currentTime } = options
+    const nowMs = new Date().getTime()
+
+    let stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ ip, videoId: video.id })
+
+    if (stats && stats.watchSections.length >= MAX_LOCAL_VIEWER_WATCH_SECTIONS) {
+      logger.warn('Too much watch section to store for a viewer, skipping this one', { currentTime, viewEvent, ...lTags(video.uuid) })
+      return
+    }
+
+    if (!stats) {
+      const country = await GeoIP.Instance.safeCountryISOLookup(ip)
+
+      stats = {
+        firstUpdated: nowMs,
+        lastUpdated: nowMs,
+
+        watchSections: [],
+
+        watchTime: 0,
+
+        country,
+        videoId: video.id
+      }
+    }
+
+    stats.lastUpdated = nowMs
+
+    if (viewEvent === 'seek' || stats.watchSections.length === 0) {
+      stats.watchSections.push({
+        start: currentTime,
+        end: currentTime
+      })
+    } else {
+      const lastSection = stats.watchSections[stats.watchSections.length - 1]
+
+      if (lastSection.start > currentTime) {
+        logger.debug('Invalid end watch section %d. Last start record was at %d. Starting a new section.', currentTime, lastSection.start)
+
+        stats.watchSections.push({
+          start: currentTime,
+          end: currentTime
+        })
+      } else {
+        lastSection.end = currentTime
+      }
+    }
+
+    stats.watchTime = this.buildWatchTimeFromSections(stats.watchSections)
+
+    logger.debug('Set local video viewer stats for video %s.', video.uuid, { stats, ...lTags(video.uuid) })
+
+    await Redis.Instance.setLocalVideoViewer(ip, video.id, stats)
+  }
+
+  async processViewerStats () {
+    if (this.processingViewersStats) return
+    this.processingViewersStats = true
+
+    if (!isTestOrDevInstance()) logger.info('Processing viewer statistics.', lTags())
+
+    const now = new Date().getTime()
+
+    try {
+      const allKeys = await Redis.Instance.listLocalVideoViewerKeys()
+
+      for (const key of allKeys) {
+        const stats: LocalViewerStats = await Redis.Instance.getLocalVideoViewer({ key })
+
+        // Process expired stats
+        if (stats.lastUpdated > now - VIEW_LIFETIME.VIEWER_STATS) {
+          continue
+        }
+
+        try {
+          await sequelizeTypescript.transaction(async t => {
+            const video = await VideoModel.load(stats.videoId, t)
+            if (!video) return
+
+            const statsModel = await this.saveViewerStats(video, stats, t)
+
+            if (video.remote) {
+              await sendCreateWatchAction(statsModel, t)
+            }
+          })
+
+          await Redis.Instance.deleteLocalVideoViewersKeys(key)
+        } catch (err) {
+          logger.error('Cannot process viewer stats for Redis key %s.', key, { err, ...lTags() })
+        }
+      }
+    } catch (err) {
+      logger.error('Error in video save viewers stats scheduler.', { err, ...lTags() })
+    }
+
+    this.processingViewersStats = false
+  }
+
+  private async saveViewerStats (video: MVideo, stats: LocalViewerStats, transaction: Transaction) {
+    const statsModel = new LocalVideoViewerModel({
+      startDate: new Date(stats.firstUpdated),
+      endDate: new Date(stats.lastUpdated),
+      watchTime: stats.watchTime,
+      country: stats.country,
+      videoId: video.id
+    })
+
+    statsModel.url = getLocalVideoViewerActivityPubUrl(statsModel)
+    statsModel.Video = video as VideoModel
+
+    await statsModel.save({ transaction })
+
+    statsModel.WatchSections = await LocalVideoViewerWatchSectionModel.bulkCreateSections({
+      localVideoViewerId: statsModel.id,
+      watchSections: stats.watchSections,
+      transaction
+    })
+
+    return statsModel
+  }
+
+  private buildWatchTimeFromSections (sections: { start: number, end: number }[]) {
+    return sections.reduce((p, current) => p + (current.end - current.start), 0)
+  }
+}

server/core/lib/views/shared/video-views.ts

@@ -0,0 +1,70 @@
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { sendView } from '@server/lib/activitypub/send/send-view.js'
+import { getCachedVideoDuration } from '@server/lib/video.js'
+import { getServerActor } from '@server/models/application/application.js'
+import { MVideo, MVideoImmutable } from '@server/types/models/index.js'
+import { buildUUID } from '@peertube/peertube-node-utils'
+import { Redis } from '../../redis.js'
+
+const lTags = loggerTagsFactory('views')
+
+export class VideoViews {
+
+  async addLocalView (options: {
+    video: MVideoImmutable
+    ip: string
+    watchTime: number
+  }) {
+    const { video, ip, watchTime } = options
+
+    logger.debug('Adding local view to video %s.', video.uuid, { watchTime, ...lTags(video.uuid) })
+
+    if (!await this.hasEnoughWatchTime(video, watchTime)) return false
+
+    const viewExists = await Redis.Instance.doesVideoIPViewExist(ip, video.uuid)
+    if (viewExists) return false
+
+    await Redis.Instance.setIPVideoView(ip, video.uuid)
+
+    await this.addView(video)
+
+    await sendView({ byActor: await getServerActor(), video, type: 'view', viewerIdentifier: buildUUID() })
+
+    return true
+  }
+
+  async addRemoteView (options: {
+    video: MVideo
+  }) {
+    const { video } = options
+
+    logger.debug('Adding remote view to video %s.', video.uuid, { ...lTags(video.uuid) })
+
+    await this.addView(video)
+
+    return true
+  }
+
+  // ---------------------------------------------------------------------------
+
+  private async addView (video: MVideoImmutable) {
+    const promises: Promise<any>[] = []
+
+    if (video.isOwned()) {
+      promises.push(Redis.Instance.addLocalVideoView(video.id))
+    }
+
+    promises.push(Redis.Instance.addVideoViewStats(video.id))
+
+    await Promise.all(promises)
+  }
+
+  private async hasEnoughWatchTime (video: MVideoImmutable, watchTime: number) {
+    const { duration, isLive } = await getCachedVideoDuration(video.id)
+
+    if (isLive || duration >= 30) return watchTime >= 30
+
+    // Check more than 50% of the video is watched
+    return duration / watchTime < 2
+  }
+}