server/server -> server/core

server/core/lib/opentelemetry/metric-helpers/bittorrent-tracker-observers-builder.ts

@@ -0,0 +1,51 @@
+import { Meter } from '@opentelemetry/api'
+
+export class BittorrentTrackerObserversBuilder {
+
+  constructor (private readonly meter: Meter, private readonly trackerServer: any) {
+
+  }
+
+  buildObservers () {
+    const activeInfohashes = this.meter.createObservableGauge('peertube_bittorrent_tracker_active_infohashes_total', {
+      description: 'Total active infohashes in the PeerTube BitTorrent Tracker'
+    })
+    const inactiveInfohashes = this.meter.createObservableGauge('peertube_bittorrent_tracker_inactive_infohashes_total', {
+      description: 'Total inactive infohashes in the PeerTube BitTorrent Tracker'
+    })
+    const peers = this.meter.createObservableGauge('peertube_bittorrent_tracker_peers_total', {
+      description: 'Total peers in the PeerTube BitTorrent Tracker'
+    })
+
+    this.meter.addBatchObservableCallback(observableResult => {
+      const infohashes = Object.keys(this.trackerServer.torrents)
+
+      const counters = {
+        activeInfohashes: 0,
+        inactiveInfohashes: 0,
+        peers: 0,
+        uncompletedPeers: 0
+      }
+
+      for (const infohash of infohashes) {
+        const content = this.trackerServer.torrents[infohash]
+
+        const peers = content.peers
+        if (peers.keys.length !== 0) counters.activeInfohashes++
+        else counters.inactiveInfohashes++
+
+        for (const peerId of peers.keys) {
+          const peer = peers.peek(peerId)
+          if (peer == null) return
+
+          counters.peers++
+        }
+      }
+
+      observableResult.observe(activeInfohashes, counters.activeInfohashes)
+      observableResult.observe(inactiveInfohashes, counters.inactiveInfohashes)
+      observableResult.observe(peers, counters.peers)
+    }, [ activeInfohashes, inactiveInfohashes, peers ])
+  }
+
+}

server/core/lib/opentelemetry/metric-helpers/index.ts

@@ -0,0 +1,7 @@
+export * from './bittorrent-tracker-observers-builder.js'
+export * from './lives-observers-builder.js'
+export * from './job-queue-observers-builder.js'
+export * from './nodejs-observers-builder.js'
+export * from './playback-metrics.js'
+export * from './stats-observers-builder.js'
+export * from './viewers-observers-builder.js'

server/core/lib/opentelemetry/metric-helpers/job-queue-observers-builder.ts

@@ -0,0 +1,24 @@
+import { Meter } from '@opentelemetry/api'
+import { JobQueue } from '@server/lib/job-queue/index.js'
+
+export class JobQueueObserversBuilder {
+
+  constructor (private readonly meter: Meter) {
+
+  }
+
+  buildObservers () {
+    this.meter.createObservableGauge('peertube_job_queue_total', {
+      description: 'Total jobs in the PeerTube job queue'
+    }).addCallback(async observableResult => {
+      const stats = await JobQueue.Instance.getStats()
+
+      for (const { jobType, counts } of stats) {
+        for (const state of Object.keys(counts)) {
+          observableResult.observe(counts[state], { jobType, state })
+        }
+      }
+    })
+  }
+
+}

server/core/lib/opentelemetry/metric-helpers/lives-observers-builder.ts

@@ -0,0 +1,21 @@
+import { Meter } from '@opentelemetry/api'
+import { VideoModel } from '@server/models/video/video.js'
+
+export class LivesObserversBuilder {
+
+  constructor (private readonly meter: Meter) {
+
+  }
+
+  buildObservers () {
+    this.meter.createObservableGauge('peertube_running_lives_total', {
+      description: 'Total running lives on the instance'
+    }).addCallback(async observableResult => {
+      const local = await VideoModel.countLives({ remote: false, mode: 'published' })
+      const remote = await VideoModel.countLives({ remote: true, mode: 'published' })
+
+      observableResult.observe(local, { liveOrigin: 'local' })
+      observableResult.observe(remote, { liveOrigin: 'remote' })
+    })
+  }
+}

server/core/lib/opentelemetry/metric-helpers/nodejs-observers-builder.ts

@@ -0,0 +1,202 @@
+import { readdir } from 'fs/promises'
+import { constants, NodeGCPerformanceDetail, PerformanceObserver } from 'perf_hooks'
+import * as process from 'process'
+import { Meter, ObservableResult } from '@opentelemetry/api'
+import { ExplicitBucketHistogramAggregation } from '@opentelemetry/sdk-metrics'
+import { View } from '@opentelemetry/sdk-metrics/build/src/view/View.js'
+import { logger } from '@server/helpers/logger.js'
+
+// Thanks to https://github.com/siimon/prom-client
+// We took their logic and adapter it for opentelemetry
+// Try to keep consistency with their metric name/description so it's easier to process (grafana dashboard template etc)
+
+export class NodeJSObserversBuilder {
+
+  constructor (private readonly meter: Meter) {
+  }
+
+  static getViews () {
+    return [
+      new View({
+        aggregation: new ExplicitBucketHistogramAggregation([ 0.001, 0.01, 0.1, 1, 2, 5 ]),
+        instrumentName: 'nodejs_gc_duration_seconds'
+      })
+    ]
+  }
+
+  buildObservers () {
+    this.buildCPUObserver()
+    this.buildMemoryObserver()
+
+    this.buildHandlesObserver()
+    this.buildFileDescriptorsObserver()
+
+    this.buildGCObserver()
+    this.buildEventLoopLagObserver()
+
+    this.buildLibUVActiveRequestsObserver()
+    this.buildActiveResourcesObserver()
+  }
+
+  private buildCPUObserver () {
+    const cpuTotal = this.meter.createObservableCounter('process_cpu_seconds_total', {
+      description: 'Total user and system CPU time spent in seconds.'
+    })
+    const cpuUser = this.meter.createObservableCounter('process_cpu_user_seconds_total', {
+      description: 'Total user CPU time spent in seconds.'
+    })
+    const cpuSystem = this.meter.createObservableCounter('process_cpu_system_seconds_total', {
+      description: 'Total system CPU time spent in seconds.'
+    })
+
+    let lastCpuUsage = process.cpuUsage()
+
+    this.meter.addBatchObservableCallback(observableResult => {
+      const cpuUsage = process.cpuUsage()
+
+      const userUsageMicros = cpuUsage.user - lastCpuUsage.user
+      const systemUsageMicros = cpuUsage.system - lastCpuUsage.system
+
+      lastCpuUsage = cpuUsage
+
+      observableResult.observe(cpuTotal, (userUsageMicros + systemUsageMicros) / 1e6)
+      observableResult.observe(cpuUser, userUsageMicros / 1e6)
+      observableResult.observe(cpuSystem, systemUsageMicros / 1e6)
+
+    }, [ cpuTotal, cpuUser, cpuSystem ])
+  }
+
+  private buildMemoryObserver () {
+    this.meter.createObservableGauge('nodejs_memory_usage_bytes', {
+      description: 'Memory'
+    }).addCallback(observableResult => {
+      const current = process.memoryUsage()
+
+      observableResult.observe(current.heapTotal, { memoryType: 'heapTotal' })
+      observableResult.observe(current.heapUsed, { memoryType: 'heapUsed' })
+      observableResult.observe(current.arrayBuffers, { memoryType: 'arrayBuffers' })
+      observableResult.observe(current.external, { memoryType: 'external' })
+      observableResult.observe(current.rss, { memoryType: 'rss' })
+    })
+  }
+
+  private buildHandlesObserver () {
+    if (typeof (process as any)._getActiveHandles !== 'function') return
+
+    this.meter.createObservableGauge('nodejs_active_handles_total', {
+      description: 'Total number of active handles.'
+    }).addCallback(observableResult => {
+      const handles = (process as any)._getActiveHandles()
+
+      observableResult.observe(handles.length)
+    })
+  }
+
+  private buildGCObserver () {
+    const kinds = {
+      [constants.NODE_PERFORMANCE_GC_MAJOR]: 'major',
+      [constants.NODE_PERFORMANCE_GC_MINOR]: 'minor',
+      [constants.NODE_PERFORMANCE_GC_INCREMENTAL]: 'incremental',
+      [constants.NODE_PERFORMANCE_GC_WEAKCB]: 'weakcb'
+    }
+
+    const histogram = this.meter.createHistogram('nodejs_gc_duration_seconds', {
+      description: 'Garbage collection duration by kind, one of major, minor, incremental or weakcb'
+    })
+
+    const obs = new PerformanceObserver(list => {
+      const entry = list.getEntries()[0]
+
+      // Node < 16 uses entry.kind
+      // Node >= 16 uses entry.detail.kind
+      // See: https://nodejs.org/docs/latest-v16.x/api/deprecations.html#deprecations_dep0152_extension_performanceentry_properties
+      const kind = entry.detail
+        ? kinds[(entry.detail as NodeGCPerformanceDetail).kind]
+        : kinds[(entry as any).kind]
+
+      // Convert duration from milliseconds to seconds
+      histogram.record(entry.duration / 1000, {
+        kind
+      })
+    })
+
+    obs.observe({ entryTypes: [ 'gc' ] })
+  }
+
+  private buildEventLoopLagObserver () {
+    const reportEventloopLag = (start: [ number, number ], observableResult: ObservableResult, res: () => void) => {
+      const delta = process.hrtime(start)
+      const nanosec = delta[0] * 1e9 + delta[1]
+      const seconds = nanosec / 1e9
+
+      observableResult.observe(seconds)
+
+      res()
+    }
+
+    this.meter.createObservableGauge('nodejs_eventloop_lag_seconds', {
+      description: 'Lag of event loop in seconds.'
+    }).addCallback(observableResult => {
+      return new Promise(res => {
+        const start = process.hrtime()
+
+        setImmediate(reportEventloopLag, start, observableResult, res)
+      })
+    })
+  }
+
+  private buildFileDescriptorsObserver () {
+    this.meter.createObservableGauge('process_open_fds', {
+      description: 'Number of open file descriptors.'
+    }).addCallback(async observableResult => {
+      try {
+        const fds = await readdir('/proc/self/fd')
+        observableResult.observe(fds.length - 1)
+      } catch (err) {
+        logger.debug('Cannot list file descriptors of current process for OpenTelemetry.', { err })
+      }
+    })
+  }
+
+  private buildLibUVActiveRequestsObserver () {
+    if (typeof (process as any)._getActiveRequests !== 'function') return
+
+    this.meter.createObservableGauge('nodejs_active_requests_total', {
+      description: 'Total number of active libuv requests.'
+    }).addCallback(observableResult => {
+      const requests = (process as any)._getActiveRequests()
+
+      observableResult.observe(requests.length)
+    })
+  }
+
+  private buildActiveResourcesObserver () {
+    if (typeof (process as any).getActiveResourcesInfo !== 'function') return
+
+    const grouped = this.meter.createObservableCounter('nodejs_active_resources', {
+      description: 'Number of active resources that are currently keeping the event loop alive, grouped by async resource type.'
+    })
+    const total = this.meter.createObservableCounter('nodejs_active_resources_total', {
+      description: 'Total number of active resources.'
+    })
+
+    this.meter.addBatchObservableCallback(observableResult => {
+      const resources = (process as any).getActiveResourcesInfo()
+
+      const data = {}
+
+      for (let i = 0; i < resources.length; i++) {
+        const resource = resources[i]
+
+        if (data[resource] === undefined) data[resource] = 0
+        data[resource] += 1
+      }
+
+      for (const type of Object.keys(data)) {
+        observableResult.observe(grouped, data[type], { type })
+      }
+
+      observableResult.observe(total, resources.length)
+    }, [ grouped, total ])
+  }
+}

server/core/lib/opentelemetry/metric-helpers/playback-metrics.ts

@@ -0,0 +1,85 @@
+import { Counter, Meter } from '@opentelemetry/api'
+import { MVideoImmutable } from '@server/types/models/index.js'
+import { PlaybackMetricCreate } from '@peertube/peertube-models'
+
+export class PlaybackMetrics {
+  private errorsCounter: Counter
+  private resolutionChangesCounter: Counter
+
+  private downloadedBytesP2PCounter: Counter
+  private uploadedBytesP2PCounter: Counter
+
+  private downloadedBytesHTTPCounter: Counter
+
+  private peersP2PPeersGaugeBuffer: {
+    value: number
+    attributes: any
+  }[] = []
+
+  constructor (private readonly meter: Meter) {
+
+  }
+
+  buildCounters () {
+    this.errorsCounter = this.meter.createCounter('peertube_playback_errors_count', {
+      description: 'Errors collected from PeerTube player.'
+    })
+
+    this.resolutionChangesCounter = this.meter.createCounter('peertube_playback_resolution_changes_count', {
+      description: 'Resolution changes collected from PeerTube player.'
+    })
+
+    this.downloadedBytesHTTPCounter = this.meter.createCounter('peertube_playback_http_downloaded_bytes', {
+      description: 'Downloaded bytes with HTTP by PeerTube player.'
+    })
+    this.downloadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_downloaded_bytes', {
+      description: 'Downloaded bytes with P2P by PeerTube player.'
+    })
+
+    this.uploadedBytesP2PCounter = this.meter.createCounter('peertube_playback_p2p_uploaded_bytes', {
+      description: 'Uploaded bytes with P2P by PeerTube player.'
+    })
+
+    this.meter.createObservableGauge('peertube_playback_p2p_peers', {
+      description: 'Total P2P peers connected to the PeerTube player.'
+    }).addCallback(observableResult => {
+      for (const gauge of this.peersP2PPeersGaugeBuffer) {
+        observableResult.observe(gauge.value, gauge.attributes)
+      }
+
+      this.peersP2PPeersGaugeBuffer = []
+    })
+  }
+
+  observe (video: MVideoImmutable, metrics: PlaybackMetricCreate) {
+    const attributes = {
+      videoOrigin: video.remote
+        ? 'remote'
+        : 'local',
+
+      playerMode: metrics.playerMode,
+
+      resolution: metrics.resolution + '',
+      fps: metrics.fps + '',
+
+      p2pEnabled: metrics.p2pEnabled,
+
+      videoUUID: video.uuid
+    }
+
+    this.errorsCounter.add(metrics.errors, attributes)
+    this.resolutionChangesCounter.add(metrics.resolutionChanges, attributes)
+
+    this.downloadedBytesHTTPCounter.add(metrics.downloadedBytesHTTP, attributes)
+    this.downloadedBytesP2PCounter.add(metrics.downloadedBytesP2P, attributes)
+
+    this.uploadedBytesP2PCounter.add(metrics.uploadedBytesP2P, attributes)
+
+    if (metrics.p2pPeers) {
+      this.peersP2PPeersGaugeBuffer.push({
+        value: metrics.p2pPeers,
+        attributes
+      })
+    }
+  }
+}

server/core/lib/opentelemetry/metric-helpers/stats-observers-builder.ts

@@ -0,0 +1,186 @@
+import memoizee from 'memoizee'
+import { Meter } from '@opentelemetry/api'
+import { MEMOIZE_TTL } from '@server/initializers/constants.js'
+import { buildAvailableActivities } from '@server/lib/activitypub/activity.js'
+import { StatsManager } from '@server/lib/stat-manager.js'
+
+export class StatsObserversBuilder {
+
+  private readonly getInstanceStats = memoizee(() => {
+    return StatsManager.Instance.getStats()
+  }, { maxAge: MEMOIZE_TTL.GET_STATS_FOR_OPEN_TELEMETRY_METRICS })
+
+  constructor (private readonly meter: Meter) {
+
+  }
+
+  buildObservers () {
+    this.buildUserStatsObserver()
+    this.buildVideoStatsObserver()
+    this.buildCommentStatsObserver()
+    this.buildPlaylistStatsObserver()
+    this.buildChannelStatsObserver()
+    this.buildInstanceFollowsStatsObserver()
+    this.buildRedundancyStatsObserver()
+    this.buildActivityPubStatsObserver()
+  }
+
+  private buildUserStatsObserver () {
+    this.meter.createObservableGauge('peertube_users_total', {
+      description: 'Total users on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalUsers)
+    })
+
+    this.meter.createObservableGauge('peertube_active_users_total', {
+      description: 'Total active users on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalDailyActiveUsers, { activeInterval: 'daily' })
+      observableResult.observe(stats.totalWeeklyActiveUsers, { activeInterval: 'weekly' })
+      observableResult.observe(stats.totalMonthlyActiveUsers, { activeInterval: 'monthly' })
+    })
+  }
+
+  private buildChannelStatsObserver () {
+    this.meter.createObservableGauge('peertube_channels_total', {
+      description: 'Total channels on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideoChannels, { channelOrigin: 'local' })
+    })
+
+    this.meter.createObservableGauge('peertube_active_channels_total', {
+      description: 'Total active channels on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalDailyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'daily' })
+      observableResult.observe(stats.totalLocalWeeklyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'weekly' })
+      observableResult.observe(stats.totalLocalMonthlyActiveVideoChannels, { channelOrigin: 'local', activeInterval: 'monthly' })
+    })
+  }
+
+  private buildVideoStatsObserver () {
+    this.meter.createObservableGauge('peertube_videos_total', {
+      description: 'Total videos on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideos, { videoOrigin: 'local' })
+      observableResult.observe(stats.totalVideos - stats.totalLocalVideos, { videoOrigin: 'remote' })
+    })
+
+    this.meter.createObservableGauge('peertube_video_views_total', {
+      description: 'Total video views made on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideoViews, { viewOrigin: 'local' })
+    })
+
+    this.meter.createObservableGauge('peertube_video_bytes_total', {
+      description: 'Total bytes of videos'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideoFilesSize, { videoOrigin: 'local' })
+    })
+  }
+
+  private buildCommentStatsObserver () {
+    this.meter.createObservableGauge('peertube_comments_total', {
+      description: 'Total comments on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalVideoComments, { accountOrigin: 'local' })
+    })
+  }
+
+  private buildPlaylistStatsObserver () {
+    this.meter.createObservableGauge('peertube_playlists_total', {
+      description: 'Total playlists on the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalLocalPlaylists, { playlistOrigin: 'local' })
+    })
+  }
+
+  private buildInstanceFollowsStatsObserver () {
+    this.meter.createObservableGauge('peertube_instance_followers_total', {
+      description: 'Total followers of the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalInstanceFollowers)
+    })
+
+    this.meter.createObservableGauge('peertube_instance_following_total', {
+      description: 'Total following of the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalInstanceFollowing)
+    })
+  }
+
+  private buildRedundancyStatsObserver () {
+    this.meter.createObservableGauge('peertube_redundancy_used_bytes_total', {
+      description: 'Total redundancy used of the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      for (const r of stats.videosRedundancy) {
+        observableResult.observe(r.totalUsed, { strategy: r.strategy })
+      }
+    })
+
+    this.meter.createObservableGauge('peertube_redundancy_available_bytes_total', {
+      description: 'Total redundancy available of the instance'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      for (const r of stats.videosRedundancy) {
+        observableResult.observe(r.totalSize, { strategy: r.strategy })
+      }
+    })
+  }
+
+  private buildActivityPubStatsObserver () {
+    const availableActivities = buildAvailableActivities()
+
+    this.meter.createObservableGauge('peertube_ap_inbox_success_total', {
+      description: 'Total inbox messages processed with success'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      for (const type of availableActivities) {
+        observableResult.observe(stats[`totalActivityPub${type}MessagesSuccesses`], { activityType: type })
+      }
+    })
+
+    this.meter.createObservableGauge('peertube_ap_inbox_error_total', {
+      description: 'Total inbox messages processed with error'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      for (const type of availableActivities) {
+        observableResult.observe(stats[`totalActivityPub${type}MessagesErrors`], { activityType: type })
+      }
+    })
+
+    this.meter.createObservableGauge('peertube_ap_inbox_waiting_total', {
+      description: 'Total inbox messages waiting for being processed'
+    }).addCallback(async observableResult => {
+      const stats = await this.getInstanceStats()
+
+      observableResult.observe(stats.totalActivityPubMessagesWaiting)
+    })
+  }
+}

server/core/lib/opentelemetry/metric-helpers/viewers-observers-builder.ts

@@ -0,0 +1,24 @@
+import { Meter } from '@opentelemetry/api'
+import { VideoScope, ViewerScope } from '@server/lib/views/shared/index.js'
+import { VideoViewsManager } from '@server/lib/views/video-views-manager.js'
+
+export class ViewersObserversBuilder {
+
+  constructor (private readonly meter: Meter) {
+
+  }
+
+  buildObservers () {
+    this.meter.createObservableGauge('peertube_viewers_total', {
+      description: 'Total viewers on the instance'
+    }).addCallback(observableResult => {
+      for (const viewerScope of [ 'local', 'remote' ] as ViewerScope[]) {
+        for (const videoScope of [ 'local', 'remote' ] as VideoScope[]) {
+          const result = VideoViewsManager.Instance.getTotalViewers({ viewerScope, videoScope })
+
+          observableResult.observe(result, { viewerOrigin: viewerScope, videoOrigin: videoScope })
+        }
+      }
+    })
+  }
+}

server/core/lib/opentelemetry/metrics.ts

@@ -0,0 +1,123 @@
+import { Application, Request, Response } from 'express'
+import { Meter, metrics } from '@opentelemetry/api'
+import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'
+import { MeterProvider } from '@opentelemetry/sdk-metrics'
+import { logger } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+import { MVideoImmutable } from '@server/types/models/index.js'
+import { PlaybackMetricCreate } from '@peertube/peertube-models'
+import {
+  BittorrentTrackerObserversBuilder,
+  JobQueueObserversBuilder,
+  LivesObserversBuilder,
+  NodeJSObserversBuilder,
+  PlaybackMetrics,
+  StatsObserversBuilder,
+  ViewersObserversBuilder
+} from './metric-helpers/index.js'
+
+class OpenTelemetryMetrics {
+
+  private static instance: OpenTelemetryMetrics
+
+  private meter: Meter
+
+  private onRequestDuration: (req: Request, res: Response) => void
+
+  private playbackMetrics: PlaybackMetrics
+
+  private constructor () {}
+
+  init (app: Application) {
+    if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return
+
+    app.use((req, res, next) => {
+      res.once('finish', () => {
+        if (!this.onRequestDuration) return
+
+        this.onRequestDuration(req as Request, res as Response)
+      })
+
+      next()
+    })
+  }
+
+  registerMetrics (options: { trackerServer: any }) {
+    if (CONFIG.OPEN_TELEMETRY.METRICS.ENABLED !== true) return
+
+    logger.info('Registering Open Telemetry metrics')
+
+    const provider = new MeterProvider({
+      views: [
+        ...NodeJSObserversBuilder.getViews()
+      ]
+    })
+
+    provider.addMetricReader(new PrometheusExporter({
+      host: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.HOSTNAME,
+      port: CONFIG.OPEN_TELEMETRY.METRICS.PROMETHEUS_EXPORTER.PORT
+    }))
+
+    metrics.setGlobalMeterProvider(provider)
+
+    this.meter = metrics.getMeter('default')
+
+    if (CONFIG.OPEN_TELEMETRY.METRICS.HTTP_REQUEST_DURATION.ENABLED === true) {
+      this.buildRequestObserver()
+    }
+
+    this.playbackMetrics = new PlaybackMetrics(this.meter)
+    this.playbackMetrics.buildCounters()
+
+    const nodeJSObserversBuilder = new NodeJSObserversBuilder(this.meter)
+    nodeJSObserversBuilder.buildObservers()
+
+    const jobQueueObserversBuilder = new JobQueueObserversBuilder(this.meter)
+    jobQueueObserversBuilder.buildObservers()
+
+    const statsObserversBuilder = new StatsObserversBuilder(this.meter)
+    statsObserversBuilder.buildObservers()
+
+    const livesObserversBuilder = new LivesObserversBuilder(this.meter)
+    livesObserversBuilder.buildObservers()
+
+    const viewersObserversBuilder = new ViewersObserversBuilder(this.meter)
+    viewersObserversBuilder.buildObservers()
+
+    const bittorrentTrackerObserversBuilder = new BittorrentTrackerObserversBuilder(this.meter, options.trackerServer)
+    bittorrentTrackerObserversBuilder.buildObservers()
+  }
+
+  observePlaybackMetric (video: MVideoImmutable, metrics: PlaybackMetricCreate) {
+    this.playbackMetrics.observe(video, metrics)
+  }
+
+  private buildRequestObserver () {
+    const requestDuration = this.meter.createHistogram('http_request_duration_ms', {
+      unit: 'milliseconds',
+      description: 'Duration of HTTP requests in ms'
+    })
+
+    this.onRequestDuration = (req: Request, res: Response) => {
+      const duration = Date.now() - res.locals.requestStart
+
+      requestDuration.record(duration, {
+        path: this.buildRequestPath(req.originalUrl),
+        method: req.method,
+        statusCode: res.statusCode + ''
+      })
+    }
+  }
+
+  private buildRequestPath (path: string) {
+    return path.split('?')[0]
+  }
+
+  static get Instance () {
+    return this.instance || (this.instance = new this())
+  }
+}
+
+export {
+  OpenTelemetryMetrics
+}

server/core/lib/opentelemetry/tracing.ts

@@ -0,0 +1,140 @@
+import type { Span, Tracer } from '@opentelemetry/api'
+import { logger } from '@server/helpers/logger.js'
+import { CONFIG } from '@server/initializers/config.js'
+
+let tracer: Tracer | TrackerMock
+
+async function registerOpentelemetryTracing () {
+  if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) {
+    tracer = new TrackerMock()
+
+    return
+  }
+
+  const { diag, DiagLogLevel, trace } = await import('@opentelemetry/api')
+  tracer = trace.getTracer('peertube')
+
+  const [
+    { JaegerExporter },
+    { registerInstrumentations },
+    DnsInstrumentation,
+    ExpressInstrumentation,
+    { FsInstrumentation },
+    { HttpInstrumentation },
+    IORedisInstrumentation,
+    PgInstrumentation,
+    { SequelizeInstrumentation },
+    Resource,
+    BatchSpanProcessor,
+    NodeTracerProvider,
+    SemanticResourceAttributes
+  ] = await Promise.all([
+    import('@opentelemetry/exporter-jaeger'),
+    import('@opentelemetry/instrumentation'),
+    import('@opentelemetry/instrumentation-dns'),
+    import('@opentelemetry/instrumentation-express'),
+    import('@opentelemetry/instrumentation-fs'),
+    import('@opentelemetry/instrumentation-http'),
+    import('@opentelemetry/instrumentation-ioredis'),
+    import('@opentelemetry/instrumentation-pg'),
+    import('opentelemetry-instrumentation-sequelize'),
+    import('@opentelemetry/resources'),
+    import('@opentelemetry/sdk-trace-base'),
+    import('@opentelemetry/sdk-trace-node'),
+    import('@opentelemetry/semantic-conventions')
+  ])
+
+  logger.info('Registering Open Telemetry tracing')
+
+  const customLogger = (level: string) => {
+    return (message: string, ...args: unknown[]) => {
+      let fullMessage = message
+
+      for (const arg of args) {
+        if (typeof arg === 'string') fullMessage += arg
+        else break
+      }
+
+      logger[level](fullMessage)
+    }
+  }
+
+  diag.setLogger({
+    error: customLogger('error'),
+    warn: customLogger('warn'),
+    info: customLogger('info'),
+    debug: customLogger('debug'),
+    verbose: customLogger('verbose')
+  }, DiagLogLevel.INFO)
+
+  const tracerProvider = new NodeTracerProvider.default.NodeTracerProvider({
+    resource: new Resource.default.Resource({
+      [SemanticResourceAttributes.default.SemanticResourceAttributes.SERVICE_NAME]: 'peertube'
+    })
+  })
+
+  registerInstrumentations({
+    tracerProvider,
+    instrumentations: [
+      new PgInstrumentation.default.PgInstrumentation({
+        enhancedDatabaseReporting: true
+      }),
+      new DnsInstrumentation.default.DnsInstrumentation(),
+      new HttpInstrumentation(),
+      new ExpressInstrumentation.default.ExpressInstrumentation(),
+      new IORedisInstrumentation.default.IORedisInstrumentation({
+        dbStatementSerializer: function (cmdName, cmdArgs) {
+          return [ cmdName, ...cmdArgs ].join(' ')
+        }
+      }),
+      new FsInstrumentation(),
+      new SequelizeInstrumentation()
+    ]
+  })
+
+  tracerProvider.addSpanProcessor(
+    new BatchSpanProcessor.default.BatchSpanProcessor(
+      new JaegerExporter({ endpoint: CONFIG.OPEN_TELEMETRY.TRACING.JAEGER_EXPORTER.ENDPOINT })
+    )
+  )
+
+  tracerProvider.register()
+}
+
+async function wrapWithSpanAndContext <T> (spanName: string, cb: () => Promise<T>) {
+  const { context, trace } = await import('@opentelemetry/api')
+
+  if (CONFIG.OPEN_TELEMETRY.TRACING.ENABLED !== true) {
+    return cb()
+  }
+
+  const span = tracer.startSpan(spanName)
+  const activeContext = trace.setSpan(context.active(), span as Span)
+
+  const result = await context.with(activeContext, () => cb())
+  span.end()
+
+  return result
+}
+
+export {
+  registerOpentelemetryTracing,
+  tracer,
+  wrapWithSpanAndContext
+}
+
+// ---------------------------------------------------------------------------
+// Private
+// ---------------------------------------------------------------------------
+
+class TrackerMock {
+  startSpan () {
+    return new SpanMock()
+  }
+}
+
+class SpanMock {
+  end () {
+
+  }
+}