From cdb861a26a5ed64d4742b096321022c29b2d76d1 Mon Sep 17 00:00:00 2001 From: Ankit lal Date: Tue, 22 Jul 2025 22:37:51 +0530 Subject: [PATCH] Add WatchDog for stalled Transcription jobs --- config/default.yaml | 1 + server/core/initializers/config.ts | 3 ++- .../core/lib/schedulers/runner-job-watch-dog-scheduler.ts | 7 ++++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 48e575a93..75add5b4a 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -455,6 +455,7 @@ remote_runners: stalled_jobs: live: '30 seconds' vod: '2 minutes' + transcription: '2 minutes' thumbnails: # When automatically generating a thumbnail from the video diff --git a/server/core/initializers/config.ts b/server/core/initializers/config.ts index 17491d2ac..485facb67 100644 --- a/server/core/initializers/config.ts +++ b/server/core/initializers/config.ts @@ -442,7 +442,8 @@ const CONFIG = { REMOTE_RUNNERS: { STALLED_JOBS: { LIVE: parseDurationToMs(config.get('remote_runners.stalled_jobs.live')), - VOD: parseDurationToMs(config.get('remote_runners.stalled_jobs.vod')) + VOD: parseDurationToMs(config.get('remote_runners.stalled_jobs.vod')), + TRANSCRIPTION: parseDurationToMs(config.get('remote_runners.stalled_jobs.transcription')) } }, THUMBNAILS: { diff --git a/server/core/lib/schedulers/runner-job-watch-dog-scheduler.ts b/server/core/lib/schedulers/runner-job-watch-dog-scheduler.ts index 7669e359b..61b77fe02 100644 --- a/server/core/lib/schedulers/runner-job-watch-dog-scheduler.ts +++ b/server/core/lib/schedulers/runner-job-watch-dog-scheduler.ts @@ -28,7 +28,12 @@ export class RunnerJobWatchDogScheduler extends AbstractScheduler { types: [ 'live-rtmp-hls-transcoding' ] }) - for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs ]) { + const transcriptionStalledJobs = await RunnerJobModel.listStalledJobs({ + staleTimeMS: CONFIG.REMOTE_RUNNERS.STALLED_JOBS.TRANSCRIPTION, + types: [ 'video-transcription' ] + }) + + for (const stalled of [ ...vodStalledJobs, ...liveStalledJobs, ...transcriptionStalledJobs ]) { logger.info('Abort stalled runner job %s (%s)', stalled.uuid, stalled.type, lTags(stalled.uuid, stalled.type)) const Handler = getRunnerJobHandlerClass(stalled)