diff --git a/apps/peertube-runner/src/server/server.ts b/apps/peertube-runner/src/server/server.ts index e6f50cac7..62a7bce64 100644 --- a/apps/peertube-runner/src/server/server.ts +++ b/apps/peertube-runner/src/server/server.ts @@ -1,10 +1,10 @@ +import { pick, shuffle, wait } from '@peertube/peertube-core-utils' +import { PeerTubeProblemDocument, RunnerJobType, ServerErrorCode } from '@peertube/peertube-models' +import { PeerTubeServer as PeerTubeServerCommand } from '@peertube/peertube-server-commands' import { ensureDir, remove } from 'fs-extra/esm' import { readdir } from 'fs/promises' import { join } from 'path' import { io, Socket } from 'socket.io-client' -import { pick, shuffle, wait } from '@peertube/peertube-core-utils' -import { PeerTubeProblemDocument, RunnerJobType, ServerErrorCode } from '@peertube/peertube-models' -import { PeerTubeServer as PeerTubeServerCommand } from '@peertube/peertube-server-commands' import { ConfigManager } from '../shared/index.js' import { IPCServer } from '../shared/ipc/index.js' import { logger } from '../shared/logger.js' @@ -95,7 +95,12 @@ export class RunnerServer { logger.info(`Registering runner ${runnerName} on ${url}...`) const serverCommand = new PeerTubeServerCommand({ url }) - const { runnerToken } = await serverCommand.runners.register({ name: runnerName, description: runnerDescription, registrationToken }) + const { runnerToken } = await serverCommand.runners.register({ + name: runnerName, + description: runnerDescription, + registrationToken, + version: process.env.PACKAGE_VERSION + }) const server: PeerTubeServer = Object.assign(serverCommand, { runnerToken, @@ -268,7 +273,9 @@ export class RunnerServer { jobTypes: this.enabledJobsArray.length !== getSupportedJobsList().length ? this.enabledJobsArray - : undefined + : undefined, + + version: process.env.PACKAGE_VERSION }) // FIXME: remove in PeerTube v8: jobTypes has been introduced in PeerTube v7, so do the filter here too diff --git a/apps/peertube-runner/src/shared/config-manager.ts b/apps/peertube-runner/src/shared/config-manager.ts index 97a70204a..1c1550f55 100644 --- a/apps/peertube-runner/src/shared/config-manager.ts +++ b/apps/peertube-runner/src/shared/config-manager.ts @@ -137,7 +137,7 @@ export class ConfigManager { // --------------------------------------------------------------------------- // Thanks: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Object/freeze - private deepFreeze (object: T) { + private deepFreeze (object: T) { const propNames = Reflect.ownKeys(object) // Freeze properties before freezing self diff --git a/client/src/app/+admin/system/runners/runner-list/runner-list.component.html b/client/src/app/+admin/system/runners/runner-list/runner-list.component.html index 2e0f57416..50b673258 100644 --- a/client/src/app/+admin/system/runners/runner-list/runner-list.component.html +++ b/client/src/app/+admin/system/runners/runner-list/runner-list.component.html @@ -18,6 +18,8 @@ {{ runner.ip }} + {{ runner.version }} + {{ runner.lastContact | ptDate: 'short' }} {{ runner.createdAt | ptDate: 'short' }} diff --git a/client/src/app/+admin/system/runners/runner-list/runner-list.component.ts b/client/src/app/+admin/system/runners/runner-list/runner-list.component.ts index 37985618e..a8a555874 100644 --- a/client/src/app/+admin/system/runners/runner-list/runner-list.component.ts +++ b/client/src/app/+admin/system/runners/runner-list/runner-list.component.ts @@ -32,6 +32,7 @@ export class RunnerListComponent implements OnInit { { id: 'name', label: $localize`Name`, sortable: false }, { id: 'description', label: $localize`Description`, sortable: false }, { id: 'ip', label: $localize`IP`, sortable: false }, + { id: 'version', label: $localize`Version`, sortable: false }, { id: 'lastContact', label: $localize`Last contact`, sortable: false }, { id: 'createdAt', label: $localize`Created`, sortable: true } ] diff --git a/packages/models/src/runners/register-runner-body.model.ts b/packages/models/src/runners/register-runner-body.model.ts index 969bb35e1..fb1aa9843 100644 --- a/packages/models/src/runners/register-runner-body.model.ts +++ b/packages/models/src/runners/register-runner-body.model.ts @@ -3,4 +3,5 @@ export interface RegisterRunnerBody { name: string description?: string + version?: string } diff --git a/packages/models/src/runners/request-runner-job-body.model.ts b/packages/models/src/runners/request-runner-job-body.model.ts index dfbd11105..f10c87632 100644 --- a/packages/models/src/runners/request-runner-job-body.model.ts +++ b/packages/models/src/runners/request-runner-job-body.model.ts @@ -3,4 +3,5 @@ import { RunnerJobType } from './runner-jobs/runner-job-type.type.js' export interface RequestRunnerJobBody { runnerToken: string jobTypes?: RunnerJobType[] + version?: string } diff --git a/packages/models/src/runners/runner.model.ts b/packages/models/src/runners/runner.model.ts index 3284f2992..f78bf86d3 100644 --- a/packages/models/src/runners/runner.model.ts +++ b/packages/models/src/runners/runner.model.ts @@ -7,6 +7,8 @@ export interface Runner { ip: string lastContact: Date | string + version: string + createdAt: Date | string updatedAt: Date | string } diff --git a/packages/server-commands/src/runners/runner-jobs-command.ts b/packages/server-commands/src/runners/runner-jobs-command.ts index 215a29937..b727c8960 100644 --- a/packages/server-commands/src/runners/runner-jobs-command.ts +++ b/packages/server-commands/src/runners/runner-jobs-command.ts @@ -82,7 +82,7 @@ export class RunnerJobsCommand extends AbstractCommand { ...options, path, - fields: pick(options, [ 'runnerToken', 'jobTypes' ]), + fields: pick(options, [ 'runnerToken', 'jobTypes', 'version' ]), implicitToken: false, defaultExpectedStatus: HttpStatusCode.OK_200 })) diff --git a/packages/server-commands/src/runners/runners-command.ts b/packages/server-commands/src/runners/runners-command.ts index 376a1dff9..0d39a654c 100644 --- a/packages/server-commands/src/runners/runners-command.ts +++ b/packages/server-commands/src/runners/runners-command.ts @@ -12,7 +12,6 @@ import { unwrapBody } from '../requests/index.js' import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js' export class RunnersCommand extends AbstractCommand { - list (options: OverrideCommandOptions & { start?: number count?: number @@ -37,7 +36,7 @@ export class RunnersCommand extends AbstractCommand { ...options, path, - fields: pick(options, [ 'name', 'registrationToken', 'description' ]), + fields: pick(options, [ 'name', 'registrationToken', 'description', 'version' ]), implicitToken: true, defaultExpectedStatus: HttpStatusCode.OK_200 })) @@ -56,9 +55,11 @@ export class RunnersCommand extends AbstractCommand { }) } - delete (options: OverrideCommandOptions & { - id: number - }) { + delete ( + options: OverrideCommandOptions & { + id: number + } + ) { const path = '/api/v1/runners/' + options.id return this.deleteRequest({ diff --git a/packages/tests/src/api/check-params/runners.ts b/packages/tests/src/api/check-params/runners.ts index bc7259660..c40a6b4b4 100644 --- a/packages/tests/src/api/check-params/runners.ts +++ b/packages/tests/src/api/check-params/runners.ts @@ -96,9 +96,7 @@ describe('Test managing runners', function () { }) describe('Managing runner registration tokens', function () { - describe('Common', function () { - it('Should fail to generate, list or delete runner registration token without oauth token', async function () { const expectedStatus = HttpStatusCode.UNAUTHORIZED_401 @@ -117,7 +115,6 @@ describe('Test managing runners', function () { }) describe('Delete', function () { - it('Should fail to delete with a bad id', async function () { await server.runnerRegistrationTokens.delete({ id: 404, expectedStatus: HttpStatusCode.NOT_FOUND_404 }) }) @@ -175,6 +172,13 @@ describe('Test managing runners', function () { await server.runners.register({ name, description: 'a'.repeat(5000), registrationToken, expectedStatus }) }) + it('Should fail with an invalid version', async function () { + const expectedStatus = HttpStatusCode.BAD_REQUEST_400 + + await server.runners.register({ name, version: '', registrationToken, expectedStatus }) + await server.runners.register({ name, version: 'a'.repeat(5000), registrationToken, expectedStatus }) + }) + it('Should succeed with the correct params', async function () { const { id } = await server.runners.register({ name, description: 'super description', registrationToken }) @@ -192,7 +196,6 @@ describe('Test managing runners', function () { }) describe('Delete', function () { - it('Should fail without oauth token', async function () { await server.runners.delete({ token: null, id: toDeleteId, expectedStatus: HttpStatusCode.UNAUTHORIZED_401 }) }) @@ -245,11 +248,9 @@ describe('Test managing runners', function () { await server.runners.list({ start: 0, count: 5, sort: '-createdAt' }) }) }) - }) describe('Runner jobs by admin', function () { - describe('Cancel', function () { let jobUUID: string @@ -356,7 +357,6 @@ describe('Test managing runners', function () { await server.runnerJobs.deleteByAdmin({ jobUUID }) }) }) - }) describe('Runner jobs by runners', function () { @@ -490,7 +490,6 @@ describe('Test managing runners', function () { }) describe('Common runner tokens validations', function () { - async function testEndpoints (options: { jobUUID: string runnerToken: string @@ -597,7 +596,6 @@ describe('Test managing runners', function () { }) describe('Unregister', function () { - it('Should fail without a runner token', async function () { await server.runners.unregister({ runnerToken: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) }) @@ -612,7 +610,6 @@ describe('Test managing runners', function () { }) describe('Request', function () { - it('Should fail without a runner token', async function () { await server.runnerJobs.request({ runnerToken: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) }) @@ -629,13 +626,16 @@ describe('Test managing runners', function () { await server.runnerJobs.request({ runnerToken, jobTypes: 'toto' as any, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) }) + it('Should fail with an invalid runner version', async function () { + await server.runnerJobs.request({ runnerToken, jobTypes: [], version: 'invalid', expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) + }) + it('Should succeed with the correct params', async function () { await server.runnerJobs.request({ runnerToken, jobTypes: [] }) }) }) describe('Accept', function () { - it('Should fail with a bad a job uuid', async function () { await server.runnerJobs.accept({ jobUUID: '', runnerToken, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) }) @@ -663,7 +663,6 @@ describe('Test managing runners', function () { }) describe('Abort', function () { - it('Should fail without a reason', async function () { await server.runnerJobs.abort({ jobUUID, jobToken, runnerToken, reason: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) }) @@ -685,9 +684,7 @@ describe('Test managing runners', function () { }) describe('Update', function () { - describe('Common', function () { - it('Should fail with an invalid progress', async function () { await server.runnerJobs.update({ jobUUID, jobToken, runnerToken, progress: 101, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) }) @@ -742,7 +739,6 @@ describe('Test managing runners', function () { }) describe('Error', function () { - it('Should fail with a missing error message', async function () { await server.runnerJobs.error({ jobUUID, jobToken, runnerToken, message: null, expectedStatus: HttpStatusCode.BAD_REQUEST_400 }) }) @@ -768,7 +764,6 @@ describe('Test managing runners', function () { let vodJobToken: string describe('Common', function () { - it('Should fail with a job not in processing state', async function () { await server.runnerJobs.success({ jobUUID: completedJobUUID, @@ -781,7 +776,6 @@ describe('Test managing runners', function () { }) describe('VOD', function () { - it('Should fail with an invalid vod web video payload', async function () { const { job } = await server.runnerJobs.autoAccept({ runnerToken, type: 'vod-web-video-transcoding' }) @@ -834,7 +828,6 @@ describe('Test managing runners', function () { }) describe('Video studio', function () { - it('Should fail with an invalid video studio transcoding payload', async function () { await server.runnerJobs.success({ jobUUID: studioAcceptedJob.uuid, @@ -848,9 +841,7 @@ describe('Test managing runners', function () { }) describe('Job files', function () { - describe('Check video param for common job file routes', function () { - async function fetchFiles (options: { videoUUID?: string expectedStatus: HttpStatusCodeType @@ -898,7 +889,6 @@ describe('Test managing runners', function () { }) describe('Video studio tasks file routes', function () { - it('Should fail with an invalid studio filename', async function () { await fetchStudioFiles({ videoUUID: videoStudioUUID, diff --git a/packages/tests/src/api/runners/runner-common.ts b/packages/tests/src/api/runners/runner-common.ts index 2119593c4..0a295e1b2 100644 --- a/packages/tests/src/api/runners/runner-common.ts +++ b/packages/tests/src/api/runners/runner-common.ts @@ -154,7 +154,8 @@ describe('Test runner common actions', function () { await server.runners.register({ name: 'runner 2', - registrationToken + registrationToken, + version: '1.0.0' }) const { total, data } = await server.runners.list({ sort: 'createdAt' }) @@ -173,9 +174,11 @@ describe('Test runner common actions', function () { expect(data[0].name).to.equal('runner 1') expect(data[0].description).to.equal('my super runner 1') + expect(data[0].version).to.not.exist expect(data[1].name).to.equal('runner 2') expect(data[1].description).to.be.null + expect(data[1].version).to.equal('1.0.0') toDelete = data[1] }) @@ -244,7 +247,6 @@ describe('Test runner common actions', function () { } describe('List jobs', function () { - it('Should not have jobs', async function () { const { total, data } = await server.runnerJobs.list() @@ -369,7 +371,6 @@ describe('Test runner common actions', function () { }) describe('Accept/update/abort/process a job', function () { - it('Should request available jobs', async function () { lastRunnerContact = new Date() @@ -395,6 +396,13 @@ describe('Test runner common actions', function () { jobUUID = webVideoJobs[0].uuid }) + it('Should update runner version', async function () { + await server.runnerJobs.request({ runnerToken, version: '2.0.0' }) + + const { data } = await server.runners.list({ sort: 'createdAt' }) + expect(data[0].version).to.equal('2.0.0') + }) + it('Should filter requested jobs', async function () { { const { availableJobs } = await server.runnerJobs.request({ runnerToken, jobTypes: [ 'vod-web-video-transcoding' ] }) @@ -526,7 +534,6 @@ describe('Test runner common actions', function () { }) describe('Error job', function () { - it('Should accept another job and post an error', async function () { await server.runnerJobs.cancelAllJobs() await server.videos.quickUpload({ name: 'video' }) @@ -588,7 +595,6 @@ describe('Test runner common actions', function () { }) describe('Cancel', function () { - it('Should cancel a pending job', async function () { await server.videos.quickUpload({ name: 'video' }) await waitJobs([ server ]) @@ -636,7 +642,6 @@ describe('Test runner common actions', function () { }) describe('Remove', function () { - it('Should remove a pending job', async function () { await server.videos.quickUpload({ name: 'video' }) await waitJobs([ server ]) @@ -663,7 +668,6 @@ describe('Test runner common actions', function () { }) describe('Stalled jobs', function () { - it('Should abort stalled jobs', async function () { this.timeout(60000) @@ -689,7 +693,6 @@ describe('Test runner common actions', function () { }) describe('Rate limit', function () { - before(async function () { this.timeout(60000) diff --git a/server/core/controllers/api/runners/jobs.ts b/server/core/controllers/api/runners/jobs.ts index cb66acd52..ea0a87ac7 100644 --- a/server/core/controllers/api/runners/jobs.ts +++ b/server/core/controllers/api/runners/jobs.ts @@ -46,6 +46,7 @@ import { getRunnerFromTokenValidator, jobOfRunnerGetValidatorFactory, listRunnerJobsValidator, + requestRunnerJobValidator, runnerJobGetValidator, successRunnerJobValidator, updateRunnerJobValidator @@ -72,13 +73,16 @@ const runnerJobsRouter = express.Router() // Controllers for runners // --------------------------------------------------------------------------- -runnerJobsRouter.post('/jobs/request', +runnerJobsRouter.post( + '/jobs/request', apiRateLimiter, + requestRunnerJobValidator, asyncMiddleware(getRunnerFromTokenValidator), asyncMiddleware(requestRunnerJob) ) -runnerJobsRouter.post('/jobs/:jobUUID/accept', +runnerJobsRouter.post( + '/jobs/:jobUUID/accept', apiRateLimiter, asyncMiddleware(runnerJobGetValidator), acceptRunnerJobValidator, @@ -86,14 +90,16 @@ runnerJobsRouter.post('/jobs/:jobUUID/accept', asyncMiddleware(acceptRunnerJob) ) -runnerJobsRouter.post('/jobs/:jobUUID/abort', +runnerJobsRouter.post( + '/jobs/:jobUUID/abort', apiRateLimiter, asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), abortRunnerJobValidator, asyncMiddleware(abortRunnerJob) ) -runnerJobsRouter.post('/jobs/:jobUUID/update', +runnerJobsRouter.post( + '/jobs/:jobUUID/update', runnerJobUpdateVideoFiles, apiRateLimiter, // Has to be after multer middleware to parse runner token asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING, RunnerJobState.COMPLETING, RunnerJobState.COMPLETED ])), @@ -101,13 +107,15 @@ runnerJobsRouter.post('/jobs/:jobUUID/update', asyncMiddleware(updateRunnerJobController) ) -runnerJobsRouter.post('/jobs/:jobUUID/error', +runnerJobsRouter.post( + '/jobs/:jobUUID/error', asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), errorRunnerJobValidator, asyncMiddleware(errorRunnerJob) ) -runnerJobsRouter.post('/jobs/:jobUUID/success', +runnerJobsRouter.post( + '/jobs/:jobUUID/success', postRunnerJobSuccessVideoFiles, apiRateLimiter, // Has to be after multer middleware to parse runner token asyncMiddleware(jobOfRunnerGetValidatorFactory([ RunnerJobState.PROCESSING ])), @@ -119,7 +127,8 @@ runnerJobsRouter.post('/jobs/:jobUUID/success', // Controllers for admins // --------------------------------------------------------------------------- -runnerJobsRouter.post('/jobs/:jobUUID/cancel', +runnerJobsRouter.post( + '/jobs/:jobUUID/cancel', authenticate, ensureUserHasRight(UserRight.MANAGE_RUNNERS), asyncMiddleware(runnerJobGetValidator), @@ -127,7 +136,8 @@ runnerJobsRouter.post('/jobs/:jobUUID/cancel', asyncMiddleware(cancelRunnerJob) ) -runnerJobsRouter.get('/jobs', +runnerJobsRouter.get( + '/jobs', authenticate, ensureUserHasRight(UserRight.MANAGE_RUNNERS), paginationValidator, @@ -138,7 +148,8 @@ runnerJobsRouter.get('/jobs', asyncMiddleware(listRunnerJobs) ) -runnerJobsRouter.delete('/jobs/:jobUUID', +runnerJobsRouter.delete( + '/jobs/:jobUUID', authenticate, ensureUserHasRight(UserRight.MANAGE_RUNNERS), asyncMiddleware(runnerJobGetValidator), @@ -172,6 +183,11 @@ async function requestRunnerJob (req: express.Request, res: express.Response) { })) } + if (body.version && runner.version !== body.version) { + runner.version = body.version + await runner.save() + } + updateLastRunnerContact(req, runner) return res.json(result) @@ -218,7 +234,10 @@ async function acceptRunnerJob (req: express.Request, res: express.Response) { updateLastRunnerContact(req, runner) logger.info( - 'Remote runner %s has accepted job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, + 'Remote runner %s has accepted job %s (%s)', + runner.name, + runnerJob.uuid, + runnerJob.type, lTags(runner.name, runnerJob.uuid, runnerJob.type) ) @@ -231,7 +250,10 @@ async function abortRunnerJob (req: express.Request, res: express.Response) { const body: AbortRunnerJobBody = req.body logger.info( - 'Remote runner %s is aborting job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, + 'Remote runner %s is aborting job %s (%s)', + runner.name, + runnerJob.uuid, + runnerJob.type, { reason: body.reason, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } ) @@ -251,7 +273,10 @@ async function errorRunnerJob (req: express.Request, res: express.Response) { runnerJob.failures += 1 logger.error( - 'Remote runner %s had an error with job %s (%s)', runner.name, runnerJob.uuid, runnerJob.type, + 'Remote runner %s had an error with job %s (%s)', + runner.name, + runnerJob.uuid, + runnerJob.type, { errorMessage: body.message, totalFailures: runnerJob.failures, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } ) @@ -294,7 +319,10 @@ async function updateRunnerJobController (req: express.Request, res: express.Res : undefined logger.debug( - 'Remote runner %s is updating job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, + 'Remote runner %s is updating job %s (%s)', + runnerJob.Runner.name, + runnerJob.uuid, + runnerJob.type, { body, updatePayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } ) @@ -367,7 +395,10 @@ async function postRunnerJobSuccess (req: express.Request, res: express.Response const resultPayload = jobSuccessPayloadBuilders[runnerJob.type](body.payload, req.files as UploadFiles) logger.info( - 'Remote runner %s is sending success result for job %s (%s)', runnerJob.Runner.name, runnerJob.uuid, runnerJob.type, + 'Remote runner %s is sending success result for job %s (%s)', + runnerJob.Runner.name, + runnerJob.uuid, + runnerJob.type, { resultPayload, ...lTags(runner.name, runnerJob.uuid, runnerJob.type) } ) diff --git a/server/core/controllers/api/runners/manage-runners.ts b/server/core/controllers/api/runners/manage-runners.ts index fb2199bea..1e01bc8d8 100644 --- a/server/core/controllers/api/runners/manage-runners.ts +++ b/server/core/controllers/api/runners/manage-runners.ts @@ -1,4 +1,3 @@ -import express from 'express' import { HttpStatusCode, ListRunnersQuery, RegisterRunnerBody, UserRight } from '@peertube/peertube-models' import { logger, loggerTagsFactory } from '@server/helpers/logger.js' import { generateRunnerToken } from '@server/helpers/token-generator.js' @@ -18,23 +17,28 @@ import { registerRunnerValidator } from '@server/middlewares/validators/runners/index.js' import { RunnerModel } from '@server/models/runner/runner.js' +import express from 'express' const lTags = loggerTagsFactory('api', 'runner') const manageRunnersRouter = express.Router() -manageRunnersRouter.post('/register', +manageRunnersRouter.post( + '/register', apiRateLimiter, asyncMiddleware(registerRunnerValidator), asyncMiddleware(registerRunner) ) -manageRunnersRouter.post('/unregister', + +manageRunnersRouter.post( + '/unregister', apiRateLimiter, asyncMiddleware(getRunnerFromTokenValidator), asyncMiddleware(unregisterRunner) ) -manageRunnersRouter.delete('/:runnerId', +manageRunnersRouter.delete( + '/:runnerId', apiRateLimiter, authenticate, ensureUserHasRight(UserRight.MANAGE_RUNNERS), @@ -42,7 +46,8 @@ manageRunnersRouter.delete('/:runnerId', asyncMiddleware(deleteRunner) ) -manageRunnersRouter.get('/', +manageRunnersRouter.get( + '/', apiRateLimiter, authenticate, ensureUserHasRight(UserRight.MANAGE_RUNNERS), @@ -72,6 +77,7 @@ async function registerRunner (req: express.Request, res: express.Response) { description: body.description, lastContact: new Date(), ip: req.ip, + version: body.version, runnerRegistrationTokenId: res.locals.runnerRegistrationToken.id }) diff --git a/server/core/helpers/custom-validators/misc.ts b/server/core/helpers/custom-validators/misc.ts index fb8ae3b37..726e2b1ae 100644 --- a/server/core/helpers/custom-validators/misc.ts +++ b/server/core/helpers/custom-validators/misc.ts @@ -171,3 +171,26 @@ export function toIntArray (value: any) { return value.map(v => validator.default.toInt(v)) } + +// --------------------------------------------------------------------------- + +export function isStableVersionValid (value: string) { + if (!exists(value)) return false + + const parts = (value + '').split('.') + + return parts.length === 3 && parts.every(p => validator.default.isInt(p)) +} + +export function isStableOrUnstableVersionValid (value: string) { + if (!exists(value)) return false + + // suffix is beta.x or alpha.x + const [ stable, suffix ] = value.split('-') + if (!isStableVersionValid(stable)) return false + + const suffixRegex = /^(rc|alpha|beta)\.\d+$/ + if (suffix && !suffixRegex.test(suffix)) return false + + return true +} diff --git a/server/core/helpers/custom-validators/plugins.ts b/server/core/helpers/custom-validators/plugins.ts index 99ee476b7..2d388f6dd 100644 --- a/server/core/helpers/custom-validators/plugins.ts +++ b/server/core/helpers/custom-validators/plugins.ts @@ -1,67 +1,46 @@ -import validator from 'validator' import { PluginPackageJSON, PluginType, PluginType_Type } from '@peertube/peertube-models' +import validator from 'validator' import { CONSTRAINTS_FIELDS } from '../../initializers/constants.js' import { isUrlValid } from './activitypub/misc.js' import { exists, isArray, isSafePath } from './misc.js' const PLUGINS_CONSTRAINTS_FIELDS = CONSTRAINTS_FIELDS.PLUGINS -function isPluginTypeValid (value: any) { +export function isPluginTypeValid (value: any) { return exists(value) && (value === PluginType.PLUGIN || value === PluginType.THEME) } -function isPluginNameValid (value: string) { +export function isPluginNameValid (value: string) { return exists(value) && validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && validator.default.matches(value, /^[a-z-0-9]+$/) } -function isNpmPluginNameValid (value: string) { +export function isNpmPluginNameValid (value: string) { return exists(value) && validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.NAME) && validator.default.matches(value, /^[a-z\-._0-9]+$/) && (value.startsWith('peertube-plugin-') || value.startsWith('peertube-theme-')) } -function isPluginDescriptionValid (value: string) { +export function isPluginDescriptionValid (value: string) { return exists(value) && validator.default.isLength(value, PLUGINS_CONSTRAINTS_FIELDS.DESCRIPTION) } -function isPluginStableVersionValid (value: string) { - if (!exists(value)) return false - - const parts = (value + '').split('.') - - return parts.length === 3 && parts.every(p => validator.default.isInt(p)) -} - -function isPluginStableOrUnstableVersionValid (value: string) { - if (!exists(value)) return false - - // suffix is beta.x or alpha.x - const [ stable, suffix ] = value.split('-') - if (!isPluginStableVersionValid(stable)) return false - - const suffixRegex = /^(rc|alpha|beta)\.\d+$/ - if (suffix && !suffixRegex.test(suffix)) return false - - return true -} - -function isPluginEngineValid (engine: any) { +export function isPluginEngineValid (engine: any) { return exists(engine) && exists(engine.peertube) } -function isPluginHomepage (value: string) { +export function isPluginHomepage (value: string) { return exists(value) && (!value || isUrlValid(value)) } -function isPluginBugs (value: string) { +export function isPluginBugs (value: string) { return exists(value) && (!value || isUrlValid(value)) } -function areStaticDirectoriesValid (staticDirs: any) { +export function areStaticDirectoriesValid (staticDirs: any) { if (!exists(staticDirs) || typeof staticDirs !== 'object') return false for (const key of Object.keys(staticDirs)) { @@ -71,14 +50,14 @@ function areStaticDirectoriesValid (staticDirs: any) { return true } -function areClientScriptsValid (clientScripts: any[]) { +export function areClientScriptsValid (clientScripts: any[]) { return isArray(clientScripts) && clientScripts.every(c => { return isSafePath(c.script) && isArray(c.scopes) }) } -function areTranslationPathsValid (translations: any) { +export function areTranslationPathsValid (translations: any) { if (!exists(translations) || typeof translations !== 'object') return false for (const key of Object.keys(translations)) { @@ -88,16 +67,16 @@ function areTranslationPathsValid (translations: any) { return true } -function areCSSPathsValid (css: any[]) { +export function areCSSPathsValid (css: any[]) { return isArray(css) && css.every(c => isSafePath(c)) } -function isThemeNameValid (name: string) { +export function isThemeNameValid (name: string) { return name && typeof name === 'string' && (isPluginNameValid(name) || name.startsWith('peertube-core-')) } -function isPackageJSONValid (packageJSON: PluginPackageJSON, pluginType: PluginType_Type) { +export function isPackageJSONValid (packageJSON: PluginPackageJSON, pluginType: PluginType_Type) { let result = true const badFields: string[] = [] @@ -159,20 +138,7 @@ function isPackageJSONValid (packageJSON: PluginPackageJSON, pluginType: PluginT return { result, badFields } } -function isLibraryCodeValid (library: any) { +export function isLibraryCodeValid (library: any) { return typeof library.register === 'function' && typeof library.unregister === 'function' } - -export { - isPluginTypeValid, - isPackageJSONValid, - isThemeNameValid, - isPluginHomepage, - isPluginStableVersionValid, - isPluginStableOrUnstableVersionValid, - isPluginNameValid, - isPluginDescriptionValid, - isLibraryCodeValid, - isNpmPluginNameValid -} diff --git a/server/core/initializers/constants.ts b/server/core/initializers/constants.ts index c69521ca9..a8582d477 100644 --- a/server/core/initializers/constants.ts +++ b/server/core/initializers/constants.ts @@ -48,7 +48,7 @@ import { CONFIG, registerConfigChangedHandler } from './config.js' // --------------------------------------------------------------------------- -export const LAST_MIGRATION_VERSION = 910 +export const LAST_MIGRATION_VERSION = 915 // --------------------------------------------------------------------------- diff --git a/server/core/initializers/migrations/0915-runner-version.ts b/server/core/initializers/migrations/0915-runner-version.ts new file mode 100644 index 000000000..c6cfea5f8 --- /dev/null +++ b/server/core/initializers/migrations/0915-runner-version.ts @@ -0,0 +1,26 @@ +import * as Sequelize from 'sequelize' + +async function up (utils: { + transaction: Sequelize.Transaction + queryInterface: Sequelize.QueryInterface + sequelize: Sequelize.Sequelize +}): Promise { + const { transaction } = utils + + { + await utils.queryInterface.addColumn('runner', 'version', { + type: Sequelize.STRING, + defaultValue: null, + allowNull: true + }, { transaction }) + } +} + +function down (options) { + throw new Error('Not implemented.') +} + +export { + down, + up +} diff --git a/server/core/lib/plugins/yarn.ts b/server/core/lib/plugins/yarn.ts index f725cf698..099474039 100644 --- a/server/core/lib/plugins/yarn.ts +++ b/server/core/lib/plugins/yarn.ts @@ -1,12 +1,13 @@ +import { isStableOrUnstableVersionValid } from '@server/helpers/custom-validators/misc.js' import { outputJSON, pathExists } from 'fs-extra/esm' import { join } from 'path' import { execShell } from '../../helpers/core-utils.js' -import { isNpmPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins.js' +import { isNpmPluginNameValid } from '../../helpers/custom-validators/plugins.js' import { logger } from '../../helpers/logger.js' import { CONFIG } from '../../initializers/config.js' import { getLatestPluginVersion } from './plugin-index.js' -async function installNpmPlugin (npmName: string, versionArg?: string) { +export async function installNpmPlugin (npmName: string, versionArg?: string) { // Security check checkNpmPluginNameOrThrow(npmName) if (versionArg) checkPluginVersionOrThrow(versionArg) @@ -21,31 +22,22 @@ async function installNpmPlugin (npmName: string, versionArg?: string) { logger.debug('Added a yarn package.', { yarnStdout: stdout }) } -async function installNpmPluginFromDisk (path: string) { +export async function installNpmPluginFromDisk (path: string) { await execYarn('add file:' + path) } -async function removeNpmPlugin (name: string) { +export async function removeNpmPlugin (name: string) { checkNpmPluginNameOrThrow(name) await execYarn('remove ' + name) } -async function rebuildNativePlugins () { +export async function rebuildNativePlugins () { await execYarn('install --pure-lockfile') } // ############################################################################ -export { - installNpmPlugin, - installNpmPluginFromDisk, - rebuildNativePlugins, - removeNpmPlugin -} - -// ############################################################################ - async function execYarn (command: string) { try { const pluginDirectory = CONFIG.STORAGE.PLUGINS_DIR @@ -69,5 +61,5 @@ function checkNpmPluginNameOrThrow (name: string) { } function checkPluginVersionOrThrow (name: string) { - if (!isPluginStableOrUnstableVersionValid(name)) throw new Error('Invalid NPM plugin version to install') + if (!isStableOrUnstableVersionValid(name)) throw new Error('Invalid NPM plugin version to install') } diff --git a/server/core/middlewares/validators/plugins.ts b/server/core/middlewares/validators/plugins.ts index 0d2b7605a..ce5c6089e 100644 --- a/server/core/middlewares/validators/plugins.ts +++ b/server/core/middlewares/validators/plugins.ts @@ -1,19 +1,21 @@ +import { HttpStatusCode, InstallOrUpdatePlugin, PluginType_Type } from '@peertube/peertube-models' import express from 'express' import { body, param, query, ValidationChain } from 'express-validator' -import { HttpStatusCode, InstallOrUpdatePlugin, PluginType_Type } from '@peertube/peertube-models' -import { exists, isBooleanValid, isSafePath, toBooleanOrNull, toIntOrNull } from '../../helpers/custom-validators/misc.js' import { - isNpmPluginNameValid, - isPluginNameValid, - isPluginStableOrUnstableVersionValid, - isPluginTypeValid -} from '../../helpers/custom-validators/plugins.js' + exists, + isBooleanValid, + isSafePath, + isStableOrUnstableVersionValid, + toBooleanOrNull, + toIntOrNull +} from '../../helpers/custom-validators/misc.js' +import { isNpmPluginNameValid, isPluginNameValid, isPluginTypeValid } from '../../helpers/custom-validators/plugins.js' import { CONFIG } from '../../initializers/config.js' import { PluginManager } from '../../lib/plugins/plugin-manager.js' import { PluginModel } from '../../models/server/plugin.js' import { areValidationErrors } from './shared/index.js' -const getPluginValidator = (pluginType: PluginType_Type, withVersion = true) => { +export const getPluginValidator = (pluginType: PluginType_Type, withVersion = true) => { const validators: (ValidationChain | express.Handler)[] = [ param('pluginName') .custom(isPluginNameValid) @@ -22,7 +24,7 @@ const getPluginValidator = (pluginType: PluginType_Type, withVersion = true) => if (withVersion) { validators.push( param('pluginVersion') - .custom(isPluginStableOrUnstableVersionValid) + .custom(isStableOrUnstableVersionValid) ) } @@ -53,7 +55,7 @@ const getPluginValidator = (pluginType: PluginType_Type, withVersion = true) => ]) } -const getExternalAuthValidator = [ +export const getExternalAuthValidator = [ param('authName') .custom(exists), @@ -82,7 +84,7 @@ const getExternalAuthValidator = [ } ] -const pluginStaticDirectoryValidator = [ +export const pluginStaticDirectoryValidator = [ param('staticEndpoint') .custom(isSafePath), @@ -93,7 +95,7 @@ const pluginStaticDirectoryValidator = [ } ] -const listPluginsValidator = [ +export const listPluginsValidator = [ query('pluginType') .optional() .customSanitizer(toIntOrNull) @@ -110,13 +112,13 @@ const listPluginsValidator = [ } ] -const installOrUpdatePluginValidator = [ +export const installOrUpdatePluginValidator = [ body('npmName') .optional() .custom(isNpmPluginNameValid), body('pluginVersion') .optional() - .custom(isPluginStableOrUnstableVersionValid), + .custom(isStableOrUnstableVersionValid), body('path') .optional() .custom(isSafePath), @@ -136,7 +138,7 @@ const installOrUpdatePluginValidator = [ } ] -const uninstallPluginValidator = [ +export const uninstallPluginValidator = [ body('npmName') .custom(isNpmPluginNameValid), @@ -147,7 +149,7 @@ const uninstallPluginValidator = [ } ] -const existingPluginValidator = [ +export const existingPluginValidator = [ param('npmName') .custom(isNpmPluginNameValid), @@ -167,7 +169,7 @@ const existingPluginValidator = [ } ] -const updatePluginSettingsValidator = [ +export const updatePluginSettingsValidator = [ body('settings') .exists(), @@ -178,7 +180,7 @@ const updatePluginSettingsValidator = [ } ] -const listAvailablePluginsValidator = [ +export const listAvailablePluginsValidator = [ query('search') .optional() .exists(), @@ -188,7 +190,7 @@ const listAvailablePluginsValidator = [ .custom(isPluginTypeValid), query('currentPeerTubeEngine') .optional() - .custom(isPluginStableOrUnstableVersionValid), + .custom(isStableOrUnstableVersionValid), (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res)) return @@ -200,17 +202,3 @@ const listAvailablePluginsValidator = [ return next() } ] - -// --------------------------------------------------------------------------- - -export { - pluginStaticDirectoryValidator, - getPluginValidator, - updatePluginSettingsValidator, - uninstallPluginValidator, - listAvailablePluginsValidator, - existingPluginValidator, - installOrUpdatePluginValidator, - listPluginsValidator, - getExternalAuthValidator -} diff --git a/server/core/middlewares/validators/runners/runners.ts b/server/core/middlewares/validators/runners/runners.ts index c58009a5a..31e102447 100644 --- a/server/core/middlewares/validators/runners/runners.ts +++ b/server/core/middlewares/validators/runners/runners.ts @@ -1,24 +1,25 @@ -import express from 'express' -import { body, param } from 'express-validator' -import { isIdValid } from '@server/helpers/custom-validators/misc.js' +import { forceNumber } from '@peertube/peertube-core-utils' +import { HttpStatusCode, RegisterRunnerBody, ServerErrorCode } from '@peertube/peertube-models' +import { isIdValid, isStableOrUnstableVersionValid } from '@server/helpers/custom-validators/misc.js' import { isRunnerDescriptionValid, isRunnerNameValid, isRunnerRegistrationTokenValid, isRunnerTokenValid } from '@server/helpers/custom-validators/runners/runners.js' -import { RunnerModel } from '@server/models/runner/runner.js' import { RunnerRegistrationTokenModel } from '@server/models/runner/runner-registration-token.js' -import { forceNumber } from '@peertube/peertube-core-utils' -import { HttpStatusCode, RegisterRunnerBody, ServerErrorCode } from '@peertube/peertube-models' +import { RunnerModel } from '@server/models/runner/runner.js' +import express from 'express' +import { body, param } from 'express-validator' import { areValidationErrors } from '../shared/utils.js' const tags = [ 'runner' ] -const registerRunnerValidator = [ +export const registerRunnerValidator = [ body('registrationToken').custom(isRunnerRegistrationTokenValid), body('name').custom(isRunnerNameValid), body('description').optional().custom(isRunnerDescriptionValid), + body('version').optional().custom(isStableOrUnstableVersionValid), async (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res, { tags })) return @@ -50,7 +51,7 @@ const registerRunnerValidator = [ } ] -const deleteRunnerValidator = [ +export const deleteRunnerValidator = [ param('runnerId').custom(isIdValid), async (req: express.Request, res: express.Response, next: express.NextFunction) => { @@ -72,9 +73,8 @@ const deleteRunnerValidator = [ } ] -const getRunnerFromTokenValidator = [ +export const getRunnerFromTokenValidator = [ body('runnerToken').custom(isRunnerTokenValid), - body('jobTypes').optional().isArray(), async (req: express.Request, res: express.Response, next: express.NextFunction) => { if (areValidationErrors(req, res, { tags })) return @@ -96,10 +96,13 @@ const getRunnerFromTokenValidator = [ } ] -// --------------------------------------------------------------------------- +export const requestRunnerJobValidator = [ + body('version').optional().custom(isStableOrUnstableVersionValid), + body('jobTypes').optional().isArray(), -export { - registerRunnerValidator, - deleteRunnerValidator, - getRunnerFromTokenValidator -} + (req: express.Request, res: express.Response, next: express.NextFunction) => { + if (areValidationErrors(req, res, { tags })) return + + return next() + } +] diff --git a/server/core/middlewares/validators/themes.ts b/server/core/middlewares/validators/themes.ts index ec7ce3b4f..c3e338a96 100644 --- a/server/core/middlewares/validators/themes.ts +++ b/server/core/middlewares/validators/themes.ts @@ -1,16 +1,16 @@ +import { HttpStatusCode } from '@peertube/peertube-models' import express from 'express' import { param } from 'express-validator' -import { HttpStatusCode } from '@peertube/peertube-models' -import { isSafePath } from '../../helpers/custom-validators/misc.js' -import { isPluginNameValid, isPluginStableOrUnstableVersionValid } from '../../helpers/custom-validators/plugins.js' +import { isSafePath, isStableOrUnstableVersionValid } from '../../helpers/custom-validators/misc.js' +import { isPluginNameValid } from '../../helpers/custom-validators/plugins.js' import { PluginManager } from '../../lib/plugins/plugin-manager.js' import { areValidationErrors } from './shared/index.js' -const serveThemeCSSValidator = [ +export const serveThemeCSSValidator = [ param('themeName') .custom(isPluginNameValid), param('themeVersion') - .custom(isPluginStableOrUnstableVersionValid), + .custom(isStableOrUnstableVersionValid), param('staticEndpoint') .custom(isSafePath), @@ -38,9 +38,3 @@ const serveThemeCSSValidator = [ return next() } ] - -// --------------------------------------------------------------------------- - -export { - serveThemeCSSValidator -} diff --git a/server/core/models/runner/runner.ts b/server/core/models/runner/runner.ts index b96d24352..924dacca8 100644 --- a/server/core/models/runner/runner.ts +++ b/server/core/models/runner/runner.ts @@ -1,10 +1,10 @@ +import { Runner } from '@peertube/peertube-models' +import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' +import { MRunner } from '@server/types/models/runners/index.js' import { FindOptions } from 'sequelize' import { AllowNull, BelongsTo, Column, CreatedAt, DataType, ForeignKey, Table, UpdatedAt } from 'sequelize-typescript' -import { MRunner } from '@server/types/models/runners/index.js' -import { Runner } from '@peertube/peertube-models' import { SequelizeModel, getSort } from '../shared/index.js' import { RunnerRegistrationTokenModel } from './runner-registration-token.js' -import { CONSTRAINTS_FIELDS } from '@server/initializers/constants.js' @Table({ tableName: 'runner', @@ -44,6 +44,10 @@ export class RunnerModel extends SequelizeModel { @Column declare ip: string + @AllowNull(true) + @Column + declare version: string + @CreatedAt declare createdAt: Date @@ -114,6 +118,7 @@ export class RunnerModel extends SequelizeModel { ip: this.ip, lastContact: this.lastContact, + version: this.version, createdAt: this.createdAt, updatedAt: this.updatedAt diff --git a/server/core/models/server/plugin.ts b/server/core/models/server/plugin.ts index c7b6fbcde..0620e7914 100644 --- a/server/core/models/server/plugin.ts +++ b/server/core/models/server/plugin.ts @@ -6,6 +6,7 @@ import { SettingValue, type PluginType_Type } from '@peertube/peertube-models' +import { isStableOrUnstableVersionValid, isStableVersionValid } from '@server/helpers/custom-validators/misc.js' import { MPlugin, MPluginFormattable } from '@server/types/models/index.js' import { FindAndCountOptions, QueryTypes, json } from 'sequelize' import { AllowNull, Column, CreatedAt, DataType, DefaultScope, Is, Table, UpdatedAt } from 'sequelize-typescript' @@ -13,8 +14,6 @@ import { isPluginDescriptionValid, isPluginHomepage, isPluginNameValid, - isPluginStableOrUnstableVersionValid, - isPluginStableVersionValid, isPluginTypeValid } from '../../helpers/custom-validators/plugins.js' import { SequelizeModel, getSort, throwIfNotValid } from '../shared/index.js' @@ -45,12 +44,12 @@ export class PluginModel extends SequelizeModel { declare type: PluginType_Type @AllowNull(false) - @Is('PluginVersion', value => throwIfNotValid(value, isPluginStableOrUnstableVersionValid, 'version')) + @Is('PluginVersion', value => throwIfNotValid(value, isStableOrUnstableVersionValid, 'version')) @Column declare version: string @AllowNull(true) - @Is('PluginLatestVersion', value => throwIfNotValid(value, isPluginStableVersionValid, 'version')) + @Is('PluginLatestVersion', value => throwIfNotValid(value, isStableVersionValid, 'latestVersion')) @Column declare latestVersion: string