Add user import/export tests

packages/server-commands/src/requests/requests.ts

@@ -3,7 +3,7 @@
 import { decode } from 'querystring'
 import request from 'supertest'
 import { URL } from 'url'
-import { pick } from '@peertube/peertube-core-utils'
+import { pick, queryParamsToObject } from '@peertube/peertube-core-utils'
 import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models'
 import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils'
 
@@ -23,23 +23,33 @@ export type CommonRequestParams = {
   expectedStatus?: HttpStatusCodeType
 }
 
-function makeRawRequest (options: {
+export function makeRawRequest (options: {
   url: string
   token?: string
   expectedStatus?: HttpStatusCodeType
+  responseType?: string
   range?: string
   query?: { [ id: string ]: string }
   method?: 'GET' | 'POST'
+  accept?: string
   headers?: { [ name: string ]: string }
+  redirects?: number
 }) {
-  const { host, protocol, pathname } = new URL(options.url)
+  const { host, protocol, pathname, searchParams } = new URL(options.url)
 
   const reqOptions = {
     url: `${protocol}//${host}`,
     path: pathname,
+
     contentType: undefined,
 
-    ...pick(options, [ 'expectedStatus', 'range', 'token', 'query', 'headers' ])
+    query: {
+      ...(options.query || {}),
+
+      ...queryParamsToObject(searchParams)
+    },
+
+    ...pick(options, [ 'expectedStatus', 'range', 'token', 'headers', 'responseType', 'accept', 'redirects' ])
   }
 
   if (options.method === 'POST') {
@@ -49,7 +59,7 @@ function makeRawRequest (options: {
   return makeGetRequest(reqOptions)
 }
 
-function makeGetRequest (options: CommonRequestParams & {
+export function makeGetRequest (options: CommonRequestParams & {
   query?: any
   rawQuery?: string
 }) {
@@ -61,7 +71,7 @@ function makeGetRequest (options: CommonRequestParams & {
   return buildRequest(req, { contentType: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options })
 }
 
-function makeHTMLRequest (url: string, path: string) {
+export function makeHTMLRequest (url: string, path: string) {
   return makeGetRequest({
     url,
     path,
@@ -70,7 +80,9 @@ function makeHTMLRequest (url: string, path: string) {
   })
 }
 
-function makeActivityPubGetRequest (url: string, path: string, expectedStatus: HttpStatusCodeType = HttpStatusCode.OK_200) {
+// ---------------------------------------------------------------------------
+
+export function makeActivityPubGetRequest (url: string, path: string, expectedStatus: HttpStatusCodeType = HttpStatusCode.OK_200) {
   return makeGetRequest({
     url,
     path,
@@ -79,7 +91,17 @@ function makeActivityPubGetRequest (url: string, path: string, expectedStatus: H
   })
 }
 
-function makeDeleteRequest (options: CommonRequestParams & {
+export function makeActivityPubRawRequest (url: string, expectedStatus: HttpStatusCodeType = HttpStatusCode.OK_200) {
+  return makeRawRequest({
+    url,
+    expectedStatus,
+    accept: 'application/activity+json,text/html;q=0.9,\\*/\\*;q=0.8'
+  })
+}
+
+// ---------------------------------------------------------------------------
+
+export function makeDeleteRequest (options: CommonRequestParams & {
   query?: any
   rawQuery?: string
 }) {
@@ -91,7 +113,7 @@ function makeDeleteRequest (options: CommonRequestParams & {
   return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options })
 }
 
-function makeUploadRequest (options: CommonRequestParams & {
+export function makeUploadRequest (options: CommonRequestParams & {
   method?: 'POST' | 'PUT'
 
   fields: { [ fieldName: string ]: any }
@@ -119,7 +141,7 @@ function makeUploadRequest (options: CommonRequestParams & {
   return req
 }
 
-function makePostBodyRequest (options: CommonRequestParams & {
+export function makePostBodyRequest (options: CommonRequestParams & {
   fields?: { [ fieldName: string ]: any }
 }) {
   const req = request(options.url).post(options.path)
@@ -128,7 +150,7 @@ function makePostBodyRequest (options: CommonRequestParams & {
   return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options })
 }
 
-function makePutBodyRequest (options: {
+export function makePutBodyRequest (options: {
   url: string
   path: string
   token?: string
@@ -142,21 +164,35 @@ function makePutBodyRequest (options: {
   return buildRequest(req, { accept: 'application/json', expectedStatus: HttpStatusCode.BAD_REQUEST_400, ...options })
 }
 
-function decodeQueryString (path: string) {
+// ---------------------------------------------------------------------------
+
+export async function getRedirectionUrl (url: string) {
+  const res = await makeRawRequest({
+    url,
+    redirects: 0,
+    expectedStatus: HttpStatusCode.FOUND_302
+  })
+
+  return res.headers['location']
+}
+
+// ---------------------------------------------------------------------------
+
+export function decodeQueryString (path: string) {
   return decode(path.split('?')[1])
 }
 
 // ---------------------------------------------------------------------------
 
-function unwrapBody <T> (test: request.Test): Promise<T> {
+export function unwrapBody <T> (test: request.Test): Promise<T> {
   return test.then(res => res.body)
 }
 
-function unwrapText (test: request.Test): Promise<string> {
+export function unwrapText (test: request.Test): Promise<string> {
   return test.then(res => res.text)
 }
 
-function unwrapBodyOrDecodeToJSON <T> (test: request.Test): Promise<T> {
+export function unwrapBodyOrDecodeToJSON <T> (test: request.Test): Promise<T> {
   return test.then(res => {
     if (res.body instanceof Buffer) {
       try {
@@ -180,28 +216,12 @@ function unwrapBodyOrDecodeToJSON <T> (test: request.Test): Promise<T> {
   })
 }
 
-function unwrapTextOrDecode (test: request.Test): Promise<string> {
+export function unwrapTextOrDecode (test: request.Test): Promise<string> {
   return test.then(res => res.text || new TextDecoder().decode(res.body))
 }
 
 // ---------------------------------------------------------------------------
-
-export {
-  makeHTMLRequest,
-  makeGetRequest,
-  decodeQueryString,
-  makeUploadRequest,
-  makePostBodyRequest,
-  makePutBodyRequest,
-  makeDeleteRequest,
-  makeRawRequest,
-  makeActivityPubGetRequest,
-  unwrapBody,
-  unwrapTextOrDecode,
-  unwrapBodyOrDecodeToJSON,
-  unwrapText
-}
-
+// Private
 // ---------------------------------------------------------------------------
 
 function buildRequest (req: request.Test, options: CommonRequestParams) {

packages/server-commands/src/server/config-command.ts

@@ -46,15 +46,15 @@ export class ConfigCommand extends AbstractCommand {
 
   // ---------------------------------------------------------------------------
 
-  disableImports () {
-    return this.setImportsEnabled(false)
+  disableVideoImports () {
+    return this.setVideoImportsEnabled(false)
   }
 
-  enableImports () {
-    return this.setImportsEnabled(true)
+  enableVideoImports () {
+    return this.setVideoImportsEnabled(true)
   }
 
-  private setImportsEnabled (enabled: boolean) {
+  private setVideoImportsEnabled (enabled: boolean) {
     return this.updateExistingSubConfig({
       newConfig: {
         import: {
@@ -118,6 +118,74 @@ export class ConfigCommand extends AbstractCommand {
 
   // ---------------------------------------------------------------------------
 
+  enableAutoBlacklist () {
+    return this.setAutoblacklistEnabled(true)
+  }
+
+  disableAutoBlacklist () {
+    return this.setAutoblacklistEnabled(false)
+  }
+
+  private setAutoblacklistEnabled (enabled: boolean) {
+    return this.updateExistingSubConfig({
+      newConfig: {
+        autoBlacklist: {
+          videos: {
+            ofUsers: {
+              enabled
+            }
+          }
+        }
+      }
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  enableUserImport () {
+    return this.setUserImportEnabled(true)
+  }
+
+  disableUserImport () {
+    return this.setUserImportEnabled(false)
+  }
+
+  private setUserImportEnabled (enabled: boolean) {
+    return this.updateExistingSubConfig({
+      newConfig: {
+        import: {
+          users: {
+            enabled
+          }
+        }
+      }
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
+  enableUserExport () {
+    return this.setUserExportEnabled(true)
+  }
+
+  disableUserExport () {
+    return this.setUserExportEnabled(false)
+  }
+
+  private setUserExportEnabled (enabled: boolean) {
+    return this.updateExistingSubConfig({
+      newConfig: {
+        export: {
+          users: {
+            enabled
+          }
+        }
+      }
+    })
+  }
+
+  // ---------------------------------------------------------------------------
+
   enableLive (options: {
     allowReplay?: boolean
     transcoding?: boolean
@@ -552,6 +620,16 @@ export class ConfigCommand extends AbstractCommand {
         videoChannelSynchronization: {
           enabled: false,
           maxPerUser: 10
+        },
+        users: {
+          enabled: true
+        }
+      },
+      export: {
+        users: {
+          enabled: true,
+          maxUserVideoQuota: 5242881,
+          exportExpiration: 1000 * 3600
         }
       },
       trending: {

packages/server-commands/src/server/server.ts

@@ -17,12 +17,14 @@ import { SocketIOCommand } from '../socket/index.js'
 import {
   AccountsCommand,
   BlocklistCommand,
+  UserExportsCommand,
   LoginCommand,
   NotificationsCommand,
   RegistrationsCommand,
   SubscriptionsCommand,
   TwoFactorCommand,
-  UsersCommand
+  UsersCommand,
+  UserImportsCommand
 } from '../users/index.js'
 import {
   BlacklistCommand,
@@ -33,7 +35,7 @@ import {
   ChaptersCommand,
   CommentsCommand,
   HistoryCommand,
-  ImportsCommand,
+  VideoImportsCommand,
   LiveCommand,
   PlaylistsCommand,
   ServicesCommand,
@@ -90,7 +92,6 @@ export class PeerTubeServer {
     user?: {
       username: string
       password: string
-      email?: string
     }
 
     channel?: VideoChannel
@@ -134,7 +135,7 @@ export class PeerTubeServer {
   changeOwnership?: ChangeOwnershipCommand
   playlists?: PlaylistsCommand
   history?: HistoryCommand
-  imports?: ImportsCommand
+  videoImports?: VideoImportsCommand
   channelSyncs?: ChannelSyncsCommand
   streamingPlaylists?: StreamingPlaylistsCommand
   channels?: ChannelsCommand
@@ -155,6 +156,9 @@ export class PeerTubeServer {
   storyboard?: StoryboardCommand
   chapters?: ChaptersCommand
 
+  userImports?: UserImportsCommand
+  userExports?: UserExportsCommand
+
   runners?: RunnersCommand
   runnerRegistrationTokens?: RunnerRegistrationTokensCommand
   runnerJobs?: RunnerJobsCommand
@@ -426,7 +430,7 @@ export class PeerTubeServer {
     this.changeOwnership = new ChangeOwnershipCommand(this)
     this.playlists = new PlaylistsCommand(this)
     this.history = new HistoryCommand(this)
-    this.imports = new ImportsCommand(this)
+    this.videoImports = new VideoImportsCommand(this)
     this.channelSyncs = new ChannelSyncsCommand(this)
     this.streamingPlaylists = new StreamingPlaylistsCommand(this)
     this.channels = new ChannelsCommand(this)
@@ -446,6 +450,9 @@ export class PeerTubeServer {
     this.storyboard = new StoryboardCommand(this)
     this.chapters = new ChaptersCommand(this)
 
+    this.userExports = new UserExportsCommand(this)
+    this.userImports = new UserImportsCommand(this)
+
     this.runners = new RunnersCommand(this)
     this.runnerRegistrationTokens = new RunnerRegistrationTokensCommand(this)
     this.runnerJobs = new RunnerJobsCommand(this)

packages/server-commands/src/server/servers.ts

@@ -21,7 +21,7 @@ function createMultipleServers (totalServers: number, configOverride?: object, o
 }
 
 function killallServers (servers: PeerTubeServer[]) {
-  return Promise.all(servers.map(s => s.kill()))
+  return Promise.all(servers.filter(s => !!s).map(s => s.kill()))
 }
 
 async function cleanupTests (servers: PeerTubeServer[]) {
@@ -33,6 +33,8 @@ async function cleanupTests (servers: PeerTubeServer[]) {
 
   let p: Promise<any>[] = []
   for (const server of servers) {
+    if (!server) continue
+
     p = p.concat(server.servers.cleanupTests())
   }
 

packages/server-commands/src/shared/abstract-command.ts

@@ -1,6 +1,7 @@
+/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
+
 import { isAbsolute } from 'path'
-import { HttpStatusCodeType } from '@peertube/peertube-models'
-import { buildAbsoluteFixturePath } from '@peertube/peertube-node-utils'
+import { buildAbsoluteFixturePath, getFileSize } from '@peertube/peertube-node-utils'
 import {
   makeDeleteRequest,
   makeGetRequest,
@@ -10,8 +11,12 @@ import {
   unwrapBody,
   unwrapText
 } from '../requests/requests.js'
+import { expect } from 'chai'
+import got, { Response as GotResponse } from 'got'
+import { HttpStatusCode, HttpStatusCodeType } from '@peertube/peertube-models'
 
 import type { PeerTubeServer } from '../server/server.js'
+import { createReadStream } from 'fs'
 
 export interface OverrideCommandOptions {
   token?: string
@@ -48,7 +53,7 @@ interface InternalDeleteCommandOptions extends InternalCommonCommandOptions {
   rawQuery?: string
 }
 
-abstract class AbstractCommand {
+export abstract class AbstractCommand {
 
   constructor (
     protected server: PeerTubeServer
@@ -218,8 +223,221 @@ abstract class AbstractCommand {
       ? { 'x-peertube-video-password': videoPassword }
       : undefined
   }
-}
 
-export {
-  AbstractCommand
+  // ---------------------------------------------------------------------------
+
+  protected async buildResumeUpload <T> (options: OverrideCommandOptions & {
+    path: string
+
+    fixture: string
+    attaches?: Record<string, string>
+    fields?: Record<string, any>
+
+    completedExpectedStatus?: HttpStatusCodeType // When the upload is finished
+  }): Promise<T> {
+    const { path, fixture, expectedStatus = HttpStatusCode.OK_200, completedExpectedStatus } = options
+
+    let size = 0
+    let videoFilePath: string
+    let mimetype = 'video/mp4'
+
+    if (fixture) {
+      videoFilePath = buildAbsoluteFixturePath(fixture)
+      size = await getFileSize(videoFilePath)
+
+      if (videoFilePath.endsWith('.mkv')) {
+        mimetype = 'video/x-matroska'
+      } else if (videoFilePath.endsWith('.webm')) {
+        mimetype = 'video/webm'
+      } else if (videoFilePath.endsWith('.zip')) {
+        mimetype = 'application/zip'
+      }
+    }
+
+    // Do not check status automatically, we'll check it manually
+    const initializeSessionRes = await this.prepareResumableUpload({
+      ...options,
+
+      path,
+      expectedStatus: null,
+
+      size,
+      mimetype
+    })
+    const initStatus = initializeSessionRes.status
+
+    if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) {
+      const locationHeader = initializeSessionRes.header['location']
+      expect(locationHeader).to.not.be.undefined
+
+      const pathUploadId = locationHeader.split('?')[1]
+
+      const result = await this.sendResumableChunks({
+        ...options,
+
+        path,
+        pathUploadId,
+        videoFilePath,
+        size,
+        expectedStatus: completedExpectedStatus
+      })
+
+      if (result.statusCode === HttpStatusCode.OK_200) {
+        await this.endResumableUpload({
+          ...options,
+
+          expectedStatus: HttpStatusCode.NO_CONTENT_204,
+          path,
+          pathUploadId
+        })
+      }
+
+      return result.body as T
+    }
+
+    const expectedInitStatus = expectedStatus === HttpStatusCode.OK_200
+      ? HttpStatusCode.CREATED_201
+      : expectedStatus
+
+    expect(initStatus).to.equal(expectedInitStatus)
+
+    return initializeSessionRes.body.video || initializeSessionRes.body
+  }
+
+  protected async prepareResumableUpload (options: OverrideCommandOptions & {
+    path: string
+
+    fixture: string
+    size: number
+    mimetype: string
+
+    attaches?: Record<string, string>
+    fields?: Record<string, any>
+
+    originalName?: string
+    lastModified?: number
+  }) {
+    const { path, attaches = {}, fields = {}, originalName, lastModified, fixture, size, mimetype } = options
+
+    const uploadOptions = {
+      ...options,
+
+      path,
+      headers: {
+        'X-Upload-Content-Type': mimetype,
+        'X-Upload-Content-Length': size.toString()
+      },
+      fields: {
+        filename: fixture,
+        originalName,
+        lastModified,
+
+        ...fields
+      },
+
+      // Fixture will be sent later
+      attaches,
+      implicitToken: true,
+
+      defaultExpectedStatus: null
+    }
+
+    if (Object.keys(attaches).length === 0) return this.postBodyRequest(uploadOptions)
+
+    return this.postUploadRequest(uploadOptions)
+  }
+
+  protected async sendResumableChunks <T> (options: OverrideCommandOptions & {
+    pathUploadId: string
+    path: string
+    videoFilePath: string
+    size: number
+    contentLength?: number
+    contentRangeBuilder?: (start: number, chunk: any) => string
+    digestBuilder?: (chunk: any) => string
+  }) {
+    const {
+      path,
+      pathUploadId,
+      videoFilePath,
+      size,
+      contentLength,
+      contentRangeBuilder,
+      digestBuilder,
+      expectedStatus = HttpStatusCode.OK_200
+    } = options
+
+    let start = 0
+
+    const token = this.buildCommonRequestToken({ ...options, implicitToken: true })
+
+    const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 })
+    const server = this.server
+    return new Promise<GotResponse<T>>((resolve, reject) => {
+      readable.on('data', async function onData (chunk) {
+        try {
+          readable.pause()
+
+          const byterangeStart = start + chunk.length - 1
+
+          const headers = {
+            'Authorization': 'Bearer ' + token,
+            'Content-Type': 'application/octet-stream',
+            'Content-Range': contentRangeBuilder
+              ? contentRangeBuilder(start, chunk)
+              : `bytes ${start}-${byterangeStart}/${size}`,
+            'Content-Length': contentLength ? contentLength + '' : chunk.length + ''
+          }
+
+          if (digestBuilder) {
+            Object.assign(headers, { digest: digestBuilder(chunk) })
+          }
+
+          const res = await got<T>({
+            url: new URL(path + '?' + pathUploadId, server.url).toString(),
+            method: 'put',
+            headers,
+            body: chunk,
+            responseType: 'json',
+            throwHttpErrors: false
+          })
+
+          start += chunk.length
+
+          // Last request, check final status
+          if (byterangeStart + 1 === size) {
+            if (res.statusCode === expectedStatus) {
+              return resolve(res)
+            }
+
+            if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) {
+              readable.off('data', onData)
+
+              // eslint-disable-next-line max-len
+              const message = `Incorrect transient behaviour sending intermediary chunks. Status code is ${res.statusCode} instead of ${expectedStatus}`
+              return reject(new Error(message))
+            }
+          }
+
+          readable.resume()
+        } catch (err) {
+          reject(err)
+        }
+      })
+    })
+  }
+
+  protected endResumableUpload (options: OverrideCommandOptions & {
+    path: string
+    pathUploadId: string
+  }) {
+    return this.deleteRequest({
+      ...options,
+
+      path: options.path,
+      rawQuery: options.pathUploadId,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+    })
+  }
 }

packages/server-commands/src/users/index.ts

@@ -1,10 +1,12 @@
 export * from './accounts-command.js'
 export * from './accounts.js'
 export * from './blocklist-command.js'
-export * from './login.js'
 export * from './login-command.js'
+export * from './login.js'
 export * from './notifications-command.js'
 export * from './registrations-command.js'
 export * from './subscriptions-command.js'
 export * from './two-factor-command.js'
+export * from './user-exports-command.js'
+export * from './user-imports-command.js'
 export * from './users-command.js'

packages/server-commands/src/users/login.ts

@@ -1,19 +1,11 @@
 import { PeerTubeServer } from '../server/server.js'
 
-function setAccessTokensToServers (servers: PeerTubeServer[]) {
-  const tasks: Promise<any>[] = []
+export function setAccessTokensToServers (servers: PeerTubeServer[]) {
+  return Promise.all(
+    servers.map(async server => {
+      const token = await server.login.getAccessToken()
 
-  for (const server of servers) {
-    const p = server.login.getAccessToken()
-                                 .then(t => { server.accessToken = t })
-    tasks.push(p)
-  }
-
-  return Promise.all(tasks)
-}
-
-// ---------------------------------------------------------------------------
-
-export {
-  setAccessTokensToServers
+      server.accessToken = token
+    })
+  )
 }

packages/server-commands/src/users/user-exports-command.ts

@@ -0,0 +1,77 @@
+import { HttpStatusCode, ResultList, UserExport, UserExportRequestResult, UserExportState } from '@peertube/peertube-models'
+import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
+import { wait } from '@peertube/peertube-core-utils'
+import { unwrapBody } from '../requests/requests.js'
+
+export class UserExportsCommand extends AbstractCommand {
+
+  request (options: OverrideCommandOptions & {
+    userId: number
+    withVideoFiles: boolean
+  }) {
+    const { userId, withVideoFiles } = options
+
+    return unwrapBody<UserExportRequestResult>(this.postBodyRequest({
+      ...options,
+
+      path: `/api/v1/users/${userId}/exports/request`,
+      fields: { withVideoFiles },
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.OK_200
+    }))
+  }
+
+  async waitForCreation (options: OverrideCommandOptions & {
+    userId: number
+  }) {
+    const { userId } = options
+
+    while (true) {
+      const { data } = await this.list({ ...options, userId })
+
+      if (data.some(e => e.state.id === UserExportState.COMPLETED)) break
+
+      await wait(250)
+    }
+  }
+
+  list (options: OverrideCommandOptions & {
+    userId: number
+  }) {
+    const { userId } = options
+
+    return this.getRequestBody<ResultList<UserExport>>({
+      ...options,
+
+      path: `/api/v1/users/${userId}/exports`,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.OK_200
+    })
+  }
+
+  async deleteAllArchives (options: OverrideCommandOptions & {
+    userId: number
+  }) {
+    const { data } = await this.list(options)
+
+    for (const { id } of data) {
+      await this.delete({ ...options, exportId: id })
+    }
+  }
+
+  delete (options: OverrideCommandOptions & {
+    exportId: number
+    userId: number
+  }) {
+    const { userId, exportId } = options
+
+    return this.deleteRequest({
+      ...options,
+
+      path: `/api/v1/users/${userId}/exports/${exportId}`,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
+    })
+  }
+
+}

packages/server-commands/src/users/user-imports-command.ts

@@ -0,0 +1,31 @@
+import { HttpStatusCode, HttpStatusCodeType, UserImport, UserImportUploadResult } from '@peertube/peertube-models'
+import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
+
+export class UserImportsCommand extends AbstractCommand {
+
+  importArchive (options: OverrideCommandOptions & {
+    userId: number
+    fixture: string
+    completedExpectedStatus?: HttpStatusCodeType
+  }) {
+    return this.buildResumeUpload<UserImportUploadResult>({
+      ...options,
+
+      path: `/api/v1/users/${options.userId}/imports/import-resumable`,
+      fixture: options.fixture,
+      completedExpectedStatus: HttpStatusCode.OK_200
+    })
+  }
+
+  getLatestImport (options: OverrideCommandOptions & {
+    userId: number
+  }) {
+    return this.getRequestBody<UserImport>({
+      ...options,
+
+      path: `/api/v1/users/${options.userId}/imports/latest`,
+      implicitToken: true,
+      defaultExpectedStatus: HttpStatusCode.OK_200
+    })
+  }
+}

packages/server-commands/src/videos/index.ts

@@ -7,7 +7,7 @@ export * from './chapters-command.js'
 export * from './channel-syncs-command.js'
 export * from './comments-command.js'
 export * from './history-command.js'
-export * from './imports-command.js'
+export * from './video-imports-command.js'
 export * from './live-command.js'
 export * from './live.js'
 export * from './playlists-command.js'

packages/server-commands/src/videos/playlists-command.ts

@@ -11,6 +11,8 @@ import {
   VideoPlaylistElementCreate,
   VideoPlaylistElementCreateResult,
   VideoPlaylistElementUpdate,
+  VideoPlaylistPrivacy,
+  VideoPlaylistPrivacyType,
   VideoPlaylistReorder,
   VideoPlaylistType_Type,
   VideoPlaylistUpdate
@@ -156,6 +158,27 @@ export class PlaylistsCommand extends AbstractCommand {
     return body.videoPlaylist
   }
 
+  async quickCreate (options: OverrideCommandOptions & {
+    displayName: string
+    privacy?: VideoPlaylistPrivacyType
+  }) {
+    const { displayName, privacy = VideoPlaylistPrivacy.PUBLIC } = options
+
+    const { videoChannels } = await this.server.users.getMyInfo({ token: options.token })
+
+    return this.create({
+      ...options,
+
+      attributes: {
+        displayName,
+        privacy,
+        videoChannelId: privacy === VideoPlaylistPrivacy.PUBLIC
+          ? videoChannels[0].id
+          : undefined
+      }
+    })
+  }
+
   update (options: OverrideCommandOptions & {
     attributes: VideoPlaylistUpdate
     playlistId: number | string

packages/server-commands/src/videos/video-imports-command.ts

@@ -2,7 +2,7 @@ import { HttpStatusCode, ResultList, VideoImport, VideoImportCreate } from '@pee
 import { unwrapBody } from '../requests/index.js'
 import { AbstractCommand, OverrideCommandOptions } from '../shared/index.js'
 
-export class ImportsCommand extends AbstractCommand {
+export class VideoImportsCommand extends AbstractCommand {
 
   importVideo (options: OverrideCommandOptions & {
     attributes: (VideoImportCreate | { torrentfile?: string, previewfile?: string, thumbnailfile?: string })

packages/server-commands/src/videos/videos-command.ts

@@ -1,9 +1,5 @@
 /* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/no-floating-promises */
 
-import { expect } from 'chai'
-import { createReadStream } from 'fs'
-import { stat } from 'fs/promises'
-import got, { Response as GotResponse } from 'got'
 import validator from 'validator'
 import { getAllPrivacies, omit, pick, wait } from '@peertube/peertube-core-utils'
 import {
@@ -429,7 +425,13 @@ export class VideosCommand extends AbstractCommand {
 
     const created = mode === 'legacy'
       ? await this.buildLegacyUpload({ ...options, attributes })
-      : await this.buildResumeUpload({ ...options, path: '/api/v1/videos/upload-resumable', attributes })
+      : await this.buildResumeVideoUpload({
+        ...options,
+        path: '/api/v1/videos/upload-resumable',
+        fixture: attributes.fixture,
+        attaches: this.buildUploadAttaches(attributes, false),
+        fields: this.buildUploadFields(attributes)
+      })
 
     // Wait torrent generation
     const expectedStatus = this.buildExpectedStatus({ ...options, defaultExpectedStatus: HttpStatusCode.OK_200 })
@@ -456,231 +458,26 @@ export class VideosCommand extends AbstractCommand {
 
       path,
       fields: this.buildUploadFields(options.attributes),
-      attaches: this.buildUploadAttaches(options.attributes),
+      attaches: this.buildUploadAttaches(options.attributes, true),
       implicitToken: true,
       defaultExpectedStatus: HttpStatusCode.OK_200
     })).then(body => body.video || body as any)
   }
 
-  async buildResumeUpload (options: OverrideCommandOptions & {
-    path: string
-    attributes: { fixture?: string } & { [id: string]: any }
-    completedExpectedStatus?: HttpStatusCodeType // When the upload is finished
-  }): Promise<VideoCreateResult> {
-    const { path, attributes, expectedStatus = HttpStatusCode.OK_200, completedExpectedStatus } = options
-
-    let size = 0
-    let videoFilePath: string
-    let mimetype = 'video/mp4'
-
-    if (attributes.fixture) {
-      videoFilePath = buildAbsoluteFixturePath(attributes.fixture)
-      size = (await stat(videoFilePath)).size
-
-      if (videoFilePath.endsWith('.mkv')) {
-        mimetype = 'video/x-matroska'
-      } else if (videoFilePath.endsWith('.webm')) {
-        mimetype = 'video/webm'
-      }
-    }
-
-    // Do not check status automatically, we'll check it manually
-    const initializeSessionRes = await this.prepareResumableUpload({
-      ...options,
-
-      path,
-      expectedStatus: null,
-      attributes,
-      size,
-      mimetype
-    })
-    const initStatus = initializeSessionRes.status
-
-    if (videoFilePath && initStatus === HttpStatusCode.CREATED_201) {
-      const locationHeader = initializeSessionRes.header['location']
-      expect(locationHeader).to.not.be.undefined
-
-      const pathUploadId = locationHeader.split('?')[1]
-
-      const result = await this.sendResumableChunks({
-        ...options,
-
-        path,
-        pathUploadId,
-        videoFilePath,
-        size,
-        expectedStatus: completedExpectedStatus
-      })
-
-      if (result.statusCode === HttpStatusCode.OK_200) {
-        await this.endResumableUpload({
-          ...options,
-
-          expectedStatus: HttpStatusCode.NO_CONTENT_204,
-          path,
-          pathUploadId
-        })
-      }
-
-      return result.body?.video || result.body as any
-    }
-
-    const expectedInitStatus = expectedStatus === HttpStatusCode.OK_200
-      ? HttpStatusCode.CREATED_201
-      : expectedStatus
-
-    expect(initStatus).to.equal(expectedInitStatus)
-
-    return initializeSessionRes.body.video || initializeSessionRes.body
-  }
-
-  async prepareResumableUpload (options: OverrideCommandOptions & {
-    path: string
-    attributes: { fixture?: string } & { [id: string]: any }
-    size: number
-    mimetype: string
-
-    originalName?: string
-    lastModified?: number
-  }) {
-    const { path, attributes, originalName, lastModified, size, mimetype } = options
-
-    const attaches = this.buildUploadAttaches(omit(options.attributes, [ 'fixture' ]))
-
-    const uploadOptions = {
-      ...options,
-
-      path,
-      headers: {
-        'X-Upload-Content-Type': mimetype,
-        'X-Upload-Content-Length': size.toString()
-      },
-      fields: {
-        filename: attributes.fixture,
-        originalName,
-        lastModified,
-
-        ...this.buildUploadFields(options.attributes)
-      },
-
-      // Fixture will be sent later
-      attaches: this.buildUploadAttaches(omit(options.attributes, [ 'fixture' ])),
-      implicitToken: true,
-
-      defaultExpectedStatus: null
-    }
-
-    if (Object.keys(attaches).length === 0) return this.postBodyRequest(uploadOptions)
-
-    return this.postUploadRequest(uploadOptions)
-  }
-
-  sendResumableChunks (options: OverrideCommandOptions & {
-    pathUploadId: string
-    path: string
-    videoFilePath: string
-    size: number
-    contentLength?: number
-    contentRangeBuilder?: (start: number, chunk: any) => string
-    digestBuilder?: (chunk: any) => string
-  }) {
-    const {
-      path,
-      pathUploadId,
-      videoFilePath,
-      size,
-      contentLength,
-      contentRangeBuilder,
-      digestBuilder,
-      expectedStatus = HttpStatusCode.OK_200
-    } = options
-
-    let start = 0
-
-    const token = this.buildCommonRequestToken({ ...options, implicitToken: true })
-
-    const readable = createReadStream(videoFilePath, { highWaterMark: 8 * 1024 })
-    const server = this.server
-    return new Promise<GotResponse<{ video: VideoCreateResult }>>((resolve, reject) => {
-      readable.on('data', async function onData (chunk) {
-        try {
-          readable.pause()
-
-          const byterangeStart = start + chunk.length - 1
-
-          const headers = {
-            'Authorization': 'Bearer ' + token,
-            'Content-Type': 'application/octet-stream',
-            'Content-Range': contentRangeBuilder
-              ? contentRangeBuilder(start, chunk)
-              : `bytes ${start}-${byterangeStart}/${size}`,
-            'Content-Length': contentLength ? contentLength + '' : chunk.length + ''
-          }
-
-          if (digestBuilder) {
-            Object.assign(headers, { digest: digestBuilder(chunk) })
-          }
-
-          const res = await got<{ video: VideoCreateResult }>({
-            url: new URL(path + '?' + pathUploadId, server.url).toString(),
-            method: 'put',
-            headers,
-            body: chunk,
-            responseType: 'json',
-            throwHttpErrors: false
-          })
-
-          start += chunk.length
-
-          // Last request, check final status
-          if (byterangeStart + 1 === size) {
-            if (res.statusCode === expectedStatus) {
-              return resolve(res)
-            }
-
-            if (res.statusCode !== HttpStatusCode.PERMANENT_REDIRECT_308) {
-              readable.off('data', onData)
-
-              // eslint-disable-next-line max-len
-              const message = `Incorrect transient behaviour sending intermediary chunks. Status code is ${res.statusCode} instead of ${expectedStatus}`
-              return reject(new Error(message))
-            }
-          }
-
-          readable.resume()
-        } catch (err) {
-          reject(err)
-        }
-      })
-    })
-  }
-
-  endResumableUpload (options: OverrideCommandOptions & {
-    path: string
-    pathUploadId: string
-  }) {
-    return this.deleteRequest({
-      ...options,
-
-      path: options.path,
-      rawQuery: options.pathUploadId,
-      implicitToken: true,
-      defaultExpectedStatus: HttpStatusCode.NO_CONTENT_204
-    })
-  }
-
   quickUpload (options: OverrideCommandOptions & {
     name: string
     nsfw?: boolean
     privacy?: VideoPrivacyType
     fixture?: string
     videoPasswords?: string[]
+    channelId?: number
   }) {
     const attributes: VideoEdit = { name: options.name }
     if (options.nsfw) attributes.nsfw = options.nsfw
     if (options.privacy) attributes.privacy = options.privacy
     if (options.fixture) attributes.fixture = options.fixture
     if (options.videoPasswords) attributes.videoPasswords = options.videoPasswords
+    if (options.channelId) attributes.channelId = options.channelId
 
     return this.upload({ ...options, attributes })
   }
@@ -713,7 +510,7 @@ export class VideosCommand extends AbstractCommand {
       ...options,
 
       path: '/api/v1/videos/' + options.videoId + '/source/replace-resumable',
-      attributes: { fixture: options.fixture }
+      fixture: options.fixture
     })
   }
 
@@ -813,19 +610,38 @@ export class VideosCommand extends AbstractCommand {
     ])
   }
 
-  private buildUploadFields (attributes: VideoEdit) {
+  buildUploadFields (attributes: VideoEdit) {
     return omit(attributes, [ 'fixture', 'thumbnailfile', 'previewfile' ])
   }
 
-  private buildUploadAttaches (attributes: VideoEdit) {
+  buildUploadAttaches (attributes: VideoEdit, includeFixture: boolean) {
     const attaches: { [ name: string ]: string } = {}
 
     for (const key of [ 'thumbnailfile', 'previewfile' ]) {
       if (attributes[key]) attaches[key] = buildAbsoluteFixturePath(attributes[key])
     }
 
-    if (attributes.fixture) attaches.videofile = buildAbsoluteFixturePath(attributes.fixture)
+    if (includeFixture && attributes.fixture) attaches.videofile = buildAbsoluteFixturePath(attributes.fixture)
 
     return attaches
   }
+
+  // Make these methods public, needed by some offensive tests
+  sendResumableVideoChunks (options: Parameters<AbstractCommand['sendResumableChunks']>[0]) {
+    return super.sendResumableChunks<{ video: VideoCreateResult }>(options)
+  }
+
+  async buildResumeVideoUpload (options: Parameters<AbstractCommand['buildResumeUpload']>[0]) {
+    const result = await super.buildResumeUpload<{ video: VideoCreateResult }>(options)
+
+    return result?.video || undefined
+  }
+
+  prepareVideoResumableUpload (options: Parameters<AbstractCommand['prepareResumableUpload']>[0]) {
+    return super.prepareResumableUpload(options)
+  }
+
+  endVideoResumableUpload (options: Parameters<AbstractCommand['endResumableUpload']>[0]) {
+    return super.endResumableUpload(options)
+  }
 }

packages/server-commands/tsconfig.json

@@ -1,7 +1,6 @@
 {
   "extends": "../../tsconfig.base.json",
   "compilerOptions": {
-    "baseUrl": "./",
     "outDir": "./dist",
     "rootDir": "src",
     "tsBuildInfoFile": "./dist/.tsbuildinfo"