yetangitu/Peertube
1
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2025-10-05 19:42:24 +02:00
Code Issues Releases Wiki Activity

Introduce redundancy command

Browse source
This commit is contained in:
Chocobozzz 2021-07-07 10:56:45 +02:00
parent ae2abfd3ae
commit dab047092b
No known key found for this signature in database
GPG key ID: 583A612D890159BE
7 changed files with 166 additions and 247 deletions
Download patch file Download diff file Expand all files Collapse all files

167
server/tests/api/redundancy/manage-redundancy.ts
View file

@ -1,21 +1,21 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import * as chai from 'chai'
import 'mocha'
import * as chai from 'chai'
import {
cleanupTests,
doubleFollow,
flushAndRunMultipleServers,
getLocalIdByUUID,
RedundancyCommand,
ServerInfo,
setAccessTokensToServers,
uploadVideo,
uploadVideoAndGetId,
waitJobs,
waitUntilLog
} from '../../../../shared/extra-utils'
import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
import { addVideoRedundancy, listVideoRedundancies, removeVideoRedundancy, updateRedundancy } from '@shared/extra-utils/server/redundancy'
import { VideoPrivacy, VideoRedundanciesTarget, VideoRedundancy } from '@shared/models'
} from '@shared/extra-utils'
import { VideoPrivacy, VideoRedundanciesTarget } from '@shared/models'
const expect = chai.expect
@ -27,6 +27,8 @@ describe('Test manage videos redundancy', function () {
let video2Server2UUID: string
let redundanciesToRemove: number[] = []
let commands: RedundancyCommand[]
before(async function () {
this.timeout(120000)
@ -55,6 +57,8 @@ describe('Test manage videos redundancy', function () {
// Get the access tokens
await setAccessTokensToServers(servers)
commands = servers.map(s => s.redundancyCommand)
{
const res = await uploadVideo(servers[1].url, servers[1].accessToken, { name: 'video 1 server 2' })
video1Server2UUID = res.body.video.uuid
@ -69,21 +73,17 @@ describe('Test manage videos redundancy', function () {
// Server 1 and server 2 follow each other
await doubleFollow(servers[0], servers[1])
await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
await commands[0].updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
await waitJobs(servers)
})
it('Should not have redundancies on server 3', async function () {
for (const target of targets) {
const res = await listVideoRedundancies({
url: servers[2].url,
accessToken: servers[2].accessToken,
target
})
const body = await commands[2].listVideos({ target })
expect(res.body.total).to.equal(0)
expect(res.body.data).to.have.lengthOf(0)
expect(body.total).to.equal(0)
expect(body.data).to.have.lengthOf(0)
}
})
@ -94,28 +94,19 @@ describe('Test manage videos redundancy', function () {
await waitUntilLog(servers[0], 'Duplicated ', 10)
await waitJobs(servers)
const res = await listVideoRedundancies({
url: servers[1].url,
accessToken: servers[1].accessToken,
target: 'remote-videos'
})
const body = await commands[1].listVideos({ target: 'remote-videos' })
expect(res.body.total).to.equal(0)
expect(res.body.data).to.have.lengthOf(0)
expect(body.total).to.equal(0)
expect(body.data).to.have.lengthOf(0)
})
it('Should have "my-videos" redundancies on server 2', async function () {
this.timeout(120000)
const res = await listVideoRedundancies({
url: servers[1].url,
accessToken: servers[1].accessToken,
target: 'my-videos'
})
const body = await commands[1].listVideos({ target: 'my-videos' })
expect(body.total).to.equal(2)
expect(res.body.total).to.equal(2)
const videos = res.body.data as VideoRedundancy[]
const videos = body.data
expect(videos).to.have.lengthOf(2)
const videos1 = videos.find(v => v.uuid === video1Server2UUID)
@ -139,28 +130,19 @@ describe('Test manage videos redundancy', function () {
})
it('Should not have "my-videos" redundancies on server 1', async function () {
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
target: 'my-videos'
})
const body = await commands[0].listVideos({ target: 'my-videos' })
expect(res.body.total).to.equal(0)
expect(res.body.data).to.have.lengthOf(0)
expect(body.total).to.equal(0)
expect(body.data).to.have.lengthOf(0)
})
it('Should have "remote-videos" redundancies on server 1', async function () {
this.timeout(120000)
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
target: 'remote-videos'
})
const body = await commands[0].listVideos({ target: 'remote-videos' })
expect(body.total).to.equal(2)
expect(res.body.total).to.equal(2)
const videos = res.body.data as VideoRedundancy[]
const videos = body.data
expect(videos).to.have.lengthOf(2)
const videos1 = videos.find(v => v.uuid === video1Server2UUID)
@ -185,47 +167,40 @@ describe('Test manage videos redundancy', function () {
it('Should correctly paginate and sort results', async function () {
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: 'name',
start: 0,
count: 2
})
const videos = res.body.data
const videos = body.data
expect(videos[0].name).to.equal('video 1 server 2')
expect(videos[1].name).to.equal('video 2 server 2')
}
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 0,
count: 2
})
const videos = res.body.data
const videos = body.data
expect(videos[0].name).to.equal('video 2 server 2')
expect(videos[1].name).to.equal('video 1 server 2')
}
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 1,
count: 1
})
const videos = res.body.data
expect(videos[0].name).to.equal('video 1 server 2')
expect(body.data[0].name).to.equal('video 1 server 2')
}
})
@ -236,30 +211,23 @@ describe('Test manage videos redundancy', function () {
await waitJobs(servers)
const videoId = await getLocalIdByUUID(servers[0].url, uuid)
await addVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
videoId
})
await commands[0].addVideo({ videoId })
await waitJobs(servers)
await waitUntilLog(servers[0], 'Duplicated ', 15)
await waitJobs(servers)
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = res.body.data
expect(videos[0].name).to.equal('video 3 server 2')
const video = body.data[0]
const video = videos[0]
expect(video.name).to.equal('video 3 server 2')
expect(video.redundancies.files).to.have.lengthOf(4)
expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
@ -276,19 +244,15 @@ describe('Test manage videos redundancy', function () {
}
}
const res = await listVideoRedundancies({
url: servers[1].url,
accessToken: servers[1].accessToken,
const body = await commands[1].listVideos({
target: 'my-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = res.body.data
expect(videos[0].name).to.equal('video 3 server 2')
const video = videos[0]
const video = body.data[0]
expect(video.name).to.equal('video 3 server 2')
expect(video.redundancies.files).to.have.lengthOf(4)
expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
@ -307,64 +271,47 @@ describe('Test manage videos redundancy', function () {
this.timeout(120000)
for (const redundancyId of redundanciesToRemove) {
await removeVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
redundancyId
})
await commands[0].removeVideo({ redundancyId })
}
{
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = res.body.data
const videos = body.data
expect(videos).to.have.lengthOf(2)
expect(videos[0].name).to.equal('video 2 server 2')
redundanciesToRemove = []
const video = videos[0]
expect(video.name).to.equal('video 2 server 2')
expect(video.redundancies.files).to.have.lengthOf(4)
expect(video.redundancies.streamingPlaylists).to.have.lengthOf(1)
const redundancies = video.redundancies.files.concat(video.redundancies.streamingPlaylists)
for (const r of redundancies) {
redundanciesToRemove.push(r.id)
}
redundanciesToRemove = redundancies.map(r => r.id)
}
})
it('Should remove another (auto) redundancy', async function () {
{
for (const redundancyId of redundanciesToRemove) {
await removeVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
redundancyId
})
}
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
target: 'remote-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = res.body.data
expect(videos[0].name).to.equal('video 1 server 2')
expect(videos).to.have.lengthOf(1)
for (const redundancyId of redundanciesToRemove) {
await commands[0].removeVideo({ redundancyId })
}
const body = await commands[0].listVideos({
target: 'remote-videos',
sort: '-name',
start: 0,
count: 5
})
const videos = body.data
expect(videos).to.have.lengthOf(1)
expect(videos[0].name).to.equal('video 1 server 2')
})
after(async function () {

28
server/tests/api/redundancy/redundancy-constraints.ts
View file

@ -1,9 +1,6 @@
/* eslint-disable @typescript-eslint/no-unused-expressions,@typescript-eslint/require-await */
import 'mocha'
import * as chai from 'chai'
import { listVideoRedundancies, updateRedundancy } from '@shared/extra-utils/server/redundancy'
import { VideoPrivacy } from '@shared/models'
import {
cleanupTests,
flushAndRunServer,
@ -13,9 +10,10 @@ import {
setAccessTokensToServers,
updateVideo,
uploadVideo,
waitJobs,
waitUntilLog
} from '../../../../shared/extra-utils'
import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
} from '@shared/extra-utils'
import { VideoPrivacy } from '@shared/models'
const expect = chai.expect
@ -50,23 +48,15 @@ describe('Test redundancy constraints', function () {
}
async function getTotalRedundanciesLocalServer () {
const res = await listVideoRedundancies({
url: localServer.url,
accessToken: localServer.accessToken,
target: 'my-videos'
})
const body = await localServer.redundancyCommand.listVideos({ target: 'my-videos' })
return res.body.total
return body.total
}
async function getTotalRedundanciesRemoteServer () {
const res = await listVideoRedundancies({
url: remoteServer.url,
accessToken: remoteServer.accessToken,
target: 'remote-videos'
})
const body = await remoteServer.redundancyCommand.listVideos({ target: 'remote-videos' })
return res.body.total
return body.total
}
before(async function () {
@ -99,7 +89,7 @@ describe('Test redundancy constraints', function () {
// Server 1 and server 2 follow each other
await remoteServer.followsCommand.follow({ targets: [ localServer.url ] })
await waitJobs(servers)
await updateRedundancy(remoteServer.url, remoteServer.accessToken, localServer.host, true)
await remoteServer.redundancyCommand.updateRedundancy({ host: localServer.host, redundancyAllowed: true })
await waitJobs(servers)
})
@ -161,7 +151,7 @@ describe('Test redundancy constraints', function () {
}
}
}
await killallServers([ localServer ])
killallServers([ localServer ])
await reRunServer(localServer, config)
await uploadWrapper('video 3 server 2')

49
server/tests/api/redundancy/redundancy.ts
View file

@ -5,7 +5,8 @@ import * as chai from 'chai'
import { readdir } from 'fs-extra'
import * as magnetUtil from 'magnet-uri'
import { join } from 'path'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
import { removeVideoRedundancy } from '@server/lib/redundancy'
import { HttpStatusCode } from '@shared/core-utils'
import {
checkSegmentHash,
checkVideoFilesWereRemoved,
@ -26,19 +27,18 @@ import {
uploadVideo,
viewVideo,
wait,
waitJobs,
waitUntilLog
} from '../../../../shared/extra-utils'
import { waitJobs } from '../../../../shared/extra-utils/server/jobs'
} from '@shared/extra-utils'
import { getStats } from '@shared/extra-utils/server/stats'
import {
addVideoRedundancy,
listVideoRedundancies,
removeVideoRedundancy,
updateRedundancy
} from '../../../../shared/extra-utils/server/redundancy'
import { getStats } from '../../../../shared/extra-utils/server/stats'
import { VideoRedundancy, VideoRedundancyStrategy, VideoRedundancyStrategyWithManual } from '../../../../shared/models/redundancy'
import { ServerStats } from '../../../../shared/models/server/server-stats.model'
import { VideoDetails, VideoPrivacy } from '../../../../shared/models/videos'
ServerStats,
VideoDetails,
VideoPrivacy,
VideoRedundancy,
VideoRedundancyStrategy,
VideoRedundancyStrategyWithManual
} from '@shared/models'
const expect = chai.expect
@ -279,7 +279,7 @@ async function findServerFollows () {
}
async function enableRedundancyOnServer1 () {
await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, true)
await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: true })
const { server2, server3 } = await findServerFollows()
@ -291,7 +291,7 @@ async function enableRedundancyOnServer1 () {
}
async function disableRedundancyOnServer1 () {
await updateRedundancy(servers[0].url, servers[0].accessToken, servers[1].host, false)
await servers[0].redundancyCommand.updateRedundancy({ host: servers[1].host, redundancyAllowed: false })
const { server2, server3 } = await findServerFollows()
@ -551,11 +551,7 @@ describe('Test videos redundancy', function () {
})
it('Should create a redundancy on first video', async function () {
await addVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
videoId: video1Server2Id
})
await servers[0].redundancyCommand.addVideo({ videoId: video1Server2Id })
})
it('Should have 2 webseeds on the first video', async function () {
@ -573,22 +569,15 @@ describe('Test videos redundancy', function () {
it('Should manually remove redundancies on server 1 and remove duplicated videos', async function () {
this.timeout(80000)
const res = await listVideoRedundancies({
url: servers[0].url,
accessToken: servers[0].accessToken,
target: 'remote-videos'
})
const body = await servers[0].redundancyCommand.listVideos({ target: 'remote-videos' })
const videos = res.body.data as VideoRedundancy[]
const videos = body.data
expect(videos).to.have.lengthOf(1)
const video = videos[0]
for (const r of video.redundancies.files.concat(video.redundancies.streamingPlaylists)) {
await removeVideoRedundancy({
url: servers[0].url,
accessToken: servers[0].accessToken,
redundancyId: r.id
})
await servers[0].redundancyCommand.removeVideo({ redundancyId: r.id })
}
await waitJobs(servers)