yetangitu/Peertube
1
0
Fork 0
mirror of https://github.com/Chocobozzz/PeerTube.git synced 2025-10-03 09:49:20 +02:00
Code Issues Releases Wiki Activity

Speed up built-in db migration

Browse source
This commit is contained in:
Chocobozzz 2024-08-05 11:35:59 +02:00
parent b8f2b9f16c
commit 486183fe62
No known key found for this signature in database
GPG key ID: 583A612D890159BE
3 changed files with 47 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

44
server/scripts/migrations/peertube-6.3.ts
View file

@ -1,12 +1,13 @@
import { ffprobePromise, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
import { VideoFileStream } from '@peertube/peertube-models'
import { initDatabaseModels } from '@server/initializers/database.js'
import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
import { buildFileMetadata } from '@server/lib/video-file.js'
import { VideoPathManager } from '@server/lib/video-path-manager.js'
import { VideoFileModel } from '@server/models/video/video-file.js'
import { VideoModel } from '@server/models/video/video.js'
import Bluebird from 'bluebird'
import { pathExists } from 'fs-extra/esm'
import { QueryTypes } from 'sequelize'
run()
.then(() => process.exit(0))
@ -16,10 +17,47 @@ run()
})
async function run () {
console.log('## Assigning metadata information to local video files ##\n')
await initDatabaseModels(true)
{
console.log('## Updating "formatFlags" column for web videos in "videoFile" table in database ##\n')
const totalQuery = 'SELECT COUNT(*) AS "total" FROM "videoFile" WHERE "videoId" IS NOT NULL AND "formatFlags" != 1'
const res = await sequelizeTypescript.query<{ total: string }>(totalQuery, { type: QueryTypes.SELECT as QueryTypes.SELECT })
const total = parseInt(res[0].total)
console.log(`Will update ${total.toLocaleString()} rows`)
if (total > 10000) {
console.log('Processing update in chunks because there are many rows to update...')
const chunkSize = 10000
let remaining = total
while (remaining > chunkSize) {
const before = new Date().getTime()
await sequelizeTypescript.query(
'UPDATE "videoFile" SET "formatFlags" = 1 WHERE id IN (' +
'SELECT id FROM "videoFile" WHERE "videoId" IS NOT NULL AND "formatFlags" != 1 LIMIT ' + chunkSize +
')'
)
remaining -= chunkSize
const ms = new Date().getTime() - before
console.log(`Processed ${chunkSize.toLocaleString()} rows in ${ms.toLocaleString()}ms. Remaining: ${remaining.toLocaleString()}`)
}
}
const query = 'UPDATE "videoFile" SET "formatFlags" = 1 WHERE "videoId" IS NOT NULL AND "formatFlags" != 1'
await sequelizeTypescript.query(query)
console.log('Rows updated!')
}
console.log('## Assigning metadata information to local video files ##\n')
const ids = await VideoModel.listLocalIds()
await Bluebird.map(ids, async id => {