From 486183fe62c234f15c2f9742519c64de96f76a37 Mon Sep 17 00:00:00 2001
From: Chocobozzz <me@florianbigard.com>
Date: Mon, 5 Aug 2024 11:35:59 +0200
Subject: [PATCH] Speed up built-in db migration

---
 .../migrations/0865-video-file-streams.ts     |  9 ++--
 server/core/models/video/video-file.ts        | 17 -------
 server/scripts/migrations/peertube-6.3.ts     | 44 +++++++++++++++++--
 3 files changed, 47 insertions(+), 23 deletions(-)

diff --git a/server/core/initializers/migrations/0865-video-file-streams.ts b/server/core/initializers/migrations/0865-video-file-streams.ts
index a063953a5..7633b8397 100644
--- a/server/core/initializers/migrations/0865-video-file-streams.ts
+++ b/server/core/initializers/migrations/0865-video-file-streams.ts
@@ -7,6 +7,11 @@ async function up (utils: {
 }): Promise<void> {
   const { transaction } = utils
 
+  {
+    await utils.sequelize.query('DROP INDEX IF EXISTS "video_file_video_id"')
+    await utils.sequelize.query('DROP INDEX IF EXISTS "video_file_video_streaming_playlist_id"')
+  }
+
   {
     await utils.queryInterface.addColumn('videoFile', 'formatFlags', {
       type: Sequelize.INTEGER,
@@ -14,9 +19,7 @@ async function up (utils: {
       allowNull: false
     }, { transaction })
 
-    // Web videos
-    const query = 'UPDATE "videoFile" SET "formatFlags" = 1 WHERE "videoId" IS NOT NULL'
-    await utils.sequelize.query(query, { transaction })
+    // Web videos will be updated in the migration script because the query can be slow
 
     await utils.queryInterface.changeColumn('videoFile', 'formatFlags', {
       type: Sequelize.INTEGER,
diff --git a/server/core/models/video/video-file.ts b/server/core/models/video/video-file.ts
index 02927b892..4b79c57c0 100644
--- a/server/core/models/video/video-file.ts
+++ b/server/core/models/video/video-file.ts
@@ -112,23 +112,6 @@ export enum ScopeNames {
 @Table({
   tableName: 'videoFile',
   indexes: [
-    {
-      fields: [ 'videoId' ],
-      where: {
-        videoId: {
-          [Op.ne]: null
-        }
-      }
-    },
-    {
-      fields: [ 'videoStreamingPlaylistId' ],
-      where: {
-        videoStreamingPlaylistId: {
-          [Op.ne]: null
-        }
-      }
-    },
-
     {
       fields: [ 'infoHash' ]
     },
diff --git a/server/scripts/migrations/peertube-6.3.ts b/server/scripts/migrations/peertube-6.3.ts
index e3e6fb0b7..9077fd458 100644
--- a/server/scripts/migrations/peertube-6.3.ts
+++ b/server/scripts/migrations/peertube-6.3.ts
@@ -1,12 +1,13 @@
 import { ffprobePromise, getVideoStreamFPS } from '@peertube/peertube-ffmpeg'
 import { VideoFileStream } from '@peertube/peertube-models'
-import { initDatabaseModels } from '@server/initializers/database.js'
+import { initDatabaseModels, sequelizeTypescript } from '@server/initializers/database.js'
 import { buildFileMetadata } from '@server/lib/video-file.js'
 import { VideoPathManager } from '@server/lib/video-path-manager.js'
 import { VideoFileModel } from '@server/models/video/video-file.js'
 import { VideoModel } from '@server/models/video/video.js'
 import Bluebird from 'bluebird'
 import { pathExists } from 'fs-extra/esm'
+import { QueryTypes } from 'sequelize'
 
 run()
   .then(() => process.exit(0))
@@ -16,10 +17,47 @@ run()
   })
 
 async function run () {
-  console.log('## Assigning metadata information to local video files ##\n')
-
   await initDatabaseModels(true)
 
+  {
+    console.log('## Updating "formatFlags" column for web videos in "videoFile" table in database ##\n')
+
+    const totalQuery = 'SELECT COUNT(*) AS "total" FROM "videoFile" WHERE "videoId" IS NOT NULL AND "formatFlags" != 1'
+    const res = await sequelizeTypescript.query<{ total: string }>(totalQuery, { type: QueryTypes.SELECT as QueryTypes.SELECT })
+    const total = parseInt(res[0].total)
+
+    console.log(`Will update ${total.toLocaleString()} rows`)
+
+    if (total > 10000) {
+      console.log('Processing update in chunks because there are many rows to update...')
+
+      const chunkSize = 10000
+      let remaining = total
+
+      while (remaining > chunkSize) {
+        const before = new Date().getTime()
+
+        await sequelizeTypescript.query(
+          'UPDATE "videoFile" SET "formatFlags" = 1 WHERE id IN (' +
+            'SELECT id FROM "videoFile" WHERE "videoId" IS NOT NULL AND "formatFlags" != 1 LIMIT ' + chunkSize +
+          ')'
+        )
+
+        remaining -= chunkSize
+
+        const ms = new Date().getTime() - before
+        console.log(`Processed ${chunkSize.toLocaleString()} rows in ${ms.toLocaleString()}ms. Remaining: ${remaining.toLocaleString()}`)
+      }
+    }
+
+    const query = 'UPDATE "videoFile" SET "formatFlags" = 1 WHERE "videoId" IS NOT NULL AND "formatFlags" != 1'
+    await sequelizeTypescript.query(query)
+
+    console.log('Rows updated!')
+  }
+
+  console.log('## Assigning metadata information to local video files ##\n')
+
   const ids = await VideoModel.listLocalIds()
 
   await Bluebird.map(ids, async id => {