Fix import timeout inconsistency

server/helpers/youtube-dl/youtube-dl-cli.ts

@@ -90,11 +90,13 @@ export class YoutubeDLCLI {
     format: string
     output: string
     processOptions: execa.NodeOptions
+    timeout: number
     additionalYoutubeDLArgs?: string[]
   }) {
     return this.run({
       url: options.url,
       processOptions: options.processOptions,
+      timeout: options.timeout,
       args: (options.additionalYoutubeDLArgs || []).concat([ '-f', options.format, '-o', options.output ])
     })
   }
@@ -145,16 +147,23 @@ export class YoutubeDLCLI {
   private async run (options: {
     url: string
     args: string[]
+    timeout?: number
     processOptions: execa.NodeOptions
   }) {
-    const { url, args, processOptions } = options
+    const { url, args, timeout, processOptions } = options
 
     let completeArgs = this.wrapWithProxyOptions(args)
     completeArgs = this.wrapWithIPOptions(completeArgs)
     completeArgs = this.wrapWithFFmpegOptions(completeArgs)
 
     const { PYTHON_PATH } = CONFIG.IMPORT.VIDEOS.HTTP.YOUTUBE_DL_RELEASE
-    const output = await execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
+    const subProcess = execa(PYTHON_PATH, [ youtubeDLBinaryPath, ...completeArgs, url ], processOptions)
+
+    if (timeout) {
+      setTimeout(() => subProcess.cancel(), timeout)
+    }
+
+    const output = await subProcess
 
     logger.debug('Runned youtube-dl command.', { command: output.command, ...lTags() })
 

server/helpers/youtube-dl/youtube-dl-wrapper.ts

@@ -77,38 +77,32 @@ class YoutubeDLWrapper {
 
     const youtubeDL = await YoutubeDLCLI.safeGet()
 
-    let timer: NodeJS.Timeout
-    const timeoutPromise = new Promise<string>((_, rej) => {
-      timer = setTimeout(() => rej(new Error('YoutubeDL download timeout.')), timeout)
-    })
-
-    const downloadPromise = youtubeDL.download({
-      url: this.url,
-      format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions),
-      output: pathWithoutExtension,
-      processOptions
-    }).then(() => clearTimeout(timer))
-      .then(async () => {
-        // If youtube-dl did not guess an extension for our file, just use .mp4 as default
-        if (await pathExists(pathWithoutExtension)) {
-          await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
-        }
-
-        return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+    try {
+      await youtubeDL.download({
+        url: this.url,
+        format: YoutubeDLCLI.getYoutubeDLVideoFormat(this.enabledResolutions),
+        output: pathWithoutExtension,
+        timeout,
+        processOptions
       })
 
-    return Promise.race([ downloadPromise, timeoutPromise ])
-      .catch(err => {
-        this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
-          .then(path => {
-            logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() })
+      // If youtube-dl did not guess an extension for our file, just use .mp4 as default
+      if (await pathExists(pathWithoutExtension)) {
+        await move(pathWithoutExtension, pathWithoutExtension + '.mp4')
+      }
 
-            return remove(path)
-          })
-          .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() }))
+      return this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+    } catch (err) {
+      this.guessVideoPathWithExtension(pathWithoutExtension, fileExt)
+        .then(path => {
+          logger.debug('Error in youtube-dl import, deleting file %s.', path, { err, ...lTags() })
 
-        throw err
-      })
+          return remove(path)
+        })
+        .catch(innerErr => logger.error('Cannot remove file in youtubeDL timeout.', { innerErr, ...lTags() }))
+
+      throw err
+    }
   }
 
   private async guessVideoPathWithExtension (tmpPath: string, sourceExt: string) {

server/initializers/constants.ts

@@ -215,7 +215,7 @@ const REQUEST_TIMEOUTS = {
 }
 
 const JOB_COMPLETED_LIFETIME = 60000 * 60 * 24 * 2 // 2 days
-const VIDEO_IMPORT_TIMEOUT = 1000 * 3600 // 1 hour
+const VIDEO_IMPORT_TIMEOUT = Math.floor(JOB_TTL['video-import'] * 0.9)
 
 const SCHEDULER_INTERVALS_MS = {
   ACTOR_FOLLOW_SCORES: 60000 * 60, // 1 hour

server/lib/job-queue/handlers/video-import.ts

@@ -29,7 +29,7 @@ import { ffprobePromise, getDurationFromVideoFile, getVideoFileFPS, getVideoFile
 import { logger } from '../../../helpers/logger'
 import { getSecureTorrentName } from '../../../helpers/utils'
 import { createTorrentAndSetInfoHash, downloadWebTorrentVideo } from '../../../helpers/webtorrent'
-import { VIDEO_IMPORT_TIMEOUT } from '../../../initializers/constants'
+import { JOB_TTL } from '../../../initializers/constants'
 import { sequelizeTypescript } from '../../../initializers/database'
 import { VideoModel } from '../../../models/video/video'
 import { VideoFileModel } from '../../../models/video/video-file'
@@ -72,7 +72,7 @@ async function processTorrentImport (job: Job, videoImport: MVideoImportDefault,
     torrentName: videoImport.torrentName ? getSecureTorrentName(videoImport.torrentName) : undefined,
     uri: videoImport.magnetUri
   }
-  return processFile(() => downloadWebTorrentVideo(target, VIDEO_IMPORT_TIMEOUT), videoImport, options)
+  return processFile(() => downloadWebTorrentVideo(target, JOB_TTL['video-import']), videoImport, options)
 }
 
 async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefault, payload: VideoImportYoutubeDLPayload) {
@@ -83,7 +83,7 @@ async function processYoutubeDLImport (job: Job, videoImport: MVideoImportDefaul
   const youtubeDL = new YoutubeDLWrapper(videoImport.targetUrl, ServerConfigManager.Instance.getEnabledResolutions('vod'))
 
   return processFile(
-    () => youtubeDL.downloadVideo(payload.fileExt, VIDEO_IMPORT_TIMEOUT),
+    () => youtubeDL.downloadVideo(payload.fileExt, JOB_TTL['video-import']),
     videoImport,
     options
   )