Add priority to transcoding jobs

(1 = highest priority)

100 for new resolutions
10 for original file optimization

Add a malus for transcoding jobs depending on how many uploads the user did in the
last 7 days
This commit is contained in:
Chocobozzz 2021-01-21 16:57:21 +01:00 committed by Chocobozzz
parent 92c871b405
commit 77d7e851dc
15 changed files with 172 additions and 79 deletions

View File

@ -40,6 +40,7 @@
<th style="width: 40px"></th>
<th style="width: calc(100% - 390px)" class="job-id" i18n>ID</th>
<th style="width: 200px" class="job-type" i18n>Type</th>
<th style="width: 200px" class="job-priority" i18n>Priority <small>(1 = highest priority)</small></th>
<th style="width: 200px" class="job-state" i18n *ngIf="jobState === 'all'">State</th>
<th style="width: 100px" class="job-progress" i18n *ngIf="hasProgress()">Progress</th>
<th style="width: 150px" class="job-date" i18n pSortableColumn="createdAt">Created <p-sortIcon field="createdAt"></p-sortIcon></th>
@ -56,12 +57,13 @@
<td class="job-id c-hand" [pRowToggler]="job" [title]="job.id">{{ job.id }}</td>
<td class="job-type c-hand" [pRowToggler]="job">{{ job.type }}</td>
<td class="job-priority c-hand" [pRowToggler]="job">{{ job.priority }}</td>
<td class="job-state c-hand" [pRowToggler]="job" *ngIf="jobState === 'all'">
<span class="badge" [ngClass]="getJobStateClass(job.state)">{{ job.state }}</span>
</td>
<td class="job-state" [pRowToggler]="job" *ngIf="hasProgress()">
<td class="job-progress c-hand" [pRowToggler]="job" *ngIf="hasProgress()">
{{ getProgress(job) }}
</td>

View File

@ -10,7 +10,8 @@
}
.job-type,
.job-state {
.job-state,
.job-priority {
width: 150px !important;
}

View File

@ -21,19 +21,19 @@ export class JobsComponent extends RestTable implements OnInit {
jobType: JobTypeClient = 'all'
jobTypes: JobTypeClient[] = [
'all',
'activitypub-follow',
'activitypub-http-broadcast',
'activitypub-http-fetcher',
'activitypub-http-unicast',
'activitypub-refresher',
'all',
'email',
'video-transcoding',
'video-file-import',
'video-import',
'videos-views',
'activitypub-refresher',
'video-live-ending',
'video-redundancy',
'video-live-ending'
'video-transcoding',
'videos-views'
]
jobs: Job[] = []
@ -75,7 +75,7 @@ export class JobsComponent extends RestTable implements OnInit {
getColspan () {
if (this.jobState === 'all' && this.hasProgress()) return 6
if (this.jobState === 'all') return 5
if (this.jobState === 'all' || this.hasProgress()) return 5
return 4
}

View File

@ -69,6 +69,7 @@ async function formatJob (job: any, state?: JobState): Promise<Job> {
type: job.queue.name as JobType,
data: job.data,
progress: await job.progress(),
priority: job.opts.priority,
error,
createdAt: new Date(job.timestamp),
finishedOn: new Date(job.finishedOn),

View File

@ -2,16 +2,16 @@ import * as express from 'express'
import { move } from 'fs-extra'
import { extname } from 'path'
import toInt from 'validator/lib/toInt'
import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { changeVideoChannelShare } from '@server/lib/activitypub/share'
import { getLocalVideoActivityPubUrl } from '@server/lib/activitypub/url'
import { LiveManager } from '@server/lib/live-manager'
import { buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
import { addOptimizeOrMergeAudioJob, buildLocalVideoFromReq, buildVideoThumbnailsFromReq, setVideoTags } from '@server/lib/video'
import { getVideoFilePath } from '@server/lib/video-paths'
import { getServerActor } from '@server/models/application/application'
import { MVideoFullLight } from '@server/types/models'
import { VideoCreate, VideoState, VideoUpdate } from '../../../../shared'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
import { VideoFilter } from '../../../../shared/models/videos/video-query.type'
import { auditLoggerFactory, getAuditIdFromRes, VideoAuditView } from '../../../helpers/audit-logger'
import { resetSequelizeInstance } from '../../../helpers/database-utils'
@ -66,7 +66,6 @@ import { liveRouter } from './live'
import { ownershipVideoRouter } from './ownership'
import { rateVideoRouter } from './rate'
import { watchingRouter } from './watching'
import { HttpStatusCode } from '../../../../shared/core-utils/miscs/http-error-codes'
const auditLogger = auditLoggerFactory('videos')
const videosRouter = express.Router()
@ -267,7 +266,7 @@ async function addVideo (req: express.Request, res: express.Response) {
Notifier.Instance.notifyOnNewVideoIfNeeded(videoCreated)
if (video.state === VideoState.TO_TRANSCODE) {
await addOptimizeOrMergeAudioJob(videoCreated, videoFile)
await addOptimizeOrMergeAudioJob(videoCreated, videoFile, res.locals.oauth.token.User)
}
Hooks.runAction('action:api.video.uploaded', { video: videoCreated })

View File

@ -4,6 +4,14 @@ import { Model } from 'sequelize-typescript'
import { logger } from './logger'
import { Transaction } from 'sequelize'
function retryTransactionWrapper <T, A, B, C, D> (
functionToRetry: (arg1: A, arg2: B, arg3: C, arg4: D) => Promise<T> | Bluebird<T>,
arg1: A,
arg2: B,
arg3: C,
arg4: D,
): Promise<T>
function retryTransactionWrapper <T, A, B, C> (
functionToRetry: (arg1: A, arg2: B, arg3: C) => Promise<T> | Bluebird<T>,
arg1: A,

View File

@ -1,20 +1,17 @@
import { Response } from 'express'
import { CONFIG } from '@server/initializers/config'
import { DEFAULT_AUDIO_RESOLUTION } from '@server/initializers/constants'
import { JobQueue } from '@server/lib/job-queue'
import {
isStreamingPlaylist,
MStreamingPlaylistVideo,
MVideo,
MVideoAccountLightBlacklistAllFiles,
MVideoFile,
MVideoFullLight,
MVideoIdThumbnail,
MVideoImmutable,
MVideoThumbnail,
MVideoWithRights
} from '@server/types/models'
import { VideoPrivacy, VideoState, VideoTranscodingPayload } from '@shared/models'
import { VideoPrivacy, VideoState } from '@shared/models'
import { VideoModel } from '../models/video/video'
type VideoFetchType = 'all' | 'only-video' | 'only-video-with-rights' | 'id' | 'none' | 'only-immutable-attributes'
@ -69,27 +66,6 @@ function getVideoWithAttributes (res: Response) {
return res.locals.videoAll || res.locals.onlyVideo || res.locals.onlyVideoWithRights
}
function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile) {
let dataInput: VideoTranscodingPayload
if (videoFile.isAudio()) {
dataInput = {
type: 'merge-audio-to-webtorrent',
resolution: DEFAULT_AUDIO_RESOLUTION,
videoUUID: video.uuid,
isNewVideo: true
}
} else {
dataInput = {
type: 'optimize-to-webtorrent',
videoUUID: video.uuid,
isNewVideo: true
}
}
return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput })
}
function extractVideo (videoOrPlaylist: MVideo | MStreamingPlaylistVideo) {
return isStreamingPlaylist(videoOrPlaylist)
? videoOrPlaylist.Video
@ -107,7 +83,6 @@ function isStateForFederation (state: VideoState) {
const castedState = parseInt(state + '', 10)
return castedState === VideoState.PUBLISHED || castedState === VideoState.WAITING_FOR_LIVE || castedState === VideoState.LIVE_ENDED
}
function getPrivaciesForFederation () {
@ -130,7 +105,6 @@ export {
fetchVideo,
getVideoWithAttributes,
fetchVideoByUrl,
addOptimizeOrMergeAudioJob,
extractVideo,
getExtFromMimetype,
isStateForFederation,

View File

@ -179,6 +179,12 @@ const REPEAT_JOBS: { [ id: string ]: EveryRepeatOptions | CronRepeatOptions } =
cron: randomInt(1, 20) + ' * * * *' // Between 1-20 minutes past the hour
}
}
const JOB_PRIORITY = {
TRANSCODING: {
OPTIMIZER: 10,
NEW_RESOLUTION: 100
}
}
const BROADCAST_CONCURRENCY = 10 // How many requests in parallel we do in activitypub-http-broadcast job
const CRAWL_REQUEST_CONCURRENCY = 1 // How many requests in parallel to fetch remote data (likes, shares...)
@ -851,6 +857,7 @@ export {
VIDEO_STATES,
QUEUE_CONCURRENCY,
VIDEO_RATE_TYPES,
JOB_PRIORITY,
VIDEO_TRANSCODING_FPS,
FFMPEG_NICE,
ABUSE_STATES,

View File

@ -3,6 +3,7 @@ import { copy, stat } from 'fs-extra'
import { extname } from 'path'
import { createTorrentAndSetInfoHash } from '@server/helpers/webtorrent'
import { getVideoFilePath } from '@server/lib/video-paths'
import { UserModel } from '@server/models/account/user'
import { MVideoFile, MVideoWithFile } from '@server/types/models'
import { VideoFileImportPayload } from '@shared/models'
import { getVideoFileFPS, getVideoFileResolution } from '../../../helpers/ffprobe-utils'
@ -24,7 +25,9 @@ async function processVideoFileImport (job: Bull.Job) {
await updateVideoFile(video, payload.filePath)
await onNewWebTorrentFileResolution(video)
const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId)
await onNewWebTorrentFileResolution(video, user)
return video
}

View File

@ -1,10 +1,10 @@
import * as Bull from 'bull'
import { move, remove, stat } from 'fs-extra'
import { extname } from 'path'
import { addOptimizeOrMergeAudioJob } from '@server/helpers/video'
import { isPostImportVideoAccepted } from '@server/lib/moderation'
import { Hooks } from '@server/lib/plugins/hooks'
import { isAbleToUploadVideo } from '@server/lib/user'
import { addOptimizeOrMergeAudioJob } from '@server/lib/video'
import { getVideoFilePath } from '@server/lib/video-paths'
import { MVideoImportDefault, MVideoImportDefaultFiles, MVideoImportVideo } from '@server/types/models/video/video-import'
import {
@ -224,7 +224,7 @@ async function processFile (downloader: () => Promise<string>, videoImport: MVid
// Create transcoding jobs?
if (video.state === VideoState.TO_TRANSCODE) {
await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile)
await addOptimizeOrMergeAudioJob(videoImportUpdated.Video, videoFile, videoImport.User)
}
} catch (err) {

View File

@ -1,8 +1,9 @@
import * as Bull from 'bull'
import { TranscodeOptionsType } from '@server/helpers/ffmpeg-utils'
import { publishAndFederateIfNeeded } from '@server/lib/video'
import { JOB_PRIORITY } from '@server/initializers/constants'
import { getJobTranscodingPriorityMalus, publishAndFederateIfNeeded } from '@server/lib/video'
import { getVideoFilePath } from '@server/lib/video-paths'
import { MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models'
import { MUser, MUserId, MVideoFullLight, MVideoUUID, MVideoWithFile } from '@server/types/models'
import {
HLSTranscodingPayload,
MergeAudioTranscodingPayload,
@ -25,8 +26,11 @@ import {
transcodeNewWebTorrentResolution
} from '../../video-transcoding'
import { JobQueue } from '../job-queue'
import { UserModel } from '@server/models/account/user'
const handlers: { [ id: string ]: (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight) => Promise<any> } = {
type HandlerFunction = (job: Bull.Job, payload: VideoTranscodingPayload, video: MVideoFullLight, user: MUser) => Promise<any>
const handlers: { [ id: string ]: HandlerFunction } = {
// Deprecated, introduced in 3.1
'hls': handleHLSJob,
'new-resolution-to-hls': handleHLSJob,
@ -55,13 +59,15 @@ async function processVideoTranscoding (job: Bull.Job) {
return undefined
}
const user = await UserModel.loadByChannelActorId(video.VideoChannel.actorId)
const handler = handlers[payload.type]
if (!handler) {
throw new Error('Cannot find transcoding handler for ' + payload.type)
}
await handler(job, payload, video)
await handler(job, payload, video, user)
return video
}
@ -90,22 +96,27 @@ async function handleHLSJob (job: Bull.Job, payload: HLSTranscodingPayload, vide
await retryTransactionWrapper(onHlsPlaylistGeneration, video)
}
async function handleNewWebTorrentResolutionJob (job: Bull.Job, payload: NewResolutionTranscodingPayload, video: MVideoFullLight) {
async function handleNewWebTorrentResolutionJob (
job: Bull.Job,
payload: NewResolutionTranscodingPayload,
video: MVideoFullLight,
user: MUserId
) {
await transcodeNewWebTorrentResolution(video, payload.resolution, payload.isPortraitMode || false, job)
await retryTransactionWrapper(onNewWebTorrentFileResolution, video, payload)
await retryTransactionWrapper(onNewWebTorrentFileResolution, video, user, payload)
}
async function handleWebTorrentMergeAudioJob (job: Bull.Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight) {
async function handleWebTorrentMergeAudioJob (job: Bull.Job, payload: MergeAudioTranscodingPayload, video: MVideoFullLight, user: MUserId) {
await mergeAudioVideofile(video, payload.resolution, job)
await retryTransactionWrapper(onNewWebTorrentFileResolution, video, payload)
await retryTransactionWrapper(onNewWebTorrentFileResolution, video, user, payload)
}
async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight) {
async function handleWebTorrentOptimizeJob (job: Bull.Job, payload: OptimizeTranscodingPayload, video: MVideoFullLight, user: MUserId) {
const transcodeType = await optimizeOriginalVideofile(video, video.getMaxQualityFile(), job)
await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType)
await retryTransactionWrapper(onVideoFileOptimizer, video, payload, transcodeType, user)
}
// ---------------------------------------------------------------------------
@ -129,7 +140,8 @@ async function onHlsPlaylistGeneration (video: MVideoFullLight) {
async function onVideoFileOptimizer (
videoArg: MVideoWithFile,
payload: OptimizeTranscodingPayload,
transcodeType: TranscodeOptionsType
transcodeType: TranscodeOptionsType,
user: MUserId
) {
if (videoArg === undefined) return undefined
@ -142,13 +154,6 @@ async function onVideoFileOptimizer (
// Video does not exist anymore
if (!videoDatabase) return undefined
// Create transcoding jobs if there are enabled resolutions
const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod')
logger.info(
'Resolutions computed for video %s and origin file resolution of %d.', videoDatabase.uuid, videoFileResolution,
{ resolutions: resolutionsEnabled }
)
let videoPublished = false
// Generate HLS version of the original file
@ -158,9 +163,9 @@ async function onVideoFileOptimizer (
// If we quick transcoded original file, force transcoding for HLS to avoid some weird playback issues
copyCodecs: transcodeType !== 'quick-transcode'
})
createHlsJobIfEnabled(originalFileHLSPayload)
await createHlsJobIfEnabled(user, originalFileHLSPayload)
const hasNewResolutions = createLowerResolutionsJobs(videoDatabase, videoFileResolution, isPortraitMode)
const hasNewResolutions = createLowerResolutionsJobs(videoDatabase, user, videoFileResolution, isPortraitMode)
if (!hasNewResolutions) {
// No transcoding to do, it's now published
@ -178,11 +183,12 @@ async function onVideoFileOptimizer (
async function onNewWebTorrentFileResolution (
video: MVideoUUID,
user: MUserId,
payload?: NewResolutionTranscodingPayload | MergeAudioTranscodingPayload
) {
await publishAndFederateIfNeeded(video)
createHlsJobIfEnabled(Object.assign({}, payload, { copyCodecs: true }))
await createHlsJobIfEnabled(user, Object.assign({}, payload, { copyCodecs: true }))
}
// ---------------------------------------------------------------------------
@ -194,22 +200,35 @@ export {
// ---------------------------------------------------------------------------
function createHlsJobIfEnabled (payload: { videoUUID: string, resolution: number, isPortraitMode?: boolean, copyCodecs: boolean }) {
// Generate HLS playlist?
if (payload && CONFIG.TRANSCODING.HLS.ENABLED) {
const hlsTranscodingPayload: HLSTranscodingPayload = {
type: 'new-resolution-to-hls',
videoUUID: payload.videoUUID,
resolution: payload.resolution,
isPortraitMode: payload.isPortraitMode,
copyCodecs: payload.copyCodecs
}
async function createHlsJobIfEnabled (user: MUserId, payload: {
videoUUID: string
resolution: number
isPortraitMode?: boolean
copyCodecs: boolean
}) {
if (!payload || CONFIG.TRANSCODING.HLS.ENABLED !== true) return
return JobQueue.Instance.createJob({ type: 'video-transcoding', payload: hlsTranscodingPayload })
const jobOptions = {
priority: JOB_PRIORITY.TRANSCODING.NEW_RESOLUTION + await getJobTranscodingPriorityMalus(user)
}
const hlsTranscodingPayload: HLSTranscodingPayload = {
type: 'new-resolution-to-hls',
videoUUID: payload.videoUUID,
resolution: payload.resolution,
isPortraitMode: payload.isPortraitMode,
copyCodecs: payload.copyCodecs
}
return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: hlsTranscodingPayload }, jobOptions)
}
function createLowerResolutionsJobs (video: MVideoFullLight, videoFileResolution: number, isPortraitMode: boolean) {
async function createLowerResolutionsJobs (
video: MVideoFullLight,
user: MUserId,
videoFileResolution: number,
isPortraitMode: boolean
) {
// Create transcoding jobs if there are enabled resolutions
const resolutionsEnabled = computeResolutionsToTranscode(videoFileResolution, 'vod')
logger.info(
@ -244,7 +263,11 @@ function createLowerResolutionsJobs (video: MVideoFullLight, videoFileResolution
}
}
JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput })
const jobOptions = {
priority: JOB_PRIORITY.TRANSCODING.NEW_RESOLUTION + await getJobTranscodingPriorityMalus(user)
}
JobQueue.Instance.createJob({ type: 'video-transcoding', payload: dataInput }, jobOptions)
}
logger.info('Transcoding jobs created for uuid %s.', video.uuid, { resolutionsEnabled })

View File

@ -47,6 +47,7 @@ type CreateJobArgument =
type CreateJobOptions = {
delay?: number
priority?: number
}
const handlers: { [id in JobType]: (job: Bull.Job) => Promise<any> } = {
@ -148,6 +149,7 @@ class JobQueue {
backoff: { delay: 60 * 1000, type: 'exponential' },
attempts: JOB_ATTEMPTS[obj.type],
timeout: JOB_TTL[obj.type],
priority: options.priority,
delay: options.delay
}

View File

@ -1,11 +1,13 @@
import { Transaction } from 'sequelize/types'
import { DEFAULT_AUDIO_RESOLUTION, JOB_PRIORITY } from '@server/initializers/constants'
import { sequelizeTypescript } from '@server/initializers/database'
import { TagModel } from '@server/models/video/tag'
import { VideoModel } from '@server/models/video/video'
import { FilteredModelAttributes } from '@server/types'
import { MTag, MThumbnail, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
import { ThumbnailType, VideoCreate, VideoPrivacy } from '@shared/models'
import { MTag, MThumbnail, MUserId, MVideo, MVideoFile, MVideoTag, MVideoThumbnail, MVideoUUID } from '@server/types/models'
import { ThumbnailType, VideoCreate, VideoPrivacy, VideoTranscodingPayload } from '@shared/models'
import { federateVideoIfNeeded } from './activitypub/videos'
import { JobQueue } from './job-queue/job-queue'
import { Notifier } from './notifier'
import { createVideoMiniatureFromExisting } from './thumbnail'
@ -104,11 +106,47 @@ async function publishAndFederateIfNeeded (video: MVideoUUID, wasLive = false) {
}
}
async function addOptimizeOrMergeAudioJob (video: MVideo, videoFile: MVideoFile, user: MUserId) {
let dataInput: VideoTranscodingPayload
if (videoFile.isAudio()) {
dataInput = {
type: 'merge-audio-to-webtorrent',
resolution: DEFAULT_AUDIO_RESOLUTION,
videoUUID: video.uuid,
isNewVideo: true
}
} else {
dataInput = {
type: 'optimize-to-webtorrent',
videoUUID: video.uuid,
isNewVideo: true
}
}
const jobOptions = {
priority: JOB_PRIORITY.TRANSCODING.OPTIMIZER + await getJobTranscodingPriorityMalus(user)
}
return JobQueue.Instance.createJobWithPromise({ type: 'video-transcoding', payload: dataInput }, jobOptions)
}
async function getJobTranscodingPriorityMalus (user: MUserId) {
const now = new Date()
const lastWeek = new Date(now.getFullYear(), now.getMonth(), now.getDate() - 7)
const videoUploadedByUser = await VideoModel.countVideosUploadedByUserSince(user.id, lastWeek)
return videoUploadedByUser
}
// ---------------------------------------------------------------------------
export {
buildLocalVideoFromReq,
publishAndFederateIfNeeded,
buildVideoThumbnailsFromReq,
setVideoTags
setVideoTags,
addOptimizeOrMergeAudioJob,
getJobTranscodingPriorityMalus
}

View File

@ -129,6 +129,7 @@ import { VideoShareModel } from './video-share'
import { VideoStreamingPlaylistModel } from './video-streaming-playlist'
import { VideoTagModel } from './video-tag'
import { VideoViewModel } from './video-view'
import { UserModel } from '../account/user'
export enum ScopeNames {
AVAILABLE_FOR_LIST_IDS = 'AVAILABLE_FOR_LIST_IDS',
@ -1198,6 +1199,39 @@ export class VideoModel extends Model {
return VideoModel.count(options)
}
static countVideosUploadedByUserSince (userId: number, since: Date) {
const options = {
include: [
{
model: VideoChannelModel.unscoped(),
required: true,
include: [
{
model: AccountModel.unscoped(),
required: true,
include: [
{
model: UserModel.unscoped(),
required: true,
where: {
id: userId
}
}
]
}
]
}
],
where: {
createdAt: {
[Op.gte]: since
}
}
}
return VideoModel.unscoped().count(options)
}
static countLivesOfAccount (accountId: number) {
const options = {
where: {

View File

@ -23,6 +23,7 @@ export interface Job {
state: JobState
type: JobType
data: any
priority: number
progress: number
error: any
createdAt: Date | string