Add ability to set proxies to youtube-dl

This commit is contained in:
Chocobozzz 2024-08-13 10:49:31 +02:00
parent 766f2cb76a
commit 5439a591e0
No known key found for this signature in database
GPG Key ID: 583A612D890159BE
9 changed files with 43 additions and 22 deletions

View File

@ -50,6 +50,7 @@ jobs:
OBJECT_STORAGE_SCALEWAY_KEY_ID: ${{ secrets.OBJECT_STORAGE_SCALEWAY_KEY_ID }}
OBJECT_STORAGE_SCALEWAY_ACCESS_KEY: ${{ secrets.OBJECT_STORAGE_SCALEWAY_ACCESS_KEY }}
YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
YOUTUBE_DL_PROXY: ${{ secrets.YOUTUBE_DL_PROXY }}
steps:
- uses: actions/checkout@v4

View File

@ -803,6 +803,12 @@ import:
# IPv6 is very strongly rate-limited on most sites supported by youtube-dl
force_ipv4: false
# By default PeerTube uses HTTP_PROXY and HTTPS_PROXY environment variables
# But you can specify custom proxies for youtube-dl because remote websites (like YouTube) may block your server IP address
# PeerTube will randomly select a proxy from the following list
proxies:
# - "https://username:password@example.com:8888"
# Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
torrent:
# We recommend to only enable magnet URI/torrent import if you trust your users

View File

@ -813,6 +813,12 @@ import:
# IPv6 is very strongly rate-limited on most sites supported by youtube-dl
force_ipv4: false
# By default PeerTube uses HTTP_PROXY and HTTPS_PROXY environment variables
# But you can specify custom proxies for youtube-dl because remote websites (like YouTube) may block your server IP address
# PeerTube will randomly select a proxy from the following list
proxies:
# - "https://username:password@example.com:8888"
# Magnet URI or torrent file (use classic TCP/UDP/WebSeed to download the file)
torrent:
# We recommend to only enable magnet URI/torrent import if you trust your users

View File

@ -1,20 +1,9 @@
import { BinaryToTextEncoding, createHash } from 'crypto'
function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
export function sha256 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
return createHash('sha256').update(str).digest(encoding)
}
function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
export function sha1 (str: string | Buffer, encoding: BinaryToTextEncoding = 'hex') {
return createHash('sha1').update(str).digest(encoding)
}
// high excluded
function randomInt (low: number, high: number) {
return Math.floor(Math.random() * (high - low) + low)
}
export {
randomInt,
sha256,
sha1
}

View File

@ -58,6 +58,7 @@ import { PluginsCommand } from './plugins-command.js'
import { RedundancyCommand } from './redundancy-command.js'
import { ServersCommand } from './servers-command.js'
import { StatsCommand } from './stats-command.js'
import merge from 'lodash-es/merge.js'
export type RunServerOptions = {
hideLogs?: boolean
@ -240,10 +241,10 @@ export class PeerTubeServer {
await this.assignCustomConfigFile()
const configOverride = this.buildConfigOverride()
let configOverride = this.buildConfigOverride()
if (configOverrideArg !== undefined) {
Object.assign(configOverride, configOverrideArg)
configOverride = merge(configOverride, configOverrideArg)
}
// Share the environment
@ -363,9 +364,15 @@ export class PeerTubeServer {
}
private buildConfigOverride () {
if (!this.parallel) return {}
const base = process.env.YOUTUBE_DL_PROXY
? { import: { videos: { http: { proxies: [ process.env.YOUTUBE_DL_PROXY ] } } } }
: {}
if (!this.parallel) return base
return {
...base,
listen: {
port: this.port
},

View File

@ -1,10 +1,11 @@
import { randomInt } from '@peertube/peertube-core-utils'
import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models'
import { CONFIG } from '@server/initializers/config.js'
import { execa, Options as ExecaNodeOptions } from 'execa'
import { ensureDir, pathExists } from 'fs-extra/esm'
import { writeFile } from 'fs/promises'
import { OptionsOfBufferResponseBody } from 'got'
import { dirname, join } from 'path'
import { VideoResolution, VideoResolutionType } from '@peertube/peertube-models'
import { CONFIG } from '@server/initializers/config.js'
import { logger, loggerTagsFactory } from '../logger.js'
import { getProxy, isProxyEnabled } from '../proxy.js'
import { isBinaryResponse, peertubeGot } from '../requests.js'
@ -231,10 +232,17 @@ export class YoutubeDLCLI {
}
private wrapWithProxyOptions (args: string[]) {
if (isProxyEnabled()) {
logger.debug('Using proxy %s for YoutubeDL', getProxy(), lTags())
const config = CONFIG.IMPORT.VIDEOS.HTTP.PROXIES
const configProxyEnabled = Array.isArray(config) && config.length !== 0
return [ '--proxy', getProxy() ].concat(args)
if (configProxyEnabled || isProxyEnabled()) {
const proxy = configProxyEnabled
? config[randomInt(0, config.length)]
: getProxy()
logger.debug('Using proxy %s for YoutubeDL', proxy, lTags())
return [ '--proxy', proxy ].concat(args)
}
return args

View File

@ -43,6 +43,7 @@ function checkMissedConfig () {
'remote_runners.stalled_jobs.vod', 'remote_runners.stalled_jobs.live',
'thumbnails.generation_from_video.frames_to_analyze', 'thumbnails.sizes',
'import.videos.http.enabled', 'import.videos.torrent.enabled', 'import.videos.concurrency', 'import.videos.timeout',
'import.videos.http.force_ipv4', 'import.videos.http.proxies',
'import.video_channel_synchronization.enabled', 'import.video_channel_synchronization.max_per_user',
'import.video_channel_synchronization.check_interval', 'import.video_channel_synchronization.videos_limit_per_synchronization',
'import.video_channel_synchronization.full_sync_videos_limit',

View File

@ -554,7 +554,9 @@ const CONFIG = {
get PYTHON_PATH () { return config.get<string>('import.videos.http.youtube_dl_release.python_path') }
},
get FORCE_IPV4 () { return config.get<boolean>('import.videos.http.force_ipv4') }
get FORCE_IPV4 () { return config.get<boolean>('import.videos.http.force_ipv4') },
get PROXIES () { return config.get<string[]>('import.videos.http.proxies') }
},
TORRENT: {
get ENABLED () { return config.get<boolean>('import.videos.torrent.enabled') }

View File

@ -76,6 +76,7 @@ Some env variables can be defined to disable/enable some tests:
* `OBJECT_STORAGE_SCALEWAY_KEY_ID` and `OBJECT_STORAGE_SCALEWAY_ACCESS_KEY`: specify Scaleway API keys to test object storage ACL (not supported by our `chocobozzz/s3-ninja` container)
* `ENABLE_FFMPEG_THUMBNAIL_PIXEL_COMPARISON_TESTS=true`: enable pixel comparison on images generated by ffmpeg. Disabled by default because a custom ffmpeg version may fails the tests
* `YOUTUBE_DL_DOWNLOAD_BEARER_TOKEN`: Bearer token to download youtube-dl binary
* `YOUTUBE_DL_PROXY`: Custom proxy URL for youtube-dl HTTP video import
### Debug server logs