ClearML/clearml-server
1
0
Fork 0
mirror of https://github.com/clearml/clearml-server synced 2025-04-29 10:23:11 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add ES http_auth credentials support (#93)

Browse Source 
Also update ES and MongoDB versions and fix nginx configuration bug

Co-authored-by: huangweixiao <huangweixiao@megvii.com>
This commit is contained in:
Weixiao Huang 2021-11-15 21:01:27 +08:00 committed by GitHub
parent 3f2b96266b
commit 3d816c747d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 40 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
apiserver/es_factory.py
View File

@ -21,10 +21,26 @@ OVERRIDE_PORT_ENV_KEY = (
"ELASTIC_SERVICE_PORT",
)
OVERRIDE_USERNAME_ENV_KEY = (
"CLEARML_ELASTIC_SERVICE_USERNAME",
)
OVERRIDE_PASSWORD_ENV_KEY = (
"CLEARML_ELASTIC_SERVICE_PASSWORD",
)
OVERRIDE_HOST = first(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY)))
if OVERRIDE_HOST:
log.info(f"Using override elastic host {OVERRIDE_HOST}")
OVERRIDE_USERNAME = first(filter(None, map(getenv, OVERRIDE_USERNAME_ENV_KEY)))
if OVERRIDE_USERNAME:
log.info(f"Using override elastic username {OVERRIDE_USERNAME}")
OVERRIDE_PASSWORD = first(filter(None, map(getenv, OVERRIDE_PASSWORD_ENV_KEY)))
if OVERRIDE_PASSWORD:
log.info("Using override elastic password ********")
OVERRIDE_PORT = first(filter(None, map(getenv, OVERRIDE_PORT_ENV_KEY)))
if OVERRIDE_PORT:
log.info(f"Using override elastic port {OVERRIDE_PORT}")
@ -65,9 +81,12 @@ class ESFactory:
if not hosts:
raise InvalidClusterConfiguration(cluster_name)
http_auth = cluster_config.get("http_auth", None)
if not cluster_config.get("secure", True):
http_auth = None
args = cluster_config.get("args", {})
_instances[cluster_name] = Elasticsearch(
hosts=hosts, transport_class=Transport, **args
hosts=hosts, transport_class=Transport, http_auth=http_auth, **args
)
return _instances[cluster_name]
@ -77,8 +96,8 @@ class ESFactory:
return list(config.get("hosts.elastic"))
@classmethod
def get_override(cls, cluster_name: str) -> Tuple[str, str]:
return OVERRIDE_HOST, OVERRIDE_PORT
def get_override(cls, cluster_name: str) -> Tuple[str, str, str, str]:
return OVERRIDE_HOST, OVERRIDE_PORT, OVERRIDE_USERNAME, OVERRIDE_PASSWORD
@classmethod
def get_cluster_config(cls, cluster_name):
@ -97,7 +116,7 @@ class ESFactory:
for entry in cluster_config.get("hosts", []):
entry[key] = value
host, port = cls.get_override(cluster_name)
host, port, username, password = cls.get_override(cluster_name)
if host:
set_host_prop("host", host)
@ -105,6 +124,9 @@ class ESFactory:
if port:
set_host_prop("port", port)
if username and password:
cluster_config.set("http_auth", (username, password))
return cluster_config
@classmethod

1
docker/build/internal_files/clearml.conf.template
View File

@ -94,3 +94,4 @@ http {
location = /50x.html {
}
}
}

4
docker/build/internal_files/final_image_preparation.sh
View File

@ -1,4 +1,8 @@
#!/usr/bin/env bash
set -o errexit
set -o nounset
set -o pipefail
yum update -y
yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm
yum install -y python36 python36-pip nginx gcc python3-devel gettext

8
docker/docker-compose-win10.yml
View File

@ -19,6 +19,7 @@ services:
environment:
CLEARML_ELASTIC_SERVICE_HOST: elasticsearch
CLEARML_ELASTIC_SERVICE_PORT: 9200
CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD}
CLEARML_MONGODB_SERVICE_HOST: mongo
CLEARML_MONGODB_SERVICE_PORT: 27017
CLEARML_REDIS_SERVICE_HOST: redis
@ -39,6 +40,7 @@ services:
container_name: clearml-elastic
environment:
ES_JAVA_OPTS: -Xms2g -Xmx2g
ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
bootstrap.memory_lock: "true"
cluster.name: clearml
cluster.routing.allocation.node_initial_primaries_recoveries: "500"
@ -60,7 +62,7 @@ services:
nofile:
soft: 65536
hard: 65536
image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
restart: unless-stopped
volumes:
- c:/opt/clearml/data/elastic_7:/usr/share/elasticsearch/data
@ -87,7 +89,7 @@ services:
networks:
- backend
container_name: clearml-mongo
image: mongo:3.6.5
image: mongo:3.6.23
restart: unless-stopped
command: --setParameter internalQueryExecMaxBlockingSortBytes=196100200
volumes:
@ -124,4 +126,4 @@ networks:
driver: bridge
frontend:
name: frontend
driver: bridge
driver: bridge

6
docker/docker-compose.yml → docker/docker-compose.yaml
View File

@ -19,6 +19,7 @@ services:
environment:
CLEARML_ELASTIC_SERVICE_HOST: elasticsearch
CLEARML_ELASTIC_SERVICE_PORT: 9200
CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD}
CLEARML_MONGODB_SERVICE_HOST: mongo
CLEARML_MONGODB_SERVICE_PORT: 27017
CLEARML_REDIS_SERVICE_HOST: redis
@ -39,6 +40,7 @@ services:
container_name: clearml-elastic
environment:
ES_JAVA_OPTS: -Xms2g -Xmx2g
ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
bootstrap.memory_lock: "true"
cluster.name: clearml
cluster.routing.allocation.node_initial_primaries_recoveries: "500"
@ -60,7 +62,7 @@ services:
nofile:
soft: 65536
hard: 65536
image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2
image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2
restart: unless-stopped
volumes:
- /opt/clearml/data/elastic_7:/usr/share/elasticsearch/data
@ -86,7 +88,7 @@ services:
networks:
- backend
container_name: clearml-mongo
image: mongo:3.6.5
image: mongo:3.6.23
restart: unless-stopped
command: --setParameter internalQueryExecMaxBlockingSortBytes=196100200
volumes: