diff --git a/apiserver/es_factory.py b/apiserver/es_factory.py index 83770e6..311706c 100644 --- a/apiserver/es_factory.py +++ b/apiserver/es_factory.py @@ -21,10 +21,26 @@ OVERRIDE_PORT_ENV_KEY = ( "ELASTIC_SERVICE_PORT", ) +OVERRIDE_USERNAME_ENV_KEY = ( + "CLEARML_ELASTIC_SERVICE_USERNAME", +) + +OVERRIDE_PASSWORD_ENV_KEY = ( + "CLEARML_ELASTIC_SERVICE_PASSWORD", +) + OVERRIDE_HOST = first(filter(None, map(getenv, OVERRIDE_HOST_ENV_KEY))) if OVERRIDE_HOST: log.info(f"Using override elastic host {OVERRIDE_HOST}") +OVERRIDE_USERNAME = first(filter(None, map(getenv, OVERRIDE_USERNAME_ENV_KEY))) +if OVERRIDE_USERNAME: + log.info(f"Using override elastic username {OVERRIDE_USERNAME}") + +OVERRIDE_PASSWORD = first(filter(None, map(getenv, OVERRIDE_PASSWORD_ENV_KEY))) +if OVERRIDE_PASSWORD: + log.info("Using override elastic password ********") + OVERRIDE_PORT = first(filter(None, map(getenv, OVERRIDE_PORT_ENV_KEY))) if OVERRIDE_PORT: log.info(f"Using override elastic port {OVERRIDE_PORT}") @@ -65,9 +81,12 @@ class ESFactory: if not hosts: raise InvalidClusterConfiguration(cluster_name) + http_auth = cluster_config.get("http_auth", None) + if not cluster_config.get("secure", True): + http_auth = None args = cluster_config.get("args", {}) _instances[cluster_name] = Elasticsearch( - hosts=hosts, transport_class=Transport, **args + hosts=hosts, transport_class=Transport, http_auth=http_auth, **args ) return _instances[cluster_name] @@ -77,8 +96,8 @@ class ESFactory: return list(config.get("hosts.elastic")) @classmethod - def get_override(cls, cluster_name: str) -> Tuple[str, str]: - return OVERRIDE_HOST, OVERRIDE_PORT + def get_override(cls, cluster_name: str) -> Tuple[str, str, str, str]: + return OVERRIDE_HOST, OVERRIDE_PORT, OVERRIDE_USERNAME, OVERRIDE_PASSWORD @classmethod def get_cluster_config(cls, cluster_name): @@ -97,7 +116,7 @@ class ESFactory: for entry in cluster_config.get("hosts", []): entry[key] = value - host, port = cls.get_override(cluster_name) + host, port, username, password = cls.get_override(cluster_name) if host: set_host_prop("host", host) @@ -105,6 +124,9 @@ class ESFactory: if port: set_host_prop("port", port) + if username and password: + cluster_config.set("http_auth", (username, password)) + return cluster_config @classmethod diff --git a/docker/build/internal_files/clearml.conf.template b/docker/build/internal_files/clearml.conf.template index 4d7130a..c1718eb 100644 --- a/docker/build/internal_files/clearml.conf.template +++ b/docker/build/internal_files/clearml.conf.template @@ -94,3 +94,4 @@ http { location = /50x.html { } } +} \ No newline at end of file diff --git a/docker/build/internal_files/final_image_preparation.sh b/docker/build/internal_files/final_image_preparation.sh index 207ecb2..54fb5b2 100644 --- a/docker/build/internal_files/final_image_preparation.sh +++ b/docker/build/internal_files/final_image_preparation.sh @@ -1,4 +1,8 @@ #!/usr/bin/env bash +set -o errexit +set -o nounset +set -o pipefail + yum update -y yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm yum install -y python36 python36-pip nginx gcc python3-devel gettext diff --git a/docker/docker-compose-win10.yml b/docker/docker-compose-win10.yml index 499f89f..704e24c 100644 --- a/docker/docker-compose-win10.yml +++ b/docker/docker-compose-win10.yml @@ -19,6 +19,7 @@ services: environment: CLEARML_ELASTIC_SERVICE_HOST: elasticsearch CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD} CLEARML_MONGODB_SERVICE_HOST: mongo CLEARML_MONGODB_SERVICE_PORT: 27017 CLEARML_REDIS_SERVICE_HOST: redis @@ -39,6 +40,7 @@ services: container_name: clearml-elastic environment: ES_JAVA_OPTS: -Xms2g -Xmx2g + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD} bootstrap.memory_lock: "true" cluster.name: clearml cluster.routing.allocation.node_initial_primaries_recoveries: "500" @@ -60,7 +62,7 @@ services: nofile: soft: 65536 hard: 65536 - image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2 + image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2 restart: unless-stopped volumes: - c:/opt/clearml/data/elastic_7:/usr/share/elasticsearch/data @@ -87,7 +89,7 @@ services: networks: - backend container_name: clearml-mongo - image: mongo:3.6.5 + image: mongo:3.6.23 restart: unless-stopped command: --setParameter internalQueryExecMaxBlockingSortBytes=196100200 volumes: @@ -124,4 +126,4 @@ networks: driver: bridge frontend: name: frontend - driver: bridge \ No newline at end of file + driver: bridge diff --git a/docker/docker-compose.yml b/docker/docker-compose.yaml similarity index 95% rename from docker/docker-compose.yml rename to docker/docker-compose.yaml index 314ef7f..f171f78 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yaml @@ -19,6 +19,7 @@ services: environment: CLEARML_ELASTIC_SERVICE_HOST: elasticsearch CLEARML_ELASTIC_SERVICE_PORT: 9200 + CLEARML_ELASTIC_SERVICE_PASSWORD: ${ELASTIC_PASSWORD} CLEARML_MONGODB_SERVICE_HOST: mongo CLEARML_MONGODB_SERVICE_PORT: 27017 CLEARML_REDIS_SERVICE_HOST: redis @@ -39,6 +40,7 @@ services: container_name: clearml-elastic environment: ES_JAVA_OPTS: -Xms2g -Xmx2g + ELASTIC_PASSWORD: ${ELASTIC_PASSWORD} bootstrap.memory_lock: "true" cluster.name: clearml cluster.routing.allocation.node_initial_primaries_recoveries: "500" @@ -60,7 +62,7 @@ services: nofile: soft: 65536 hard: 65536 - image: docker.elastic.co/elasticsearch/elasticsearch:7.6.2 + image: docker.elastic.co/elasticsearch/elasticsearch:7.10.2 restart: unless-stopped volumes: - /opt/clearml/data/elastic_7:/usr/share/elasticsearch/data @@ -86,7 +88,7 @@ services: networks: - backend container_name: clearml-mongo - image: mongo:3.6.5 + image: mongo:3.6.23 restart: unless-stopped command: --setParameter internalQueryExecMaxBlockingSortBytes=196100200 volumes: