diff --git a/docker/k8s-glue/README.md b/docker/k8s-glue/README.md index c980792..98db050 100644 --- a/docker/k8s-glue/README.md +++ b/docker/k8s-glue/README.md @@ -12,4 +12,3 @@ To run, you'll need to: ```bash kubectl -n clearml apply -f k8s-glue.yml ``` - \ No newline at end of file diff --git a/docker/k8s-glue/glue-build/clearml.conf b/docker/k8s-glue/build-resources/clearml.conf similarity index 100% rename from docker/k8s-glue/glue-build/clearml.conf rename to docker/k8s-glue/build-resources/clearml.conf diff --git a/docker/k8s-glue/glue-build/entrypoint.sh b/docker/k8s-glue/build-resources/entrypoint.sh similarity index 72% rename from docker/k8s-glue/glue-build/entrypoint.sh rename to docker/k8s-glue/build-resources/entrypoint.sh index f17ea0b..2b895e9 100644 --- a/docker/k8s-glue/glue-build/entrypoint.sh +++ b/docker/k8s-glue/build-resources/entrypoint.sh @@ -24,4 +24,13 @@ QUEUE=${K8S_GLUE_QUEUE:-k8s_glue} MAX_PODS=${K8S_GLUE_MAX_PODS:-2} EXTRA_ARGS=${K8S_GLUE_EXTRA_ARGS:-} +# shellcheck disable=SC2129 +echo "api.credentials.access_key: ${CLEARML_API_ACCESS_KEY}" >> ~/clearml.conf +echo "api.credentials.secret_key: ${CLEARML_API_SECRET_KEY}" >> ~/clearml.conf +echo "api.api_server: ${CLEARML_API_HOST}" >> ~/clearml.conf +echo "api.web_server: ${CLEARML_WEB_HOST}" >> ~/clearml.conf +echo "api.files_server: ${CLEARML_FILES_HOST}" >> ~/clearml.conf + +./provider_entrypoint.sh + python3 k8s_glue_example.py --queue ${QUEUE} --max-pods ${MAX_PODS} ${EXTRA_ARGS} diff --git a/docker/k8s-glue/glue-build/k8s_glue_example.py b/docker/k8s-glue/build-resources/k8s_glue_example.py similarity index 100% rename from docker/k8s-glue/glue-build/k8s_glue_example.py rename to docker/k8s-glue/build-resources/k8s_glue_example.py diff --git a/docker/k8s-glue/build-resources/setup.sh b/docker/k8s-glue/build-resources/setup.sh new file mode 100644 index 0000000..41083ab --- /dev/null +++ b/docker/k8s-glue/build-resources/setup.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +chmod +x /root/entrypoint.sh + +apt-get update -y +apt-get dist-upgrade -y +apt-get install -y curl unzip less locales + +locale-gen en_US.UTF-8 + +apt-get install -y curl python3-pip git +python3 -m pip install -U pip +python3 -m pip install clearml-agent +python3 -m pip install -U "cryptography>=2.9" diff --git a/docker/k8s-glue/glue-build-aws/Dockerfile b/docker/k8s-glue/glue-build-aws/Dockerfile new file mode 100644 index 0000000..7762333 --- /dev/null +++ b/docker/k8s-glue/glue-build-aws/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:18.04 + +USER root +WORKDIR /root + +ENV LC_ALL=en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV PYTHONIOENCODING=UTF-8 + +COPY ../build-resources/setup.sh /root/setup.sh +RUN /root/setup.sh + +COPY ./setup_aws.sh /root/setup_aws.sh +RUN /root/setup_aws.sh + +COPY ../build-resources/entrypoint.sh /root/entrypoint.sh +COPY ./provider_entrypoint.sh /root/provider_entrypoint.sh +COPY ./build-resources/k8s_glue_example.py /root/k8s_glue_example.py +COPY ./clearml.conf /root/clearml.conf + +ENTRYPOINT ["/root/entrypoint.sh"] \ No newline at end of file diff --git a/docker/k8s-glue/glue-build-aws/provider_entrypoint.sh b/docker/k8s-glue/glue-build-aws/provider_entrypoint.sh new file mode 100644 index 0000000..cd1140f --- /dev/null +++ b/docker/k8s-glue/glue-build-aws/provider_entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash -x + +source /root/.bashrc +export PATH=$PATH:$HOME/bin diff --git a/docker/k8s-glue/glue-build-aws/setup_aws.sh b/docker/k8s-glue/glue-build-aws/setup_aws.sh new file mode 100644 index 0000000..dd54281 --- /dev/null +++ b/docker/k8s-glue/glue-build-aws/setup_aws.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip awscliv2.zip +./aws/install + +curl -o kubectl https://amazon-eks.s3-us-west-2.amazonaws.com/1.21.2/2021-07-05/bin/linux/amd64/kubectl +#curl -o kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl +chmod +x ./kubectl && mkdir -p $HOME/bin && cp ./kubectl $HOME/bin/kubectl && export PATH=$PATH:$HOME/bin + +curl -o aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.21.2/2021-07-05/bin/linux/amd64/aws-iam-authenticator +#curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/aws-iam-authenticator +chmod +x ./aws-iam-authenticator && mkdir -p $HOME/bin && cp ./aws-iam-authenticator $HOME/bin/aws-iam-authenticator && export PATH=$PATH:$HOME/bin +echo 'export PATH=$PATH:$HOME/bin' >> ~/.bashrc diff --git a/docker/k8s-glue/glue-build-gcp/Dockerfile b/docker/k8s-glue/glue-build-gcp/Dockerfile new file mode 100644 index 0000000..7e730a5 --- /dev/null +++ b/docker/k8s-glue/glue-build-gcp/Dockerfile @@ -0,0 +1,22 @@ +FROM ubuntu:18.04 + +USER root +WORKDIR /root + +ENV LC_ALL=en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 +ENV PYTHONIOENCODING=UTF-8 + +COPY ../build-resources/setup.sh /root/setup.sh +RUN /root/setup.sh + +COPY ./setup_gcp.sh /root/setup_gcp.sh +RUN /root/setup_gcp.sh + +COPY ../build-resources/entrypoint.sh /root/entrypoint.sh +COPY ./provider_entrypoint.sh /root/provider_entrypoint.sh +COPY ./build-resources/k8s_glue_example.py /root/k8s_glue_example.py +COPY ./clearml.conf /root/clearml.conf + +ENTRYPOINT ["/root/entrypoint.sh"] \ No newline at end of file diff --git a/docker/k8s-glue/glue-build-gcp/provider_entrypoint.sh b/docker/k8s-glue/glue-build-gcp/provider_entrypoint.sh new file mode 100644 index 0000000..979d8ce --- /dev/null +++ b/docker/k8s-glue/glue-build-gcp/provider_entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/bash -x + +gcloud auth activate-service-account ${CLEARML_SERVICE_ACC} --key-file=/root/keys/${SERVICE_ACC_KEY_JSON} +gcloud container clusters get-credentials ${CLUSTER_CRED} diff --git a/docker/k8s-glue/glue-build-gcp/setup_gcp.sh b/docker/k8s-glue/glue-build-gcp/setup_gcp.sh new file mode 100644 index 0000000..5d245be --- /dev/null +++ b/docker/k8s-glue/glue-build-gcp/setup_gcp.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +curl -LO https://dl.k8s.io/release/v1.21.0/bin/linux/amd64/kubectl + +install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl + +sudo apt-get install -y apt-transport-https ca-certificates gnupg + +echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + +curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + +apt-get update -y +apt-get install -y google-cloud-sdk diff --git a/docker/k8s-glue/glue-build/Dockerfile b/docker/k8s-glue/glue-build/Dockerfile deleted file mode 100644 index 33560f9..0000000 --- a/docker/k8s-glue/glue-build/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -FROM ubuntu:18.04 - -USER root -WORKDIR /root - -ENV LC_ALL=en_US.UTF-8 -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US.UTF-8 -ENV PYTHONIOENCODING=UTF-8 - -COPY ./entrypoint.sh /root/entrypoint.sh -COPY ./k8s_glue_example.py /root/k8s_glue_example.py -COPY ./setup.sh /root/setup.sh -COPY ./clearml.conf /root/clearml.conf - -RUN /root/setup.sh - -ENTRYPOINT ["/root/entrypoint.sh"] \ No newline at end of file diff --git a/docker/k8s-glue/glue-build/setup.sh b/docker/k8s-glue/glue-build/setup.sh deleted file mode 100644 index d0459d6..0000000 --- a/docker/k8s-glue/glue-build/setup.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -chmod +x /root/entrypoint.sh - -apt-get update -y -apt-get dist-upgrade -y -apt-get install -y curl unzip less locales - -locale-gen en_US.UTF-8 - -apt-get install -y curl python3-pip git -python3 -m pip install -U pip -python3 -m pip install clearml-agent -python3 -m pip install -U "cryptography>=2.9" - -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -unzip awscliv2.zip -./aws/install - -curl -o kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/kubectl -chmod +x ./kubectl && mkdir -p $HOME/bin && cp ./kubectl $HOME/bin/kubectl && export PATH=$PATH:$HOME/bin - -curl -o aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.19.6/2021-01-05/bin/linux/amd64/aws-iam-authenticator -chmod +x ./aws-iam-authenticator && mkdir -p $HOME/bin && cp ./aws-iam-authenticator $HOME/bin/aws-iam-authenticator && export PATH=$PATH:$HOME/bin -echo 'export PATH=$PATH:$HOME/bin' >> ~/.bashrc diff --git a/docker/k8s-glue/k8s-glue.yml b/docker/k8s-glue/k8s-glue-aws.yml similarity index 63% rename from docker/k8s-glue/k8s-glue.yml rename to docker/k8s-glue/k8s-glue-aws.yml index 4b3698d..11c1473 100644 --- a/docker/k8s-glue/k8s-glue.yml +++ b/docker/k8s-glue/k8s-glue-aws.yml @@ -3,22 +3,15 @@ kind: Pod metadata: name: k8s-glue spec: + serviceAccountName: "" containers: - name: k8s-glue-container - image: allegroai/clearml-agent-k8s:test + image: allegroai/clearml-agent-k8s:aws-latest-1.21 imagePullPolicy: Always command: [ "/bin/bash", "-c", - "echo \"api.credentials.access_key: $CLEARML_API_ACCESS_KEY\" >> ~/clearml.conf \ - && echo \"api.credentials.secret_key: $CLEARML_API_SECRET_KEY\" >> ~/clearml.conf \ - && echo \"api.api_server: $CLEARML_API_HOST\" >> ~/clearml.conf \ - && echo \"api.web_server: $CLEARML_WEB_HOST\" >> ~/clearml.conf \ - && echo \"api.files_server: $CLEARML_FILES_HOST\" >> ~/clearml.conf \ - && source /root/.bashrc \ - && export PATH=$PATH:$HOME/bin \ - && /root/entrypoint.sh - " + "source /root/.bashrc && /root/entrypoint.sh" ] volumeMounts: - name: pod-template @@ -41,7 +34,7 @@ spec: - name: CLEARML_API_SECRET_KEY value: "" - name: CLEARML_WORKER_ID - value: "k8s-glue-agent" + value: "k8s-glue-agent" - name: CLEARML_AGENT_UPDATE_REPO value: "" - name: FORCE_CLEARML_AGENT_REPO diff --git a/docker/k8s-glue/k8s-glue-gcp.yml b/docker/k8s-glue/k8s-glue-gcp.yml new file mode 100644 index 0000000..0233a73 --- /dev/null +++ b/docker/k8s-glue/k8s-glue-gcp.yml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: Pod +metadata: + name: k8s-glue +spec: + serviceAccountName: "" + containers: + - name: k8s-glue-container + image: allegroai/clearml-agent-k8s:gcp-latest-1.21 + imagePullPolicy: Always + command: [ + "/bin/bash", + "-c", + "source /root/.bashrc && /root/entrypoint.sh" + ] + volumeMounts: + - name: pod-template + mountPath: /root/template + - name: service-acc-key + mountPath: /root/keys + env: + - name: CLEARML_API_HOST + value: "" + - name: CLEARML_WEB_HOST + value: "" + - name: CLEARML_FILES_HOST + value: "" +# - name: K8S_GLUE_MAX_PODS +# value: "2" + - name: K8S_GLUE_QUEUE + value: "k8s-glue" + - name: K8S_GLUE_EXTRA_ARGS + value: "--template-yaml /root/template/pod_template.yml" + - name: CLEARML_API_ACCESS_KEY + value: "" + - name: CLEARML_API_SECRET_KEY + value: "" + - name: CLEARML_WORKER_ID + value: "k8s-glue-agent" + - name: CLEARML_AGENT_UPDATE_REPO + value: "" + - name: FORCE_CLEARML_AGENT_REPO + value: "" + - name: CLEARML_DOCKER_IMAGE + value: "ubuntu:18.04" + - name: CLEARML_SERVICE_ACC + value: "" + - name: SERVICE_ACC_KEY_JSON + value: service-account-key.json + - name: CLUSTER_CRED + value: "" + volumes: + - name: pod-template + secret: + secretName: k8s-glue-pod-template + - name: service-acc-key + secret: + secretName: k8s-glue-service-acc-key diff --git a/docker/k8s-glue/task-pod-build/Dockerfile b/docker/k8s-glue/task-pod-build/Dockerfile new file mode 100644 index 0000000..72c48e3 --- /dev/null +++ b/docker/k8s-glue/task-pod-build/Dockerfile @@ -0,0 +1,7 @@ +FROM ubuntu:18.04 + +USER root +WORKDIR /root +COPY ./setup.sh /root/setup.sh + +RUN /root/setup.sh diff --git a/docker/k8s-glue/task-pod-build/setup.sh b/docker/k8s-glue/task-pod-build/setup.sh new file mode 100644 index 0000000..7356e6b --- /dev/null +++ b/docker/k8s-glue/task-pod-build/setup.sh @@ -0,0 +1,10 @@ +echo 'Binary::apt::APT::Keep-Downloaded-Packages \"true\";' > /etc/apt/apt.conf.d/docker-clean +chown -R root /root/.cache/pip + +apt-get update -y +apt-get dist-upgrade -y +apt-get install -y git libsm6 libxext6 libxrender-dev libglib2.0-0 curl python3-pip + +python3 -m pip install -U pip +python3 -m pip install clearml-agent +python3 -m pip install -U "cryptography>=2.9"