From d05555b01a8dc5c6e4390a45bdafcec968b05bee Mon Sep 17 00:00:00 2001 From: pollfly <75068813+pollfly@users.noreply.github.com> Date: Thu, 21 Oct 2021 09:44:30 +0300 Subject: [PATCH] Exemplify configuration file environment variable access (#90) --- docs/integrations/storage.md | 44 ++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/docs/integrations/storage.md b/docs/integrations/storage.md index 6d6918f9..040a73e8 100644 --- a/docs/integrations/storage.md +++ b/docs/integrations/storage.md @@ -18,6 +18,8 @@ Once uploading an object to a storage medium, each machine that uses the object Configuration for storage is done by editing the [clearml.conf](../configs/clearml_conf.md). +The ClearML configuration file uses [HOCON](https://github.com/lightbend/config/blob/main/HOCON.md) format, which supports runtime environment variable access. + ### Configuring AWS S3 Modify these parts of the clearml.conf file and add the key, secret, and region of the s3 bucket. @@ -49,6 +51,20 @@ aws { } ``` +AWS's S3 access parameters can be specified by referencing the standard environment variables if already defined. + +For example: +``` +s3 { + ... + # default, used for any bucket not specified below + key: "${AWS_ACCESS_KEY_ID}" + secret: "${AWS_SECRET_ACCESS_KEY}" + region: "${AWS_DEFAULT_REGION}" + ... +} +``` + ClearML also supports [MinIO](https://github.com/minio/minio) by adding this configuration: ``` # { @@ -60,6 +76,7 @@ ClearML also supports [MinIO](https://github.com/minio/minio) by adding this con # } ``` + ### Configuring Azure To configure Azure blob storage specify the account name and key. @@ -75,6 +92,20 @@ To configure Azure blob storage specify the account name and key. } ``` +Azure's storage access parameters can be specified by referencing the standard environment variables if already defined. + +For example: +``` +... +containers: [ + { + account_name: "${AZURE_STORAGE_ACCOUNT}" + account_key: "${AZURE_STORAGE_KEY}" + # container_name: + } + ] +``` + ### Configuring Google Storage To configure Google Storage, specify the project and the path to the credentials json file. It's also possible to specify credentials for a specific bucket. @@ -98,6 +129,19 @@ It's also possible to specify credentials for a specific bucket. } ``` +GCP's storage access parameters can be specified by referencing the standard environment variables if already defined. + +``` +... +credentials = [ + { + bucket: "my-bucket" + ... + credentials_json: "${GOOGLE_APPLICATION_CREDENTIALS}" + } + +``` + ## Storage Manager ClearML offers the [StorageManager](../references/sdk/storage.md) class to manage downloading, uploading, and caching of