ClearML/clearml-agent
1
0
Fork 0
mirror of https://github.com/clearml/clearml-agent synced 2025-02-07 05:19:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix AWS dynamic cluster management service use of AWS credentials

Browse Source
This commit is contained in:
allegroai 2020-01-05 13:57:29 +02:00
parent 41d75df40c
commit d6cc2be653
1 changed files with 23 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
examples/dynamic_cloud_cluster.ipynb
View File

@ -138,7 +138,7 @@
"metadata": {},
"outputs": [],
"source": [
"# AWS credentials\n",
"# AWS credentials (leave empty to use credentials set using the aws cli)\n",
"CLOUD_CREDENTIALS_KEY = \"\"\n",
"CLOUD_CREDENTIALS_SECRET = \"\"\n",
"CLOUD_CREDENTIALS_REGION = \"us-east-1\"\n",
@ -278,7 +278,12 @@
" git_pass=TRAINS_GIT_PASS,\n",
" )\n",
"\n",
" ec2 = boto3.client(\"ec2\", region_name=CLOUD_CREDENTIALS_REGION)\n",
" ec2 = boto3.client(\n",
" \"ec2\",\n",
" aws_access_key_id=CLOUD_CREDENTIALS_KEY or None,\n",
" aws_secret_access_key=CLOUD_CREDENTIALS_SECRET or None,\n",
" region_name=CLOUD_CREDENTIALS_REGION\n",
" )\n",
"\n",
" if resource_conf[\"is_spot\"]:\n",
" # Create a request for a spot instance in AWS\n",
@ -310,9 +315,12 @@
" SpotInstanceRequestIds=[request_id]\n",
" )\n",
" instance_id = response[\"SpotInstanceRequests\"][0][\"InstanceId\"]\n",
" instance = boto3.resource(\"ec2\", region_name=CLOUD_CREDENTIALS_REGION).Instance(\n",
" instance_id\n",
" )\n",
" instance = boto3.resource(\n",
" \"ec2\",\n",
" aws_access_key_id=CLOUD_CREDENTIALS_KEY or None,\n",
" aws_secret_access_key=CLOUD_CREDENTIALS_SECRET or None,\n",
" region_name=CLOUD_CREDENTIALS_REGION\n",
" ).Instance(instance_id)\n",
"\n",
" else:\n",
" # Create a new EC2 instance\n",
@ -352,9 +360,12 @@
" (currently, only AWS EC2 is supported)\n",
" \"\"\"\n",
" try:\n",
" boto3.resource(\"ec2\", region_name=CLOUD_CREDENTIALS_REGION).instances.filter(\n",
" InstanceIds=[instance_id]\n",
" ).terminate()\n",
" boto3.resource(\n",
" \"ec2\",\n",
" aws_access_key_id=CLOUD_CREDENTIALS_KEY or None,\n",
" aws_secret_access_key=CLOUD_CREDENTIALS_SECRET or None,\n",
" region_name=CLOUD_CREDENTIALS_REGION\n",
" ).instances.filter(InstanceIds=[instance_id]).terminate()\n",
" except Exception as ex:\n",
" raise ex"
]
@ -501,7 +512,8 @@
" try:\n",
" supervisor()\n",
" except Exception as ex:\n",
" print(\"Warning! exception occurred: {ex}\".format(ex=ex))"
" print(\"Warning! exception occurred: {ex}\\nRetry in 15 seconds\".format(ex=ex))\n",
" sleep(15)"
]
}
],
@ -526,10 +538,10 @@
"pycharm": {
"stem_cell": {
"cell_type": "raw",
"source": [],
"metadata": {
"collapsed": false
},
"source": []
}
}
}
},