mirror of
https://github.com/clearml/clearml
synced 2025-01-31 00:56:57 +00:00
Support path to certificate files to be specified as a downloadable URL
This commit is contained in:
allegroai
parent
33e4c24b79
commit
2811931d69
@ -55,6 +55,7 @@ class DownloadError(Exception):
|
|||||||
|
|
||||||
@six.add_metaclass(ABCMeta)
|
@six.add_metaclass(ABCMeta)
|
||||||
class _Driver(object):
|
class _Driver(object):
|
||||||
|
_certs_cache_context = "certs"
|
||||||
_file_server_hosts = None
|
_file_server_hosts = None
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@ -116,6 +117,28 @@ class _Driver(object):
|
|||||||
cls._file_server_hosts = hosts
|
cls._file_server_hosts = hosts
|
||||||
return cls._file_server_hosts
|
return cls._file_server_hosts
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def download_cert(cls, cert_url):
|
||||||
|
# import here to avoid circular imports
|
||||||
|
from .manager import StorageManager
|
||||||
|
|
||||||
|
cls.get_logger().info("Attempting to download remote certificate '{}'".format(cert_url))
|
||||||
|
potential_exception = None
|
||||||
|
downloaded_verify = None
|
||||||
|
try:
|
||||||
|
downloaded_verify = StorageManager.get_local_copy(cert_url, cache_context=cls._certs_cache_context)
|
||||||
|
except Exception as e:
|
||||||
|
potential_exception = e
|
||||||
|
if not downloaded_verify:
|
||||||
|
cls.get_logger().error(
|
||||||
|
"Failed downloading remote certificate '{}'{}".format(
|
||||||
|
cert_url, "Error is: {}".format(potential_exception) if potential_exception else ""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
cls.get_logger().info("Successfully downloaded remote certificate '{}'".format(cert_url))
|
||||||
|
return downloaded_verify
|
||||||
|
|
||||||
|
|
||||||
class _HttpDriver(_Driver):
|
class _HttpDriver(_Driver):
|
||||||
""" LibCloud http/https adapter (simple, enough for now) """
|
""" LibCloud http/https adapter (simple, enough for now) """
|
||||||
@ -447,6 +470,8 @@ class _Boto3Driver(_Driver):
|
|||||||
# True is a non-documented value for boto3, use None instead (which means verify)
|
# True is a non-documented value for boto3, use None instead (which means verify)
|
||||||
print("Using boto3 verify=None instead of true")
|
print("Using boto3 verify=None instead of true")
|
||||||
verify = None
|
verify = None
|
||||||
|
elif isinstance(verify, str) and not os.path.exists(verify) and verify.split("://")[0] in driver_schemes:
|
||||||
|
verify = _Boto3Driver.download_cert(verify)
|
||||||
|
|
||||||
# boto3 client creation isn't thread-safe (client itself is)
|
# boto3 client creation isn't thread-safe (client itself is)
|
||||||
with self._creation_lock:
|
with self._creation_lock:
|
||||||
|
|||||||
@ -1,185 +1,185 @@
|
|||||||
{
|
{
|
||||||
"cells": [
|
"cells": [
|
||||||
{
|
{
|
||||||
"cell_type": "markdown",
|
"cell_type": "markdown",
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"colab_type": "text",
|
"colab_type": "text",
|
||||||
"id": "view-in-github"
|
"id": "view-in-github"
|
||||||
},
|
},
|
||||||
"source": [
|
"source": [
|
||||||
"<a href=\"https://colab.research.google.com/github/allegroai/clearml/blob/master/examples/clearml_agent/clearml_colab_agent.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
"<a href=\"https://colab.research.google.com/github/allegroai/clearml/blob/master/examples/clearml_agent/clearml_colab_agent.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
|
||||||
]
|
]
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {
|
|
||||||
"id": "mC3A4rio6Y--"
|
|
||||||
},
|
|
||||||
"source": [
|
|
||||||
"# Google Colab Used to Launch ClearML Agent\n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {
|
|
||||||
"id": "EPYjmFLy1zuC"
|
|
||||||
},
|
|
||||||
"source": [
|
|
||||||
"## Step 1: Install all necessary packages"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"id": "DwFC3fL8JAP3"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"# If you don't have ClearML and ClearML Agent installed then uncomment these line\n",
|
|
||||||
"# !pip install clearml\n",
|
|
||||||
"# !pip install clearml-agent"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {
|
|
||||||
"id": "gb41kM8i1-7T"
|
|
||||||
},
|
|
||||||
"source": [
|
|
||||||
"## Step 2: Export this environment variable\n",
|
|
||||||
"\n",
|
|
||||||
"This environment variable makes Matplotlib work in headless mode, so it won't output graphs to the screen"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"id": "3-Bm4811VMLK"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"! export MPLBACKEND=TkAg"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {},
|
|
||||||
"source": [
|
|
||||||
"### Step 2b (OPTIONAL): Enter your github credentials (only for private repositories)\n",
|
|
||||||
"In order for the agent to pull your code, it needs access to your repositories. If these are private, you'll have to supply the agent with github credentials to log in. Github/Bitbucket will no longer let you log in using username/password combinations. Instead, you have to use a personal token, read more about it [here for Github](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and [here for Bitbucket](https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/)\n",
|
|
||||||
"\n",
|
|
||||||
"We can let the agent know which credentials to use by setting the following env variables"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"#@title Insert your Git Credentials\n",
|
|
||||||
"\n",
|
|
||||||
"import os\n",
|
|
||||||
"os.environ[\"CLEARML_AGENT_GIT_USER\"] = \"username-goes-here\" #@param {type:\"string\"}\n",
|
|
||||||
"os.environ[\"CLEARML_AGENT_GIT_PASS\"] = \"git-personal-access-token\" #@param {type:\"string\"}"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {
|
|
||||||
"id": "chuUzBaU2NyB"
|
|
||||||
},
|
|
||||||
"source": [
|
|
||||||
"## Step 3: Create new credentials: \n",
|
|
||||||
"1. Go to your [ClearML WebApp **Settings**](https://app.clear.ml/settings/workspace-configuration). \n",
|
|
||||||
"1. Under the **WORKSPACES** section, go to **App Credentials**, and click **+ Create new credentials**\n",
|
|
||||||
"1. Copy your credentials"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {
|
|
||||||
"id": "1Kloxwfj0Vnq"
|
|
||||||
},
|
|
||||||
"source": [
|
|
||||||
"## Step 4: Set your ClearML Credentials\n",
|
|
||||||
"\n",
|
|
||||||
"Insert the credentials you created in Step 3. \n",
|
|
||||||
"\n",
|
|
||||||
"If you aren't using the ClearML hosted server, make sure to modify the server variables. "
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"id": "YBPdmP4sJHnQ"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"#@title Insert your own Credentials\n",
|
|
||||||
"\n",
|
|
||||||
"from clearml import Task\n",
|
|
||||||
"\n",
|
|
||||||
"web_server = 'https://app.clear.ml'#@param {type:\"string\"} \n",
|
|
||||||
"api_server = 'https://api.clear.ml'#@param {type:\"string\"} \n",
|
|
||||||
"files_server = 'https://files.clear.ml'#@param {type:\"string\"}\n",
|
|
||||||
"access_key = ''#@param {type:\"string\"}\n",
|
|
||||||
"secret_key = ''#@param {type:\"string\"}\n",
|
|
||||||
"\n",
|
|
||||||
"Task.set_credentials(web_host=web_server,\n",
|
|
||||||
" api_host=api_server,\n",
|
|
||||||
" files_host=files_server,\n",
|
|
||||||
" key=access_key,\n",
|
|
||||||
" secret=secret_key\n",
|
|
||||||
" )\n",
|
|
||||||
" \n"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "markdown",
|
|
||||||
"metadata": {
|
|
||||||
"id": "Er3HUBty4m7i"
|
|
||||||
},
|
|
||||||
"source": [
|
|
||||||
"## Step 4: Run clearml-agent"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"cell_type": "code",
|
|
||||||
"execution_count": null,
|
|
||||||
"metadata": {
|
|
||||||
"id": "QcczeU7OJ9G-"
|
|
||||||
},
|
|
||||||
"outputs": [],
|
|
||||||
"source": [
|
|
||||||
"!clearml-agent daemon --queue default"
|
|
||||||
]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metadata": {
|
|
||||||
"colab": {
|
|
||||||
"authorship_tag": "ABX9TyMKeTOHrlQb98s9lDHxWFts",
|
|
||||||
"collapsed_sections": [],
|
|
||||||
"include_colab_link": true,
|
|
||||||
"name": "clearml_colab_agent.ipynb",
|
|
||||||
"provenance": [],
|
|
||||||
"toc_visible": true
|
|
||||||
},
|
|
||||||
"interpreter": {
|
|
||||||
"hash": "d75e902da2bbfe9f41879fcf2334f5819447e02a7f656a079df344fef4e78809"
|
|
||||||
},
|
|
||||||
"kernelspec": {
|
|
||||||
"display_name": "Python 3.7.12 64-bit ('.env')",
|
|
||||||
"name": "python3"
|
|
||||||
},
|
|
||||||
"language_info": {
|
|
||||||
"name": "python",
|
|
||||||
"version": ""
|
|
||||||
}
|
|
||||||
},
|
},
|
||||||
"nbformat": 4,
|
{
|
||||||
"nbformat_minor": 0
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"id": "mC3A4rio6Y--"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"# Google Colab Used to Launch ClearML Agent\n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"id": "EPYjmFLy1zuC"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Step 1: Install all necessary packages"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"id": "DwFC3fL8JAP3"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"# If you don't have ClearML and ClearML Agent installed then uncomment these line\n",
|
||||||
|
"# !pip install clearml\n",
|
||||||
|
"# !pip install clearml-agent"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"id": "gb41kM8i1-7T"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Step 2: Export this environment variable\n",
|
||||||
|
"\n",
|
||||||
|
"This environment variable makes Matplotlib work in headless mode, so it won't output graphs to the screen"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"id": "3-Bm4811VMLK"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"! export MPLBACKEND=TkAg"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {},
|
||||||
|
"source": [
|
||||||
|
"### Step 2b (OPTIONAL): Enter your github credentials (only for private repositories)\n",
|
||||||
|
"In order for the agent to pull your code, it needs access to your repositories. If these are private, you'll have to supply the agent with github credentials to log in. Github/Bitbucket will no longer let you log in using username/password combinations. Instead, you have to use a personal token, read more about it [here for Github](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) and [here for Bitbucket](https://support.atlassian.com/bitbucket-cloud/docs/app-passwords/)\n",
|
||||||
|
"\n",
|
||||||
|
"We can let the agent know which credentials to use by setting the following env variables"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"#@title Insert your Git Credentials\n",
|
||||||
|
"\n",
|
||||||
|
"import os\n",
|
||||||
|
"os.environ[\"CLEARML_AGENT_GIT_USER\"] = \"username-goes-here\" #@param {type:\"string\"}\n",
|
||||||
|
"os.environ[\"CLEARML_AGENT_GIT_PASS\"] = \"git-personal-access-token\" #@param {type:\"string\"}"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"id": "chuUzBaU2NyB"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Step 3: Create new credentials: \n",
|
||||||
|
"1. Go to your [ClearML WebApp **Settings**](https://app.clear.ml/settings/workspace-configuration). \n",
|
||||||
|
"1. Under the **WORKSPACES** section, go to **App Credentials**, and click **+ Create new credentials**\n",
|
||||||
|
"1. Copy your credentials"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"id": "1Kloxwfj0Vnq"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Step 4: Set your ClearML Credentials\n",
|
||||||
|
"\n",
|
||||||
|
"Insert the credentials you created in Step 3. \n",
|
||||||
|
"\n",
|
||||||
|
"If you aren't using the ClearML hosted server, make sure to modify the server variables. "
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"id": "YBPdmP4sJHnQ"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"#@title Insert your own Credentials\n",
|
||||||
|
"\n",
|
||||||
|
"from clearml import Task\n",
|
||||||
|
"\n",
|
||||||
|
"web_server = 'https://app.clear.ml'#@param {type:\"string\"} \n",
|
||||||
|
"api_server = 'https://api.clear.ml'#@param {type:\"string\"} \n",
|
||||||
|
"files_server = 'https://files.clear.ml'#@param {type:\"string\"}\n",
|
||||||
|
"access_key = ''#@param {type:\"string\"}\n",
|
||||||
|
"secret_key = ''#@param {type:\"string\"}\n",
|
||||||
|
"\n",
|
||||||
|
"Task.set_credentials(web_host=web_server,\n",
|
||||||
|
" api_host=api_server,\n",
|
||||||
|
" files_host=files_server,\n",
|
||||||
|
" key=access_key,\n",
|
||||||
|
" secret=secret_key\n",
|
||||||
|
" )\n",
|
||||||
|
" \n"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "markdown",
|
||||||
|
"metadata": {
|
||||||
|
"id": "Er3HUBty4m7i"
|
||||||
|
},
|
||||||
|
"source": [
|
||||||
|
"## Step 4: Run clearml-agent"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cell_type": "code",
|
||||||
|
"execution_count": null,
|
||||||
|
"metadata": {
|
||||||
|
"id": "QcczeU7OJ9G-"
|
||||||
|
},
|
||||||
|
"outputs": [],
|
||||||
|
"source": [
|
||||||
|
"!clearml-agent daemon --queue default"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metadata": {
|
||||||
|
"colab": {
|
||||||
|
"authorship_tag": "ABX9TyMKeTOHrlQb98s9lDHxWFts",
|
||||||
|
"collapsed_sections": [],
|
||||||
|
"include_colab_link": true,
|
||||||
|
"name": "clearml_colab_agent.ipynb",
|
||||||
|
"provenance": [],
|
||||||
|
"toc_visible": true
|
||||||
|
},
|
||||||
|
"interpreter": {
|
||||||
|
"hash": "d75e902da2bbfe9f41879fcf2334f5819447e02a7f656a079df344fef4e78809"
|
||||||
|
},
|
||||||
|
"kernelspec": {
|
||||||
|
"display_name": "Python 3.7.12 64-bit ('.env')",
|
||||||
|
"name": "python3"
|
||||||
|
},
|
||||||
|
"language_info": {
|
||||||
|
"name": "python",
|
||||||
|
"version": ""
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"nbformat": 4,
|
||||||
|
"nbformat_minor": 0
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user