From dc1c9b584434f064b423cb9fe8bdda156c28ae46 Mon Sep 17 00:00:00 2001 From: allegroai <> Date: Wed, 9 Nov 2022 11:29:46 +0200 Subject: [PATCH] Add clearml.browser_login to authenticate browser online sessions such as CoLab, Jupyter Notebooks etc. --- clearml/__init__.py | 2 + clearml/backend_api/__init__.py | 4 +- clearml/backend_api/session/__init__.py | 5 +- clearml/backend_api/session/session.py | 102 ++++++++++++++++++++++-- 4 files changed, 101 insertions(+), 12 deletions(-) diff --git a/clearml/__init__.py b/clearml/__init__.py index 58b9bb72..7d7e91f7 100644 --- a/clearml/__init__.py +++ b/clearml/__init__.py @@ -8,6 +8,7 @@ from .logger import Logger from .storage import StorageManager from .errors import UsageError from .datasets import Dataset +from .backend_api import browser_login TaskTypes = Task.TaskTypes @@ -27,6 +28,7 @@ if not PY2: "Dataset", "PipelineController", "PipelineDecorator", + "browser_login", ] else: __all__ = [ diff --git a/clearml/backend_api/__init__.py b/clearml/backend_api/__init__.py index 2e2b2d0f..93b3da4e 100644 --- a/clearml/backend_api/__init__.py +++ b/clearml/backend_api/__init__.py @@ -1,4 +1,4 @@ -from .session import Session, CallResult, TimeoutExpiredError, ResultNotReadyError +from .session import Session, CallResult, TimeoutExpiredError, ResultNotReadyError, browser_login from .config import load as load_config -__all__ = ["Session", "CallResult", "TimeoutExpiredError", "ResultNotReadyError", "load_config"] +__all__ = ["Session", "CallResult", "TimeoutExpiredError", "ResultNotReadyError", "load_config", "browser_login"] diff --git a/clearml/backend_api/session/__init__.py b/clearml/backend_api/session/__init__.py index dd10808c..007107cb 100644 --- a/clearml/backend_api/session/__init__.py +++ b/clearml/backend_api/session/__init__.py @@ -1,4 +1,4 @@ -from .session import Session +from .session import Session, browser_login from .datamodel import DataModel, NonStrictDataModel, schema_property, StringEnum from .request import Request, BatchRequest, CompoundRequest from .response import Response @@ -7,4 +7,5 @@ from .errors import TimeoutExpiredError, ResultNotReadyError from .callresult import CallResult __all__ = ["Session", "DataModel", "NonStrictDataModel", "schema_property", "StringEnum", "Request", "BatchRequest", - "CompoundRequest", "Response", "TokenManager", "TimeoutExpiredError", "ResultNotReadyError", "CallResult"] + "CompoundRequest", "Response", "TokenManager", "TimeoutExpiredError", "ResultNotReadyError", "CallResult", + "browser_login"] diff --git a/clearml/backend_api/session/session.py b/clearml/backend_api/session/session.py index 144cb3e7..bd0c55ff 100644 --- a/clearml/backend_api/session/session.py +++ b/clearml/backend_api/session/session.py @@ -10,7 +10,7 @@ import requests import six from requests.auth import HTTPBasicAuth from six.moves.urllib.parse import urlparse, urlunparse -from typing import List +from typing import List, Optional from .callresult import CallResult from .defs import ( @@ -164,13 +164,6 @@ class Session(TokenManager): default=(self.config.get("api.credentials.secret_key", None) or self.default_secret) ) - # check if we can login to existing browser session - if not self.access_key and not self.secret_key: - self.__auth_token = \ - self.__get_browser_token(self.get_app_server_host(config=self.config)) or None - if self.__auth_token: - token_expiration_threshold_sec = max(token_expiration_threshold_sec, 3600) - # init the token manager super(Session, self).__init__( token_expiration_threshold_sec=token_expiration_threshold_sec, **kwargs @@ -839,3 +832,96 @@ class Session(TokenManager): if not self.logger: self.logger = get_logger() return self.logger + + +def browser_login(clearml_server=None): + # type: (Optional[str]) -> () + """ + Alternative authentication / login method, (instead of configuring ~/clearml.conf or Environment variables) + ** Only applicable when running inside a browser session, + for example Google Colab, Kaggle notebook, Jupyter Notebooks etc. ** + + Notice: If called inside a python script, or when running with an agent, this function is ignored + + :param clearml_server: Optional, set the clearml server address, default: https://app.clear.ml + """ + + # check if we are running inside a Jupyter notebook of a sort + if not os.environ.get("JPY_PARENT_PID"): + return + + # if we are running remotely or in offline mode, skip login + from clearml.config import running_remotely + # noinspection PyProtectedMember + if running_remotely(): + return + + # if we have working local configuration, nothing to do + try: + Session() + return + except: # noqa + pass + + # conform clearml_server address + if clearml_server: + if not clearml_server.lower().startswith("http"): + clearml_server = "http://{}".format(clearml_server) + + parsed = urlparse(clearml_server) + if parsed.port: + parsed = parsed._replace(netloc=parsed.netloc.replace(':%d' % parsed.port, ':8008', 1)) + + if parsed.netloc.startswith('demoapp.'): + parsed = parsed._replace(netloc=parsed.netloc.replace('demoapp.', 'demoapi.', 1)) + elif parsed.netloc.startswith('app.'): + parsed = parsed._replace(netloc=parsed.netloc.replace('app.', 'api.', 1)) + elif parsed.netloc.startswith('api.'): + pass + else: + parsed = parsed._replace(netloc='api.' + parsed.netloc) + + clearml_server = urlunparse(parsed) + + # set for later usage + ENV_HOST.set(clearml_server) + + token = None + counter = 0 + clearml_app_server = Session.get_app_server_host() + while not token: + # try to get authentication toke + try: + # noinspection PyProtectedMember + token = Session._Session__get_browser_token(clearml_app_server) + except ValueError: + token = None + except Exception: # noqa + token = None + # if we could not get a token, instruct the user to login + if not token: + if not counter: + print( + f"ClearML automatic browser login failed, please login or create a new account\n" + f"To get started with ClearML: setup your own `clearml-server`, " + f"or create a free account at {clearml_app_server}\n" + ) + print(f"Please login to {clearml_app_server} , then press [Enter] to connect ", end="") + else: + print("Oh no we failed to connect \N{worried face}, " + "try to logout and login again - Press [Enter] to retry ", end="") + + input() + counter += 1 + + print("") + if counter: + print("\nHurrah! \N{FACE WITH PARTY HORN AND PARTY HAT} \N{CONFETTI BALL} \N{party popper}") + + if token: + # set Token + ENV_AUTH_TOKEN.set(token) + # verify token + Session() + # success + print("\N{robot face} ClearML connected successfully - let's build something! \N{rocket}")