diff --git a/docs/deploying_clearml/enterprise_deploy/appgw.md b/docs/deploying_clearml/enterprise_deploy/appgw.md index 2679df85..647c575a 100644 --- a/docs/deploying_clearml/enterprise_deploy/appgw.md +++ b/docs/deploying_clearml/enterprise_deploy/appgw.md @@ -30,12 +30,12 @@ their instances: * [Embedding Model Deployment](../../webapp/applications/apps_embed_model_deployment.md) * [Llama.cpp Model Deployment](../../webapp/applications/apps_llama_deployment.md) -The AI Application Gateway is provided through an additional component to the ClearML Server deployment: The ClearML Task Traffic Router. -If your ClearML Deployment does not have the Task Traffic Router properly installed, these application instances may not be accessible. +The AI Application Gateway requires an additional component to the ClearML Server deployment: the **ClearML App Gateway Router**. +If your ClearML Deployment does not have the App Gateway Router properly installed, these application instances may not be accessible. #### Installation -The Task Traffic Router supports two deployment options: +The App Gateway Router supports two deployment options: * [Docker Compose](appgw_install_compose.md) * [Kubernetes](appgw_install_k8s.md) diff --git a/docs/deploying_clearml/enterprise_deploy/appgw_install_compose.md b/docs/deploying_clearml/enterprise_deploy/appgw_install_compose.md index c77f4113..de16511f 100644 --- a/docs/deploying_clearml/enterprise_deploy/appgw_install_compose.md +++ b/docs/deploying_clearml/enterprise_deploy/appgw_install_compose.md @@ -40,77 +40,72 @@ This is an example of the `docker-compose` file you will need: ``` version: '3.5' services: -task_traffic_webserver: - image: allegroai/task-traffic-router-webserver:${TASK-TRAFFIC-ROUTER-WEBSERVER-TAG} - ports: - - "80:8080" - restart: unless-stopped - container_name: task_traffic_webserver - volumes: - - ./task_traffic_router/config/nginx:/etc/nginx/conf.d:ro - - ./task_traffic_router/config/lua:/usr/local/openresty/nginx/lua:ro -task_traffic_router: - image: allegroai/task-traffic-router:${TASK-TRAFFIC-ROUTER-TAG} - restart: unless-stopped - container_name: task_traffic_router - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./task_traffic_router/config/nginx:/etc/nginx/conf.d:rw - - ./task_traffic_router/config/lua:/usr/local/openresty/nginx/lua:rw - environment: - - LOGGER_LEVEL=INFO - - CLEARML_API_HOST=${CLEARML_API_HOST:?err} - - CLEARML_API_ACCESS_KEY=${CLEARML_API_ACCESS_KEY:?err} - - CLEARML_API_SECRET_KEY=${CLEARML_API_SECRET_KEY:?err} - - ROUTER_URL=${ROUTER_URL:?err} - - ROUTER_NAME=${ROUTER_NAME:?err} - - AUTH_ENABLED=${AUTH_ENABLED:?err} - - SSL_VERIFY=${SSL_VERIFY:?err} - - AUTH_COOKIE_NAME=${AUTH_COOKIE_NAME:?err} - - AUTH_BASE64_JWKS_KEY=${AUTH_BASE64_JWKS_KEY:?err} - - LISTEN_QUEUE_NAME=${LISTEN_QUEUE_NAME} - - EXTRA_BASH_COMMAND=${EXTRA_BASH_COMMAND} - - TCP_ROUTER_ADDRESS=${TCP_ROUTER_ADDRESS} - - TCP_PORT_START=${TCP_PORT_START} - - TCP_PORT_END=${TCP_PORT_END} - + task_traffic_webserver: + image: clearml/ai-gateway-proxy:${PROXY_TAG:?err} + network_mode: "host" + restart: unless-stopped + container_name: task_traffic_webserver + volumes: + - ./task_traffic_router/config/nginx:/etc/nginx/conf.d:ro + - ./task_traffic_router/config/lua:/usr/local/openresty/nginx/lua:ro + task_traffic_router: + image: clearml/ai-gateway-router:${ROUTER_TAG:?err} + restart: unless-stopped + container_name: task_traffic_router + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ./task_traffic_router/config/nginx:/etc/nginx/conf.d:rw + - ./task_traffic_router/config/lua:/usr/local/openresty/nginx/lua:rw + environment: + - ROUTER_NAME=${ROUTER_NAME:?err} + - ROUTER__WEBSERVER__SERVER_PORT=${ROUTER__WEBSERVER__SERVER_PORT:?err} + - ROUTER_URL=${ROUTER_URL:?err} + - CLEARML_API_HOST=${CLEARML_API_HOST:?err} + - CLEARML_API_ACCESS_KEY=${CLEARML_API_ACCESS_KEY:?err} + - CLEARML_API_SECRET_KEY=${CLEARML_API_SECRET_KEY:?err} + - AUTH_COOKIE_NAME=${AUTH_COOKIE_NAME:?err} + - AUTH_SECURE_ENABLED=${AUTH_SECURE_ENABLED} + - TCP_ROUTER_ADDRESS=${TCP_ROUTER_ADDRESS} + - TCP_PORT_START=${TCP_PORT_START} + - TCP_PORT_END=${TCP_PORT_END} ``` -Create a *runtime.env* file containing the following entries: +Create a `runtime.env` file containing the following entries: ``` -TASK-TRAFFIC-ROUTER-WEBSERVER-TAG= -TASK-TRAFFIC-ROUTER-TAG= -CLEARML_API_HOST=https://api. +PROXY_TAG= +ROUTER_TAG= +ROUTER_NAME=main-router +ROUTER__WEBSERVER__SERVER_PORT=8010 +ROUTER_URL= +CLEARML_API_HOST= CLEARML_API_ACCESS_KEY= CLEARML_API_SECRET_KEY= -ROUTER_URL= -ROUTER_NAME=main-router -AUTH_ENABLED=true -SSL_VERIFY=true AUTH_COOKIE_NAME= -AUTH_BASE64_JWKS_KEY= -LISTEN_QUEUE_NAME= -EXTRA_BASH_COMMAND= +AUTH_SECURE_ENABLED=true TCP_ROUTER_ADDRESS= TCP_PORT_START= TCP_PORT_END= ``` Edit it according to the following guidelines: - -* `CLEARML_API_HOST`: URL usually starting with `https://api.` -* `CLEARML_API_ACCESS_KEY`: ClearML server api key -* `CLEARML_API_SECRET_KEY`: ClearML server secret key -* `ROUTER_URL`: URL for this router that was previously configured in the load balancer starting with `https://` -* `ROUTER_NAME`: Unique name for this router -* `AUTH_ENABLED`: Enable or disable http calls authentication when the router is communicating with the ClearML server -* `SSL_VERIFY`: Enable or disable SSL certificate validation when the router is communicating with the ClearML server -* `AUTH_COOKIE_NAME`: Cookie name used by the ClearML server to store the ClearML authentication cookie. This can usually be found in the `value_prefix` key starting with `allegro_token` in `envoy.yaml` file in the ClearML server installation (`/opt/allegro/config/envoy/envoy.yaml`) (see below) -* `AUTH_SECURE_ENABLED`: Enable the Set-Cookie `secure` parameter -* `AUTH_BASE64_JWKS_KEY`: Value form `k` key in the `jwks.json` file in the ClearML server installation -* `LISTEN_QUEUE_NAME`: (*optional*) Name of queue to check for tasks (if none, every task is checked) -* `EXTRA_BASH_COMMAND`: Command to be launched before starting the router +* `PROXY_TAG`: AI Application Gateway proxy tag. The Docker image tag for the proxy component, which needs to be + specified during installation. This tag is provided by ClearML to ensure compatibility with the recommended version. +* `ROUTER_TAG`: App Gateway Router tag. The Docker image tag for the router component. It defines the specific version + to be installed and is provided by ClearML as part of the setup process. +* `ROUTER_NAME`: In the case of [multiple routers on the same tenant](#multiple-router-in-the-same-tenant), each router + needs to have a unique name. +* `ROUTER__WEBSERVER__SERVER_PORT`: Webserver port. The default port is 8080, but it can be adjusted to meet specific network requirements. +* `ROUTER_URL`: External address to access the router. This can be the IP address or DNS of the node where the router + is running, or the address of a load balancer if the router operates behind a proxy/load balancer. This URL is used + to access AI workload applications (e.g. remote IDE, model deployment, etc.), so it must be reachable and resolvable for them. +* `CLEARML_API_HOST`: ClearML API server URL starting with `https://api.` +* `CLEARML_API_ACCESS_KEY`: ClearML server API key. +* `CLEARML_API_SECRET_KEY`: ClearML server secret key. +* `AUTH_COOKIE_NAME`: Cookie used by the ClearML server to store the ClearML authentication cookie. This can usually be + found in the `envoy.yaml` file in the ClearML server installation (`/opt/allegro/config/envoy/envoy.yaml`), under the + `value_prefix` key starting with `allegro_token` +* `AUTH_SECURE_ENABLED`: Enable the Set-Cookie `secure` parameter. Set to `false` in case services are exposed with `http`. * `TCP_ROUTER_ADDRESS`: Router external address, can be an IP or the host machine or a load balancer hostname, depends on network configuration * `TCP_PORT_START`: Start port for the TCP Session feature * `TCP_PORT_END`: End port for the TCP Session feature @@ -121,12 +116,42 @@ Run the following command to start the router: sudo docker compose --env-file runtime.env up -d ``` -:::note How to find my jwkskey +### Advanced Configuration -The *JSON Web Key Set* (*JWKS*) is a set of keys containing the public keys used to verify any JSON Web Token (JWT). +#### Using Open HTTP -In a `docker-compose` server installation, this can be found in the `CLEARML__secure__auth__token_secret` env var in the apiserver server component. +To deploy the App Gateway Router on open HTTP (without a certificate), set the `AUTH_SECURE_ENABLED` entry +to `false` in the `runtime.env` file. -::: +#### Multiple Router in the Same Tenant +If you have workloads running in separate networks that cannot communicate with each other, you need to deploy multiple +routers, one for each isolated environment. Each router will only process tasks from designated queues, ensuring that +tasks are correctly routed to agents within the same network. +For example: +* If Agent A and Agent B are in separate networks, each must have its own router to receive tasks. +* Router A will handle tasks from Agent A’s queues. Router B will handle tasks from Agent B’s queues. + +To achieve this, each router must be configured with: +* A unique `ROUTER_NAME` +* A distinct set of queues defined in `LISTEN_QUEUE_NAME`. + +##### Example Configuration +Each router's `runtime.env` file should include: + +* Router A: + + ``` + ROUTER_NAME=router-a + LISTEN_QUEUE_NAME=queue1,queue2 + ``` + +* Router B: + + ``` + ROUTER_NAME=router-b + LISTEN_QUEUE_NAME=queue3,queue4 + ``` + +Make sure `LISTEN_QUEUE_NAME` is set in the [`docker-compose` environment variables](#docker-compose-file) for each router instance. diff --git a/docs/deploying_clearml/enterprise_deploy/appgw_install_k8s.md b/docs/deploying_clearml/enterprise_deploy/appgw_install_k8s.md index 906429c6..3a7e546f 100644 --- a/docs/deploying_clearml/enterprise_deploy/appgw_install_k8s.md +++ b/docs/deploying_clearml/enterprise_deploy/appgw_install_k8s.md @@ -3,17 +3,26 @@ title: Kubernetes Deployment --- :::important Enterprise Feature -The Application Gateway is available under the ClearML Enterprise plan. +The AI Application Gateway is available under the ClearML Enterprise plan. +::: + +This guide details the installation of the ClearML App Gateway Router. +The App Gateway Router enables access to your AI workload applications (e.g. remote IDEs like VSCode and Jupyter, model API interface, etc.). +It acts as a proxy, identifying ClearML Tasks running within its [K8s namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) +and making them available for network access. + +:::important +The App Gateway Router must be installed in the same K8s namespace as a dedicated ClearML Agent. +It can only configure access for ClearML Tasks within its own namespace. ::: -This guide details the installation of the ClearML AI Application Gateway, specifically the ClearML Task Router Component. ## Requirements * Kubernetes cluster: `>= 1.21.0-0 < 1.32.0-0` * Helm installed and configured -* Helm token to access `allegroai` helm-chart repo -* Credentials for `allegroai` docker repo +* Helm token to access `clearml` helm-chart repo +* Credentials for `clearml` docker repo * A valid ClearML Server installation ## Optional for HTTPS @@ -26,62 +35,55 @@ This guide details the installation of the ClearML AI Application Gateway, speci ### Login ``` -helm repo add allegroai-enterprise \ +helm repo add clearml-enterprise \ https://raw.githubusercontent.com/clearml/clearml-enterprise-helm-charts/gh-pages \ --username \ --password ``` +Replace `` with your valid GitHub token that has access to the ClearML Enterprise Helm charts repository. + ### Prepare Values -Before installing the TTR, create a `helm-override` files named `task-traffic-router.values-override.yaml`: +Before installing the App Gateway Router, create a Helm override file: ``` imageCredentials: - password: "" + password: "" clearml: - apiServerKey: "" - apiServerSecret: "" - apiServerUrlReference: "https://api." - jwksKey: "" - authCookieName: "" + apiServerKey: "" + apiServerSecret: "" + apiServerUrlReference: "" + authCookieName: "" + sslVerify: true ingress: - enabled: true - hostName: "task-router.dev" + enabled: true + hostName: "" tcpSession: - routerAddress: "" - portRange: - start: - end: + routerAddress: "" + service: + type: LoadBalancer + portRange: + start: + end: ``` -Edit it accordingly to these guidelines: +Configuration options: -* `clearml.apiServerUrlReference`: URL usually starting with `https://api.` -* `clearml.apiServerKey`: ClearML server api key -* `clearml.apiServerSecret`: ClearML server secret key -* `ingress.hostName`: URL of router we configured previously for load balancer starting with `https://` -* `clearml.sslVerify`: Enable or disable SSL certificate validation on apiserver calls check -* `clearml.authCookieName`: Value from `value_prefix` key starting with `allegro_token` in `envoy.yaml` file in ClearML server installation. -* `clearml.jwksKey`: Value form `k` key in `jwks.json` file in ClearML server installation (see below) -* `tcpSession.routerAddress`: Router external address can be an IP or the host machine or a load balancer hostname, depends on the network configuration -* `tcpSession.portRange.start`: Start port for the TCP Session feature -* `tcpSession.portRange.end`: End port for the TCP Session feature - -:::note How to find my jwkskey - -The *JSON Web Key Set* (*JWKS*) is a set of keys containing the public keys used to verify any JSON Web Token (JWT). - -``` -kubectl -n clearml get secret clearml-conf \ --o jsonpath='{.data.secure_auth_token_secret}' \ -| base64 -d && echo -``` - -::: +* `imageCredentials.password`: ClearML DockerHub Access Token. +* `clearml.apiServerKey`: ClearML server API key. +* `clearml.apiServerSecret`: ClearML server secret key. +* `clearml.apiServerUrlReference`: ClearML API server URL starting with `https://api.`. +* `clearml.authCookieName`: Cookie used by the ClearML server to store the ClearML authentication cookie. +* `clearml.sslVerify`: Enable or disable SSL certificate validation on `apiserver` calls check. +* `ingress.hostName`: Hostname of router used by the ingress controller to access it. +* `tcpSession.routerAddress`: The external router address (can be an IP, hostname, or load balancer address) depending on your network setup. Ensure this address is accessible for TCP connections. +* `tcpSession.service.type`: Service type used to expose TCP functionality, default is `NodePort`. +* `tcpSession.portRange.start`: Start port for the TCP Session feature. +* `tcpSession.portRange.end`: End port for the TCP Session feature. -The whole list of supported configuration is available with the command: +The full list of supported configuration is available with the command: ``` helm show readme allegroai-enterprise/clearml-enterprise-task-traffic-router @@ -94,9 +96,22 @@ To install the TTR component via Helm use the following command: ``` helm upgrade --install \ \ --n \ +-n \ allegroai-enterprise/clearml-enterprise-task-traffic-router \ ---version \ --f task-traffic-router.values-override.yaml +--version \ +-f override.yaml ``` +Replace the placeholders with the following values: + +* `` - Unique name for the App Gateway Router within the K8s namespace. This is a required parameter in + Helm, which identifies a specific installation of the chart. The release name also defines the router’s name and + appears in the UI within AI workload application URLs (e.g. Remote IDE URLs). This can be customized to support multiple installations within the same + namespace by assigning different release names. +* `` - [Kubernetes Namespace](https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/) + where workloads will be executed. This namespace must be shared between a dedicated ClearML Agent and an App + Gateway Router. The agent is responsible for monitoring its assigned task queues and spawning workloads within this + namespace. The router monitors the same namespace for AI workloads (e.g. remote IDE applications). The router has a + namespace-limited scope, meaning it can only detect and manage tasks within its + assigned namespace. +* `` - Version recommended by the ClearML Support Team. \ No newline at end of file diff --git a/docs/deploying_clearml/enterprise_deploy/multi_tenant_k8s.md b/docs/deploying_clearml/enterprise_deploy/multi_tenant_k8s.md index 7ab41ef4..60650bf2 100644 --- a/docs/deploying_clearml/enterprise_deploy/multi_tenant_k8s.md +++ b/docs/deploying_clearml/enterprise_deploy/multi_tenant_k8s.md @@ -513,31 +513,30 @@ Create a `NetworkPolicy` in the tenant namespace with the following configuratio - podSelector: {} ``` -### Install Task Traffic Router Chart +### Install the App Gateway Router Chart -Install the [Task Traffic Router](appgw.md) in your Kubernetes cluster, allowing it to manage and route tasks: +Install the App Gateway Router in your Kubernetes cluster, allowing it to manage and route tasks: 1. Prepare the `overrides.yaml` file with the following content: ``` imageCredentials: - password: "" + password: "" clearml: apiServerUrlReference: "" apiserverKey: "" apiserverSecret: "" - jwksKey: "" ingress: enabled: true hostName: "" ``` -2. Install Task Traffic Router in the specified tenant namespace: +2. Install App Gateway Router in the specified tenant namespace: ``` helm install -n \\ clearml-ttr \\ - allegroai-enterprise/clearml-task-traffic-router \\ + clearml-enterprise/clearml-task-traffic-router \\ --create-namespace \\ -f overrides.yaml ``` diff --git a/docs/webapp/applications/apps_embed_model_deployment.md b/docs/webapp/applications/apps_embed_model_deployment.md index ee8be58c..40d57048 100644 --- a/docs/webapp/applications/apps_embed_model_deployment.md +++ b/docs/webapp/applications/apps_embed_model_deployment.md @@ -13,7 +13,7 @@ running, it serves your embedding model through a secure, publicly accessible ne endpoint activity and shuts down if the model remains inactive for a specified maximum idle time. :::info AI Application Gateway -The Embedding Model Deployment app makes use of the ClearML Traffic Router which implements a secure, authenticated +The Embedding Model Deployment app makes use of the App Gateway Router which implements a secure, authenticated network endpoint for the model. If the ClearML AI application Gateway is not available, the model endpoint might not be accessible. diff --git a/docs/webapp/applications/apps_gradio.md b/docs/webapp/applications/apps_gradio.md index 31e48b92..e8bb9a54 100644 --- a/docs/webapp/applications/apps_gradio.md +++ b/docs/webapp/applications/apps_gradio.md @@ -16,7 +16,7 @@ The Gradio launcher monitors the Gradio app activity and shuts down if it is ina :::important AI Application Gateway -The Gradio Launcher relies on the ClearML Traffic Router which implements user authentication, and redirects requests +The Gradio Launcher relies on the ClearML App Gateway Router which implements user authentication, and redirects requests to the IP/port served by the Gradio app. If the ClearML AI application Gateway is not available, the Gradio app might not be accessible. diff --git a/docs/webapp/applications/apps_llama_deployment.md b/docs/webapp/applications/apps_llama_deployment.md index 442640b9..9cc5d37d 100644 --- a/docs/webapp/applications/apps_llama_deployment.md +++ b/docs/webapp/applications/apps_llama_deployment.md @@ -12,7 +12,7 @@ running, it serves your model through a secure, publicly accessible network endp and shuts down if the model remains inactive for a specified maximum idle time. :::important AI Application Gateway -The llama.cpp Model Deployment app makes use of the ClearML Traffic Router which implements a secure, authenticated +The llama.cpp Model Deployment app makes use of the App Gateway Router which implements a secure, authenticated network endpoint for the model. If the ClearML AI application Gateway is not available, the model endpoint might not be accessible. diff --git a/docs/webapp/applications/apps_model_deployment.md b/docs/webapp/applications/apps_model_deployment.md index eba05532..0936db39 100644 --- a/docs/webapp/applications/apps_model_deployment.md +++ b/docs/webapp/applications/apps_model_deployment.md @@ -13,7 +13,7 @@ it serves your model through a secure, publicly accessible network endpoint. The shuts down if the model remains inactive for a specified maximum idle time. :::info AI Application Gateway -The vLLM Model Deployment app makes use of the ClearML Traffic Router which implements a secure, authenticated +The vLLM Model Deployment app makes use of the App Gateway Router which implements a secure, authenticated network endpoint for the model. If the ClearML AI application Gateway is not available, the model endpoint might not be accessible. diff --git a/docs/webapp/applications/apps_streamlit.md b/docs/webapp/applications/apps_streamlit.md index 4741a3e5..199e28e0 100644 --- a/docs/webapp/applications/apps_streamlit.md +++ b/docs/webapp/applications/apps_streamlit.md @@ -17,7 +17,7 @@ time. :::important AI Application Gateway -The Streamlit Launcher relies on the ClearML Traffic Router which implements user authentication, and redirects requests +The Streamlit Launcher relies on the ClearML App Gateway Router which implements user authentication, and redirects requests to the IP/port served by the Streamlit app. If the ClearML AI application Gateway is not available, the Streamlit app might not be accessible.