diff --git a/docs/clearml_data/clearml_data_sdk.md b/docs/clearml_data/clearml_data_sdk.md index 8db5d204..60e39120 100644 --- a/docs/clearml_data/clearml_data_sdk.md +++ b/docs/clearml_data/clearml_data_sdk.md @@ -246,7 +246,7 @@ dataset.get_logger().report_table( title="Raw Dataset Metadata", series="Raw Dataset Metadata", csv="path/to/csv" ) -# Attach a historgram to the table +# Attach a histogram to the table dataset.get_logger().report_histogram( title="Class distribution", series="Class distribution", @@ -261,7 +261,7 @@ dataset.get_logger().report_histogram( To upload the dataset files to network storage, use [`Dataset.upload()`](../references/sdk/dataset.md#upload). -Use the `output_url` parameter to specify storage target, such as S3 / GS / Azure. For example: +Use the `output_url` parameter to specify a storage target, such as S3 / GS / Azure. For example: * A shared folder: `/mnt/share/folder` * S3: `s3://bucket/folder` * Non-AWS S3-like services (such as MinIO): `s3://host_addr:port/bucket`. **Note that port specification is required**. diff --git a/docs/clearml_data/data_management_examples/data_man_simple.md b/docs/clearml_data/data_management_examples/data_man_simple.md index 03f4fa1c..066fa909 100644 --- a/docs/clearml_data/data_management_examples/data_man_simple.md +++ b/docs/clearml_data/data_management_examples/data_man_simple.md @@ -30,7 +30,7 @@ the needed files. ``` 1. Add a folder. File addition is recursive, so it's enough to point at the folder -to captures all files and subfolders: +to capture all files and subfolders: ```bash clearml-data add --files data_samples diff --git a/docs/community.md b/docs/community.md index 31dbd5dd..29472c9e 100644 --- a/docs/community.md +++ b/docs/community.md @@ -8,7 +8,7 @@ For feature requests or bug reports, see **ClearML** [GitHub issues](https://git If you have any questions, post on the **ClearML** [Slack channel](https://joinslack.clear.ml). -Or, tag your questions on [stackoverflow](https://stackoverflow.com/questions/tagged/clearml) with the **clearml** tag. +Or, tag your questions on [Stack Overflow](https://stackoverflow.com/questions/tagged/clearml) with the **clearml** tag. You can always find us at [support@clear.ml](mailto:support@clear.ml?subject=ClearML). diff --git a/docs/deploying_clearml/clearml_server_linux_mac.md b/docs/deploying_clearml/clearml_server_linux_mac.md index 75cedeb7..c817f557 100644 --- a/docs/deploying_clearml/clearml_server_linux_mac.md +++ b/docs/deploying_clearml/clearml_server_linux_mac.md @@ -127,7 +127,13 @@ Deploying the server requires a minimum of 8 GB of memory, 16 GB is recommended. ``` sudo chown -R 1000:1000 /opt/clearml ``` - + + :::note + This assumes the container processes run as UID 1000 and GID 1000. The ownership of `/opt/clearml` must match the + UID and GID used inside the container. If the container runs as a different user or group, update the ownership + accordingly to ensure they can access the mounted directories. + ::: + * macOS: ``` diff --git a/docs/deploying_clearml/clearml_server_security.md b/docs/deploying_clearml/clearml_server_security.md index 73e00a13..bb7f87ed 100644 --- a/docs/deploying_clearml/clearml_server_security.md +++ b/docs/deploying_clearml/clearml_server_security.md @@ -33,7 +33,7 @@ has been configured. It is recommended to use an [object storage solution](../in ## Server Credentials and Secrets -By default, ClearML Server comes with default values that are designed to allow to set it up quickly and to start working +By default, ClearML Server comes with default values designed to allow you to quickly set it up and to start working with the ClearML SDK. However, this also means that the **server must be secured** by either preventing any external access, or by changing diff --git a/docs/deploying_clearml/enterprise_deploy/k8s.md b/docs/deploying_clearml/enterprise_deploy/k8s.md index 67127b24..d4052918 100644 --- a/docs/deploying_clearml/enterprise_deploy/k8s.md +++ b/docs/deploying_clearml/enterprise_deploy/k8s.md @@ -58,83 +58,46 @@ that will have records pointing to the cluster’s ingress controller (see ingre ::: -``` +```yaml imageCredentials: - password: "" - + password: "" clearml: - cookieDomain: "" - # Set values for improved security - apiserverKey: "" - apiserverSecret: "" - fileserverKey: "" - fileserverSecret: "" - secureAuthTokenSecret: "" - testUserKey: "" - testUserSecret: "" - + cookieDomain: "" + # Set values for improved security + apiserverKey: "" + apiserverSecret: "" + fileserverKey: "" + fileserverSecret: "" + secureAuthTokenSecret: "" + testUserKey: "" + testUserSecret: "" apiserver: - ingress: - enabled: true - hostName: "api." - service: - type: ClusterIP - extraEnvs: - - name: CLEARML__services__organization__features__user_management_advanced - value: "true" - - name: CLEARML__services__auth__ui_features_per_role__user__show_datasets - value: "false" - - name: CLEARML__services__auth__ui_features_per_role__user__show_orchestration - value: "false" - - name: CLEARML__services__workers__resource_usages__supervisor_company - value: "" - - name: CLEARML__secure__credentials__supervisor__role - value: "system" - - name: CLEARML__secure__credentials__supervisor__allow_login - value: "true" - - name: CLEARML__secure__credentials__supervisor__user_key - value: "" - - name: CLEARML__secure__credentials__supervisor__user_secret - value: "" - - name: CLEARML__secure__credentials__supervisor__sec_groups - value: "[\"users\", \"admins\", \"queue_admins\"]" - - name: CLEARML__secure__credentials__supervisor__email - value: "\"\"" - - name: CLEARML__apiserver__company__unique_names - value: "true" - + ingress: + enabled: true + hostName: "api." + service: + type: ClusterIP fileserver: - ingress: - enabled: true - hostName: "file." - service: - type: ClusterIP - + ingress: + enabled: true + hostName: "file." + service: + type: ClusterIP webserver: - ingress: - enabled: true - hostName: "app." - service: - type: ClusterIP - + ingress: + enabled: true + hostName: "app." + service: + type: ClusterIP clearmlApplications: - enabled: true + enabled: true ``` - -The credentials specified in `` and `` can be used to log in as the -supervisor user from the ClearML Web UI accessible using the URL `app.`. - - -Note that the `` value must be explicitly quoted. To do so, put `\"` around the quoted value. -For example `"\"email@example.com\""`. - - #### Additional Configuration Options ##### Fixed Users (Simple Login) @@ -181,7 +144,7 @@ Substitute all ``s with the correct value for your configuration. ##### Auth0 Identity Provider -``` +```yaml apiserver: extraEnvs: - name: CLEARML__secure__login__sso__oauth_client__auth0__client_id @@ -202,7 +165,7 @@ apiserver: ##### Keycloak Identity Provider -``` +```yaml apiserver: extraEnvs: - name: CLEARML__secure__login__sso__oauth_client__keycloak__client_id @@ -217,8 +180,6 @@ apiserver: value: "/realms//protocol/openid-connect/token" - name: CLEARML__services__login__sso__oauth_client__keycloak__idp_logout value: "true" - - ``` @@ -247,24 +208,24 @@ To configure the agent you will need to choose a Redis password and use that whe The Helm Chart must be installed with `overrides.yaml`: -``` +```yaml imageCredentials: - password: "" + password: "" clearml: - agentk8sglueKey: "" - agentk8sglueSecret: "" + agentk8sglueKey: "" + agentk8sglueSecret: "" agentk8sglue: - apiServerUrlReference: "https://api." - fileServerUrlReference: "https://files." - webServerUrlReference: "https://app." - defaultContainerImage: "python:3.9" + apiServerUrlReference: "https://api." + fileServerUrlReference: "https://files." + webServerUrlReference: "https://app." + defaultContainerImage: "python:3.9" ``` #### Installing the Chart -``` +```bash helm install -n \ clearml-agent \ clearml-enterprise/clearml-enterprise-agent \ @@ -276,7 +237,7 @@ helm install -n \ To create a queue by API: -``` +```bash curl $APISERVER_URL/queues.create \ -H "Content-Type: application/json" \ -H "X-Clearml-Impersonate-As:" \ @@ -294,22 +255,22 @@ curl $APISERVER_URL/queues.create \ The Helm Chart must be installed with `overrides.yaml`: -``` +```yaml imageCredentials: - password: "" + password: "" clearml: - apiServerKey: "" - apiServerSecret: "" - apiServerUrlReference: "https://api." - authCookieName: "" + apiServerKey: "" + apiServerSecret: "" + apiServerUrlReference: "https://api." + authCookieName: "" ingress: - enabled: true - hostName: "task-router.dev" + enabled: true + hostName: "task-router.dev" tcpSession: - routerAddress: "" - portRange: - start: - end: + routerAddress: "" + portRange: + start: + end: ``` @@ -330,7 +291,7 @@ tcpSession: ### Installing the Chart -``` +```bash helm install -n \ clearml-ttr \ clearml-enterprise/clearml-enterprise-task-traffic-router \ @@ -429,20 +390,20 @@ This example configures a specific queue, but you can include this setting in th apply it to all tasks. -``` +```yaml agentk8sglue: - queues: - GPUshm: - templateOverrides: - env: - - name: VLLM_SKIP_P2P_CHECK - value: "1" - volumeMounts: - - name: dshm - mountPath: /dev/shm - volumes: - - name: dshm - emptyDir: - medium: Memory - sizeLimit: Gi + queues: + GPUshm: + templateOverrides: + env: + - name: VLLM_SKIP_P2P_CHECK + value: "1" + volumeMounts: + - name: dshm + mountPath: /dev/shm + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: Gi ``` diff --git a/docs/fundamentals/models.md b/docs/fundamentals/models.md index 16ae3546..360895d1 100644 --- a/docs/fundamentals/models.md +++ b/docs/fundamentals/models.md @@ -58,7 +58,7 @@ ClearML provides flexibility for explicitly connecting input models and experime ## WebApp Interface -In the ClearML's web UI, model information can be located through a project's Model Table or through the model's creating +In the ClearML web UI, model information can be located through a project's Model Table or through the model's creating task. Models associated with a task appear in the task's **ARTIFACTS** tab. To see further model details, including design, diff --git a/docs/guides/reporting/clearml_logging_example.md b/docs/guides/reporting/clearml_logging_example.md index 6bbd0c7d..9b71789b 100644 --- a/docs/guides/reporting/clearml_logging_example.md +++ b/docs/guides/reporting/clearml_logging_example.md @@ -17,7 +17,7 @@ In the ``clearml`` GitHub repository, this example includes a clickable icon to ## Scalars -To reports scalars, call [`Logger.report_scalar()`](../../references/sdk/logger.md#report_scalar). +To report scalars, call [`Logger.report_scalar()`](../../references/sdk/logger.md#report_scalar). The scalar plots appear in the **web UI** in **SCALARS**. ```python diff --git a/docs/guides/reporting/hyper_parameters.md b/docs/guides/reporting/hyper_parameters.md index 0fe63a8a..1ecdaca1 100644 --- a/docs/guides/reporting/hyper_parameters.md +++ b/docs/guides/reporting/hyper_parameters.md @@ -49,7 +49,7 @@ flags.DEFINE_string('echo5', '5', 'Text to echo.', module_name='test') ``` -TensorFlow Definitions appear in **HYPEPARAMETERS** **>** **TF_DEFINE**. +TensorFlow Definitions appear in **HYPERPARAMETERS** **>** **TF_DEFINE**. ![TF Defines](../../img/examples_reporting_hyper_param_03.png#light-mode-only) ![TF Defines](../../img/examples_reporting_hyper_param_03_dark.png#dark-mode-only) diff --git a/docs/guides/reporting/scalar_reporting.md b/docs/guides/reporting/scalar_reporting.md index bee2d8cf..e4e4234d 100644 --- a/docs/guides/reporting/scalar_reporting.md +++ b/docs/guides/reporting/scalar_reporting.md @@ -7,7 +7,8 @@ demonstrates explicit scalar reporting. ClearML reports scalars in the **ClearML When the script runs, it creates a task named `scalar reporting` in the `examples` project. -To reports scalars, call [`Logger.report_scalar()`](../../references/sdk/logger.md#report_scalar). +## Reporting Scalar Series +To report scalar series, call [`Logger.report_scalar()`](../../references/sdk/logger.md#report_scalar). To report more than one series on the same plot, use the same `title` argument. For different plots, use different `title` arguments. @@ -31,4 +32,20 @@ for i in range(100): ) ``` -![image](../../img/examples_reporting_14.png) \ No newline at end of file +![Scalars series](../../img/examples_reporting_14.png#light-mode-only) +![Scalars series](../../img/examples_reporting_14_dark.png#dark-mode-only) + +## Reporting Single Scalar Values + +To report single scalar values (individual metrics, not part of a series), use [`Logger.report_single_value()`](../../references/sdk/logger.md#report_single_value). + +```python +# Report individual scalar values +Logger.current_logger().report_single_value(name="metric A", value=486) +Logger.current_logger().report_single_value(name="metric B", value=305.95) +``` + +Single value scalars are shown in the UI in the task's **SCALARS** tab under the `Summary` table. + +![Single scalars](../../img/examples_reporting_14a.png#light-mode-only) +![Single scalars](../../img/examples_reporting_14a_dark.png#dark-mode-only) diff --git a/docs/guides/reporting/text_reporting.md b/docs/guides/reporting/text_reporting.md index 1b7eba72..16aa4d05 100644 --- a/docs/guides/reporting/text_reporting.md +++ b/docs/guides/reporting/text_reporting.md @@ -3,15 +3,46 @@ title: Text Reporting --- The [text_reporting.py](https://github.com/clearml/clearml/blob/master/examples/reporting/text_reporting.py) script -demonstrates reporting explicit text by calling [`Logger.report_text()`](../../references/sdk/logger.md#report_text). - -ClearML reports the text in the **ClearML Web UI**, in the task's **CONSOLE** tab. +demonstrates reporting text output and samples. When the script runs, it creates a task named `text reporting` in the `examples` project. + + +## Reporting Text to Console +To report text to the task console, call [`Logger.report_text()`](../../references/sdk/logger.md#report_text): + ```python # report text Logger.current_logger().report_text("hello, this is plain text") ``` -![image](../../img/examples_reporting_text.png) \ No newline at end of file +Text reported with `Logger.report_text()` appears in the task's **CONSOLE** tab in the ClearML Web UI. + +![Text to console](../../img/examples_reporting_text.png#light-mode-only) +![Text to console](../../img/examples_reporting_text_dark.png#dark-mode-only) + +## Reporting Text as Debug Samples +To report longer text as a debug sample (e.g., logs, large text outputs, or structured text files), +use [`Logger.report_media()`](../../references/sdk/logger.md#report_media) with a text stream and `.txt` file extension: + +```python +text_to_send = """ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. +Suspendisse ac justo ut dolor scelerisque posuere. +... +""" + +Logger.current_logger().report_media( + title="text title", + series="text series", + iteration=1, + stream=six.StringIO(text_to_send), + file_extension=".txt", +) +``` + +Text samples appear in the task's **DEBUG SAMPLES** tab in the ClearML Web UI. + +![Text debug sample](../../img/examples_reporting_text_debug.png#light-mode-only) +![Text debug sample](../../img/examples_reporting_text_debug_dark.png#dark-mode-only) diff --git a/docs/hyperdatasets/dataviews.md b/docs/hyperdatasets/dataviews.md index 1f26a841..30c26675 100644 --- a/docs/hyperdatasets/dataviews.md +++ b/docs/hyperdatasets/dataviews.md @@ -380,7 +380,7 @@ list_of_frames = myDataView.to_list() ##### Frame Query for Time Interval This example demonstrates a frame query filtering for frames containing the meta key `updated` with any value between -`08:000` and `09:00` on October 20th, 2024: +`08:00` and `09:00` on October 20th, 2024: ```python # Add a frame query for frames with the meta key's value between 08:00:00 and 09:00:00 on 2024-10-20 diff --git a/docs/hyperdatasets/webapp/webapp_datasets_versioning.md b/docs/hyperdatasets/webapp/webapp_datasets_versioning.md index 6e7e2068..a2e5d361 100644 --- a/docs/hyperdatasets/webapp/webapp_datasets_versioning.md +++ b/docs/hyperdatasets/webapp/webapp_datasets_versioning.md @@ -248,7 +248,7 @@ Filter by date/time metadata fields using Lucene queries. * Open a frame in the frame viewer to see its metadata. - ![Frame datetimee metadata in frame viewer](../../img/hyperdatasets/frame_filtering_14.png#light-mode-only) + ![Frame datetime metadata in frame viewer](../../img/hyperdatasets/frame_filtering_14.png#light-mode-only) ![Frame datetime metadata in frame viewer](../../img/hyperdatasets/frame_filtering_14_dark.png#dark-mode-only) diff --git a/docs/img/examples_reporting_14.png b/docs/img/examples_reporting_14.png index e7d49358..dbe9a00b 100644 Binary files a/docs/img/examples_reporting_14.png and b/docs/img/examples_reporting_14.png differ diff --git a/docs/img/examples_reporting_14_dark.png b/docs/img/examples_reporting_14_dark.png new file mode 100644 index 00000000..54a46e7e Binary files /dev/null and b/docs/img/examples_reporting_14_dark.png differ diff --git a/docs/img/examples_reporting_14a.png b/docs/img/examples_reporting_14a.png new file mode 100644 index 00000000..ad989ab5 Binary files /dev/null and b/docs/img/examples_reporting_14a.png differ diff --git a/docs/img/examples_reporting_14a_dark.png b/docs/img/examples_reporting_14a_dark.png new file mode 100644 index 00000000..dd2a9d67 Binary files /dev/null and b/docs/img/examples_reporting_14a_dark.png differ diff --git a/docs/img/examples_reporting_text.png b/docs/img/examples_reporting_text.png index 1328f2b4..7e4cd4ac 100644 Binary files a/docs/img/examples_reporting_text.png and b/docs/img/examples_reporting_text.png differ diff --git a/docs/img/examples_reporting_text_dark.png b/docs/img/examples_reporting_text_dark.png new file mode 100644 index 00000000..45bf37cc Binary files /dev/null and b/docs/img/examples_reporting_text_dark.png differ diff --git a/docs/img/examples_reporting_text_debug.png b/docs/img/examples_reporting_text_debug.png new file mode 100644 index 00000000..341f8238 Binary files /dev/null and b/docs/img/examples_reporting_text_debug.png differ diff --git a/docs/img/examples_reporting_text_debug_dark.png b/docs/img/examples_reporting_text_debug_dark.png new file mode 100644 index 00000000..be0c3742 Binary files /dev/null and b/docs/img/examples_reporting_text_debug_dark.png differ diff --git a/docs/model_registry.md b/docs/model_registry.md index 2bb78762..9386f7a9 100644 --- a/docs/model_registry.md +++ b/docs/model_registry.md @@ -101,7 +101,7 @@ input_model_2 = InputModel.import_model( ) ``` -After instantiating an InputModel instance, you can connect it to a task object, so the model can be traced to n +After instantiating an InputModel instance, you can connect it to a task object, so the model can be traced to a task. ```python @@ -138,7 +138,7 @@ output_model.update_weights(weights_filename='models/model.pth') ``` ## Analyzing Models -While experimenting, you build up your model catalog. In the ClearML's web UI, model information can be located through +While experimenting, you build up your model catalog. In the ClearML web UI, model information can be located through a project's Model Table or through the model's associated task. Models associated with a task appear in the task's **ARTIFACTS** tab. To see further model details, including design, diff --git a/docs/overview.md b/docs/overview.md index d6ddb186..3c0995d5 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -77,6 +77,6 @@ For detailed instructions on each step, refer to the respective sections in this ## Support For feature requests or bug reports, see ClearML on [GitHub](https://github.com/clearml/clearml/issues). -If you have any questions, join the discussion on the **ClearML** [Slack channel](https://joinslack.clear.ml), or tag your questions on [stackoverflow](https://stackoverflow.com/questions/tagged/clearml) with the **clearml** tag. +If you have any questions, join the discussion on the **ClearML** [Slack channel](https://joinslack.clear.ml), or tag your questions on [Stack Overflow](https://stackoverflow.com/questions/tagged/clearml) with the **clearml** tag. Lastly, you can always find us at [support@clearml.ai](mailto:support@clearml.ai?subject=ClearML). \ No newline at end of file diff --git a/docs/release_notes/clearml_server/enterprise/ver_3_23.md b/docs/release_notes/clearml_server/enterprise/ver_3_23.md index c3e42865..1d967d64 100644 --- a/docs/release_notes/clearml_server/enterprise/ver_3_23.md +++ b/docs/release_notes/clearml_server/enterprise/ver_3_23.md @@ -82,5 +82,5 @@ user access to these objects or their containing project. * Fix pipeline run version not set when re-executed via the UI * Fix UI DataView successfully created with identical IDs in label enumeration * Fix UI DataView created successfully when "Limit Frames" repetition option is selected, but the maximum number of frames is not set -* Fix moving a enqueued experiment to a new UI queue results in error +* Fix moving an enqueued experiment to a new UI queue results in error * Fix metric and hyperparameter group string not searchable in UI table diff --git a/docs/release_notes/clearml_server/enterprise/ver_3_24.md b/docs/release_notes/clearml_server/enterprise/ver_3_24.md index 2cbc0ae0..1845f116 100644 --- a/docs/release_notes/clearml_server/enterprise/ver_3_24.md +++ b/docs/release_notes/clearml_server/enterprise/ver_3_24.md @@ -2,6 +2,17 @@ title: Version 3.24 --- +### Enterprise Server 3.24.7 + +**Bug Fix** + +* Fix duplicate emails can be added to user list + +### Enterprise Server 3.24.6 + +**Bug Fix** +* Fix setting a service account as admin navigates to incorrect page + ### Enterprise Server 3.24.5 **New Feature** diff --git a/docs/release_notes/clearml_server/open_source/ver_0_10.md b/docs/release_notes/clearml_server/open_source/ver_0_10.md index 414ac03d..d3cbda80 100644 --- a/docs/release_notes/clearml_server/open_source/ver_0_10.md +++ b/docs/release_notes/clearml_server/open_source/ver_0_10.md @@ -15,7 +15,7 @@ title: Version 0.10 * Replace the Python web-server with NGINX. -* Improve sub-domain support ([GitHub trains-server Issue 9](https://github.com/clearml/clearml-server/issues/9)). +* Improve subdomain support ([GitHub trains-server Issue 9](https://github.com/clearml/clearml-server/issues/9)). * Extend configuration options diff --git a/docs/release_notes/sdk/open_source/ver_0_10.md b/docs/release_notes/sdk/open_source/ver_0_10.md index 2f6662ce..af644f74 100644 --- a/docs/release_notes/sdk/open_source/ver_0_10.md +++ b/docs/release_notes/sdk/open_source/ver_0_10.md @@ -68,7 +68,7 @@ title: Version 0.10 * Add scikit-learn support (load/store using joblib) ([GitHub Issue #20](https://github.com/clearml/clearml/issues/20)). * Add xgboost support ([GitHub Issue #10](https://github.com/clearml/clearml/issues/10)). * Add loguru support ([GitHub Issue #29](https://github.com/clearml/clearml/issues/29)). -* Add sub-domain support [trains.conf](https://github.com/clearml/clearml/blob/master/docs/trains.conf#L3) ([GitHub Issue #27](https://github.com/clearml/clearml/issues/27)). +* Add subdomain support [trains.conf](https://github.com/clearml/clearml/blob/master/docs/trains.conf#L3) ([GitHub Issue #27](https://github.com/clearml/clearml/issues/27)). * Fix sub-process support. * Fix multiple TensorBoard writers ([GitHub Issue #26](https://github.com/clearml/clearml/issues/26)). diff --git a/package-lock.json b/package-lock.json index 02ea0827..6b69add4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10127,7 +10127,9 @@ } }, "node_modules/http-proxy-middleware": { - "version": "2.0.7", + "version": "2.0.9", + "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-2.0.9.tgz", + "integrity": "sha512-c1IyJYLYppU574+YI7R4QyX2ystMtVXZwIdzazUIPIJsHuWNd+mho2j+bKoHftndicGj9yh+xjd+l0yj7VeT1Q==", "license": "MIT", "dependencies": { "@types/http-proxy": "^1.17.8",