mirror of
https://github.com/clearml/clearml
synced 2025-01-31 09:07:00 +00:00
28 lines
896 B
Python
28 lines
896 B
Python
|
from clearml import StorageManager, Dataset
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
manager = StorageManager()
|
||
|
|
|
||
|
|
print("STEP1 : Downloading CSV dataset")
|
||
|
|
csv_file_path = manager.get_local_copy(
|
||
|
|
remote_url="https://allegro-datasets.s3.amazonaws.com/datasets/Iris_Species.csv")
|
||
|
|
|
||
|
|
print("STEP2 : Creating a dataset")
|
||
|
|
# By default, clearml data uploads to the clearml fileserver. Adding output_uri argument to the create() method
|
||
|
|
# allows you to specify custom storage like s3 \ gcs \ azure \ local storage
|
||
|
|
simple_dataset = Dataset.create(dataset_project="dataset_examples", dataset_name="CSV_Dataset")
|
||
|
|
|
||
|
|
print("STEP3 : Adding CSV file to the Dataset")
|
||
|
|
simple_dataset.add_files(path=csv_file_path)
|
||
|
|
|
||
|
|
print("STEP4 : Upload and finalize")
|
||
|
|
simple_dataset.upload()
|
||
|
|
simple_dataset.finalize()
|
||
|
|
|
||
|
|
print("We are done, have a great day :)")
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
main()
|