mirror of
https://github.com/clearml/clearml-web
synced 2025-03-13 07:08:17 +00:00
Implement artifact upload via UI
This commit is contained in:
Brian Hood
parent
d4f9424589
commit
e4af78213d
54
package-lock.json
generated
54
package-lock.json
generated
@ -9,16 +9,16 @@
|
||||
"version": "1.15.0",
|
||||
"dependencies": {
|
||||
"@angular/animations": "^17.1.1",
|
||||
"@angular/cdk": "^17.1.1",
|
||||
"@angular/cdk": "^17.3.6",
|
||||
"@angular/common": "^17.1.1",
|
||||
"@angular/compiler": "^17.1.1",
|
||||
"@angular/core": "^17.1.1",
|
||||
"@angular/forms": "^17.1.1",
|
||||
"@angular/material": "^17.1.1",
|
||||
"@angular/core": "^17.3.5",
|
||||
"@angular/forms": "^17.3.6",
|
||||
"@angular/material": "^17.3.6",
|
||||
"@angular/platform-browser": "^17.1.1",
|
||||
"@angular/platform-browser-dynamic": "^17.1.1",
|
||||
"@angular/platform-server": "^17.1.1",
|
||||
"@angular/router": "^17.1.1",
|
||||
"@angular/router": "^17.3.5",
|
||||
"@angular/service-worker": "^17.1.1",
|
||||
"@angular/youtube-player": "^17.1.1",
|
||||
"@aws-sdk/client-s3": "^3.499.0",
|
||||
@ -999,9 +999,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@angular/cdk": {
|
||||
"version": "17.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@angular/cdk/-/cdk-17.1.1.tgz",
|
||||
"integrity": "sha512-Q5qC6VUyT7N/hj8eETdh0bkmBmsXm0JZikhXdBvcDUl8yPbhMPKQCkx4UJzBrZJg/+78XyI9FI/q8w/yQAJZJA==",
|
||||
"version": "17.3.6",
|
||||
"resolved": "https://sres.web.boeing.com/artifactory/api/npm/npm-releases/@angular/cdk/-/cdk-17.3.6.tgz",
|
||||
"integrity": "sha512-7eKrC61/6pmMAxllU/vYKadZRF7x7GxUYpA5G70fNaQsIUUiZvxx/SJN9AuZEoPGAtF6atKlJD8QVmFoDzv/Lw==",
|
||||
"dependencies": {
|
||||
"tslib": "^2.3.0"
|
||||
},
|
||||
@ -1126,9 +1126,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@angular/core": {
|
||||
"version": "17.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@angular/core/-/core-17.1.1.tgz",
|
||||
"integrity": "sha512-JtNYM9eHr8eUSrGPq/kn0+/F+TSZ7EBWxZhM1ZndOlGu1gA4fGhrDid4ZXIHIs07DbM4NZjMn+LhRyx02YDsSA==",
|
||||
"version": "17.3.5",
|
||||
"resolved": "https://sres.web.boeing.com/artifactory/api/npm/npm-releases/@angular/core/-/core-17.3.5.tgz",
|
||||
"integrity": "sha512-y6P27lcrKy3yMx/rtMuGsAnDyVEsS3BdyArTXcD0TOImVGHhVIaB0L95DUCam3ajTe2f2x39eozJZDh7QSpJaw==",
|
||||
"dependencies": {
|
||||
"tslib": "^2.3.0"
|
||||
},
|
||||
@ -1141,9 +1141,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@angular/forms": {
|
||||
"version": "17.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@angular/forms/-/forms-17.1.1.tgz",
|
||||
"integrity": "sha512-rqHVzaJDV8+VbnfC6mDgzX6ooa0X0hmnd+XfuOZaEJ7MtyOmqQ8qas2PAKXU7nMIImYXfYc4O4XWbSc1pRy1Hw==",
|
||||
"version": "17.3.6",
|
||||
"resolved": "https://sres.web.boeing.com/artifactory/api/npm/npm-releases/@angular/forms/-/forms-17.3.6.tgz",
|
||||
"integrity": "sha512-WXxWhwvgRfYLNP2dB4Qe83tavEh2LnS4H0uoiecWHXijW2R9z8304X1vEyS1EtQK7o/s8fCVDVDjeY+hxLnCLw==",
|
||||
"dependencies": {
|
||||
"tslib": "^2.3.0"
|
||||
},
|
||||
@ -1151,9 +1151,9 @@
|
||||
"node": "^18.13.0 || >=20.9.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@angular/common": "17.1.1",
|
||||
"@angular/core": "17.1.1",
|
||||
"@angular/platform-browser": "17.1.1",
|
||||
"@angular/common": "17.3.6",
|
||||
"@angular/core": "17.3.6",
|
||||
"@angular/platform-browser": "17.3.6",
|
||||
"rxjs": "^6.5.3 || ^7.4.0"
|
||||
}
|
||||
},
|
||||
@ -1167,9 +1167,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@angular/material": {
|
||||
"version": "17.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@angular/material/-/material-17.1.1.tgz",
|
||||
"integrity": "sha512-Ngh/4MY3MAPd4Fe2kb9W8j8Ix+hA9MVPvppYTlSsYzvlhV8YhOEaH2nuv9hJLrOiurlRWt7VlW13YkufK4VBgg==",
|
||||
"version": "17.3.6",
|
||||
"resolved": "https://sres.web.boeing.com/artifactory/api/npm/npm-releases/@angular/material/-/material-17.3.6.tgz",
|
||||
"integrity": "sha512-sttN0JNvd2QvCCFIsxb5noiy7tgQdWrwvmrkJ+3KguHh5X84jDliA/d8N7Xgy2IBLnS/q/Hl9DdRCOiItWG1bw==",
|
||||
"dependencies": {
|
||||
"@material/animation": "15.0.0-canary.7f224ddd4.0",
|
||||
"@material/auto-init": "15.0.0-canary.7f224ddd4.0",
|
||||
@ -1222,7 +1222,7 @@
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@angular/animations": "^17.0.0 || ^18.0.0",
|
||||
"@angular/cdk": "17.1.1",
|
||||
"@angular/cdk": "17.3.6",
|
||||
"@angular/common": "^17.0.0 || ^18.0.0",
|
||||
"@angular/core": "^17.0.0 || ^18.0.0",
|
||||
"@angular/forms": "^17.0.0 || ^18.0.0",
|
||||
@ -1288,9 +1288,9 @@
|
||||
}
|
||||
},
|
||||
"node_modules/@angular/router": {
|
||||
"version": "17.1.1",
|
||||
"resolved": "https://registry.npmjs.org/@angular/router/-/router-17.1.1.tgz",
|
||||
"integrity": "sha512-pPIRX0v8agij2dRSU25iwj9qFy0S25cztsy7bGfZ+M510jwRCqu1JsitqXtQ85XSv/bdFqiNiFU0UbwVFl+QiQ==",
|
||||
"version": "17.3.5",
|
||||
"resolved": "https://sres.web.boeing.com/artifactory/api/npm/npm-releases/@angular/router/-/router-17.3.5.tgz",
|
||||
"integrity": "sha512-KsIIs3t9IpxsdMSrJDZzO5WgIWkVE6Ep5WWiSyPIgEfA+ndGpJLmyv0d/r1yKKlYUJxz7Hde55o4thgT2n2x/A==",
|
||||
"dependencies": {
|
||||
"tslib": "^2.3.0"
|
||||
},
|
||||
@ -1298,9 +1298,9 @@
|
||||
"node": "^18.13.0 || >=20.9.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@angular/common": "17.1.1",
|
||||
"@angular/core": "17.1.1",
|
||||
"@angular/platform-browser": "17.1.1",
|
||||
"@angular/common": "17.3.5",
|
||||
"@angular/core": "17.3.5",
|
||||
"@angular/platform-browser": "17.3.5",
|
||||
"rxjs": "^6.5.3 || ^7.4.0"
|
||||
}
|
||||
},
|
||||
|
||||
12
package.json
12
package.json
@ -20,16 +20,16 @@
|
||||
"private": true,
|
||||
"dependencies": {
|
||||
"@angular/animations": "^17.1.1",
|
||||
"@angular/cdk": "^17.1.1",
|
||||
"@angular/cdk": "^17.3.6",
|
||||
"@angular/common": "^17.1.1",
|
||||
"@angular/compiler": "^17.1.1",
|
||||
"@angular/core": "^17.1.1",
|
||||
"@angular/forms": "^17.1.1",
|
||||
"@angular/material": "^17.1.1",
|
||||
"@angular/core": "^17.3.5",
|
||||
"@angular/forms": "^17.3.6",
|
||||
"@angular/material": "^17.3.6",
|
||||
"@angular/platform-browser": "^17.1.1",
|
||||
"@angular/platform-browser-dynamic": "^17.1.1",
|
||||
"@angular/platform-server": "^17.1.1",
|
||||
"@angular/router": "^17.1.1",
|
||||
"@angular/router": "^17.3.5",
|
||||
"@angular/service-worker": "^17.1.1",
|
||||
"@angular/youtube-player": "^17.1.1",
|
||||
"@aws-sdk/client-s3": "^3.499.0",
|
||||
@ -56,9 +56,9 @@
|
||||
"date-fns": "^3.3.1",
|
||||
"diff": "^5.1.0",
|
||||
"dom-to-image": "^2.6.0",
|
||||
"dompurify": "^3.0.8",
|
||||
"export-to-csv": "^1.2.2",
|
||||
"filesize": "^10.1.0",
|
||||
"dompurify": "^3.0.8",
|
||||
"has-ansi": "^5.0.1",
|
||||
"hocon-parser": "^1.0.1",
|
||||
"localforage": "^1.10.0",
|
||||
|
||||
@ -87,6 +87,7 @@ import {MatInputModule} from '@angular/material/input';
|
||||
import {MatSelectModule} from '@angular/material/select';
|
||||
import {HesitateDirective} from '@common/shared/ui-components/directives/hesitate.directive';
|
||||
import {ShowTooltipIfEllipsisDirective} from '@common/shared/ui-components/indicators/tooltip/show-tooltip-if-ellipsis.directive';
|
||||
import {MatButtonModule} from '@angular/material/button';
|
||||
|
||||
|
||||
@NgModule({
|
||||
@ -149,7 +150,8 @@ import {ShowTooltipIfEllipsisDirective} from '@common/shared/ui-components/indic
|
||||
MatInputModule,
|
||||
MatSelectModule,
|
||||
HesitateDirective,
|
||||
ShowTooltipIfEllipsisDirective
|
||||
ShowTooltipIfEllipsisDirective,
|
||||
MatButtonModule
|
||||
],
|
||||
declarations: [
|
||||
ExperimentsComponent,
|
||||
|
||||
@ -75,6 +75,7 @@ export const getSignedUrl = createAction(
|
||||
disableCache?: number;
|
||||
dprsUrl?: string | boolean;
|
||||
error?: boolean;
|
||||
method?: string;
|
||||
}}>()
|
||||
);
|
||||
export const setSignedUrl = createAction(
|
||||
|
||||
@ -22,6 +22,9 @@
|
||||
*ngIf="(!minimized) && shared"
|
||||
><i class="al-icon al-ico-link sm-md"></i></div>
|
||||
<sm-id-badge class="me-3" [id]="experiment?.id" (copied)="copyToClipboard()" *ngIf="!viewId"></sm-id-badge>
|
||||
<div class="line-item pointer" smTooltip="Upload Artifacts">
|
||||
<i class="al-icon al-ico-upload line-item pointer" (click)="openUploadDialog()"></i>
|
||||
</div>
|
||||
<span class="comment line-item"
|
||||
[delay]="1000" [action]="'leave'"
|
||||
(smHesitate)="menuHesitate.hesitateStatus && menu.closed.emit()"
|
||||
|
||||
@ -26,6 +26,8 @@ import {
|
||||
import {addMessage} from '@common/core/actions/layout.actions';
|
||||
import {MatMenuTrigger} from '@angular/material/menu';
|
||||
import { selectExperimentsTags } from '@common/experiments/reducers';
|
||||
import { UploadArtifactDialogComponent } from '@common/shared/ui-components/overlay/upload-artifact-dialog/upload-artifact-dialog.component';
|
||||
import { MatDialog, MatDialogRef } from '@angular/material/dialog';
|
||||
|
||||
@Component({
|
||||
selector: 'sm-experiment-info-header',
|
||||
@ -43,6 +45,7 @@ export class ExperimentInfoHeaderComponent implements OnDestroy {
|
||||
public shared: boolean;
|
||||
public isPipeline: boolean;
|
||||
public selectedDisableAvailable = {};
|
||||
private upUrl: string;
|
||||
|
||||
@Input() editable: boolean = true;
|
||||
@Input() infoData;
|
||||
@ -57,7 +60,7 @@ export class ExperimentInfoHeaderComponent implements OnDestroy {
|
||||
@ViewChild('tagsMenuTrigger') tagMenuTrigger: MatMenuTrigger;
|
||||
@ViewChild(TagsMenuComponent) tagMenu: TagsMenuComponent;
|
||||
|
||||
constructor(private store: Store, private router: Router, private activatedRoute: ActivatedRoute) {
|
||||
constructor(private store: Store, private router: Router, private activatedRoute: ActivatedRoute, private dialog: MatDialog) {
|
||||
this.tagsFilterByProject$ = this.store.select(selectTagsFilterByProject);
|
||||
this.projectTags$ = this.store.select(selectExperimentsTags);
|
||||
this.companyTags$ = this.store.select(selectCompanyTags);
|
||||
@ -152,4 +155,23 @@ export class ExperimentInfoHeaderComponent implements OnDestroy {
|
||||
copyToClipboard() {
|
||||
this.store.dispatch(addMessage('success', 'Copied to clipboard'));
|
||||
}
|
||||
|
||||
enableUploadButton() {
|
||||
return this.getStatusLabel() === 'Draft';
|
||||
}
|
||||
|
||||
public openUploadDialog() {
|
||||
if(this.enableUploadButton()) {
|
||||
const uploadDialogRef: MatDialogRef<any> = this.dialog.open(UploadArtifactDialogComponent, {
|
||||
data: {
|
||||
uploadUrl: this.upUrl,
|
||||
task: this.experiment.id,
|
||||
title: 'Upload Artifacts',
|
||||
iconClass: 'al-icon al-ico-upload al-color blue-300',
|
||||
}
|
||||
});
|
||||
} else {
|
||||
this.store.dispatch(addMessage('info', 'Upload only available in \'Draft\' state.'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@ import {Store} from '@ngrx/store';
|
||||
import {from, fromEvent, Observable, of, Subject} from 'rxjs';
|
||||
import {fromFetch} from 'rxjs/fetch';
|
||||
import {catchError, debounceTime, filter, map, skip} from 'rxjs/operators';
|
||||
import {DeleteObjectsCommand, GetObjectCommand, ObjectIdentifier, S3Client, S3ClientConfig} from '@aws-sdk/client-s3';
|
||||
import {DeleteObjectsCommand, GetObjectCommand, ObjectIdentifier, S3Client, S3ClientConfig, PutObjectCommand} from '@aws-sdk/client-s3';
|
||||
import {getSignedUrl} from '@aws-sdk/s3-request-presigner';
|
||||
import {convertToReverseProxy, isFileserverUrl} from '~/shared/utils/url';
|
||||
import {
|
||||
@ -63,9 +63,9 @@ export class BaseAdminService {
|
||||
this.store.dispatch(showLocalFilePopUp({url}));
|
||||
}
|
||||
|
||||
signUrlIfNeeded(url: string, config?: { skipLocalFile?: boolean; skipFileServer?: boolean; disableCache?: number }, previousSignedUrl?: { signed: string; expires: number }):
|
||||
Observable<SignResponse> {
|
||||
config = {...{skipLocalFile: true, skipFileServer: this.confService.getStaticEnvironment().production, disableCache: null}, ...config};
|
||||
signUrlIfNeeded(url: string, config?: { skipLocalFile?: boolean; skipFileServer?: boolean; disableCache?: number; method?: string; },
|
||||
previousSignedUrl?: { signed: string; expires: number }): Observable<SignResponse> {
|
||||
config = {...{skipLocalFile: true, skipFileServer: this.confService.getStaticEnvironment().production, disableCache: null, method: 'GET'}, ...config};
|
||||
|
||||
if (isFileserverUrl(url)) {
|
||||
if (this.environment.communityServer) {
|
||||
@ -97,7 +97,7 @@ export class BaseAdminService {
|
||||
return of({type: 'popup', bucket: bucketKeyEndpoint, provider: 'azure'});
|
||||
}
|
||||
const s3 = this.findOrInitBucketS3(bucketKeyEndpoint);
|
||||
if (s3) {
|
||||
if (s3 && (config.method === 'GET' || config.method === '')) {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const command = new GetObjectCommand({
|
||||
Key: bucketKeyEndpoint.Key,
|
||||
@ -109,6 +109,18 @@ export class BaseAdminService {
|
||||
unsignableHeaders: new Set(['x-amz-content-sha256', 'x-id']),
|
||||
}))
|
||||
.pipe(map(signed => ({type: 'sign', signed, expires: (new Date()).getTime() + FOUR_DAYS * 1000})));
|
||||
} else if(s3 && config.method === 'PUT') {
|
||||
// eslint-disable-next-line @typescript-eslint/naming-convention
|
||||
const command = new PutObjectCommand({
|
||||
Key: bucketKeyEndpoint.Key,
|
||||
Bucket: bucketKeyEndpoint.Bucket/*, ResponseContentType: 'image/jpeg'*/
|
||||
});
|
||||
return from(getSignedUrl(s3, command, {
|
||||
expiresIn: FOUR_DAYS,
|
||||
unhoistableHeaders: new Set(['x-amz-content-sha256', 'x-id']),
|
||||
unsignableHeaders: new Set(['x-amz-content-sha256', 'x-id'])
|
||||
}))
|
||||
.pipe(map(signed => ({type: 'sign', signed, expires: (new Date()).getTime() + FOUR_DAYS * 1000})));
|
||||
} else if (isGoogleCloudUrl(url) && !previousSignedUrl?.signed) {
|
||||
return of({type: 'sign', signed: this.signGoogleCloudUrl(url), expires: Number.MAX_VALUE});
|
||||
} else {
|
||||
|
||||
@ -0,0 +1,43 @@
|
||||
<sm-dialog-template [displayX]="displayX && !dialogRef.disableClose" [header]="title"
|
||||
(xClicked)="closeDialog()" [closeOnX]="false" [iconClass]="iconClass"
|
||||
[iconData]="iconData">
|
||||
<div *ngIf="body" class="body" [class.text-center]="this.centerText" [innerHTML]="body | safe: 'html'"></div>
|
||||
<form [formGroup]="uploadForm">
|
||||
<div>
|
||||
<mat-form-field class="mat-mdc-form-field w-100 mat-light mat-primary" appearance="outline">
|
||||
<mat-label>Artifact(s)</mat-label>
|
||||
<input matInput name="fileName" formControlName="fName" placeholder="Select File(s)" required readonly/>
|
||||
<input type="file" id="fileInput" (change)="selectFiles($event)" formControlName="fInput" name="fileInput" multiple/>
|
||||
<mat-error *ngIf="uploadForm.get('fName').hasError('required')">File selection is required.</mat-error>
|
||||
</mat-form-field>
|
||||
</div>
|
||||
<div *ngIf="this.filesSelected">
|
||||
<mat-label>Artifact name(s)</mat-label>
|
||||
<mat-form-field *ngFor="let name of this.upKeys; let i = index" class="mat-mdc-form-field w-100 mat-light mat-primary" appearance="outline">
|
||||
<input matInput value={{name}} (change)="updateName(i, $event)"/>
|
||||
</mat-form-field>
|
||||
</div>
|
||||
<mat-form-field class="mat-mdc-form-field w-100 mat-light mat-primary" appearance="outline">
|
||||
<mat-label>Artifact Type</mat-label>
|
||||
<mat-select formControlName="artType" name="artType" required>
|
||||
<mat-option *ngFor="let type of artifactType" [value]="type.value"><span class="option">{{type.viewValue}}</span></mat-option>
|
||||
</mat-select>
|
||||
<mat-error *ngIf="uploadForm.get('artType').hasError('required')">Type selection is required.</mat-error>
|
||||
</mat-form-field>
|
||||
<mat-form-field class="mat-mdc-form-field w-100 mat-light mat-primary" appearance="outline">
|
||||
<mat-label>Bucket URL</mat-label>
|
||||
<input matInput name="bucketUrl" formControlName="upDest" placeholder="s3://bucket/folder" />
|
||||
</mat-form-field>
|
||||
<mat-radio-group formControlName="mode">
|
||||
<mat-label>Artifact Mode</mat-label>
|
||||
<mat-radio-button value="output">Output</mat-radio-button>
|
||||
<mat-radio-button value="input">Input</mat-radio-button>
|
||||
</mat-radio-group>
|
||||
</form>
|
||||
<div><strong>*NOTE*</strong> These settings apply to <strong><u>ALL FILES</u></strong> selected for upload.</div>
|
||||
|
||||
<div class="buttons">
|
||||
<button (click)="closeDialog()" class="btn btn-outline-neon">CANCEL</button>
|
||||
<button [disabled]="uploadForm.invalid" (click)="uploadFiles()" class="btn btn-neon">UPLOAD</button>
|
||||
</div>
|
||||
</sm-dialog-template>
|
||||
@ -0,0 +1,46 @@
|
||||
.body {
|
||||
margin-bottom: 3rem !important;
|
||||
}
|
||||
|
||||
.buttons {
|
||||
display: inline-block;
|
||||
left: 50%;
|
||||
position: relative;
|
||||
transform: translateX(-50%);
|
||||
margin-top: 41px;
|
||||
|
||||
button {
|
||||
text-transform: uppercase;
|
||||
min-width: 126px;
|
||||
}
|
||||
}
|
||||
|
||||
#fileInput {
|
||||
position: absolute;
|
||||
cursor: pointer;
|
||||
z-index: 10;
|
||||
opacity: 0;
|
||||
height: 100%;
|
||||
width: 100%;
|
||||
left: 0px;
|
||||
top: 0px;
|
||||
}
|
||||
|
||||
.mat-toolbar-single-row {
|
||||
height: auto !important;
|
||||
background: transparent;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.mat-toolbar-single-row button {
|
||||
width: 100px;
|
||||
}
|
||||
|
||||
.mat-form-field {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.option {
|
||||
color: #000;
|
||||
}
|
||||
|
||||
@ -0,0 +1,265 @@
|
||||
import { Component, Inject, Input, OnDestroy, OnInit, TemplateRef } from '@angular/core';
|
||||
import { MAT_DIALOG_DATA, MatDialog, MatDialogRef } from '@angular/material/dialog';
|
||||
import { FormControl, FormGroup, Validators } from '@angular/forms';
|
||||
import { Store } from '@ngrx/store';
|
||||
import { addMessage } from '@common/core/actions/layout.actions';
|
||||
import { UploadArtifactDialogConfig, ArtifactType } from './upload-artifact-dialog.model';
|
||||
import { ApiTasksService } from '~/business-logic/api-services/tasks.service';
|
||||
import { Artifact } from '~/business-logic/model/tasks/artifact';
|
||||
import { IExecutionForm } from '~/features/experiments/shared/experiment-execution.model';
|
||||
import { Sha256 } from '@aws-crypto/sha256-browser';
|
||||
import { HttpClient } from '@angular/common/http';
|
||||
import { Observable, Subscription, filter, map, take } from 'rxjs';
|
||||
import { selectSignedUrl } from '@common/core/reducers/common-auth-reducer';
|
||||
import { getSignedUrl } from '@common/core/actions/common-auth.actions';
|
||||
import { selectExperimentExecutionInfoData } from '@common/experiments/reducers';
|
||||
import { ArtifactModeEnum } from '~/business-logic/model/tasks/artifactModeEnum';
|
||||
import { ConfirmDialogComponent } from '../confirm-dialog/confirm-dialog.component';
|
||||
|
||||
@Component({
|
||||
selector: 'sm-upload-artifact-dialog',
|
||||
templateUrl: './upload-artifact-dialog.component.html',
|
||||
styleUrls: ['./upload-artifact-dialog.component.scss']
|
||||
})
|
||||
export class UploadArtifactDialogComponent implements OnInit, OnDestroy {
|
||||
artifactType: ArtifactType[] = [
|
||||
{value: 'input-model', viewValue: 'Input Model'},
|
||||
{value: 'output-model', viewValue: 'Output Model'},
|
||||
{value: 'data-audit', viewValue: 'Data Audit'},
|
||||
{value: 'other', viewValue: 'Other'},
|
||||
];
|
||||
uploadForm = new FormGroup({
|
||||
fName: new FormControl({value: '', disabled: false}, Validators.required),
|
||||
fInput: new FormControl(''),
|
||||
artType: new FormControl({value: this.artifactType[3].value, disabled: true}),
|
||||
upDest: new FormControl({value: '', disabled: false}),
|
||||
mode: new FormControl({value: 'output', disabled: false})
|
||||
});
|
||||
|
||||
@Input() displayX: boolean = true;
|
||||
title: string;
|
||||
body?: string;
|
||||
task?: string;
|
||||
currentFileList?: Array<File>;
|
||||
currentFile: File;
|
||||
template?: TemplateRef<any>;
|
||||
iconClass = '';
|
||||
iconData = '';
|
||||
centerText: boolean;
|
||||
message: string;
|
||||
uploadUrl: string;
|
||||
hashList: Array<any>;
|
||||
upCount = 0;
|
||||
filesSelected: boolean = false;
|
||||
upKeys: Array<string>;
|
||||
|
||||
public executionInfo$: Observable<IExecutionForm>;
|
||||
public executionData: IExecutionForm;
|
||||
private executionDataSubscription: Subscription;
|
||||
|
||||
constructor(
|
||||
@Inject(MAT_DIALOG_DATA) public data: UploadArtifactDialogConfig,
|
||||
public dialogRef: MatDialogRef<UploadArtifactDialogComponent>,
|
||||
private store: Store,
|
||||
private dialog: MatDialog,
|
||||
private tasksApi: ApiTasksService,
|
||||
private http: HttpClient,
|
||||
) {
|
||||
this.executionInfo$ = this.store.select(selectExperimentExecutionInfoData);
|
||||
this.title = data.title || '';
|
||||
this.body = data.body || '';
|
||||
this.task = data.task || '';
|
||||
this.template = data.template;
|
||||
this.iconClass = data.iconClass || '';
|
||||
this.iconData = data.iconData || '';
|
||||
this.centerText = data.centerText ?? false;
|
||||
}
|
||||
|
||||
ngOnInit() {
|
||||
this.executionDataSubscription = this.executionInfo$.subscribe(formData => {
|
||||
this.executionData = formData;
|
||||
});
|
||||
this.uploadUrl = this.executionData.output.destination;
|
||||
this.uploadForm.controls['upDest'].setValue(this.uploadUrl);
|
||||
}
|
||||
|
||||
ngOnDestroy(): void {
|
||||
this.executionDataSubscription.unsubscribe();
|
||||
}
|
||||
|
||||
closeDialog() {
|
||||
this.dialogRef.close();
|
||||
}
|
||||
|
||||
selectFiles(event: any) {
|
||||
let fileName = '';
|
||||
this.currentFileList = [];
|
||||
this.upKeys = [];
|
||||
if (event.target.files && event.target.files.length > 0) {
|
||||
this.filesSelected = true;
|
||||
for(const index in event.target.files) {
|
||||
let idx = Number(index);
|
||||
this.currentFileList[idx] = event.target.files[idx];
|
||||
}
|
||||
const finalIndex = (this.currentFileList.length - 1);
|
||||
for(const [index, element] of this.currentFileList.entries()){
|
||||
this.upKeys.push(element.name.substring(0, element.name.lastIndexOf('.')));
|
||||
fileName += element.name;
|
||||
if(finalIndex >= 1 && index !== finalIndex){
|
||||
fileName += ', ';
|
||||
}
|
||||
}
|
||||
this.createHash(this.currentFileList);
|
||||
this.uploadForm.controls['fName'].setValue(fileName);
|
||||
} else {
|
||||
this.uploadForm.controls['fName'].setValue('');
|
||||
this.uploadForm.controls['fName'].setErrors({required: true});
|
||||
}
|
||||
}
|
||||
|
||||
updateName(index: number, event: any) {
|
||||
this.upKeys[index] = event.target.value;
|
||||
}
|
||||
|
||||
async uploadFiles() {
|
||||
let upUrl = this.uploadForm.controls['upDest'].value;
|
||||
let signedUrl = '';
|
||||
|
||||
if(this.currentFileList && upUrl) {
|
||||
if(!upUrl.endsWith('/')) {
|
||||
upUrl += '/uploaded-artifacts/';
|
||||
} else {
|
||||
upUrl += 'uploaded-artifacts/';
|
||||
}
|
||||
this.uploadUrl = upUrl;
|
||||
for(const file of this.currentFileList) {
|
||||
const urlToSign = this.uploadUrl + file.name;
|
||||
this.signUrl(urlToSign).subscribe({
|
||||
next: (res) => {
|
||||
signedUrl = res;
|
||||
this.upload(signedUrl, file);
|
||||
},
|
||||
error: (err) => {
|
||||
this.store.dispatch(addMessage('error', `${file.name} upload failed: ${err.error?.meta?.result_msg}`));
|
||||
throw new Error(`upload failed - sign url ${err.error?.meta?.result_msg}`);
|
||||
}
|
||||
});
|
||||
}
|
||||
} else if(this.currentFileList) {
|
||||
const confirmDialogRef: MatDialogRef<any, boolean> = this.dialog.open(ConfirmDialogComponent, {
|
||||
data: {
|
||||
title: 'Missing S3 Bucket',
|
||||
body: 'Continue without S3 bucket storage?',
|
||||
yes: 'Continue',
|
||||
no: 'Cancel',
|
||||
iconClass: 'al-icon al-ico-question-mark al-color blue-300',
|
||||
}
|
||||
});
|
||||
|
||||
confirmDialogRef.afterClosed().pipe(take(1)).subscribe((confirmed) => {
|
||||
if (confirmed) {
|
||||
this.updateArtifact(this.currentFileList);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
this.store.dispatch(addMessage('error', 'No files selected.'));
|
||||
}
|
||||
}
|
||||
|
||||
protected signUrl(url: string) {
|
||||
this.store.dispatch(getSignedUrl({url, config: {method: "PUT"}}));
|
||||
return this.store.select(selectSignedUrl(url))
|
||||
.pipe(
|
||||
map((res => res?.signed)),
|
||||
filter(signedUrl => !!signedUrl),
|
||||
take(1)
|
||||
);
|
||||
}
|
||||
|
||||
protected upload(url: string, file: File) {
|
||||
let error = false;
|
||||
if(url && file) {
|
||||
this.http.request("PUT", url, {
|
||||
body: file,
|
||||
reportProgress: true,
|
||||
observe: 'events',
|
||||
}).subscribe({
|
||||
next: () => {},
|
||||
error: (err) => {
|
||||
this.store.dispatch(addMessage('error', `${file.name} upload to bucket failed with status: ${err.status}\nError body: ${err.error}\nCheck bucket credentials.`));
|
||||
throw new Error(`upload failed - http request ${err.error}`);
|
||||
},
|
||||
complete: () => {
|
||||
this.upCount++;
|
||||
if(this.upCount === this.currentFileList.length) {
|
||||
this.updateArtifact(this.currentFileList);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
updateArtifact(fileList: Array<File>) {
|
||||
let updateList: Array<Artifact> = [];
|
||||
let fileHash = '';
|
||||
let artName = '';
|
||||
for(const [index, item] of fileList.entries()) {
|
||||
const ts = (new Date().getTime())/1000.0; // Convert to seconds so the Python SDK can read it
|
||||
artName = this.upKeys[index];
|
||||
for(const hash of this.hashList) {
|
||||
if(item.name === hash.fileName) {
|
||||
fileHash = hash.fileHashHex;
|
||||
}
|
||||
}
|
||||
updateList.push({
|
||||
key: artName,
|
||||
type: this.uploadForm.controls['artType'].value,
|
||||
content_size: item.size,
|
||||
timestamp: ts,
|
||||
hash: 'SHA256: ' + fileHash,
|
||||
uri: this.uploadUrl + item.name,
|
||||
mode: this.uploadForm.controls['mode'].value as ArtifactModeEnum
|
||||
});
|
||||
}
|
||||
|
||||
this.tasksApi.tasksAddOrUpdateArtifacts({
|
||||
task: this.task,
|
||||
artifacts: updateList,
|
||||
}, null, 'body', true).subscribe({
|
||||
next: () => {},
|
||||
error: err => this.store.dispatch(addMessage('error', `Error ${err.error?.meta?.result_msg}`)),
|
||||
complete: () => {
|
||||
this.store.dispatch(addMessage('success', 'Artifact(s) uploaded successfully.'));
|
||||
this.upCount = 0;
|
||||
this.closeDialog();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async createHash(fileList: File[]) {
|
||||
this.hashList = [];
|
||||
let fileBuff: ArrayBuffer;
|
||||
let fileHashSha256: any;
|
||||
let fileHashHex: any;
|
||||
const hash = new Sha256();
|
||||
for(const element of fileList){
|
||||
element.arrayBuffer().then(async buff => {
|
||||
let fileName = element.name;
|
||||
fileBuff = buff;
|
||||
hash.update(fileBuff);
|
||||
fileHashSha256 = await hash.digest();
|
||||
fileHashHex = this.createHexStr(fileHashSha256);
|
||||
this.hashList.push({fileName, fileHashHex});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
createHexStr(arr: Uint8Array) {
|
||||
let result = '';
|
||||
for (const element of arr) {
|
||||
result += element.toString(16);
|
||||
}
|
||||
return result.toString();
|
||||
}
|
||||
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
import {TemplateRef} from '@angular/core';
|
||||
|
||||
export interface UploadArtifactDialogConfig {
|
||||
width?: number;
|
||||
centerText?: boolean;
|
||||
title?: string;
|
||||
body?: string;
|
||||
task?: string;
|
||||
fileName?: string;
|
||||
template?: TemplateRef<any>;
|
||||
iconClass?: string; // the icon class (see icons.scss).
|
||||
iconData?: string; // the icon class (see icons.scss).
|
||||
}
|
||||
|
||||
export interface ArtifactType {
|
||||
value: string;
|
||||
viewValue: string;
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user