From 82efc8b3a9758c265ecc88dc816d2c5028243db1 Mon Sep 17 00:00:00 2001 From: Tanishq Gandhi Date: Fri, 28 Aug 2026 14:26:13 -0700 Subject: [PATCH 1/6] feat(frontend): add model file upload and version creation --- frontend/src/app/app.module.ts | 4 +- .../settings/admin-settings.component.html | 34 +- .../settings/admin-settings.component.spec.ts | 183 +-- .../settings/admin-settings.component.ts | 148 ++- .../files-uploader.component.spec.ts | 187 ++- .../files-uploader.component.ts | 57 +- .../staged-objects-list.component.html} | 8 +- .../staged-objects-list.component.scss} | 0 .../staged-objects-list.component.spec.ts} | 75 +- .../staged-objects-list.component.ts} | 58 +- .../dataset-detail.component.html | 167 +-- .../dataset-detail.component.scss | 85 -- .../dataset-detail.component.spec.ts | 1033 +-------------- .../dataset-detail.component.ts | 441 +------ .../model-detail.component.html | 99 +- .../model-detail.component.scss | 61 + .../model-detail.component.spec.ts | 199 ++- .../model-detail.component.ts | 149 ++- .../version-uploader.component.html | 176 +++ .../version-uploader.component.scss | 114 ++ .../version-uploader.component.spec.ts | 1121 +++++++++++++++++ .../version-uploader.component.ts | 469 +++++++ .../user/dataset/dataset.service.spec.ts | 58 +- .../service/user/dataset/dataset.service.ts | 336 +---- .../file-resource/file-resource-endpoint.ts | 58 + .../multipart-upload.service.spec.ts | 313 +++++ .../file-resource/multipart-upload.service.ts | 371 ++++++ .../file-resource/staged-file.service.spec.ts | 91 ++ .../user/file-resource/staged-file.service.ts | 62 + .../testing/fake-xml-http-request.ts | 90 ++ .../service/user/model/model.service.spec.ts | 31 + .../service/user/model/model.service.ts | 38 +- 32 files changed, 3970 insertions(+), 2346 deletions(-) rename frontend/src/app/dashboard/component/user/{user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.html => staged-objects-list/staged-objects-list.component.html} (91%) rename frontend/src/app/dashboard/component/user/{user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.scss => staged-objects-list/staged-objects-list.component.scss} (100%) rename frontend/src/app/dashboard/component/user/{user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.spec.ts => staged-objects-list/staged-objects-list.component.spec.ts} (73%) rename frontend/src/app/dashboard/component/user/{user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.ts => staged-objects-list/staged-objects-list.component.ts} (71%) create mode 100644 frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.html create mode 100644 frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.scss create mode 100644 frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts create mode 100644 frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.ts create mode 100644 frontend/src/app/dashboard/service/user/file-resource/file-resource-endpoint.ts create mode 100644 frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.spec.ts create mode 100644 frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.ts create mode 100644 frontend/src/app/dashboard/service/user/file-resource/staged-file.service.spec.ts create mode 100644 frontend/src/app/dashboard/service/user/file-resource/staged-file.service.ts create mode 100644 frontend/src/app/dashboard/service/user/file-resource/testing/fake-xml-http-request.ts diff --git a/frontend/src/app/app.module.ts b/frontend/src/app/app.module.ts index 0dfbd0c5cea..ecbcda33ad7 100644 --- a/frontend/src/app/app.module.ts +++ b/frontend/src/app/app.module.ts @@ -182,7 +182,7 @@ import { } from "@abacritt/angularx-social-login"; import { catchError, firstValueFrom, lastValueFrom, of } from "rxjs"; import { HubSearchResultComponent } from "./hub/component/hub-search-result/hub-search-result.component"; -import { UserDatasetStagedObjectsListComponent } from "./dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component"; +import { StagedObjectsListComponent } from "./dashboard/component/user/staged-objects-list/staged-objects-list.component"; import { NzEmptyModule } from "ng-zorro-antd/empty"; import { NzDividerModule } from "ng-zorro-antd/divider"; import { NzProgressModule } from "ng-zorro-antd/progress"; @@ -322,7 +322,7 @@ registerLocaleData(en); DatasetDetailComponent, UserDatasetVersionFiletreeComponent, UserDatasetFileRendererComponent, - UserDatasetStagedObjectsListComponent, + StagedObjectsListComponent, NzModalCommentBoxComponent, LeftPanelComponent, ContextMenuComponent, diff --git a/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.html b/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.html index 2bfc1daaf00..fcd41e2a5d1 100644 --- a/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.html +++ b/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.html @@ -252,9 +252,11 @@

General Settings

- - - Dataset + + + {{ group.label }} General Settings
Configuration Guidelines
- • Concurrent parts × part size ≈ in-flight data per file
- • Keep usage within your client memory/bandwidth limits
- • Use larger part size for large files to avoid >10,000 parts
- • Out-of-range inputs will be auto-adjusted to the nearest valid value

+ • Concurrent parts × part size ≈ in-flight data per file
+ • Keep usage within your client memory/bandwidth limits
+ • Use larger part size for large files to avoid >10,000 parts
+ • Out-of-range inputs will be auto-adjusted to the nearest valid value

Learn more: General Settings
Concurrent Files: General Settings File Size:
General Settings
Concurrent Parts: General Settings
Number of parts uploaded in parallel. Range: 1 - {{ MAX_TOTAL_PARTS | number }}. Current configuration will use - approximately {{ partsAtMax | number }} parts for maximum file size. + approximately {{ partsAtMax(group.form) | number }} parts for maximum file size.
Part Size:
General Settings
Size of each chunk during multipart upload. Range: {{ MIN_PART_SIZE_MiB }} MiB - {{ MAX_PART_SIZE_MiB | number }} - MiB (5 GiB). Minimum required for current configuration: {{ requiredMinPartSizeMiB | number }} MiB. + MiB (5 GiB). Minimum required for current configuration: {{ requiredMinPartSizeMiB(group.form) | number }} MiB.
- In-flight data per file: ~{{ maxConcurrentChunks * chunkSizeMiB | number }} MiB + In-flight data per file: ~{{ group.form.maxConcurrentChunks * group.form.chunkSizeMiB | number }} MiB
diff --git a/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.spec.ts b/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.spec.ts index 637a516ee8a..cbba96d505e 100644 --- a/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.spec.ts +++ b/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.spec.ts @@ -18,7 +18,7 @@ */ import { ComponentFixture, TestBed } from "@angular/core/testing"; -import { AdminSettingsComponent } from "./admin-settings.component"; +import { AdminSettingsComponent, UploadSettingsForm } from "./admin-settings.component"; import { HttpClientTestingModule, HttpTestingController } from "@angular/common/http/testing"; import { NzCardModule } from "ng-zorro-antd/card"; import { NzMessageService } from "ng-zorro-antd/message"; @@ -32,6 +32,10 @@ describe("AdminSettingsComponent", () => { const SETTINGS_URL = "/api/config/settings"; + const formOf = (label: string): UploadSettingsForm => component.uploadGroups.find(g => g.label === label)!.form; + const datasetForm = () => formOf("Dataset"); + const modelForm = () => formOf("Model"); + beforeEach(async () => { await TestBed.configureTestingModule({ imports: [AdminSettingsComponent, HttpClientTestingModule, NzCardModule], @@ -49,9 +53,10 @@ describe("AdminSettingsComponent", () => { expect(component).toBeTruthy(); }); - it("renders MiB unit beside both size-based inputs", () => { + it("renders MiB unit beside every size-based input", () => { const units = fixture.nativeElement.querySelectorAll(".input-with-unit .unit"); - expect(units.length).toBe(2); + // File size and part size, once per upload card. + expect(units.length).toBe(2 * component.uploadGroups.length); units.forEach((el: HTMLElement) => { expect(el.textContent?.trim()).toBe("MiB"); }); @@ -70,6 +75,10 @@ describe("AdminSettingsComponent", () => { dataset_single_file_upload_max_size_mib: "128", dataset_max_number_of_concurrent_uploading_file_chunks: "7", dataset_multipart_upload_chunk_size_mib: "64", + model_max_number_of_concurrent_uploading_file: "2", + model_single_file_upload_max_size_mib: "4096", + model_max_number_of_concurrent_uploading_file_chunks: "4", + model_multipart_upload_chunk_size_mib: "32", csv_parser_max_columns: "4096", }); @@ -78,10 +87,15 @@ describe("AdminSettingsComponent", () => { expect(component.faviconData).toBe("fav.ico"); expect(component.sidebarTabs.hub_enabled).toBe(true); expect(component.sidebarTabs.home_enabled).toBe(false); - expect(component.maxConcurrentFiles).toBe(5); - expect(component.maxFileSizeMiB).toBe(128); - expect(component.maxConcurrentChunks).toBe(7); - expect(component.chunkSizeMiB).toBe(64); + expect(datasetForm().maxConcurrentFiles).toBe(5); + expect(datasetForm().maxFileSizeMiB).toBe(128); + expect(datasetForm().maxConcurrentChunks).toBe(7); + expect(datasetForm().chunkSizeMiB).toBe(64); + // Each family reads only its own keys, so the model card cannot inherit the dataset ceiling. + expect(modelForm().maxConcurrentFiles).toBe(2); + expect(modelForm().maxFileSizeMiB).toBe(4096); + expect(modelForm().maxConcurrentChunks).toBe(4); + expect(modelForm().chunkSizeMiB).toBe(32); expect(component.csvMaxColumns).toBe(4096); }); @@ -91,8 +105,10 @@ describe("AdminSettingsComponent", () => { }); expect(component.logoData).toBeNull(); - expect(component.maxFileSizeMiB).toBe(20); - expect(component.maxConcurrentFiles).toBe(3); + expect(datasetForm().maxFileSizeMiB).toBe(20); + expect(datasetForm().maxConcurrentFiles).toBe(3); + // The model default is the 2 GiB ceiling #8000 gave models, not the dataset's 20 MiB. + expect(modelForm().maxFileSizeMiB).toBe(2048); expect(component.csvMaxColumns).toBe(512); }); @@ -111,7 +127,7 @@ describe("AdminSettingsComponent", () => { csv_parser_max_columns: "0", }); - expect(component.maxConcurrentFiles).toBe(0); + expect(datasetForm().maxConcurrentFiles).toBe(0); expect(component.csvMaxColumns).toBe(0); }); @@ -287,11 +303,22 @@ describe("AdminSettingsComponent", () => { }); }); - describe("dataset settings", () => { - it("saveDatasetSettings PUTs the four upload settings and notifies success", () => { - completeLoad(); // defaults (20 / 3 / 10 / 50) are valid + describe.each([ + ["Dataset", "dataset", 20], + ["Model", "model", 2048], + ])("%s upload settings", (label, prefix, defaultMaxFileSizeMiB) => { + const group = () => component.uploadGroups.find(g => g.label === label)!; + const keys = [ + `${prefix}_max_number_of_concurrent_uploading_file`, + `${prefix}_single_file_upload_max_size_mib`, + `${prefix}_max_number_of_concurrent_uploading_file_chunks`, + `${prefix}_multipart_upload_chunk_size_mib`, + ]; + + it("PUTs the four upload settings of its own family and notifies success", () => { + completeLoad(); // the initializer defaults are valid - component.saveDatasetSettings(); + component.saveUploadSettings(group()); const expectPut = (key: string, value: string) => { const req = httpTestingController.expectOne(updateUrl(key)); @@ -299,109 +326,97 @@ describe("AdminSettingsComponent", () => { expect(req.request.body).toEqual({ value }); req.flush(null); }; - expectPut("dataset_max_number_of_concurrent_uploading_file", "3"); - expectPut("dataset_single_file_upload_max_size_mib", "20"); - expectPut("dataset_max_number_of_concurrent_uploading_file_chunks", "10"); - expectPut("dataset_multipart_upload_chunk_size_mib", "50"); + expectPut(keys[0], "3"); + expectPut(keys[1], String(defaultMaxFileSizeMiB)); + expectPut(keys[2], "10"); + expectPut(keys[3], "50"); - expect(msgSuccess).toHaveBeenCalledWith("Dataset upload settings saved successfully."); + // Saving one family must not touch the other's keys. + httpTestingController.expectNone((req: { method: string }) => req.method === "PUT"); + expect(msgSuccess).toHaveBeenCalledWith(`${label} upload settings saved successfully.`); }); - it("saveDatasetSettings refuses to save before the bulk load completes", () => { + it("refuses to save before the bulk load completes", () => { // The ngOnInit GET is left outstanding on purpose: settingsLoaded is still false. const pending = httpTestingController.expectOne(SETTINGS_URL); - component.saveDatasetSettings(); + component.saveUploadSettings(group()); httpTestingController.expectNone((req: { method: string }) => req.method === "PUT"); expect(msgError).toHaveBeenCalledWith("Settings have not loaded; refresh before saving."); pending.flush({}); }); - it("saveDatasetSettings rejects non-positive values without saving", () => { + it("rejects non-positive values without saving", () => { completeLoad(); - component.maxFileSizeMiB = 0; + group().form.maxFileSizeMiB = 0; - component.saveDatasetSettings(); + component.saveUploadSettings(group()); httpTestingController.expectNone((req: { method: string }) => req.method === "PUT"); expect(msgError).toHaveBeenCalledWith("Please enter valid integer values."); }); - it("saveDatasetSettings rejects a configuration that would exceed the 10,000-part limit", () => { + it("rejects a configuration that would exceed the 10,000-part limit", () => { completeLoad(); - component.maxFileSizeMiB = 100000; - component.chunkSizeMiB = 1; + group().form.maxFileSizeMiB = 100000; + group().form.chunkSizeMiB = 1; - component.saveDatasetSettings(); + component.saveUploadSettings(group()); httpTestingController.expectNone((req: { method: string }) => req.method === "PUT"); expect(msgError).toHaveBeenCalled(); }); - it("saveDatasetSettings notifies an error when a request fails", () => { + it("notifies an error when a request fails", () => { completeLoad(); - component.saveDatasetSettings(); + component.saveUploadSettings(group()); // Fail the last of the four PUTs so forkJoin errors with every request flushed. - const keys = [ - "dataset_max_number_of_concurrent_uploading_file", - "dataset_single_file_upload_max_size_mib", - "dataset_max_number_of_concurrent_uploading_file_chunks", - "dataset_multipart_upload_chunk_size_mib", - ]; keys.forEach((key, i) => { const req = httpTestingController.expectOne(updateUrl(key)); if (i === keys.length - 1) req.flush("boom", HTTP_ERROR); else req.flush(null); }); - expect(msgError).toHaveBeenCalledWith("Failed to save dataset settings."); + expect(msgError).toHaveBeenCalledWith(`Failed to save ${prefix} settings.`); }); - it("resetDatasetSettings POSTs a reset for all four upload settings", () => { + it("POSTs a reset for all four upload settings", () => { completeLoad(); - component.resetDatasetSettings(); + component.resetUploadSettings(group()); - [ - "dataset_max_number_of_concurrent_uploading_file", - "dataset_single_file_upload_max_size_mib", - "dataset_max_number_of_concurrent_uploading_file_chunks", - "dataset_multipart_upload_chunk_size_mib", - ].forEach(key => httpTestingController.expectOne(resetUrl(key)).flush(null)); - expect(msgInfo).toHaveBeenCalledWith("Resetting dataset settings..."); + keys.forEach(key => httpTestingController.expectOne(resetUrl(key)).flush(null)); + expect(msgInfo).toHaveBeenCalledWith(`Resetting ${prefix} settings...`); }); }); - // The issue labels these lines as `resetTabs`; they are actually the two computed - // getters that sit just below it, so the tests target the getters. + // The issue labels these lines as `resetTabs`; they are actually the two part-size + // computations that sit just below it, so the tests target those. describe("computed part-size properties", () => { it("partsAtMax is 0 unless both the total size and the chunk size are set", () => { completeLoad(); - component.maxFileSizeMiB = 0; - component.chunkSizeMiB = 8; - expect(component.partsAtMax).toBe(0); - - component.maxFileSizeMiB = 100; - component.chunkSizeMiB = 0; - expect(component.partsAtMax).toBe(0); - - component.maxFileSizeMiB = 100; - component.chunkSizeMiB = 8; - expect(component.partsAtMax).toBe(13); + expect(component.partsAtMax({ ...datasetForm(), maxFileSizeMiB: 0, chunkSizeMiB: 8 })).toBe(0); + expect(component.partsAtMax({ ...datasetForm(), maxFileSizeMiB: 100, chunkSizeMiB: 0 })).toBe(0); + expect(component.partsAtMax({ ...datasetForm(), maxFileSizeMiB: 100, chunkSizeMiB: 8 })).toBe(13); }); it("requiredMinPartSizeMiB falls back to the floor when no total size is set", () => { completeLoad(); - component.maxFileSizeMiB = 0; - expect(component.requiredMinPartSizeMiB).toBe(component.MIN_PART_SIZE_MiB); + expect(component.requiredMinPartSizeMiB({ ...datasetForm(), maxFileSizeMiB: 0 })).toBe( + component.MIN_PART_SIZE_MiB + ); // Above the floor the parts limit takes over: 10,000 parts must cover the total. - component.maxFileSizeMiB = component.MIN_PART_SIZE_MiB * component.MAX_TOTAL_PARTS * 2; - expect(component.requiredMinPartSizeMiB).toBe(component.MIN_PART_SIZE_MiB * 2); + expect( + component.requiredMinPartSizeMiB({ + ...datasetForm(), + maxFileSizeMiB: component.MIN_PART_SIZE_MiB * component.MAX_TOTAL_PARTS * 2, + }) + ).toBe(component.MIN_PART_SIZE_MiB * 2); }); }); @@ -568,14 +583,12 @@ describe("AdminSettingsComponent wiring", () => { "about_enabled", ] as const; - /** Numeric inputs in the order the template renders them. */ - const NUMBER_FIELDS = [ - "maxConcurrentFiles", - "maxFileSizeMiB", - "maxConcurrentChunks", - "chunkSizeMiB", - "csvMaxColumns", - ] as const; + /** Numeric inputs in the order the template renders them: each upload card, then the CSV card. */ + const UPLOAD_FIELDS = ["maxConcurrentFiles", "maxFileSizeMiB", "maxConcurrentChunks", "chunkSizeMiB"] as const; + const numberFields = (): Array<() => number> => [ + ...component.uploadGroups.flatMap(group => UPLOAD_FIELDS.map(field => () => group.form[field])), + () => component.csvMaxColumns, + ]; beforeEach(async () => { TestBed.resetTestingModule(); @@ -685,17 +698,18 @@ describe("AdminSettingsComponent wiring", () => { describe("numeric settings", () => { it("gives every number input its own field, in template order", () => { - expect(numberInputs().length).toBe(NUMBER_FIELDS.length); + const fields = numberFields(); + expect(numberInputs().length).toBe(fields.length); - NUMBER_FIELDS.forEach((field, i) => { + fields.forEach((_, i) => { numberInputs()[i].triggerEventHandler("ngModelChange", 42 + i); - - expect((component as any)[field]).toBe(42 + i); }); + fields.forEach((read, i) => expect(read()).toBe(42 + i)); - // Distinct values, so a shared target would have collapsed them. - const values = NUMBER_FIELDS.map(f => (component as any)[f]); - expect(new Set(values).size).toBe(NUMBER_FIELDS.length); + // Distinct values, so a shared target — including one shared across the two upload + // cards — would have collapsed them. + const values = fields.map(read => read()); + expect(new Set(values).size).toBe(fields.length); }); }); @@ -704,17 +718,18 @@ describe("AdminSettingsComponent wiring", () => { const spies = { saveLogos: vi.spyOn(component, "saveLogos").mockImplementation(() => {}), saveTabs: vi.spyOn(component, "saveTabs").mockImplementation(() => {}), - saveDatasetSettings: vi.spyOn(component, "saveDatasetSettings").mockImplementation(() => {}), + saveUploadSettings: vi.spyOn(component, "saveUploadSettings").mockImplementation(() => {}), saveCsvSettings: vi.spyOn(component, "saveCsvSettings").mockImplementation(() => {}), }; const saves = buttonsLabelled("Save"); - expect(saves.length).toBe(4); + expect(saves.length).toBe(2 + component.uploadGroups.length + 1); saves.forEach(b => b.click()); expect(spies.saveLogos).toHaveBeenCalledTimes(1); expect(spies.saveTabs).toHaveBeenCalledTimes(1); - expect(spies.saveDatasetSettings).toHaveBeenCalledTimes(1); + // Each upload card passes its own group, so the two cards cannot save each other's keys. + expect(spies.saveUploadSettings.mock.calls.map(call => call[0])).toEqual(component.uploadGroups); expect(spies.saveCsvSettings).toHaveBeenCalledTimes(1); }); @@ -722,17 +737,17 @@ describe("AdminSettingsComponent wiring", () => { const spies = { resetBranding: vi.spyOn(component, "resetBranding").mockImplementation(() => {}), resetTabs: vi.spyOn(component, "resetTabs").mockImplementation(() => {}), - resetDatasetSettings: vi.spyOn(component, "resetDatasetSettings").mockImplementation(() => {}), + resetUploadSettings: vi.spyOn(component, "resetUploadSettings").mockImplementation(() => {}), resetCsvSettings: vi.spyOn(component, "resetCsvSettings").mockImplementation(() => {}), }; const resets = buttonsLabelled("Reset"); - expect(resets.length).toBe(4); + expect(resets.length).toBe(2 + component.uploadGroups.length + 1); resets.forEach(b => b.click()); expect(spies.resetBranding).toHaveBeenCalledTimes(1); expect(spies.resetTabs).toHaveBeenCalledTimes(1); - expect(spies.resetDatasetSettings).toHaveBeenCalledTimes(1); + expect(spies.resetUploadSettings.mock.calls.map(call => call[0])).toEqual(component.uploadGroups); expect(spies.resetCsvSettings).toHaveBeenCalledTimes(1); }); }); diff --git a/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.ts b/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.ts index 100ba45d036..104a70578a9 100644 --- a/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.ts +++ b/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.ts @@ -24,19 +24,56 @@ import { NotificationService } from "../../../../common/service/notification/not import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; import { SidebarTabs } from "../../../../common/type/gui-config"; import { parseIntOrDefault } from "../../../../common/util/format.util"; +import { + DATASET_FILE_RESOURCE_ENDPOINT, + FileResourceEndpoint, + MODEL_FILE_RESOURCE_ENDPOINT, +} from "../../../service/user/file-resource/file-resource-endpoint"; import { forkJoin } from "rxjs"; import { NzCardComponent } from "ng-zorro-antd/card"; import { NzSpaceCompactItemDirective } from "ng-zorro-antd/space"; import { NzButtonComponent } from "ng-zorro-antd/button"; import { NzWaveDirective } from "ng-zorro-antd/core/wave"; import { ɵNzTransitionPatchDirective } from "ng-zorro-antd/core/transition-patch"; -import { NgIf, DecimalPipe } from "@angular/common"; +import { NgFor, NgIf, DecimalPipe } from "@angular/common"; import { NzIconDirective } from "ng-zorro-antd/icon"; import { NzSwitchComponent } from "ng-zorro-antd/switch"; import { FormsModule } from "@angular/forms"; import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; import { NzInputNumberComponent } from "ng-zorro-antd/input-number"; +/** The four multipart-upload settings one resource family exposes on this page. */ +export interface UploadSettingsForm { + maxConcurrentFiles: number; + maxFileSizeMiB: number; + maxConcurrentChunks: number; + chunkSizeMiB: number; +} + +/** One upload card: a resource family, the site_settings keys it owns, and the edited values. */ +export interface UploadSettingsGroup { + readonly label: string; + readonly endpoint: FileResourceEndpoint; + readonly form: UploadSettingsForm; +} + +// Shared across families in default.conf; only the per-file ceiling differs, and the endpoint +// descriptor already carries that. +const DEFAULT_MAX_CONCURRENT_FILES = 3; +const DEFAULT_MAX_CONCURRENT_CHUNKS = 10; +const DEFAULT_CHUNK_SIZE_MIB = 50; + +const uploadGroup = (label: string, endpoint: FileResourceEndpoint): UploadSettingsGroup => ({ + label, + endpoint, + form: { + maxConcurrentFiles: DEFAULT_MAX_CONCURRENT_FILES, + maxFileSizeMiB: endpoint.defaultMaxFileSizeMiB, + maxConcurrentChunks: DEFAULT_MAX_CONCURRENT_CHUNKS, + chunkSizeMiB: DEFAULT_CHUNK_SIZE_MIB, + }, +}); + @UntilDestroy() @Component({ selector: "texera-settings", @@ -49,6 +86,7 @@ import { NzInputNumberComponent } from "ng-zorro-antd/input-number"; NzWaveDirective, ɵNzTransitionPatchDirective, NgIf, + NgFor, NzIconDirective, NzSwitchComponent, FormsModule, @@ -77,10 +115,11 @@ export class AdminSettingsComponent implements OnInit { about_enabled: false, }; - maxConcurrentFiles: number = 3; - maxFileSizeMiB: number = 20; - maxConcurrentChunks: number = 10; - chunkSizeMiB: number = 50; + // One card per resource family; adding a family here adds its card, with no new save/reset code. + readonly uploadGroups: UploadSettingsGroup[] = [ + uploadGroup("Dataset", DATASET_FILE_RESOURCE_ENDPOINT), + uploadGroup("Model", MODEL_FILE_RESOURCE_ENDPOINT), + ]; csvMaxColumns: number = 512; @@ -123,19 +162,18 @@ export class AdminSettingsComponent implements OnInit { (Object.keys(this.sidebarTabs) as (keyof SidebarTabs)[]).forEach( tab => (this.sidebarTabs[tab] = settings[tab] === "true") ); - this.maxConcurrentFiles = parseIntOrDefault( - settings["dataset_max_number_of_concurrent_uploading_file"], - this.maxConcurrentFiles - ); - this.maxFileSizeMiB = parseIntOrDefault( - settings["dataset_single_file_upload_max_size_mib"], - this.maxFileSizeMiB - ); - this.maxConcurrentChunks = parseIntOrDefault( - settings["dataset_max_number_of_concurrent_uploading_file_chunks"], - this.maxConcurrentChunks - ); - this.chunkSizeMiB = parseIntOrDefault(settings["dataset_multipart_upload_chunk_size_mib"], this.chunkSizeMiB); + this.uploadGroups.forEach(({ endpoint, form }) => { + form.maxConcurrentFiles = parseIntOrDefault( + settings[endpoint.maxConcurrentFilesSettingKey], + form.maxConcurrentFiles + ); + form.maxFileSizeMiB = parseIntOrDefault(settings[endpoint.maxFileSizeSettingKey], form.maxFileSizeMiB); + form.maxConcurrentChunks = parseIntOrDefault( + settings[endpoint.maxConcurrentChunksSettingKey], + form.maxConcurrentChunks + ); + form.chunkSizeMiB = parseIntOrDefault(settings[endpoint.chunkSizeSettingKey], form.chunkSizeMiB); + }); this.csvMaxColumns = parseIntOrDefault(settings["csv_parser_max_columns"], this.csvMaxColumns); this.settingsLoaded = true; }, @@ -227,73 +265,69 @@ export class AdminSettingsComponent implements OnInit { } // Computed properties - get partsAtMax(): number { - if (!this.maxFileSizeMiB || !this.chunkSizeMiB) return 0; - return Math.ceil(this.maxFileSizeMiB / this.chunkSizeMiB); + partsAtMax(form: UploadSettingsForm): number { + if (!form.maxFileSizeMiB || !form.chunkSizeMiB) return 0; + return Math.ceil(form.maxFileSizeMiB / form.chunkSizeMiB); } - get requiredMinPartSizeMiB(): number { - if (!this.maxFileSizeMiB) return this.MIN_PART_SIZE_MiB; - const byPartsLimit = Math.ceil(this.maxFileSizeMiB / this.MAX_TOTAL_PARTS); + requiredMinPartSizeMiB(form: UploadSettingsForm): number { + if (!form.maxFileSizeMiB) return this.MIN_PART_SIZE_MiB; + const byPartsLimit = Math.ceil(form.maxFileSizeMiB / this.MAX_TOTAL_PARTS); return Math.max(this.MIN_PART_SIZE_MiB, byPartsLimit); } - saveDatasetSettings(): void { + // The four key/value pairs of one group, in the order the card lists them. + private settingEntries({ endpoint, form }: UploadSettingsGroup): Array<[string, number]> { + return [ + [endpoint.maxConcurrentFilesSettingKey, form.maxConcurrentFiles], + [endpoint.maxFileSizeSettingKey, form.maxFileSizeMiB], + [endpoint.maxConcurrentChunksSettingKey, form.maxConcurrentChunks], + [endpoint.chunkSizeSettingKey, form.chunkSizeMiB], + ]; + } + + saveUploadSettings(group: UploadSettingsGroup): void { if (!this.settingsLoaded) { this.message.error("Settings have not loaded; refresh before saving."); return; } + const { form } = group; if ( - this.maxFileSizeMiB < 1 || - this.maxConcurrentFiles < 1 || - this.maxConcurrentChunks < 1 || - this.chunkSizeMiB < 1 + form.maxFileSizeMiB < 1 || + form.maxConcurrentFiles < 1 || + form.maxConcurrentChunks < 1 || + form.chunkSizeMiB < 1 ) { this.message.error("Please enter valid integer values."); return; } - if (this.partsAtMax > this.MAX_TOTAL_PARTS) { + if (this.partsAtMax(form) > this.MAX_TOTAL_PARTS) { this.message.error( - `This setting would create ${this.partsAtMax.toLocaleString()} parts (exceeds 10,000 limit). ` + - `Increase "Part Size" to at least ${this.requiredMinPartSizeMiB} MiB or reduce "File Size".` + `This setting would create ${this.partsAtMax(form).toLocaleString()} parts (exceeds 10,000 limit). ` + + `Increase "Part Size" to at least ${this.requiredMinPartSizeMiB(form)} MiB or reduce "File Size".` ); return; } - const saveRequests = [ - this.adminSettingsService.updateSetting( - "dataset_max_number_of_concurrent_uploading_file", - this.maxConcurrentFiles.toString() - ), - this.adminSettingsService.updateSetting( - "dataset_single_file_upload_max_size_mib", - this.maxFileSizeMiB.toString() - ), - this.adminSettingsService.updateSetting( - "dataset_max_number_of_concurrent_uploading_file_chunks", - this.maxConcurrentChunks.toString() - ), - this.adminSettingsService.updateSetting("dataset_multipart_upload_chunk_size_mib", this.chunkSizeMiB.toString()), - ]; + const saveRequests = this.settingEntries(group).map(([key, value]) => + this.adminSettingsService.updateSetting(key, value.toString()) + ); forkJoin(saveRequests) .pipe(untilDestroyed(this)) .subscribe({ - next: () => this.message.success("Dataset upload settings saved successfully."), - error: () => this.message.error("Failed to save dataset settings."), + next: () => this.message.success(`${group.label} upload settings saved successfully.`), + error: () => this.message.error(`Failed to save ${group.label.toLowerCase()} settings.`), }); } - resetDatasetSettings(): void { - [ - "dataset_max_number_of_concurrent_uploading_file", - "dataset_single_file_upload_max_size_mib", - "dataset_max_number_of_concurrent_uploading_file_chunks", - "dataset_multipart_upload_chunk_size_mib", - ].forEach(setting => this.adminSettingsService.resetSetting(setting).pipe(untilDestroyed(this)).subscribe({})); + resetUploadSettings(group: UploadSettingsGroup): void { + this.settingEntries(group).forEach(([key]) => + this.adminSettingsService.resetSetting(key).pipe(untilDestroyed(this)).subscribe({}) + ); - this.message.info("Resetting dataset settings..."); + this.message.info(`Resetting ${group.label.toLowerCase()} settings...`); setTimeout(() => window.location.reload(), this.RELOAD_DELAY); } diff --git a/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.spec.ts b/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.spec.ts index f0f9eeed191..c72bfa6feb6 100644 --- a/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.spec.ts @@ -27,7 +27,11 @@ import { NzAlertComponent } from "ng-zorro-antd/alert"; import { NzModalService } from "ng-zorro-antd/modal"; import { commonTestProviders } from "../../../../common/testing/test-utils"; import { AdminSettingsService } from "../../../service/admin/settings/admin-settings.service"; -import { DatasetService } from "../../../service/user/dataset/dataset.service"; +import { MultipartUploadService } from "../../../service/user/file-resource/multipart-upload.service"; +import { + DATASET_FILE_RESOURCE_ENDPOINT, + FileResourceEndpoint, +} from "../../../service/user/file-resource/file-resource-endpoint"; import { NotificationService } from "../../../../common/service/notification/notification.service"; import { FileUploadItem } from "../../../type/dashboard-file.interface"; import { FilesUploaderComponent } from "./files-uploader.component"; @@ -61,14 +65,31 @@ const droppedFile = (relativePath: string, file: File): NgxFileDropEntry => }, }) as unknown as NgxFileDropEntry; +/** A resource family that is neither dataset nor model, to prove the uploader is parameterized. */ +const WIDGET_ENDPOINT: FileResourceEndpoint = { + baseUrl: "widget", + nameParamKey: "widgetName", + maxFileSizeSettingKey: "widget_single_file_upload_max_size_mib", + defaultMaxFileSizeMiB: 64, + chunkSizeSettingKey: "widget_multipart_upload_chunk_size_mib", + maxConcurrentChunksSettingKey: "widget_max_number_of_concurrent_uploading_file_chunks", + maxConcurrentFilesSettingKey: "widget_max_number_of_concurrent_uploading_file", +}; + describe("FilesUploaderComponent", () => { let component: FilesUploaderComponent; let modals: CapturedModal[]; - let datasetService: { + let uploadService: { listMultipartUploads: ReturnType; findExistingUploadFiles: ReturnType; }; + /** Builds an initialized uploader; the size ceiling is only read in ngOnInit. */ + let buildUploader: ( + adminSettingsService: AdminSettingsService, + notificationService?: NotificationService + ) => FilesUploaderComponent; + beforeEach(() => { modals = []; const modal = { @@ -80,30 +101,37 @@ describe("FilesUploaderComponent", () => { const adminSettingsService = { getPublicSetting: vi.fn().mockReturnValue(of("20")), } as unknown as AdminSettingsService; - datasetService = { + uploadService = { listMultipartUploads: vi.fn().mockReturnValue(of(["failed.csv"])), findExistingUploadFiles: vi.fn().mockReturnValue(of(["done.csv"])), }; + buildUploader = (settings, notify = { error: vi.fn() } as unknown as NotificationService) => { + const uploader = new FilesUploaderComponent( + notify, + settings, + uploadService as unknown as MultipartUploadService, + { create: vi.fn() } as unknown as NzModalService + ); + uploader.ngOnInit(); + return uploader; + }; + component = new FilesUploaderComponent( { error: vi.fn() } as unknown as NotificationService, adminSettingsService, - datasetService as unknown as DatasetService, + uploadService as unknown as MultipartUploadService, modal ); + component.ngOnInit(); component.ownerEmail = "owner@example.com"; - component.datasetName = "dataset"; - component.did = 7; + component.resourceName = "dataset"; + component.resourceId = 7; }); it("keeps the default upload size limit when the public setting is missing, and parses it when present", () => { const build = (value: string | null) => - new FilesUploaderComponent( - { error: vi.fn() } as unknown as NotificationService, - { getPublicSetting: vi.fn().mockReturnValue(of(value)) } as unknown as AdminSettingsService, - datasetService as unknown as DatasetService, - { create: vi.fn() } as unknown as NzModalService - ); + buildUploader({ getPublicSetting: vi.fn().mockReturnValue(of(value)) } as unknown as AdminSettingsService); expect(build(null).singleFileUploadMaxSizeMiB).toBe(20); expect(build("128").singleFileUploadMaxSizeMiB).toBe(128); @@ -114,16 +142,26 @@ describe("FilesUploaderComponent", () => { it("keeps the default upload size limit when the setting request fails", () => { // The component swallows the error on purpose so a settings outage cannot stop uploads. + const uploader = buildUploader({ + getPublicSetting: vi.fn().mockReturnValue(throwError(() => new Error("settings unavailable"))), + } as unknown as AdminSettingsService); + + expect(uploader.singleFileUploadMaxSizeMiB).toBe(20); + }); + + it("takes the ceiling key and fallback from the configured endpoint", () => { + const getPublicSetting = vi.fn().mockReturnValue(of(null)); const uploader = new FilesUploaderComponent( { error: vi.fn() } as unknown as NotificationService, - { - getPublicSetting: vi.fn().mockReturnValue(throwError(() => new Error("settings unavailable"))), - } as unknown as AdminSettingsService, - datasetService as unknown as DatasetService, + { getPublicSetting } as unknown as AdminSettingsService, + uploadService as unknown as MultipartUploadService, { create: vi.fn() } as unknown as NzModalService ); + uploader.endpoint = WIDGET_ENDPOINT; + uploader.ngOnInit(); - expect(uploader.singleFileUploadMaxSizeMiB).toBe(20); + expect(getPublicSetting).toHaveBeenCalledWith("widget_single_file_upload_max_size_mib"); + expect(uploader.singleFileUploadMaxSizeMiB).toBe(64); }); it("asks to resume failed multipart files and skip completed matching files in one retry batch", async () => { @@ -149,8 +187,8 @@ describe("FilesUploaderComponent", () => { }); it("asks both questions when the same file has an active upload session and an existing match", async () => { - datasetService.listMultipartUploads.mockReturnValue(of(["same.csv"])); - datasetService.findExistingUploadFiles.mockReturnValue(of(["same.csv"])); + uploadService.listMultipartUploads.mockReturnValue(of(["same.csv"])); + uploadService.findExistingUploadFiles.mockReturnValue(of(["same.csv"])); const emitted = new Promise(resolve => component.uploadedFiles.subscribe(resolve)); component.fileDropped([droppedFile("same.csv", new File(["same"], "same.csv"))]); @@ -176,7 +214,7 @@ describe("FilesUploaderComponent", () => { * session the user asked to discard. */ it("marks a file for force-restart when Restart is chosen", async () => { - datasetService.findExistingUploadFiles.mockReturnValue(of([])); + uploadService.findExistingUploadFiles.mockReturnValue(of([])); const emitted = new Promise(resolve => component.uploadedFiles.subscribe(resolve)); component.fileDropped([droppedFile("failed.csv", new File(["half"], "failed.csv"))]); @@ -192,7 +230,7 @@ describe("FilesUploaderComponent", () => { it("leaves the restart flag unset when Resume is chosen", async () => { // The counterpart of the test above: same file, other button. Without this pair, a // markForceRestart call added to the Resume branch would go unnoticed. - datasetService.findExistingUploadFiles.mockReturnValue(of([])); + uploadService.findExistingUploadFiles.mockReturnValue(of([])); const emitted = new Promise(resolve => component.uploadedFiles.subscribe(resolve)); component.fileDropped([droppedFile("failed.csv", new File(["half"], "failed.csv"))]); @@ -206,8 +244,8 @@ describe("FilesUploaderComponent", () => { }); it("restarts every remaining conflicting file after one Restart For All choice", async () => { - datasetService.listMultipartUploads.mockReturnValue(of(["one.csv", "two.csv"])); - datasetService.findExistingUploadFiles.mockReturnValue(of([])); + uploadService.listMultipartUploads.mockReturnValue(of(["one.csv", "two.csv"])); + uploadService.findExistingUploadFiles.mockReturnValue(of([])); const emitted = new Promise(resolve => component.uploadedFiles.subscribe(resolve)); component.fileDropped([ @@ -227,8 +265,8 @@ describe("FilesUploaderComponent", () => { }); it("resumes every remaining conflicting file after one Resume For All choice", async () => { - datasetService.listMultipartUploads.mockReturnValue(of(["one.csv", "two.csv"])); - datasetService.findExistingUploadFiles.mockReturnValue(of([])); + uploadService.listMultipartUploads.mockReturnValue(of(["one.csv", "two.csv"])); + uploadService.findExistingUploadFiles.mockReturnValue(of([])); const emitted = new Promise(resolve => component.uploadedFiles.subscribe(resolve)); component.fileDropped([ @@ -247,8 +285,8 @@ describe("FilesUploaderComponent", () => { }); it("passes a non-conflicting file straight through without prompting", async () => { - datasetService.listMultipartUploads.mockReturnValue(of(["other.csv"])); - datasetService.findExistingUploadFiles.mockReturnValue(of([])); + uploadService.listMultipartUploads.mockReturnValue(of(["other.csv"])); + uploadService.findExistingUploadFiles.mockReturnValue(of([])); const emitted = new Promise(resolve => component.uploadedFiles.subscribe(resolve)); component.fileDropped([droppedFile("clean.csv", new File(["clean"], "clean.csv"))]); @@ -260,8 +298,8 @@ describe("FilesUploaderComponent", () => { }); it("skips all matching files after one Skip For All choice", async () => { - datasetService.listMultipartUploads.mockReturnValue(of([])); - datasetService.findExistingUploadFiles.mockReturnValue(of(["one.csv", "two.csv"])); + uploadService.listMultipartUploads.mockReturnValue(of([])); + uploadService.findExistingUploadFiles.mockReturnValue(of(["one.csv", "two.csv"])); const emitted = new Promise(resolve => component.uploadedFiles.subscribe(resolve)); component.fileDropped([ @@ -280,8 +318,8 @@ describe("FilesUploaderComponent", () => { }); it("uploads all matching files after one Upload For All choice", async () => { - datasetService.listMultipartUploads.mockReturnValue(of([])); - datasetService.findExistingUploadFiles.mockReturnValue(of(["one.csv", "two.csv"])); + uploadService.listMultipartUploads.mockReturnValue(of([])); + uploadService.findExistingUploadFiles.mockReturnValue(of(["one.csv", "two.csv"])); const emitted = new Promise(resolve => component.uploadedFiles.subscribe(resolve)); component.fileDropped([ @@ -343,11 +381,9 @@ describe("FilesUploaderComponent", () => { it("rejects a single oversized file and reports it in the banner", async () => { const notify = { error: vi.fn() }; - component = new FilesUploaderComponent( - notify as unknown as NotificationService, + component = buildUploader( { getPublicSetting: vi.fn().mockReturnValue(of("0")) } as unknown as AdminSettingsService, - datasetService as unknown as DatasetService, - { create: vi.fn() } as unknown as NzModalService + notify as unknown as NotificationService ); const emitted = emissionOf(); @@ -360,12 +396,9 @@ describe("FilesUploaderComponent", () => { }); it("pluralises the failure banner for more than one rejected file", async () => { - component = new FilesUploaderComponent( - { error: vi.fn() } as unknown as NotificationService, - { getPublicSetting: vi.fn().mockReturnValue(of("0")) } as unknown as AdminSettingsService, - datasetService as unknown as DatasetService, - { create: vi.fn() } as unknown as NzModalService - ); + component = buildUploader({ + getPublicSetting: vi.fn().mockReturnValue(of("0")), + } as unknown as AdminSettingsService); const emitted = emissionOf(); component.fileDropped([ @@ -406,23 +439,56 @@ describe("FilesUploaderComponent", () => { const emissionOf = (): Promise => new Promise(resolve => component.uploadedFiles.subscribe(resolve)); - it("skips both lookups when the uploader has no dataset context", async () => { - // The standalone (dataset-creation) usage: no owner/name and no did yet. + it("skips both lookups when the uploader has no resource context", async () => { + // The standalone (dataset-creation) usage: no owner/name and no id yet. component.ownerEmail = ""; - component.datasetName = ""; - component.did = undefined; + component.resourceName = ""; + component.resourceId = undefined; const emitted = emissionOf(); component.fileDropped([droppedFile("fresh.csv", new File(["new"], "fresh.csv"))]); expect((await emitted).map(item => item.name)).toEqual(["fresh.csv"]); - expect(datasetService.listMultipartUploads).not.toHaveBeenCalled(); - expect(datasetService.findExistingUploadFiles).not.toHaveBeenCalled(); + expect(uploadService.listMultipartUploads).not.toHaveBeenCalled(); + expect(uploadService.findExistingUploadFiles).not.toHaveBeenCalled(); + }); + + it("addresses both lookups through the configured endpoint", async () => { + component.endpoint = WIDGET_ENDPOINT; + uploadService.listMultipartUploads.mockReturnValue(of([])); + uploadService.findExistingUploadFiles.mockReturnValue(of([])); + const emitted = emissionOf(); + + component.fileDropped([droppedFile("w.bin", new File(["w"], "w.bin"))]); + await emitted; + + expect(uploadService.listMultipartUploads).toHaveBeenCalledWith(WIDGET_ENDPOINT, "owner@example.com", "dataset"); + expect(uploadService.findExistingUploadFiles).toHaveBeenCalledWith(WIDGET_ENDPOINT, 7, [ + { path: "w.bin", sizeBytes: 1 }, + ]); + }); + + it("defaults to the dataset endpoint when the embedder sets none", async () => { + uploadService.listMultipartUploads.mockReturnValue(of([])); + uploadService.findExistingUploadFiles.mockReturnValue(of([])); + const emitted = emissionOf(); + + component.fileDropped([droppedFile("d.csv", new File(["d"], "d.csv"))]); + await emitted; + + expect(uploadService.listMultipartUploads).toHaveBeenCalledWith( + DATASET_FILE_RESOURCE_ENDPOINT, + "owner@example.com", + "dataset" + ); + expect(uploadService.findExistingUploadFiles).toHaveBeenCalledWith(DATASET_FILE_RESOURCE_ENDPOINT, 7, [ + { path: "d.csv", sizeBytes: 1 }, + ]); }); it("treats a failed lookup as nothing to reconcile", async () => { - datasetService.listMultipartUploads.mockReturnValue(throwError(() => new Error("offline"))); - datasetService.findExistingUploadFiles.mockReturnValue(throwError(() => new Error("offline"))); + uploadService.listMultipartUploads.mockReturnValue(throwError(() => new Error("offline"))); + uploadService.findExistingUploadFiles.mockReturnValue(throwError(() => new Error("offline"))); const emitted = emissionOf(); component.fileDropped([droppedFile("failed.csv", new File(["half"], "failed.csv"))]); @@ -433,8 +499,8 @@ describe("FilesUploaderComponent", () => { }); it("treats a null lookup result as nothing to reconcile", async () => { - datasetService.listMultipartUploads.mockReturnValue(of(null)); - datasetService.findExistingUploadFiles.mockReturnValue(of(null)); + uploadService.listMultipartUploads.mockReturnValue(of(null)); + uploadService.findExistingUploadFiles.mockReturnValue(of(null)); const emitted = emissionOf(); component.fileDropped([droppedFile("failed.csv", new File(["half"], "failed.csv"))]); @@ -444,7 +510,7 @@ describe("FilesUploaderComponent", () => { }); it("reports an unexpected failure of the whole drop", async () => { - datasetService.listMultipartUploads.mockImplementation(() => { + uploadService.listMultipartUploads.mockImplementation(() => { throw new Error("lookup exploded"); }); @@ -456,7 +522,7 @@ describe("FilesUploaderComponent", () => { }); it("reports an unexpected failure that carries no message", async () => { - datasetService.listMultipartUploads.mockImplementation(() => { + uploadService.listMultipartUploads.mockImplementation(() => { throw "lookup exploded"; }); @@ -471,12 +537,9 @@ describe("FilesUploaderComponent", () => { // @UntilDestroy() supplies the ngOnDestroy that ends the `untilDestroyed(this)` // subscription; without it a late setting would still be applied to a dead component. const setting = new Subject(); - const uploader = new FilesUploaderComponent( - { error: vi.fn() } as unknown as NotificationService, - { getPublicSetting: vi.fn().mockReturnValue(setting) } as unknown as AdminSettingsService, - datasetService as unknown as DatasetService, - { create: vi.fn() } as unknown as NzModalService - ); + const uploader = buildUploader({ + getPublicSetting: vi.fn().mockReturnValue(setting), + } as unknown as AdminSettingsService); setting.next("50"); expect(uploader.singleFileUploadMaxSizeMiB).toBe(50); @@ -489,8 +552,8 @@ describe("FilesUploaderComponent", () => { describe("dialog titles for paths without a file name", () => { it("falls back to the whole path when the conflicting path ends in a separator", async () => { - datasetService.listMultipartUploads.mockReturnValue(of(["folder/"])); - datasetService.findExistingUploadFiles.mockReturnValue(of(["folder/"])); + uploadService.listMultipartUploads.mockReturnValue(of(["folder/"])); + uploadService.findExistingUploadFiles.mockReturnValue(of(["folder/"])); const emitted = new Promise(resolve => component.uploadedFiles.subscribe(resolve)); component.fileDropped([droppedFile("folder/", new File(["x"], "x"))]); @@ -524,7 +587,7 @@ describe("FilesUploaderComponent rendered", () => { { provide: NotificationService, useValue: { error: vi.fn() } }, { provide: AdminSettingsService, useValue: { getPublicSetting: vi.fn().mockReturnValue(of("20")) } }, { - provide: DatasetService, + provide: MultipartUploadService, useValue: { listMultipartUploads: vi.fn().mockReturnValue(of([])), findExistingUploadFiles: vi.fn().mockReturnValue(of([])), diff --git a/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts b/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts index 10aed3dfd84..806c9e3123f 100644 --- a/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts +++ b/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts @@ -17,7 +17,7 @@ * under the License. */ -import { Component, EventEmitter, Input, Output } from "@angular/core"; +import { Component, EventEmitter, Input, OnInit, Output } from "@angular/core"; import { firstValueFrom } from "rxjs"; import { NgxFileDropEntry, NgxFileDropModule } from "ngx-file-drop"; import { NzModalRef, NzModalService } from "ng-zorro-antd/modal"; @@ -26,7 +26,11 @@ import { DatasetFileNode } from "../../../../common/type/datasetVersionFileTree" import { NotificationService } from "../../../../common/service/notification/notification.service"; import { AdminSettingsService } from "../../../service/admin/settings/admin-settings.service"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; -import { DatasetService } from "../../../service/user/dataset/dataset.service"; +import { MultipartUploadService } from "../../../service/user/file-resource/multipart-upload.service"; +import { + DATASET_FILE_RESOURCE_ENDPOINT, + FileResourceEndpoint, +} from "../../../service/user/file-resource/file-resource-endpoint"; import { formatSize } from "../../../../common/util/size-formatter.util"; import { parseIntOrDefault } from "../../../../common/util/format.util"; import { @@ -55,18 +59,18 @@ import { ɵNzTransitionPatchDirective } from "ng-zorro-antd/core/transition-patc ɵNzTransitionPatchDirective, ], }) -export class FilesUploaderComponent { +export class FilesUploaderComponent implements OnInit { @Input() showUploadAlert: boolean = false; /** - * Optional context fields supplied by the embedding component. When the - * uploader is used inside `DatasetDetailComponent`, the parent passes - * `ownerEmail` and `datasetName` so the uploader can address staged files - * under the right owner/dataset path. When used standalone (e.g. dataset - * creation flow), they default to empty. + * Optional context supplied by the embedding component so the uploader can address staged files + * under the right owner/resource path. When used standalone (e.g. dataset creation flow) they + * default to empty and the conflict lookups are skipped. */ @Input() ownerEmail: string = ""; - @Input() datasetName: string = ""; - @Input() did: number | undefined; + @Input() resourceName: string = ""; + @Input() resourceId: number | undefined; + /** Which resource family the ids above belong to. */ + @Input() endpoint: FileResourceEndpoint = DATASET_FILE_RESOURCE_ENDPOINT; @Output() uploadedFiles = new EventEmitter(); @@ -75,17 +79,21 @@ export class FilesUploaderComponent { fileUploadingFinished: boolean = false; fileUploadBannerType: "error" | "success" | "info" | "warning" = "success"; fileUploadBannerMessage: string = ""; - singleFileUploadMaxSizeMiB: number = 20; + singleFileUploadMaxSizeMiB: number = DATASET_FILE_RESOURCE_ENDPOINT.defaultMaxFileSizeMiB; constructor( private notificationService: NotificationService, private adminSettingsService: AdminSettingsService, - private datasetService: DatasetService, + private multipartUploadService: MultipartUploadService, private modal: NzModalService - ) { - // A missing key or failed fetch keeps the initializer default above. + ) {} + + // The ceiling is read here rather than in the constructor because `endpoint` is an @Input, and it + // decides both the setting key and the fallback. A missing key or failed fetch keeps the fallback. + ngOnInit(): void { + this.singleFileUploadMaxSizeMiB = this.endpoint.defaultMaxFileSizeMiB; this.adminSettingsService - .getPublicSetting("dataset_single_file_upload_max_size_mib") + .getPublicSetting(this.endpoint.maxFileSizeSettingKey) .pipe(untilDestroyed(this)) .subscribe({ next: value => (this.singleFileUploadMaxSizeMiB = parseIntOrDefault(value, this.singleFileUploadMaxSizeMiB)), @@ -273,10 +281,10 @@ export class FilesUploaderComponent { this.fileUploadBannerMessage = bannerMessage; } - private getOwnerAndName(): { ownerEmail: string; datasetName: string } { + private getOwnerAndName(): { ownerEmail: string; resourceName: string } { return { ownerEmail: this.ownerEmail, - datasetName: this.datasetName, + resourceName: this.resourceName, }; } @@ -314,7 +322,7 @@ export class FilesUploaderComponent { Promise.allSettled(filePromises) .then(async results => { - const { ownerEmail, datasetName } = this.getOwnerAndName(); + const { ownerEmail, resourceName } = this.getOwnerAndName(); const successfulUploads = results .filter((r): r is PromiseFulfilledResult => r.status === "fulfilled") @@ -322,13 +330,16 @@ export class FilesUploaderComponent { .filter((item): item is FileUploadItem => item !== null); const activePathsPromise: Promise = - ownerEmail && datasetName - ? firstValueFrom(this.datasetService.listMultipartUploads(ownerEmail, datasetName)).catch(() => []) + ownerEmail && resourceName + ? firstValueFrom( + this.multipartUploadService.listMultipartUploads(this.endpoint, ownerEmail, resourceName) + ).catch(() => []) : Promise.resolve([]); - const existingPathsPromise: Promise = this.did + const existingPathsPromise: Promise = this.resourceId ? firstValueFrom( - this.datasetService.findExistingUploadFiles( - this.did, + this.multipartUploadService.findExistingUploadFiles( + this.endpoint, + this.resourceId, successfulUploads.map(item => ({ path: item.name, sizeBytes: item.file.size })) ) ).catch(() => []) diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.html b/frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.html similarity index 91% rename from frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.html rename to frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.html index a6785abd50d..200cbaa3ac2 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.html +++ b/frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.html @@ -19,12 +19,12 @@
+ [class.has-list]="stagedObjects.length > 0"> + *cdkVirtualFor="let obj of stagedObjects; trackBy: trackByStagedObject"> {{ obj.diffType }} @@ -65,6 +65,6 @@
diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.scss b/frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.scss similarity index 100% rename from frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.scss rename to frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.scss diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.spec.ts b/frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.spec.ts similarity index 73% rename from frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.spec.ts rename to frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.spec.ts index 54c25e3bb2e..d476e389da8 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.spec.ts @@ -23,20 +23,21 @@ import { By } from "@angular/platform-browser"; import { CdkVirtualScrollViewport } from "@angular/cdk/scrolling"; import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; import { of, throwError } from "rxjs"; -import { UserDatasetStagedObjectsListComponent } from "./user-dataset-staged-objects-list.component"; -import { DatasetService } from "../../../../../service/user/dataset/dataset.service"; -import { NotificationService } from "../../../../../../common/service/notification/notification.service"; -import { DatasetStagedObject } from "../../../../../../common/type/dataset-staged-object"; -import { commonTestImports, commonTestProviders } from "../../../../../../common/testing/test-utils"; - -describe("UserDatasetStagedObjectsListComponent", () => { - let fixture: ComponentFixture; - let component: UserDatasetStagedObjectsListComponent; - let getDatasetDiffSpy: ReturnType; - let resetDatasetFileDiffSpy: ReturnType; +import { StagedObjectsListComponent } from "./staged-objects-list.component"; +import { StagedFileService } from "../../../service/user/file-resource/staged-file.service"; +import { DATASET_FILE_RESOURCE_ENDPOINT } from "../../../service/user/file-resource/file-resource-endpoint"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; +import { DatasetStagedObject } from "../../../../common/type/dataset-staged-object"; +import { commonTestImports, commonTestProviders } from "../../../../common/testing/test-utils"; + +describe("StagedObjectsListComponent", () => { + let fixture: ComponentFixture; + let component: StagedObjectsListComponent; + let getDiffSpy: ReturnType; + let resetFileDiffSpy: ReturnType; const renderList = async () => { - component.did = 1; + component.resourceId = 1; fixture.detectChanges(); await fixture.whenStable(); fixture.detectChanges(); @@ -48,44 +49,44 @@ describe("UserDatasetStagedObjectsListComponent", () => { ]; beforeEach(() => { - getDatasetDiffSpy = vi.fn(() => of(stagedObjects)); - resetDatasetFileDiffSpy = vi.fn(() => of({})); + getDiffSpy = vi.fn(() => of(stagedObjects)); + resetFileDiffSpy = vi.fn(() => of({})); TestBed.configureTestingModule({ - imports: [UserDatasetStagedObjectsListComponent, ...commonTestImports], + imports: [StagedObjectsListComponent, ...commonTestImports], providers: [ { - provide: DatasetService, - useValue: { getDatasetDiff: getDatasetDiffSpy, resetDatasetFileDiff: resetDatasetFileDiffSpy }, + provide: StagedFileService, + useValue: { getDiff: getDiffSpy, resetFileDiff: resetFileDiffSpy }, }, { provide: NotificationService, useValue: { success: vi.fn(), error: vi.fn() } }, ...commonTestProviders, ], }); - fixture = TestBed.createComponent(UserDatasetStagedObjectsListComponent); + fixture = TestBed.createComponent(StagedObjectsListComponent); component = fixture.componentInstance; }); it("fetches staged objects on init and emits them", () => { - component.did = 1; + component.resourceId = 1; const emitted: DatasetStagedObject[][] = []; component.stagedObjectsChanged.subscribe((objects: DatasetStagedObject[]) => emitted.push(objects)); component.ngOnInit(); - expect(getDatasetDiffSpy).toHaveBeenCalledWith(1); - expect(component.datasetStagedObjects).toEqual(stagedObjects); + expect(getDiffSpy).toHaveBeenCalledWith(DATASET_FILE_RESOURCE_ENDPOINT, 1); + expect(component.stagedObjects).toEqual(stagedObjects); expect(emitted).toEqual([stagedObjects]); }); - it("does not fetch staged objects when did is undefined", () => { - component.did = undefined; + it("does not fetch staged objects when resourceId is undefined", () => { + component.resourceId = undefined; component.ngOnInit(); - expect(getDatasetDiffSpy).not.toHaveBeenCalled(); - expect(component.datasetStagedObjects).toEqual([]); + expect(getDiffSpy).not.toHaveBeenCalled(); + expect(component.stagedObjects).toEqual([]); }); // #5586: one change event per finished file must not mean one dataset-diff @@ -93,17 +94,17 @@ describe("UserDatasetStagedObjectsListComponent", () => { it("coalesces bursts of change events into one refetch per audit window", () => { vi.useFakeTimers(); try { - component.did = 1; + component.resourceId = 1; const changes = new EventEmitter(); component.userMakeChangesEvent = changes; for (let i = 0; i < 50; i++) { changes.emit(); } - expect(getDatasetDiffSpy).not.toHaveBeenCalled(); + expect(getDiffSpy).not.toHaveBeenCalled(); - vi.advanceTimersByTime(UserDatasetStagedObjectsListComponent.REFRESH_AUDIT_TIME_MS); - expect(getDatasetDiffSpy).toHaveBeenCalledTimes(1); + vi.advanceTimersByTime(StagedObjectsListComponent.REFRESH_AUDIT_TIME_MS); + expect(getDiffSpy).toHaveBeenCalledTimes(1); } finally { vi.useRealTimers(); } @@ -130,8 +131,8 @@ describe("UserDatasetStagedObjectsListComponent", () => { (fixture.nativeElement.querySelector(".delete-button") as HTMLButtonElement).click(); - expect(resetDatasetFileDiffSpy).toHaveBeenCalledWith(1, "dir/a.txt"); - expect(getDatasetDiffSpy).toHaveBeenCalledTimes(2); + expect(resetFileDiffSpy).toHaveBeenCalledWith(DATASET_FILE_RESOURCE_ENDPOINT, 1, "dir/a.txt"); + expect(getDiffSpy).toHaveBeenCalledTimes(2); }); it("shows the full path and upload time in the row tooltip", async () => { @@ -159,21 +160,21 @@ describe("UserDatasetStagedObjectsListComponent", () => { describe("branch coverage", () => { it("ignores a userMakeChangesEvent that is not provided", () => { // The setter guards on the event; assigning nothing must not subscribe or refetch. - component.did = 1; + component.resourceId = 1; component.userMakeChangesEvent = undefined as unknown as EventEmitter; - expect(getDatasetDiffSpy).not.toHaveBeenCalled(); + expect(getDiffSpy).not.toHaveBeenCalled(); }); it("does not revert an object when no dataset id is set", () => { - component.did = undefined; + component.resourceId = undefined; component.onObjectReverted(stagedObjects[0]); - expect(resetDatasetFileDiffSpy).not.toHaveBeenCalled(); + expect(resetFileDiffSpy).not.toHaveBeenCalled(); }); it("notifies when reverting a staged object fails", () => { - resetDatasetFileDiffSpy.mockReturnValue(throwError(() => new Error("boom"))); + resetFileDiffSpy.mockReturnValue(throwError(() => new Error("boom"))); const notificationService = TestBed.inject(NotificationService); - component.did = 1; + component.resourceId = 1; component.onObjectReverted(stagedObjects[0]); diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.ts b/frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.ts similarity index 71% rename from frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.ts rename to frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.ts index f92ba316f0e..a69e928c08f 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/user-dataset-staged-objects-list/user-dataset-staged-objects-list.component.ts +++ b/frontend/src/app/dashboard/component/user/staged-objects-list/staged-objects-list.component.ts @@ -19,9 +19,13 @@ import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from "@angular/core"; import { auditTime } from "rxjs/operators"; -import { DatasetStagedObject } from "../../../../../../common/type/dataset-staged-object"; -import { DatasetService } from "../../../../../service/user/dataset/dataset.service"; -import { NotificationService } from "../../../../../../common/service/notification/notification.service"; +import { DatasetStagedObject } from "../../../../common/type/dataset-staged-object"; +import { StagedFileService } from "../../../service/user/file-resource/staged-file.service"; +import { + DATASET_FILE_RESOURCE_ENDPOINT, + FileResourceEndpoint, +} from "../../../service/user/file-resource/file-resource-endpoint"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; import { formatTime } from "src/app/common/util/format.util"; import { NgIf } from "@angular/common"; @@ -37,9 +41,9 @@ import { CdkFixedSizeVirtualScroll, CdkVirtualForOf, CdkVirtualScrollViewport } @UntilDestroy() @Component({ - selector: "texera-dataset-staged-objects-list", - templateUrl: "./user-dataset-staged-objects-list.component.html", - styleUrls: ["./user-dataset-staged-objects-list.component.scss"], + selector: "texera-staged-objects-list", + templateUrl: "./staged-objects-list.component.html", + styleUrls: ["./staged-objects-list.component.scss"], imports: [ NgIf, NzListComponent, @@ -56,26 +60,26 @@ import { CdkFixedSizeVirtualScroll, CdkVirtualForOf, CdkVirtualScrollViewport } NzEmptyComponent, ], }) -export class UserDatasetStagedObjectsListComponent implements OnInit { +export class StagedObjectsListComponent implements OnInit { // Coalesces change events so a bulk upload refetches the diff at most once // per window, not once per finished file. static readonly REFRESH_AUDIT_TIME_MS = 1000; - @Input() did?: number; // Dataset ID + @Input() resourceId?: number; + /** Which resource family `resourceId` belongs to. */ + @Input() endpoint: FileResourceEndpoint = DATASET_FILE_RESOURCE_ENDPOINT; @Input() set userMakeChangesEvent(event: EventEmitter) { if (event) { - event - .pipe(auditTime(UserDatasetStagedObjectsListComponent.REFRESH_AUDIT_TIME_MS), untilDestroyed(this)) - .subscribe(() => { - this.fetchDatasetStagedObjects(); - }); + event.pipe(auditTime(StagedObjectsListComponent.REFRESH_AUDIT_TIME_MS), untilDestroyed(this)).subscribe(() => { + this.fetchStagedObjects(); + }); } } @Input() uploadTimeMap?: Map; @Output() stagedObjectsChanged = new EventEmitter(); // Emits staged objects list - datasetStagedObjects: DatasetStagedObject[] = []; + stagedObjects: DatasetStagedObject[] = []; formatTime = formatTime; // Row height must match .staged-object-row in the SCSS. @@ -85,7 +89,7 @@ export class UserDatasetStagedObjectsListComponent implements OnInit { @ViewChild(CdkVirtualScrollViewport) private viewport?: CdkVirtualScrollViewport; get stagedListHeightPx(): number { - return Math.min(this.datasetStagedObjects.length * this.STAGED_ROW_HEIGHT_PX, this.STAGED_LIST_MAX_HEIGHT_PX); + return Math.min(this.stagedObjects.length * this.STAGED_ROW_HEIGHT_PX, this.STAGED_LIST_MAX_HEIGHT_PX); } // The viewport measures height 0 when created inside a hidden ancestor @@ -95,36 +99,36 @@ export class UserDatasetStagedObjectsListComponent implements OnInit { } constructor( - private datasetService: DatasetService, + private stagedFileService: StagedFileService, private notificationService: NotificationService ) {} ngOnInit(): void { - this.fetchDatasetStagedObjects(); + this.fetchStagedObjects(); } - private fetchDatasetStagedObjects(): void { - if (this.did != undefined) { - this.datasetService - .getDatasetDiff(this.did) + private fetchStagedObjects(): void { + if (this.resourceId != undefined) { + this.stagedFileService + .getDiff(this.endpoint, this.resourceId) .pipe(untilDestroyed(this)) .subscribe(diffs => { - this.datasetStagedObjects = diffs; + this.stagedObjects = diffs; // Emit the updated staged objects list - this.stagedObjectsChanged.emit(this.datasetStagedObjects); + this.stagedObjectsChanged.emit(this.stagedObjects); }); } } onObjectReverted(objDiff: DatasetStagedObject) { - if (this.did) { - this.datasetService - .resetDatasetFileDiff(this.did, objDiff.path) + if (this.resourceId) { + this.stagedFileService + .resetFileDiff(this.endpoint, this.resourceId, objDiff.path) .pipe(untilDestroyed(this)) .subscribe({ next: (res: Response) => { this.notificationService.success(`"${objDiff.diffType} ${objDiff.path}" is successfully reverted`); - this.fetchDatasetStagedObjects(); + this.fetchStagedObjects(); }, error: (err: unknown) => { this.notificationService.error("Failed to delete the file"); diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.html b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.html index f47b8d43f97..777a1537654 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.html +++ b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.html @@ -448,165 +448,16 @@
Choose a Version:
- - + - - - - - - - - -
- {{ fileName }} - -
- - - - - - -
-
-
- {{ task.status }}: {{ task.filePath }} - -
- -
- - - {{ formatSpeed(task.uploadSpeed) }} - - {{ formatTime(task.totalTime ?? 0) }} elapsed, - {{ formatTime(task.estimatedTimeRemaining ?? 0) }} left - - - - Upload time: {{ formatTime(task.totalTime ?? 0) }} - -
-
-
- - - - - - - - - - - - - -
-
- - -
-
- -
-
- - + [resourceId]="did" + [ownerEmail]="ownerEmail" + [resourceName]="datasetName" + [endpoint]="datasetEndpoint" + [createVersion]="createDatasetVersion" + (versionCreated)="onVersionCreated()"> +
diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss index b9378d661ef..1cb00a8cf7c 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss +++ b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss @@ -17,22 +17,6 @@ * under the License. */ -.create-dataset-version-button { - display: flex; /* Use flexbox for centering */ - align-items: center; /* Center vertically */ - justify-content: center; /* Center horizontally */ - color: white; - border: none; - padding: 12px 40px; /* Increase padding for a wider button */ - border-radius: 25px; - cursor: pointer; - transition: background-color 0.3s; - margin: 18px auto 0 auto; /* Auto margins for horizontal centering */ - width: 200px; /* Adjust width as needed */ - font-size: 18px; /* Make text slightly bigger */ - font-weight: bold; /* Optional: Make text bold */ -} - .version-storage { padding: 0 15px; margin-bottom: 25px; @@ -143,78 +127,9 @@ nz-select { margin-top: 15%; } -.upload-progress-wrapper { - max-height: 25vh; - overflow-y: auto; - padding-right: 4px; -} - // Rows must stay exactly PENDING_ROW_HEIGHT_PX tall for the fixed-size scroll // strategy. The gutter lives on the rows: padding on the viewport never // reaches its absolutely positioned content wrapper. -.upload-progress-wrapper-pending { - .pending-file-row { - height: 32px; - display: flex; - align-items: center; - justify-content: space-between; - padding-right: 4px; - - .pending-file-name { - overflow: hidden; - white-space: nowrap; - text-overflow: ellipsis; - } - } -} - -.version-creator { - margin-top: 20px; - padding: 20px; -} - -.version-input-container label { - font-size: 15px; -} - -.version-input-container { - display: flex; - align-items: center; - gap: 10px; -} - -.version-input { - padding: 6px; -} - -.upload-stats { - font-size: 13px; - margin-bottom: 20px; - nz-progress { - width: 97%; - display: inline-block; - } -} - -:host ::ng-deep .upload-stats .ant-tag { - border: none; -} - -.fixed-width-speed { - display: inline-block; - min-width: 5ch; - text-align: right; -} - -.fixed-width-time { - display: inline-block; - min-width: 2ch; - text-align: right; -} - -.section-divider { - margin: 8px 0; -} .status-tag-row { margin-top: 16px; diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.spec.ts b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.spec.ts index 7aae86e1033..aeaa8392582 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.spec.ts @@ -27,12 +27,9 @@ import { NzModalService } from "ng-zorro-antd/modal"; import { NzResizableDirective } from "ng-zorro-antd/resizable"; import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; import { MarkdownService } from "ngx-markdown"; -import { - DatasetDetailComponent, - ABORT_RETRY_BACKOFF_BASE_MS, - ABORT_RETRY_MAX_ATTEMPTS, -} from "./dataset-detail.component"; +import { DatasetDetailComponent } from "./dataset-detail.component"; import { DatasetService, MultipartUploadProgress } from "../../../../service/user/dataset/dataset.service"; +import { VersionUploaderComponent } from "../../version-uploader/version-uploader.component"; import { NotificationService } from "../../../../../common/service/notification/notification.service"; import { DownloadService } from "../../../../service/user/download/download.service"; import { UserService } from "../../../../../common/service/user/user.service"; @@ -50,37 +47,10 @@ import { NzResizeEvent } from "ng-zorro-antd/resizable"; import { format } from "date-fns"; import { USER_DATASET } from "../../../../../app-routing.constant"; -describe("DatasetDetailComponent upload queue", () => { +describe("DatasetDetailComponent rendered explorer", () => { let fixture: ComponentFixture; let component: DatasetDetailComponent; - let uploadSubjects: Subject[]; - let uploadedPaths: string[]; - let multipartUploadSpy: ReturnType; - - const makeFileItem = (name: string): FileUploadItem => ({ - file: new File(["x"], name), - name, - description: "", - uploadProgress: 0, - isUploadingFlag: false, - restart: false, - }); - - const dropFiles = (...names: string[]) => component.onNewUploadFilesChanged(names.map(makeFileItem)); - - const finishUpload = (index: number, filePath: string, totalTime = 1) => - uploadSubjects[index].next({ filePath, percentage: 100, status: "finished", totalTime }); - beforeEach(() => { - uploadSubjects = []; - uploadedPaths = []; - multipartUploadSpy = vi.fn((_ownerEmail: string, _datasetName: string, filePath: string) => { - const progress = new Subject(); - uploadSubjects.push(progress); - uploadedPaths.push(filePath); - return progress.asObservable(); - }); - TestBed.configureTestingModule({ imports: [DatasetDetailComponent, ...commonTestImports], providers: [ @@ -89,8 +59,6 @@ describe("DatasetDetailComponent upload queue", () => { { provide: DatasetService, useValue: { - multipartUpload: multipartUploadSpy, - finalizeMultipartUpload: vi.fn(() => of({})), getDataset: vi.fn(() => of({ dataset: { name: "test-dataset", description: "", isPublic: false, isDownloadable: true }, @@ -145,377 +113,6 @@ describe("DatasetDetailComponent upload queue", () => { * A failed upload has to tell the user why, mark the task failed without leaving its bar at * 100%, and free the concurrency slot — otherwise the queue stalls behind a dead upload. */ - describe("a failed upload", () => { - const notification = () => TestBed.inject(NotificationService) as unknown as { error: ReturnType }; - - /** Fails the in-flight upload of `name` with the given HTTP status. */ - const failUpload = (index: number, status: number) => - uploadSubjects[index].error(new HttpErrorResponse({ status })); - - it("names the 409 conflict so the user knows to retry", () => { - dropFiles("a.csv"); - - failUpload(0, HttpStatusCode.Conflict); - - expect(notification().error).toHaveBeenCalledWith(expect.stringContaining("Upload blocked (409)")); - }); - - it("falls back to a generic message for any other failure", () => { - dropFiles("a.csv"); - - failUpload(0, HttpStatusCode.InternalServerError); - - expect(notification().error).toHaveBeenCalledWith("Upload failed. Please retry."); - }); - - it("marks the task failed and keeps its progress rather than showing it complete", () => { - dropFiles("a.csv"); - // a partially-uploaded file: the bar must not jump to 100 when it fails - uploadSubjects[0].next({ filePath: "a.csv", percentage: 42, status: "uploading" }); - - failUpload(0, HttpStatusCode.InternalServerError); - - const task = component.uploadTasks.find(t => t.filePath === "a.csv"); - expect(task?.status).toBe("failed"); - expect(task?.percentage).toBe(42); - }); - - it("frees the concurrency slot so a queued upload can start", () => { - // maxConcurrentFiles is 3, so a fourth file waits for a slot - dropFiles("a.csv", "b.csv", "c.csv", "d.csv"); - expect(uploadedPaths).toEqual(["a.csv", "b.csv", "c.csv"]); - - failUpload(0, HttpStatusCode.InternalServerError); - - expect(uploadedPaths).toContain("d.csv"); - }); - - it("still reports the failure when the task is no longer in the list", () => { - dropFiles("a.csv"); - component.uploadTasks = []; // the taskIndex === -1 arm - - expect(() => failUpload(0, HttpStatusCode.InternalServerError)).not.toThrow(); - expect(notification().error).toHaveBeenCalled(); - }); - }); - - /** - * A progress event and the five-second hide timer both address a row by its index in - * `uploadTasks`, and that row can already be gone — dismissed by the user — by the time - * either arrives, so both lookups have to survive the miss. The completion path also has - * to pick a key for `uploadTimeMap` out of a name that may carry directories. - */ - describe("progress bookkeeping", () => { - beforeEach(() => { - // The completion path arms a 5s row-hide timer; keep it off the real clock. - vi.useFakeTimers(); - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - it("ignores progress for a row that is no longer listed", () => { - dropFiles("a.csv"); - component.uploadTasks = []; // dismissed while a chunk was still in flight - - expect(() => uploadSubjects[0].next({ filePath: "a.csv", percentage: 50, status: "uploading" })).not.toThrow(); - // The late event must not resurrect the row or write a phantom index into the list. - expect(component.uploadTasks).toEqual([]); - expect(Object.keys(component.uploadTasks)).toHaveLength(0); - }); - - it("keys the upload time by the last path segment, falling back to the whole name", () => { - // Taking the segment after the last "/" yields "" for a name that ends in one, and - // keying the map under "" would collide every such upload onto one entry; the - // fallback keeps the name the caller gave instead. - dropFiles("nested/dir/"); - - finishUpload(0, "nested/dir/", 7); - - expect(component.uploadTimeMap.get("nested/dir/")).toBe(7); - expect(component.uploadTimeMap.has("")).toBe(false); - - // The reader of this map (user-dataset-staged-objects-list) looks a row up by - // `filePath.split("/").pop() || filePath`, so an ordinary nested name has to be - // keyed by its last segment here or the per-file time silently stops rendering. - dropFiles("dir/sub/a.csv"); - - finishUpload(1, "dir/sub/a.csv", 9); - - expect(component.uploadTimeMap.get("a.csv")).toBe(9); - expect(component.uploadTimeMap.has("dir/sub/a.csv")).toBe(false); - }); - - it("ignores a hide request for a row that is gone", () => { - // Every one of scheduleHide's call sites already checks the index, so the -1 arm - // pins a defensive no-op rather than a reachable scenario: without the guard the - // lookup would read `filePath` off undefined and throw. The valid-index call that - // follows keeps a scheduleHide which does nothing at all from passing this test. - dropFiles("a.csv"); - const before = [...component.uploadTasks]; - - expect(() => (component as any).scheduleHide(-1)).not.toThrow(); - expect(component.uploadTasks).toEqual(before); - - (component as any).scheduleHide(0); - vi.advanceTimersByTime(5000); - - expect(component.uploadTasks).toEqual([]); - }); - }); - - /** - * Aborting an in-flight upload has to survive the backend still finalizing the previous attempt: - * the abort call is retried on 409 up to ABORT_RETRY_MAX_ATTEMPTS, a 404 means it is already gone, - * and the caller's callback must fire exactly once down every one of those paths. - */ - describe("aborting an upload", () => { - let finalize: ReturnType; - - beforeEach(() => { - vi.useFakeTimers(); - finalize = TestBed.inject(DatasetService).finalizeMultipartUpload as unknown as ReturnType; - }); - - afterEach(() => { - vi.useRealTimers(); - }); - - /** Starts an upload and reports progress, leaving one task in flight. */ - function inFlight(name = "a.txt") { - dropFiles(name); - uploadSubjects[0].next({ filePath: name, percentage: 10, status: "uploading", totalTime: 0 }); - return component.uploadTasks.find(t => t.filePath === name)!; - } - - const conflict = () => throwError(() => ({ status: 409 }) as any); - const gone = () => throwError(() => ({ status: 404 }) as any); - - it("marks the task aborted and tells the caller once", () => { - const task = inFlight(); - const onAborted = vi.fn(); - - component.onClickAbortUploadProgress(task as any, onAborted); - - expect(finalize).toHaveBeenCalledWith("owner@texera.com", "test-dataset", "a.txt", true); - expect(component.uploadTasks.find(t => t.filePath === "a.txt")!.status).toBe("aborted"); - expect(onAborted).toHaveBeenCalledTimes(1); - - // The aborted row goes on the same five-second hide timer a finished one does, so - // it clears itself out of the list instead of sitting there for the rest of the session. - vi.advanceTimersByTime(5000); - - expect(component.uploadTasks.find(t => t.filePath === "a.txt")).toBeUndefined(); - }); - - it("stops listening to the upload it aborted", () => { - const task = inFlight(); - - component.onClickAbortUploadProgress(task as any); - - // The progress stream is unsubscribed, so a late event cannot resurrect the task. - uploadSubjects[0].next({ filePath: "a.txt", percentage: 100, status: "finished", totalTime: 1 }); - expect(component.uploadTasks.find(t => t.filePath === "a.txt")!.status).toBe("aborted"); - }); - - it("treats a 404 as already aborted rather than an error", () => { - finalize.mockReturnValueOnce(gone()); - const task = inFlight(); - const onAborted = vi.fn(); - - component.onClickAbortUploadProgress(task as any, onAborted); - - expect(onAborted).toHaveBeenCalledTimes(1); - expect(finalize).toHaveBeenCalledTimes(1); - }); - - it("retries a 409 after a backoff and finishes once the server catches up", () => { - // The server is still finalizing the previous attempt; the abort has to wait it out. - finalize.mockReturnValueOnce(conflict()); - const task = inFlight(); - const onAborted = vi.fn(); - - component.onClickAbortUploadProgress(task as any, onAborted); - expect(onAborted).not.toHaveBeenCalled(); - - vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS); - - expect(finalize).toHaveBeenCalledTimes(2); - expect(onAborted).toHaveBeenCalledTimes(1); - }); - - it("backs off further on each successive conflict", () => { - finalize.mockReturnValue(conflict()); - const task = inFlight(); - - component.onClickAbortUploadProgress(task as any); - expect(finalize).toHaveBeenCalledTimes(1); - - // First wait is BASE * 1, the second BASE * 2, so BASE alone is not enough for the third call. - vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS); - expect(finalize).toHaveBeenCalledTimes(2); - - vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS); - expect(finalize).toHaveBeenCalledTimes(2); - - vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS); - expect(finalize).toHaveBeenCalledTimes(3); - }); - - it("gives up after the attempt limit but still reports the abort", () => { - // Without the bound this would retry forever against a permanently conflicted server. - finalize.mockReturnValue(conflict()); - const task = inFlight(); - const onAborted = vi.fn(); - - component.onClickAbortUploadProgress(task as any, onAborted); - vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS * ABORT_RETRY_MAX_ATTEMPTS * (ABORT_RETRY_MAX_ATTEMPTS + 1)); - - expect(finalize).toHaveBeenCalledTimes(ABORT_RETRY_MAX_ATTEMPTS + 1); - expect(onAborted).toHaveBeenCalledTimes(1); - }); - - it("reports the abort once even on an error the retry does not cover", () => { - finalize.mockReturnValueOnce(throwError(() => ({ status: 500 }) as any)); - const task = inFlight(); - const onAborted = vi.fn(); - - component.onClickAbortUploadProgress(task as any, onAborted); - - expect(onAborted).toHaveBeenCalledTimes(1); - expect(finalize).toHaveBeenCalledTimes(1); - }); - - it("frees the concurrency slot so a queued upload can start", () => { - // Aborting has to release the slot as an ordinary completion would; otherwise the queue - // stalls behind an upload that is no longer running. - dropFiles("a.txt", "b.txt", "c.txt", "d.txt"); - expect(uploadedPaths).toEqual(["a.txt", "b.txt", "c.txt"]); - uploadSubjects[0].next({ filePath: "a.txt", percentage: 10, status: "uploading", totalTime: 0 }); - const task = component.uploadTasks.find(t => t.filePath === "a.txt")!; - - component.onClickAbortUploadProgress(task as any); - - expect(uploadedPaths).toContain("d.txt"); - }); - - it("cancelExistingUpload aborts an upload that is still running", () => { - inFlight("b.txt"); - const onCanceled = vi.fn(); - - component.cancelExistingUpload("b.txt", onCanceled); - - expect(finalize).toHaveBeenCalledWith("owner@texera.com", "test-dataset", "b.txt", true); - expect(onCanceled).toHaveBeenCalledTimes(1); - }); - - it("frees the slot of an upload aborted before its first part went out", () => { - // A task sits at "initializing" until the service reports its first progress. - // Cancelling in that window still has to hand the slot to whatever is queued - // behind it, or the queue stalls on an upload that never started. - dropFiles("a.txt", "b.txt", "c.txt", "d.txt"); - expect(uploadedPaths).toEqual(["a.txt", "b.txt", "c.txt"]); - const initializing = component.uploadTasks.find(t => t.filePath === "a.txt")!; - expect(initializing.status).toBe("initializing"); - - component.onClickAbortUploadProgress(initializing as any); - - expect(uploadedPaths).toContain("d.txt"); - expect(component.activeCount).toBe(3); - }); - - it("does not free a second slot when a finished upload's row is dismissed", () => { - // The row's button becomes "Close" once the upload is done, and the slot was - // already released by the completion; releasing it a second time would let a - // fourth upload run past the concurrency cap. - dropFiles("a.txt", "b.txt", "c.txt", "d.txt"); - finishUpload(0, "a.txt"); - expect(uploadedPaths).toContain("d.txt"); - const finished = component.uploadTasks.find(t => t.filePath === "a.txt")!; - expect(finished.status).toBe("finished"); - - component.onClickAbortUploadProgress(finished as any); - - expect(component.activeCount).toBe(3); - const dismissed = component.uploadTasks.find(t => t.filePath === "a.txt")!; - expect(dismissed.status).toBe("aborted"); - - // The row lingers for five seconds after being dismissed, so the same X is - // still there to be clicked again — and that click must not release either. - component.onClickAbortUploadProgress(dismissed as any); - - expect(component.activeCount).toBe(3); - }); - - it("does not free a second slot when a failed upload's row is dismissed", () => { - // The failure handler already released this upload's slot and let the queued - // fourth file start; dismissing the row it left behind must not release a - // second slot, or a fifth upload would run past the cap of three. - dropFiles("a.txt", "b.txt", "c.txt", "d.txt"); - uploadSubjects[0].error(new HttpErrorResponse({ status: 500 })); - expect(uploadedPaths).toContain("d.txt"); - const failed = component.uploadTasks.find(t => t.filePath === "a.txt")!; - expect(failed.status).toBe("failed"); - - component.onClickAbortUploadProgress(failed as any); - - expect(component.activeCount).toBe(3); - expect(multipartUploadSpy).toHaveBeenCalledTimes(4); - }); - - it("cancelExistingUpload aborts an upload whose first part has not gone out", () => { - // Until the service reports a first chunk the task sits at "initializing". - // A re-drop in that window has to abort that attempt rather than fall - // through and race a second multipart upload against it for the same path, - // which is exactly the 409 the upload error handler warns about. - dropFiles("b.txt"); - expect(component.uploadTasks.find(t => t.filePath === "b.txt")!.status).toBe("initializing"); - const onCanceled = vi.fn(); - - component.cancelExistingUpload("b.txt", onCanceled); - - expect(finalize).toHaveBeenCalledWith("owner@texera.com", "test-dataset", "b.txt", true); - expect(component.uploadTasks.find(t => t.filePath === "b.txt")!.status).toBe("aborted"); - // The slot goes back to the queue instead of being held by an attempt that - // is no longer running. - expect(component.activeCount).toBe(0); - expect(onCanceled).toHaveBeenCalledTimes(1); - }); - - it("tells the caller once even when the abort call reports more than once", () => { - // The callback is latched so that it fires exactly once no matter how many of the - // subscription's handlers reach it. HttpClient itself delivers a single response, - // so this drives the latch directly: a response followed by a stream failure runs - // the next handler and then the error handler, and both of them report done. - finalize.mockReturnValueOnce( - concat( - of({}), - throwError(() => ({ status: 500 }) as any) - ) - ); - const task = inFlight(); - const onAborted = vi.fn(); - - component.onClickAbortUploadProgress(task as any, onAborted); - - expect(onAborted).toHaveBeenCalledTimes(1); - }); - - it("aborts a task whose row was already dropped without resurrecting it", () => { - const task = inFlight(); - component.uploadTasks = []; // the row was dismissed before the abort was clicked - - component.onClickAbortUploadProgress(task as any); - - expect(finalize).toHaveBeenCalledWith("owner@texera.com", "test-dataset", "a.txt", true); - // Writing "aborted" back at a missing index would leave a phantom "-1" property on - // the array, which neither a throw nor `.length` would reveal. - expect(component.uploadTasks).toEqual([]); - expect(Object.keys(component.uploadTasks)).toHaveLength(0); - }); - }); - /** * The explorer's toolbar and upload panel are template-only: whether a download is offered at all, * which of the maximize/minimize pair is showing, and what an in-flight upload reports. The suite @@ -678,64 +275,15 @@ describe("DatasetDetailComponent upload queue", () => { }); }); - describe("upload progress", () => { - /** - * Puts one task on the panel in the given state and opens it. The panel is gated on the - * separate activeUploads counter rather than on uploadTasks, and ng-zorro collapses it by - * default, so both have to be arranged before its body exists. - */ - function withTask(over: Record): HTMLElement { - const el = render(c => { - (c as any).activeUploads = 1; - (c as any).uploadTasks = [ - { - filePath: "big.csv", - percentage: 40, - status: "uploading", - uploadSpeed: 1024, - totalTime: 12, - estimatedTimeRemaining: 30, - ...over, - }, - ]; - }); - const header = Array.from(el.querySelectorAll(".ant-collapse-header")).find(h => - (h.textContent || "").includes("Uploading:") - ); - header!.click(); - fixture.detectChanges(); - return el; - } - - it("shows no statistics while an upload is still initializing", () => { - // There is nothing to report yet; showing a 0 B/s row reads as a stalled upload. - const el = withTask({ status: "initializing" }); - - expect(el.querySelector(".upload-stats")).toBeNull(); - }); - - it("reports speed and both timings while an upload runs", () => { - const el = withTask({ status: "uploading" }); - - const stats = el.querySelector(".upload-stats")!; - expect(stats.textContent).toContain("elapsed"); - expect(stats.textContent).toContain("left"); - expect(stats.querySelector(".fixed-width-speed")).not.toBeNull(); - }); - - it("replaces the live figures with a total once the upload finishes", () => { - const el = withTask({ status: "finished" }); - - const stats = el.querySelector(".upload-stats")!; - expect(stats.textContent).toContain("Upload time:"); - expect(stats.textContent).not.toContain("left"); - }); + it("stages a file deletion and lets the panel count it", () => { + render(); // the panel lives in the Versions & Files tab, which nz-tabs renders lazily + const node: DatasetFileNode = { name: "a.txt", type: "file", parentDir: "/owner@texera.com/test-dataset/v1" }; - it("reports a total for an aborted upload too", () => { - const el = withTask({ status: "aborted" }); + component.onPreviouslyUploadedFileDeleted(node); - expect(el.querySelector(".upload-stats")!.textContent).toContain("Upload time:"); - }); + const panel = fixture.debugElement.query(By.directive(VersionUploaderComponent)) + .componentInstance as VersionUploaderComponent; + expect(panel.pendingChangesCount).toBe(1); }); }); @@ -788,247 +336,6 @@ describe("DatasetDetailComponent upload queue", () => { expect(onAdd).toHaveBeenCalledTimes(1); }); }); - - it("starts at most maxConcurrentFiles uploads immediately and queues the rest", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); - - expect(multipartUploadSpy).toHaveBeenCalledTimes(3); - expect(uploadedPaths).toEqual(["f1.txt", "f2.txt", "f3.txt"]); - expect(component.activeCount).toBe(3); - expect(component.queuedCount).toBe(2); - expect(component.queuedFileNames).toEqual(["f4.txt", "f5.txt"]); - }); - - it("does nothing when an empty file list is dropped", () => { - dropFiles(); - - expect(multipartUploadSpy).not.toHaveBeenCalled(); - expect(component.activeCount).toBe(0); - expect(component.queuedCount).toBe(0); - expect(component.queuedFileNames).toEqual([]); - }); - - it("starts no upload at all when the route carried no dataset id", () => { - // Every multipart call is addressed to a dataset, so without one there is - // nowhere to upload into: the drop is refused outright rather than leaving - // rows on the panel for uploads that were never started. - component.did = undefined; - - dropFiles("f1.txt", "f2.txt"); - - expect(multipartUploadSpy).not.toHaveBeenCalled(); - expect(component.uploadTasks).toEqual([]); - expect(component.activeCount).toBe(0); - }); - - it("starts the next queued upload when an active upload finishes", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); - - finishUpload(0, "f1.txt"); - - expect(multipartUploadSpy).toHaveBeenCalledTimes(4); - expect(uploadedPaths[3]).toBe("f4.txt"); - expect(component.activeCount).toBe(3); - expect(component.queuedCount).toBe(1); - expect(component.queuedFileNames).toEqual(["f5.txt"]); - }); - - it("removes a cancelled file from the pending queue without starting it", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); - - component.cancelExistingUpload("f4.txt"); - - expect(multipartUploadSpy).toHaveBeenCalledTimes(3); - expect(component.queuedCount).toBe(1); - expect(component.queuedFileNames).toEqual(["f5.txt"]); - }); - - it("ignores cancellation of a file that is neither active nor queued", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); - - component.cancelExistingUpload("missing.txt"); - - expect(component.activeCount).toBe(3); - expect(component.queuedCount).toBe(1); - expect(component.queuedFileNames).toEqual(["f4.txt"]); - }); - - // #5586: the template reads queuedFileNames on every change-detection pass, - // so it must not allocate a new array unless the queue changed. - it("keeps the same queuedFileNames array reference while the queue is unchanged", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); - - const firstRead = component.queuedFileNames; - - expect(component.queuedFileNames).toBe(firstRead); - }); - - it("exposes a new queuedFileNames array after the queue changes", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); - const beforeCancel = component.queuedFileNames; - - component.cancelExistingUpload("f4.txt"); - - expect(component.queuedFileNames).not.toBe(beforeCancel); - expect(component.queuedFileNames).toEqual(["f5.txt"]); - }); - - it("identifies pending queue entries by file name in trackByPendingFile", () => { - expect(component.trackByPendingFile(0, "dir/a.txt")).toBe("dir/a.txt"); - }); - - // A resumed upload with no missing parts finishes with totalTime exactly 0; - // the slot must still be released. - it("releases the concurrency slot when a finished upload reports totalTime 0", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); - - finishUpload(0, "f1.txt", 0); - - expect(multipartUploadSpy).toHaveBeenCalledTimes(4); - expect(uploadedPaths[3]).toBe("f4.txt"); - expect(component.activeCount).toBe(3); - expect(component.queuedCount).toBe(0); - }); - - // The Pending header updates per file, so the Finished header must too — it - // cannot wait for the throttled staged-objects refetch. - it("updates the Finished count immediately when uploads finish", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); - expect(component.pendingChangesCount).toBe(0); - - finishUpload(0, "f1.txt"); - expect(component.pendingChangesCount).toBe(1); - - finishUpload(1, "f2.txt"); - expect(component.pendingChangesCount).toBe(2); - }); - - it("reconciles the optimistic Finished count with a diff response", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt"); - finishUpload(0, "f1.txt"); - finishUpload(1, "f2.txt"); - - const diff: DatasetStagedObject[] = [{ path: "f1.txt", pathType: "file", diffType: "added", sizeBytes: 1 }]; - component.onStagedObjectsUpdated(diff); - - // f1 is confirmed by the response; f2 stays counted until a response includes it. - expect(component.pendingChangesCount).toBe(2); - - component.onStagedObjectsUpdated([...diff, { path: "f2.txt", pathType: "file", diffType: "added", sizeBytes: 1 }]); - expect(component.pendingChangesCount).toBe(2); - }); - - it("keeps an in-progress upload's slot while progress events stream in", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); - - uploadSubjects[0].next({ filePath: "f1.txt", percentage: 50, status: "uploading" }); - - expect(component.uploadTasks.find(t => t.filePath === "f1.txt")?.percentage).toBe(50); - expect(component.activeCount).toBe(3); - expect(component.queuedCount).toBe(1); - }); - - it("does not double-count a finished upload already confirmed by a diff response", () => { - dropFiles("f1.txt"); - finishUpload(0, "f1.txt"); - component.onStagedObjectsUpdated([{ path: "f1.txt", pathType: "file", diffType: "added", sizeBytes: 1 }]); - expect(component.pendingChangesCount).toBe(1); - - dropFiles("f1.txt"); // re-upload the already-staged file - finishUpload(1, "f1.txt"); - - expect(component.pendingChangesCount).toBe(1); - }); - - it("does not start queued uploads beyond a lowered concurrency limit", () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); - component.maxConcurrentFiles = 1; - - finishUpload(0, "f1.txt"); - - expect(component.activeCount).toBe(2); - expect(component.queuedCount).toBe(1); - expect(multipartUploadSpy).toHaveBeenCalledTimes(3); - }); - - it("clears the Finished count when a version is created", () => { - dropFiles("f1.txt"); - finishUpload(0, "f1.txt"); - expect(component.pendingChangesCount).toBe(1); - - component.versionName = "v1"; - component.onClickOpenVersionCreator(); - - expect(component.pendingChangesCount).toBe(0); - }); - - it("does not remove a re-uploaded file's active task when hiding its finished predecessor", () => { - vi.useFakeTimers(); - try { - dropFiles("a.txt"); - finishUpload(0, "a.txt"); // schedules the finished row to hide in 5s - - dropFiles("a.txt"); // re-upload the same name within the 5s window - vi.advanceTimersByTime(5000); - - expect(component.uploadTasks).toHaveLength(1); - expect(component.uploadTasks[0].status).not.toBe("finished"); - expect(component.activeCount).toBe(1); - - finishUpload(1, "a.txt"); - expect(component.activeCount).toBe(0); - } finally { - vi.useRealTimers(); - } - }); - - it("renders the virtualized pending list and re-measures viewports on panel expand", async () => { - dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); - - // The upload UI lives in the "Versions & Files" tab; nz-tabs does not render a - // tab's content into the DOM until it has been selected at least once. - const tabButtons: NodeListOf = fixture.nativeElement.querySelectorAll(".ant-tabs-tab-btn"); - const versionsTab = Array.from(tabButtons).find(tab => tab.textContent?.includes("Versions & Files")); - expect(versionsTab).toBeTruthy(); - (versionsTab as HTMLElement).click(); - fixture.detectChanges(); - - // Flush the viewport's init microtask, then render the rows. - await Promise.resolve(); - fixture.detectChanges(); - - expect(component.pendingListHeightPx).toBe(2 * component.PENDING_ROW_HEIGHT_PX); - const rows = fixture.nativeElement.querySelectorAll(".pending-file-row"); - expect(rows.length).toBe(2); - - // Expand the Pending / Uploading / Finished panels. - const headers: NodeListOf = fixture.nativeElement.querySelectorAll( - ".upload-status-panels .ant-collapse-header" - ); - expect(headers.length).toBe(3); - headers.forEach(header => header.click()); - fixture.detectChanges(); - // Flush the checkViewportSize timers. - await new Promise(resolve => setTimeout(resolve)); - - // Collapsing again must be a no-op for the re-measure handler. - headers.forEach(header => header.click()); - fixture.detectChanges(); - - // Cancel a queued file from its row. - const cancelButton = fixture.nativeElement.querySelector(".pending-file-row button") as HTMLButtonElement; - cancelButton.click(); - expect(component.queuedCount).toBe(1); - expect(component.queuedFileNames).toEqual(["f5.txt"]); - }); - - it("counts a staged file deletion immediately", () => { - const node: DatasetFileNode = { name: "a.txt", type: "file", parentDir: "/owner@texera.com/test-dataset/v1" }; - - component.onPreviouslyUploadedFileDeleted(node); - - expect(component.pendingChangesCount).toBe(1); - }); }); describe("DatasetDetailComponent behavior", () => { @@ -1164,10 +471,9 @@ describe("DatasetDetailComponent behavior", () => { expect(component.likeCount).toBe(7); expect(component.viewCount).toBe(42); expect(hubServiceStub.isLiked).not.toHaveBeenCalled(); - expect(adminSettingsServiceStub.getPublicSetting).not.toHaveBeenCalled(); }); - it("fetches liked status and upload settings for a logged-in user", () => { + it("fetches liked status for a logged-in user", () => { hubServiceStub.isLiked.mockReturnValue(of([{ isLiked: true }])); createComponent({ did: 5 }); @@ -1176,19 +482,6 @@ describe("DatasetDetailComponent behavior", () => { expect(hubServiceStub.isLiked).toHaveBeenCalled(); expect(component.isLiked).toBe(true); - expect(adminSettingsServiceStub.getPublicSetting).toHaveBeenCalled(); - }); - - it("keeps the default upload settings when the public settings are missing", () => { - adminSettingsServiceStub.getPublicSetting.mockReturnValue(of(null)); - - createComponent({ did: 5 }); - login(); - fixture.detectChanges(); - - expect(component.chunkSizeMiB).toBe(50); - expect(component.maxConcurrentChunks).toBe(10); - expect(component.maxConcurrentFiles).toBe(3); }); it("makes no hub calls when the route carries no did", () => { @@ -1222,46 +515,6 @@ describe("DatasetDetailComponent behavior", () => { // or some other action's: nothing else in the suite pins these arguments. expect(hubServiceStub.getCounts).toHaveBeenCalledWith([EntityType.Dataset], [5], [ActionType.Like]); }); - - it("leaves the chunk size untouched when only that setting fails to load", () => { - // A distinct value per key, so a setting that lands in the wrong field is - // visible: 7 chunks and 2 files cannot stand in for one another. - adminSettingsServiceStub.getPublicSetting.mockImplementation((key: string) => - key === "dataset_multipart_upload_chunk_size_mib" - ? throwError(() => new Error("boom")) - : of(key === "dataset_max_number_of_concurrent_uploading_file_chunks" ? "7" : "2") - ); - - createComponent({ did: 5 }); - // A sentinel the class default cannot supply, so "the failed fetch wrote - // nothing" is distinguishable from "it wrote the default back". - component.chunkSizeMiB = 42; - login(); - fixture.detectChanges(); - - expect(component.chunkSizeMiB).toBe(42); - expect(component.maxConcurrentChunks).toBe(7); - expect(component.maxConcurrentFiles).toBe(2); - }); - - it("leaves both concurrency limits untouched when their settings fail to load", () => { - adminSettingsServiceStub.getPublicSetting.mockImplementation((key: string) => - key === "dataset_multipart_upload_chunk_size_mib" ? of("128") : throwError(() => new Error("boom")) - ); - - createComponent({ did: 5 }); - // Sentinels again. A failed fetch that wrote anything here — a reset or a - // NaN — would stall the queue outright, since `activeUploads < NaN` is - // never true, and a plain default-valued assertion could not see it. - component.maxConcurrentChunks = 41; - component.maxConcurrentFiles = 40; - login(); - fixture.detectChanges(); - - expect(component.chunkSizeMiB).toBe(128); - expect(component.maxConcurrentChunks).toBe(41); - expect(component.maxConcurrentFiles).toBe(40); - }); }); describe("retrieveDatasetInfo", () => { @@ -1747,49 +1000,27 @@ describe("DatasetDetailComponent behavior", () => { }); }); - describe("onClickOpenVersionCreator", () => { - it("creates a version, clears the name, refreshes the list and emits a change on success", () => { + describe("creating a version", () => { + it("hands the panel a call that commits through DatasetService", () => { datasetServiceStub.createDatasetVersion.mockReturnValue(of(makeVersion())); - datasetServiceStub.retrieveDatasetVersionList.mockReturnValue(of([])); createComponent(); component.did = 5; - component.versionName = "v2"; - const emit = vi.fn(); - component.userMakeChanges.subscribe(emit); - component.onClickOpenVersionCreator(); + component.createDatasetVersion("v2").subscribe(); expect(datasetServiceStub.createDatasetVersion).toHaveBeenCalledWith(5, "v2"); - expect(notificationServiceStub.success).toHaveBeenCalledWith("Version Created"); - expect(component.versionName).toBe(""); - expect(component.isCreatingVersion).toBe(false); - expect(datasetServiceStub.retrieveDatasetVersionList).toHaveBeenCalled(); - expect(datasetServiceStub.retrieveDatasetLatestVersion).toHaveBeenCalled(); - expect(emit).toHaveBeenCalled(); - }); - - it("surfaces the backend message and resets the in-progress flag on failure", () => { - datasetServiceStub.createDatasetVersion.mockReturnValue(throwError(() => ({ error: { message: "boom" } }))); - createComponent(); - component.did = 5; - component.versionName = "v2"; - - component.onClickOpenVersionCreator(); - - expect(notificationServiceStub.error).toHaveBeenCalledWith("Version creation failed: boom"); - expect(component.isCreatingVersion).toBe(false); }); - it("ignores a second click while a version creation is already in progress", () => { - datasetServiceStub.createDatasetVersion.mockReturnValue(new Subject()); + it("reloads the version list and the latest-version facts once the panel reports one", () => { + datasetServiceStub.retrieveDatasetVersionList.mockClear(); + datasetServiceStub.retrieveDatasetLatestVersion.mockClear(); createComponent(); component.did = 5; - component.onClickOpenVersionCreator(); - component.onClickOpenVersionCreator(); + component.onVersionCreated(); - expect(datasetServiceStub.createDatasetVersion).toHaveBeenCalledTimes(1); - expect(component.isCreatingVersion).toBe(true); + expect(datasetServiceStub.retrieveDatasetVersionList).toHaveBeenCalled(); + expect(datasetServiceStub.retrieveDatasetLatestVersion).toHaveBeenCalled(); }); }); @@ -1852,23 +1083,9 @@ describe("DatasetDetailComponent behavior", () => { }); }); - describe("staged objects and view flags", () => { + describe("view flags", () => { beforeEach(() => createComponent()); - it("tracks the pending-change count from staged objects", () => { - const staged: DatasetStagedObject[] = [ - { path: "a", pathType: "file", diffType: "added", sizeBytes: 1 }, - { path: "b", pathType: "file", diffType: "added", sizeBytes: 1 }, - ]; - component.onStagedObjectsUpdated(staged); - expect(component.pendingChangesCount).toBe(2); - expect(component.userHasPendingChanges).toBe(true); - - component.onStagedObjectsUpdated([]); - expect(component.pendingChangesCount).toBe(0); - expect(component.userHasPendingChanges).toBe(false); - }); - it("toggles the maximize, right-bar and precise-view-count flags", () => { expect(component.isMaximized).toBe(false); component.onClickScaleTheView(); @@ -2182,15 +1399,7 @@ describe("DatasetDetailComponent behavior", () => { }); }); - describe("upload status, version-node selection, and trackBy", () => { - it("getUploadStatus maps the upload status to a progress state", () => { - expect(component.getUploadStatus("uploading")).toBe("active"); - expect(component.getUploadStatus("initializing")).toBe("active"); - expect(component.getUploadStatus("aborted")).toBe("exception"); - expect(component.getUploadStatus("failed")).toBe("exception"); - expect(component.getUploadStatus("finished")).toBe("success"); - }); - + describe("version-node selection", () => { it("onVersionFileTreeNodeSelected loads the selected node's content", () => { const node = { name: "file.csv", type: "file" } as unknown as Parameters< typeof component.onVersionFileTreeNodeSelected @@ -2203,11 +1412,6 @@ describe("DatasetDetailComponent behavior", () => { expect(loadSpy).toHaveBeenCalledWith(node); }); - - it("trackByTask returns the task's file path", () => { - const task = { filePath: "owner/data/file.csv" } as unknown as Parameters[1]; - expect(component.trackByTask(0, task)).toBe("owner/data/file.csv"); - }); }); describe("onPreviouslyUploadedFileDeleted", () => { @@ -2230,35 +1434,24 @@ describe("DatasetDetailComponent behavior", () => { datasetServiceStub.deleteDatasetFile.mockReturnValue(throwError(() => new Error("boom"))); createComponent(); component.did = 5; - const emit = vi.fn(); - component.userMakeChanges.subscribe(emit); component.onPreviouslyUploadedFileDeleted(node); expect(notificationServiceStub.error).toHaveBeenCalledWith("Failed to delete the file"); - // A file the backend still holds is not a staged change, so counting it would - // offer a version to create out of a deletion that never happened. - expect(component.pendingChangesCount).toBe(0); - expect(component.userHasPendingChanges).toBe(false); - expect(emit).not.toHaveBeenCalled(); + // A file the backend still holds is not a staged change, so reporting it to the panel + // would offer a version to create out of a deletion that never happened. + expect(notificationServiceStub.success).not.toHaveBeenCalled(); }); - it("stages the deletion under the same path the next diff response confirms", () => { + it("deletes by the path relative to the version root, which is what the diff reports", () => { createComponent(); component.did = 5; component.onPreviouslyUploadedFileDeleted(node); + // Only an exact match retires the locally staged entry the panel counts, so any + // other form of this path would leave the Finished header double-counting. expect(datasetServiceStub.deleteDatasetFile).toHaveBeenCalledWith(5, "nested/a.txt"); - expect(component.pendingChangesCount).toBe(1); - - // The diff response reports staged paths relative to the version root, and - // only an exact match retires the locally staged entry. A key in any other - // form never reconciles, so the Finished header counts this one deletion - // twice until the next version is created. - component.onStagedObjectsUpdated([{ path: "nested/a.txt", pathType: "file", diffType: "removed", sizeBytes: 0 }]); - - expect(component.pendingChangesCount).toBe(1); }); }); @@ -3281,176 +2474,6 @@ describe("DatasetDetailComponent rendered template", () => { }); }); - describe("upload panel", () => { - beforeEach(() => render({ did: 5, userDatasetAccessLevel: "WRITE" })); - - it("starts an upload for a file the uploader hands over", () => { - const el = openTab("Versions & Files"); - const uploader = fixture.debugElement.query(By.css("texera-user-files-uploader")); - - uploader.triggerEventHandler("uploadedFiles", [makeFileItem("new.csv")]); - fixture.detectChanges(); - - // The chunk size and the chunk concurrency are both plain numbers, so - // asserting their exact values is the only way to notice them exchanged: - // 10-byte chunks, or 52 million parallel requests, would look identical to - // expect.any(Number). Nobody is signed in here, so the component keeps its - // built-in defaults rather than the admin settings. - expect(component.chunkSizeMiB).toBe(50); - expect(component.maxConcurrentChunks).toBe(10); - expect(datasetService.multipartUpload).toHaveBeenCalledWith( - OWNER, - "ds", - "new.csv", - expect.anything(), - 50 * 1024 * 1024, - 10, - false - ); - expect(text(el)).toContain("Uploading: 1 file(s)"); - }); - - /** Renders the given in-flight tasks and expands the "Uploading" panel. */ - const withTasks = (...tasks: Array>): HTMLElement => { - const el = render({ - uploadTasks: tasks.map(t => ({ - percentage: 40, - status: "uploading", - uploadSpeed: 1024, - totalTime: 12, - estimatedTimeRemaining: 30, - ...t, - })) as never, - }); - (component as unknown as { activeUploads: number }).activeUploads = tasks.length; - openTab("Versions & Files"); - openPanel("Uploading:"); - return el; - }; - - it("aborts the upload whose own row button was clicked", () => { - withTasks({ filePath: "first.csv" }, { filePath: "second.csv" }); - - const rows = fixture.debugElement.queryAll(By.css(".upload-progress-wrapper > div")); - expect(rows.length).toBe(2); - // Each row has to name its own task: identifying the row by position alone - // would not notice every row rendering the first task's name and status. - expect(rows.map(row => text(row.query(By.css(".progress-header")).nativeElement))).toEqual([ - "uploading: first.csv", - "uploading: second.csv", - ]); - - const abort = rows[1].query(By.css(".progress-header button")); - // A live upload is cancelled, not dismissed; the finished row below says "Close". - expect(abort.injector.get(NzTooltipDirective).directiveTitle).toBe("Cancel the upload"); - - abort.nativeElement.click(); - fixture.detectChanges(); - - expect(datasetService.finalizeMultipartUpload).toHaveBeenCalledTimes(1); - expect(datasetService.finalizeMultipartUpload).toHaveBeenCalledWith(OWNER, "ds", "second.csv", true); - }); - - it("reports the elapsed time, the time remaining and the speed in their own slots", () => { - // Distinguishable timings, so the two spans cannot stand in for each other: - // showing 90s elapsed on a 12s-old upload is the defect this guards. - const el = withTasks({ - filePath: "big.csv", - totalTime: 12, - estimatedTimeRemaining: 90, - uploadSpeed: 5 * 1024 * 1024, - }); - const stats = q(el, ".upload-stats"); - - expect(Array.from(stats.querySelectorAll(".fixed-width-time")).map(text)).toEqual(["12s", "1m30s left"]); - expect(text(q(stats, ".fixed-width-speed"))).toBe("5.0 MB/s"); - }); - - it("floors both live timings at one second while an upload reports none", () => { - const el = withTasks({ filePath: "big.csv", totalTime: undefined, estimatedTimeRemaining: undefined }); - - const times = Array.from(q(el, ".upload-stats").querySelectorAll(".fixed-width-time")).map(text); - expect(times).toEqual(["1s", "1s left"]); - }); - - it("reports the total time of a finished upload", () => { - const el = withTasks({ filePath: "big.csv", status: "finished", totalTime: 75 }); - - expect(text(q(el, ".upload-stats"))).toContain("Upload time: 1m15s"); - // A finished row is dismissed rather than cancelled. - const button = fixture.debugElement.query(By.css(".upload-progress-wrapper > div .progress-header button")); - expect(button.injector.get(NzTooltipDirective).directiveTitle).toBe("Close"); - }); - - it("floors the total of a finished upload that timed nothing", () => { - const el = withTasks({ filePath: "big.csv", status: "finished", totalTime: undefined }); - - expect(text(q(el, ".upload-stats"))).toContain("Upload time: 1s"); - }); - }); - - describe("version creator", () => { - /** Renders the creator, which only appears with staged changes to commit. */ - const withPendingChanges = (state: Partial = {}): HTMLElement => { - const el = render({ did: 5, userDatasetAccessLevel: "WRITE", userHasPendingChanges: true, ...state }); - openTab("Versions & Files"); - return el; - }; - - const typeName = (el: HTMLElement, value: string): HTMLInputElement => { - const input = q(el, ".version-input"); - input.value = value; - input.dispatchEvent(new Event("input")); - fixture.detectChanges(); - return input; - }; - - it("offers the creator only once there is something to commit", () => { - const el = render({ did: 5, userDatasetAccessLevel: "WRITE", userHasPendingChanges: false }); - openTab("Versions & Files"); - expect(el.querySelector(".version-creator")).toBeNull(); - - render({ userHasPendingChanges: true }); - - expect(el.querySelector(".version-creator")).not.toBeNull(); - expect(text(q(el, ".create-dataset-version-button"))).toBe("Submit"); - }); - - it("creates a version named by the creator's own input", () => { - const el = withPendingChanges(); - - typeName(el, "second cut"); - q(el, ".create-dataset-version-button").click(); - - expect(datasetService.createDatasetVersion).toHaveBeenCalledWith(5, "second cut"); - }); - - it("submits the version straight from the name field with Enter", () => { - const el = withPendingChanges(); - - typeName(el, "from the keyboard").dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true })); - - expect(datasetService.createDatasetVersion).toHaveBeenCalledWith(5, "from the keyboard"); - }); - - it("spins the submit button and locks the name field while a version is being created", async () => { - const el = withPendingChanges(); - expect(q(el, ".create-dataset-version-button").classList).not.toContain("ant-btn-loading"); - expect(q(el, ".version-input").disabled).toBe(false); - - render({ isCreatingVersion: true }); - // NgModel routes the input's `disabled` binding through control.disable(), - // which it defers to a microtask, so the DOM lags the render by one turn. - await Promise.resolve(); - fixture.detectChanges(); - - expect(q(el, ".create-dataset-version-button").classList).toContain("ant-btn-loading"); - // Renaming a version mid-creation would be applied to nothing, so the - // field is locked for as long as the request is in flight. - expect(q(el, ".version-input").disabled).toBe(true); - }); - }); - describe("settings tab", () => { it("persists a description edited on the Settings tab", () => { render({ did: 5, userDatasetAccessLevel: "WRITE", datasetDescription: "old" }); diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts index 2f2e8b209ec..01dd8b3d84f 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts +++ b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts @@ -17,16 +17,12 @@ * under the License. */ -import { Component, EventEmitter, OnInit, Output, ViewChild } from "@angular/core"; +import { Component, OnInit, ViewChild } from "@angular/core"; import { ActivatedRoute, Router } from "@angular/router"; import { USER_DATASET } from "../../../../../app-routing.constant"; import { extractErrorMessage } from "../../../../../common/util/error"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; -import { - DatasetService, - MultipartUploadProgress, - validateDatasetName, -} from "../../../../service/user/dataset/dataset.service"; +import { DatasetService, validateDatasetName } from "../../../../service/user/dataset/dataset.service"; import { NzResizeEvent, NzResizableDirective, NzResizeHandleComponent } from "ng-zorro-antd/resizable"; import { DatasetFileNode, @@ -41,13 +37,10 @@ import { formatSize } from "src/app/common/util/size-formatter.util"; import { UserService } from "../../../../../common/service/user/user.service"; import { isDefined } from "../../../../../common/util/predicate"; import { ActionType, EntityType, HubService, LikedStatus } from "../../../../../hub/service/hub.service"; -import { FileUploadItem } from "../../../../type/dashboard-file.interface"; -import { DatasetStagedObject } from "../../../../../common/type/dataset-staged-object"; import { NzModalService } from "ng-zorro-antd/modal"; -import { AdminSettingsService } from "../../../../service/admin/settings/admin-settings.service"; -import { HttpErrorResponse, HttpStatusCode } from "@angular/common/http"; -import { EMPTY, Subscription } from "rxjs"; -import { formatCount, formatSpeed, formatTime, parseIntOrDefault } from "src/app/common/util/format.util"; +import { HttpErrorResponse } from "@angular/common/http"; +import { EMPTY, Observable, Subscription } from "rxjs"; +import { formatCount, formatSpeed, formatTime } from "src/app/common/util/format.util"; import { replaceOneImmutable } from "src/app/common/util/array-utils"; import { format } from "date-fns"; import { NgIf, NgClass, NgFor } from "@angular/common"; @@ -74,15 +67,11 @@ import { NzCollapseComponent, NzCollapsePanelComponent } from "ng-zorro-antd/col import { NzSelectComponent, NzOptionComponent } from "ng-zorro-antd/select"; import { UserDatasetVersionFiletreeComponent } from "./user-dataset-version-filetree/user-dataset-version-filetree.component"; import { NzDividerComponent } from "ng-zorro-antd/divider"; -import { FilesUploaderComponent } from "../../files-uploader/files-uploader.component"; -import { NzProgressComponent } from "ng-zorro-antd/progress"; -import { UserDatasetStagedObjectsListComponent } from "./user-dataset-staged-objects-list/user-dataset-staged-objects-list.component"; +import { VersionUploaderComponent } from "../../version-uploader/version-uploader.component"; +import { DATASET_FILE_RESOURCE_ENDPOINT } from "../../../../service/user/file-resource/file-resource-endpoint"; import { NzInputDirective } from "ng-zorro-antd/input"; -import { CdkFixedSizeVirtualScroll, CdkVirtualForOf, CdkVirtualScrollViewport } from "@angular/cdk/scrolling"; export const THROTTLE_TIME_MS = 1000; -export const ABORT_RETRY_MAX_ATTEMPTS = 10; -export const ABORT_RETRY_BACKOFF_BASE_MS = 100; @UntilDestroy() @Component({ @@ -120,13 +109,9 @@ export const ABORT_RETRY_BACKOFF_BASE_MS = 100; NzOptionComponent, UserDatasetVersionFiletreeComponent, NzDividerComponent, - FilesUploaderComponent, - NzProgressComponent, - UserDatasetStagedObjectsListComponent, + VersionUploaderComponent, NzInputDirective, - CdkVirtualScrollViewport, - CdkFixedSizeVirtualScroll, - CdkVirtualForOf, + VersionUploaderComponent, NzDropdownDirective, NzDropdownMenuComponent, NzMenuDirective, @@ -177,47 +162,9 @@ export class DatasetDetailComponent implements OnInit { public viewCount: number = 0; public displayPreciseViewCount = false; - userHasPendingChanges: boolean = false; - pendingChangesCount: number = 0; - // Staged paths from the last diff response, plus locally staged paths not yet - // in one: counted together so the Finished header keeps pace with the - // real-time Pending header between throttled refetches. - private confirmedStagedPaths = new Set(); - private unconfirmedStagedPaths = new Set(); - - // Uploading setting - chunkSizeMiB: number = 50; - maxConcurrentChunks: number = 10; - private uploadSubscriptions = new Map(); - uploadTimeMap = new Map(); - - // Cap number of concurrent files uploads - maxConcurrentFiles: number = 3; - private activeUploads: number = 0; - // FIFO queue of uploads waiting for a concurrency slot, keyed by file name. - private pendingQueue = new Map void>(); - private pendingQueueDirty = false; - private queuedFileNamesSnapshot: string[] = []; - - // Row height must match .pending-file-row in the SCSS. - readonly PENDING_ROW_HEIGHT_PX = 32; - readonly PENDING_LIST_MAX_HEIGHT_PX = 160; + readonly datasetEndpoint = DATASET_FILE_RESOURCE_ENDPOINT; - @ViewChild(CdkVirtualScrollViewport) private pendingViewport?: CdkVirtualScrollViewport; - - versionName: string = ""; - isCreatingVersion: boolean = false; - - public activeMultipartFilePaths: string[] = []; - - // List of upload tasks – each task tracked by its filePath - public uploadTasks: Array< - MultipartUploadProgress & { - filePath: string; - } - > = []; - - @Output() userMakeChanges = new EventEmitter(); + @ViewChild(VersionUploaderComponent) private versionUploader?: VersionUploaderComponent; constructor( private route: ActivatedRoute, @@ -227,8 +174,7 @@ export class DatasetDetailComponent implements OnInit { private notificationService: NotificationService, private downloadService: DownloadService, private userService: UserService, - private hubService: HubService, - private adminSettingsService: AdminSettingsService + private hubService: HubService ) { this.userService .userChanged() @@ -294,37 +240,15 @@ export class DatasetDetailComponent implements OnInit { .subscribe((isLiked: LikedStatus[]) => { this.isLiked = isLiked.length > 0 ? isLiked[0].isLiked : false; }); - - this.loadUploadSettings(); } - public onClickOpenVersionCreator() { - if (this.did && !this.isCreatingVersion) { - this.isCreatingVersion = true; + /** Commits the staged files; the panel owns the rest of the version flow. */ + createDatasetVersion = (versionName: string): Observable => + this.datasetService.createDatasetVersion(this.did!, versionName); - this.datasetService - .createDatasetVersion(this.did, this.versionName?.trim() || "") - .pipe(untilDestroyed(this)) - .subscribe({ - next: res => { - this.notificationService.success("Version Created"); - this.isCreatingVersion = false; - this.versionName = ""; - // A new version consumes all staged changes. - this.confirmedStagedPaths.clear(); - this.unconfirmedStagedPaths.clear(); - this.refreshPendingChanges(); - this.retrieveDatasetVersionList(); - this.retrieveLatestVersionFile(); - this.userMakeChanges.emit(); - }, - error: (res: unknown) => { - const err = res as HttpErrorResponse; - this.notificationService.error(`Version creation failed: ${err.error.message}`); - this.isCreatingVersion = false; - }, - }); - } + onVersionCreated(): void { + this.retrieveDatasetVersionList(); + this.retrieveLatestVersionFile(); } public onClickDownloadVersionAsZip() { @@ -501,28 +425,6 @@ export class DatasetDetailComponent implements OnInit { this.isRightBarCollapsed = !this.isRightBarCollapsed; } - onStagedObjectsUpdated(stagedObjects: DatasetStagedObject[]) { - this.confirmedStagedPaths = new Set(stagedObjects.map(obj => obj.path)); - for (const path of this.confirmedStagedPaths) { - this.unconfirmedStagedPaths.delete(path); - } - this.refreshPendingChanges(); - } - - // Reflects a locally staged change (finished upload or file deletion) in the - // Finished header immediately, ahead of the next diff response. - private markPathStaged(path: string): void { - if (!this.confirmedStagedPaths.has(path)) { - this.unconfirmedStagedPaths.add(path); - } - this.refreshPendingChanges(); - } - - private refreshPendingChanges(): void { - this.pendingChangesCount = this.confirmedStagedPaths.size + this.unconfirmedStagedPaths.size; - this.userHasPendingChanges = this.pendingChangesCount > 0; - } - onVersionSelected(version: DatasetVersion): void { this.selectedVersion = version; if (this.did && this.selectedVersion.dvid) @@ -572,319 +474,18 @@ export class DatasetDetailComponent implements OnInit { return this.datasetIsDownloadable && (this.datasetIsPublic || this.userDatasetAccessLevel !== "NONE"); } - // Track multiple file by unique key - trackByTask(_: number, task: MultipartUploadProgress & { filePath: string }): string { - return task.filePath; - } - - trackByPendingFile(_: number, fileName: string): string { - return fileName; - } - - // A missing key or failed fetch keeps the field defaults; NaN here would - // silently stall the upload queue (`activeUploads < NaN` is always false). - private loadUploadSettings(): void { - this.adminSettingsService - .getPublicSetting("dataset_multipart_upload_chunk_size_mib") - .pipe(untilDestroyed(this)) - .subscribe({ - next: value => (this.chunkSizeMiB = parseIntOrDefault(value, this.chunkSizeMiB)), - error: () => {}, - }); - this.adminSettingsService - .getPublicSetting("dataset_max_number_of_concurrent_uploading_file_chunks") - .pipe(untilDestroyed(this)) - .subscribe({ - next: value => (this.maxConcurrentChunks = parseIntOrDefault(value, this.maxConcurrentChunks)), - error: () => {}, - }); - this.adminSettingsService - .getPublicSetting("dataset_max_number_of_concurrent_uploading_file") - .pipe(untilDestroyed(this)) - .subscribe({ - next: value => (this.maxConcurrentFiles = parseIntOrDefault(value, this.maxConcurrentFiles)), - error: () => {}, - }); - } - - onNewUploadFilesChanged(files: FileUploadItem[]) { - if (this.did) { - files.forEach(file => { - // Check if currently uploading - const continueWithUpload = () => { - // Create upload function - const startUpload = () => { - this.removeFromPendingQueue(file.name); - - // Add an initializing task placeholder to uploadTasks - this.uploadTasks.unshift({ - filePath: file.name, - percentage: 0, - status: "initializing", - }); - // Start multipart upload - const subscription = this.datasetService - .multipartUpload( - this.ownerEmail, - this.datasetName, - file.name, - file.file, - this.chunkSizeMiB * 1024 * 1024, - this.maxConcurrentChunks, - file.restart - ) - .pipe(untilDestroyed(this)) - .subscribe({ - next: progress => { - // Find the task - const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); - - if (taskIndex !== -1) { - // Update the task with new progress info - this.uploadTasks[taskIndex] = { - ...this.uploadTasks[taskIndex], - ...progress, - percentage: progress.percentage ?? this.uploadTasks[taskIndex].percentage ?? 0, - }; - - // totalTime may be exactly 0 (resumed upload with no missing - // parts); a truthiness check would leak the concurrency slot. - if (progress.status === "finished" && progress.totalTime !== undefined) { - const filename = file.name.split("/").pop() || file.name; - this.uploadTimeMap.set(filename, progress.totalTime); - this.markPathStaged(file.name); - this.userMakeChanges.emit(); - this.scheduleHide(taskIndex); - this.onUploadComplete(); - } - } - }, - error: (res: unknown) => { - const err = res as HttpErrorResponse; - - if (err?.status === HttpStatusCode.Conflict) { - this.notificationService.error( - "Upload blocked (409). Another upload is likely in progress for this file (another tab/browser), or the server is finalizing a previous upload. Please retry in a moment." - ); - } else { - this.notificationService.error("Upload failed. Please retry."); - } - // Handle upload error - const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); - - if (taskIndex !== -1) { - this.uploadTasks[taskIndex] = { - ...this.uploadTasks[taskIndex], - percentage: this.uploadTasks[taskIndex].percentage ?? 0, // was 100 - status: "failed", - }; - this.scheduleHide(taskIndex); - } - this.onUploadComplete(); - }, - complete: () => { - const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); - if (taskIndex !== -1 && this.uploadTasks[taskIndex].status !== "finished") { - this.uploadTasks[taskIndex].status = "finished"; - this.markPathStaged(file.name); - this.userMakeChanges.emit(); - this.scheduleHide(taskIndex); - this.onUploadComplete(); - } - }, - }); - // Store the subscription for later cleanup - this.uploadSubscriptions.set(file.name, subscription); - }; - - // Queue management - if (this.activeUploads < this.maxConcurrentFiles) { - this.activeUploads++; - startUpload(); - } else { - this.pendingQueue.set(file.name, startUpload); - this.pendingQueueDirty = true; - } - }; - - // Check if currently uploading - this.cancelExistingUpload(file.name, continueWithUpload); - }); - } - } - - cancelExistingUpload(fileName: string, onCanceled?: () => void): void { - const task = this.uploadTasks.find(t => t.filePath === fileName); - if (task) { - if (task.status === "uploading" || task.status === "initializing") { - this.onClickAbortUploadProgress(task, onCanceled); - return; - } - } - // Remove from pending queue if present - this.removeFromPendingQueue(fileName); - if (onCanceled) { - onCanceled(); - } - } - - private processNextQueuedUpload(): void { - if (this.activeUploads < this.maxConcurrentFiles) { - const next = this.pendingQueue.entries().next(); - if (!next.done) { - const [fileName, startUpload] = next.value; - this.pendingQueue.delete(fileName); - this.pendingQueueDirty = true; - this.activeUploads++; - startUpload(); - } - } - } - - private onUploadComplete(): void { - this.activeUploads--; - this.processNextQueuedUpload(); - } - - private removeFromPendingQueue(fileName: string): void { - if (this.pendingQueue.delete(fileName)) { - this.pendingQueueDirty = true; - } - } - - // Stable array for the template: rebuilt at most once per queue change so - // change detection does not allocate a new array per pass (#5586). - get queuedFileNames(): string[] { - if (this.pendingQueueDirty) { - this.queuedFileNamesSnapshot = Array.from(this.pendingQueue.keys()); - this.pendingQueueDirty = false; - } - return this.queuedFileNamesSnapshot; - } - - get queuedCount(): number { - return this.pendingQueue.size; - } - - get pendingListHeightPx(): number { - return Math.min(this.queuedCount * this.PENDING_ROW_HEIGHT_PX, this.PENDING_LIST_MAX_HEIGHT_PX); - } - - // The viewport initializes inside the collapsed (display: none) panel and - // measures height 0; the CDK only re-measures on window resize. - onPendingPanelActiveChange(active: boolean): void { - if (active) { - setTimeout(() => this.pendingViewport?.checkViewportSize()); - } - } - - get activeCount(): number { - return this.activeUploads; - } - - get hasAnyActivity(): boolean { - return this.pendingChangesCount > 0 || this.activeCount > 0 || this.queuedCount > 0; - } - - // Hide a task row after 5s - private scheduleHide(idx: number) { - if (idx === -1) { - return; - } - const task = this.uploadTasks[idx]; - this.uploadSubscriptions.delete(task.filePath); - // Remove by identity, not filePath: a same-named re-upload within the - // window has its own row, which must survive this timer. - setTimeout(() => { - this.uploadTasks = this.uploadTasks.filter(t => t !== task); - }, 5000); - } - - onClickAbortUploadProgress(task: MultipartUploadProgress & { filePath: string }, onAborted?: () => void) { - const subscription = this.uploadSubscriptions.get(task.filePath); - if (subscription) { - subscription.unsubscribe(); - this.uploadSubscriptions.delete(task.filePath); - } - - if (task.status === "uploading" || task.status === "initializing") { - this.onUploadComplete(); - } - - let doneCalled = false; - const done = () => { - if (doneCalled) { - return; - } - doneCalled = true; - if (onAborted) { - onAborted(); - } - }; - - const abortWithRetry = (attempt: number) => { - this.datasetService - .finalizeMultipartUpload( - this.ownerEmail, - this.datasetName, - task.filePath, - true // abort flag - ) - .pipe(untilDestroyed(this)) - .subscribe({ - next: () => { - this.notificationService.info(`${task.filePath} uploading has been terminated`); - done(); - }, - error: (res: unknown) => { - const err = res as HttpErrorResponse; - // Already gone, treat as done - if (err.status === 404) { - done(); - return; - } - - // Backend is still finalizing/aborting; retry with a tiny backoff - if (err.status === HttpStatusCode.Conflict && attempt < ABORT_RETRY_MAX_ATTEMPTS) { - setTimeout(() => abortWithRetry(attempt + 1), ABORT_RETRY_BACKOFF_BASE_MS * (attempt + 1)); - return; - } - - // Keep current UX: still consider it "aborted" client-side - done(); - }, - }); - }; - - abortWithRetry(0); - - const idx = this.uploadTasks.findIndex(t => t.filePath === task.filePath); - if (idx !== -1) { - this.uploadTasks[idx] = { ...this.uploadTasks[idx], status: "aborted" }; - this.scheduleHide(idx); - } - } - - getUploadStatus(status: MultipartUploadProgress["status"]): "active" | "exception" | "success" { - return status === "uploading" || status === "initializing" - ? "active" - : status === "aborted" || status === "failed" - ? "exception" - : "success"; - } - onPreviouslyUploadedFileDeleted(node: DatasetFileNode) { if (this.did) { + const relativePath = getRelativePathFromDatasetFileNode(node); this.datasetService - .deleteDatasetFile(this.did, getRelativePathFromDatasetFileNode(node)) + .deleteDatasetFile(this.did, relativePath) .pipe(untilDestroyed(this)) .subscribe({ next: (res: Response) => { this.notificationService.success( `File ${node.name} is successfully deleted. You may finalize it or revert it at the "Create Version" panel` ); - this.markPathStaged(getRelativePathFromDatasetFileNode(node)); - this.userMakeChanges.emit(); + this.versionUploader?.notePathStaged(relativePath); }, error: (err: unknown) => { this.notificationService.error("Failed to delete the file"); diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.html b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.html index e394517d293..f6387ae0f5b 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.html +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.html @@ -345,13 +345,110 @@
Choose a Version:
+ [isTreeNodeDeletable]="userHasWriteAccess()" + (selectedTreeNode)="onVersionFileTreeNodeSelected($event)" + (deletedTreeNode)="onPreviouslyUploadedFileDeleted($event)"> + + + +
+ +
+ +

General

+ +
+
+ +

Letters, numbers, underscores and hyphens.

+
+
+ + +
+
+ + + +
+ +
Shown on cards.
+ + +
+ + + +

Model type

+ +
+
+ +

The library this model was trained with.

+
+
+ + + +
+
+ + + +
+
+ +

How the model weights are serialized.

+
+
+ + + +
+
+ +
+ diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.scss b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.scss index ec85131c60c..edea7d38b54 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.scss +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.scss @@ -294,3 +294,64 @@ nz-tabs { text-align: right; word-break: break-word; } + +.settings-tab-content { + height: 100%; + overflow-y: auto; + padding-bottom: 24px; +} + +.settings-general-card { + max-width: 700px; + margin: 24px auto 0; + border-radius: 16px; + border: 1px solid #d9d9d9; + + .settings-card-title { + font-weight: 600; + margin-bottom: 20px; + } + + .settings-field-label { + display: block; + font-weight: 600; + } + + .settings-field-hint { + color: rgba(0, 0, 0, 0.45); + font-size: 12px; + margin-bottom: 8px; + } + + .settings-name-row { + display: flex; + justify-content: space-between; + align-items: center; + gap: 16px; + + .settings-name-label label { + font-weight: 600; + } + + .settings-hint { + color: rgba(0, 0, 0, 0.45); + font-size: 12px; + margin: 4px 0 0; + } + + .settings-name-controls { + display: flex; + gap: 8px; + flex-shrink: 0; + + input, + button { + border-radius: 8px; + } + } + } + + .settings-select { + min-width: 200px; + } +} diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts index 017e980b8d1..71f7963743b 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts @@ -22,12 +22,19 @@ import { NoopAnimationsModule } from "@angular/platform-browser/animations"; import { ActivatedRoute } from "@angular/router"; import { of, throwError } from "rxjs"; import { MarkdownService } from "ngx-markdown"; +import { NzModalService } from "ng-zorro-antd/modal"; +import { By } from "@angular/platform-browser"; import { commonTestImports, commonTestProviders } from "../../../../../common/testing/test-utils"; import { NotificationService } from "../../../../../common/service/notification/notification.service"; import { UserService } from "../../../../../common/service/user/user.service"; import { StubUserService } from "../../../../../common/service/user/stub-user.service"; -import { ModelService } from "../../../../service/user/model/model.service"; +import { MODEL_FORMATS, MODEL_FRAMEWORKS, ModelService } from "../../../../service/user/model/model.service"; import { DownloadService } from "../../../../service/user/download/download.service"; +import { AdminSettingsService } from "../../../../service/admin/settings/admin-settings.service"; +import { MultipartUploadService } from "../../../../service/user/file-resource/multipart-upload.service"; +import { StagedFileService } from "../../../../service/user/file-resource/staged-file.service"; +import { MODEL_FILE_RESOURCE_ENDPOINT } from "../../../../service/user/file-resource/file-resource-endpoint"; +import { VersionUploaderComponent } from "../../version-uploader/version-uploader.component"; import { DatasetFileNode } from "../../../../../common/type/datasetVersionFileTree"; import { ModelVersion } from "../../../../../common/type/model"; import { ModelDetailComponent } from "./model-detail.component"; @@ -58,6 +65,9 @@ describe("ModelDetailComponent", () => { let modelService: Record>; let downloadService: Record>; let notificationService: Record>; + let multipartUploadService: Record>; + let stagedFileService: Record>; + let adminSettingsService: Record>; const dashboardModel = (overrides: Partial> = {}) => ({ isOwner: true, @@ -104,6 +114,19 @@ describe("ModelDetailComponent", () => { downloadModelVersion: vi.fn(() => of(new Blob())), }; notificationService = { success: vi.fn(), error: vi.fn(), info: vi.fn() }; + multipartUploadService = { + multipartUpload: vi.fn(() => of({ filePath: "f", percentage: 100, status: "finished", totalTime: 1 })), + listMultipartUploads: vi.fn(() => of([])), + findExistingUploadFiles: vi.fn(() => of([])), + finalizeMultipartUpload: vi.fn(() => of({})), + }; + stagedFileService = { + getDiff: vi.fn(() => of([])), + resetFileDiff: vi.fn(() => of({})), + deleteFile: vi.fn(() => of({})), + }; + // Every model upload key is absent in this stub, so the component keeps its own defaults. + adminSettingsService = { getPublicSetting: vi.fn(() => of("")) }; TestBed.configureTestingModule({ imports: [ModelDetailComponent, NoopAnimationsModule, ...commonTestImports], @@ -114,6 +137,10 @@ describe("ModelDetailComponent", () => { { provide: NotificationService, useValue: notificationService }, { provide: UserService, useClass: StubUserService }, { provide: MarkdownService, useValue: { parse: vi.fn(() => "") } }, + { provide: NzModalService, useValue: {} }, + { provide: MultipartUploadService, useValue: multipartUploadService }, + { provide: StagedFileService, useValue: stagedFileService }, + { provide: AdminSettingsService, useValue: adminSettingsService }, ...commonTestProviders, ], }); @@ -456,6 +483,176 @@ describe("ModelDetailComponent", () => { expect(q(root, ".model-cover-image").src).toContain("http://cover"); }); + // ─── uploading files and cutting a version ────────────────────────────────── + // + // The panel itself is covered by version-uploader.component.spec.ts; what matters here is that + // the page hands it the model's own addressing, and what the page still owns around it. + + it("hands the version uploader the model endpoint and the model's identity", () => { + create(); + const root = openTab("Versions & Files"); + + const panel = q(root, "texera-version-uploader"); + const uploader = fixture.debugElement.query(By.directive(VersionUploaderComponent)) + .componentInstance as VersionUploaderComponent; + + expect(panel).toBeTruthy(); + expect(uploader.endpoint).toBe(MODEL_FILE_RESOURCE_ENDPOINT); + expect(uploader.resourceId).toBe(MID); + expect(uploader.resourceName).toBe("resnet-50"); + expect(uploader.ownerEmail).toBe(OWNER); + }); + + it("reads its upload settings from the model keys, not the dataset ones", () => { + create(); + openTab("Versions & Files"); + const keys = new Set(adminSettingsService["getPublicSetting"].mock.calls.map(call => call[0])); + + // Three from the panel's tuning, plus the file-picker's per-file ceiling — the 2 GiB model + // limit added by #8000, which models used to inherit from the dataset's 20 MiB. + expect(keys).toEqual( + new Set([ + MODEL_FILE_RESOURCE_ENDPOINT.chunkSizeSettingKey, + MODEL_FILE_RESOURCE_ENDPOINT.maxConcurrentChunksSettingKey, + MODEL_FILE_RESOURCE_ENDPOINT.maxConcurrentFilesSettingKey, + MODEL_FILE_RESOURCE_ENDPOINT.maxFileSizeSettingKey, + ]) + ); + expect([...keys].every(key => key.startsWith("model_"))).toBe(true); + }); + + it("shows the upload panel only to a user who can write", () => { + create(); + expect(openTab("Versions & Files").querySelector("texera-version-uploader")).toBeTruthy(); + + modelService["getModel"] = vi.fn(() => of(dashboardModel({ accessPrivilege: "READ", isOwner: false }))); + create(); + expect(openTab("Versions & Files").querySelector("texera-version-uploader")).toBeNull(); + }); + + it("commits a version through ModelService", () => { + modelService["createModelVersion"] = vi.fn(() => of(aVersion(1, "v1"))); + create(); + + component.createModelVersion("v1").subscribe(); + + expect(modelService["createModelVersion"]).toHaveBeenCalledWith(MID, "v1"); + }); + + it("reloads the version list once the panel reports a new version", () => { + create(); + expect(modelService["retrieveModelVersionList"]).toHaveBeenCalledTimes(1); + + component.onVersionCreated(); + + expect(modelService["retrieveModelVersionList"]).toHaveBeenCalledTimes(2); + }); + + it("stages a deletion of an already-committed file", () => { + create(); + openTab("Versions & Files"); + + component.onPreviouslyUploadedFileDeleted(aFile("model.pt", `/model/${OWNER}/resnet-50/v1`)); + + expect(stagedFileService["deleteFile"]).toHaveBeenCalledWith(MODEL_FILE_RESOURCE_ENDPOINT, MID, "model.pt"); + expect(notificationService["success"]).toHaveBeenCalled(); + }); + + it("stages nothing and reports the failure when the deletion is rejected", () => { + stagedFileService["deleteFile"] = vi.fn(() => throwError(() => new Error("boom"))); + create(); + openTab("Versions & Files"); + + component.onPreviouslyUploadedFileDeleted(aFile("model.pt", `/model/${OWNER}/resnet-50/v1`)); + + expect(notificationService["error"]).toHaveBeenCalledWith("Failed to delete the file"); + }); + + it("deletes nothing without a model id", () => { + create(); + component.mid = undefined; + + component.onPreviouslyUploadedFileDeleted(aFile("model.pt", `/model/${OWNER}/resnet-50/v1`)); + + expect(stagedFileService["deleteFile"]).not.toHaveBeenCalled(); + }); + + // ─── the Settings tab ─────────────────────────────────────────────────────── + + it("hides the Settings tab from a reader", () => { + modelService["getModel"] = vi.fn(() => of(dashboardModel({ accessPrivilege: "READ", isOwner: false }))); + create(); + + const titles = Array.from( + (fixture.nativeElement as HTMLElement).querySelectorAll(".ant-tabs-tab") + ).map(el => el.textContent ?? ""); + expect(titles.some(title => title.includes("Settings"))).toBe(false); + }); + + it("renames the model from the Settings tab", () => { + modelService["updateModelName"] = vi.fn(() => of({})); + create(); + component.editedModelName = "resnet-101"; + + component.onSaveModelName(); + + expect(modelService["updateModelName"]).toHaveBeenCalledWith(MID, "resnet-101"); + expect(component.modelName).toBe("resnet-101"); + }); + + it("rejects an invalid name without calling the server", () => { + modelService["updateModelName"] = vi.fn(() => of({})); + create(); + component.editedModelName = "not a valid name"; + + component.onSaveModelName(); + + expect(modelService["updateModelName"]).not.toHaveBeenCalled(); + expect(notificationService["error"]).toHaveBeenCalled(); + expect(component.modelName).toBe("resnet-50"); + }); + + it("restores the previous description when the update fails", () => { + modelService["updateModelDescription"] = vi.fn(() => throwError(() => new Error("boom"))); + create(); + + component.onModelDescriptionChange("a new description"); + + expect(component.modelDescription).toBe("a description"); + expect(notificationService["error"]).toHaveBeenCalled(); + }); + + it("skips the description update when nothing changed", () => { + modelService["updateModelDescription"] = vi.fn(() => of({})); + create(); + + component.onModelDescriptionChange("a description"); + + expect(modelService["updateModelDescription"]).not.toHaveBeenCalled(); + }); + + it("saves the framework and format, and rolls back a rejected one", () => { + modelService["updateModelFramework"] = vi.fn(() => of({})); + modelService["updateModelFormat"] = vi.fn(() => throwError(() => new Error("boom"))); + create(); + + component.onFrameworkChange("onnx"); + component.onFormatChange("safetensors"); + + expect(modelService["updateModelFramework"]).toHaveBeenCalledWith(MID, "onnx"); + expect(component.modelFramework).toBe("onnx"); + expect(component.modelFormat).toBe("torchscript"); + }); + + it("offers exactly the frameworks and formats the backend accepts", () => { + create(); + const root = openTab("Settings"); + + expect(root.querySelectorAll("nz-select").length).toBe(2); + expect(component.frameworks).toEqual(MODEL_FRAMEWORKS); + expect(component.formats).toEqual(MODEL_FORMATS); + }); + it("collapses and restores the right sider, and maximizes the preview", () => { create(); const root = openTab("Versions & Files"); diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts index d1f918ff6d0..4faa098a497 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts @@ -17,10 +17,11 @@ * under the License. */ -import { Component, OnInit } from "@angular/core"; +import { Component, OnInit, ViewChild } from "@angular/core"; import { ActivatedRoute } from "@angular/router"; import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; import { switchMap } from "rxjs/operators"; +import { Observable } from "rxjs"; import { format } from "date-fns"; import { NgIf, NgClass, NgFor } from "@angular/common"; import { FormsModule } from "@angular/forms"; @@ -37,9 +38,18 @@ import { NzEmptyComponent } from "ng-zorro-antd/empty"; import { NzCollapseComponent, NzCollapsePanelComponent } from "ng-zorro-antd/collapse"; import { NzSelectComponent, NzOptionComponent } from "ng-zorro-antd/select"; import { NzTabsComponent, NzTabComponent } from "ng-zorro-antd/tabs"; +import { NzDividerComponent } from "ng-zorro-antd/divider"; +import { NzInputDirective } from "ng-zorro-antd/input"; -import { ModelService } from "../../../../service/user/model/model.service"; +import { + MODEL_FORMATS, + MODEL_FRAMEWORKS, + ModelService, + validateModelName, +} from "../../../../service/user/model/model.service"; import { DownloadService } from "../../../../service/user/download/download.service"; +import { StagedFileService } from "../../../../service/user/file-resource/staged-file.service"; +import { MODEL_FILE_RESOURCE_ENDPOINT } from "../../../../service/user/file-resource/file-resource-endpoint"; import { NotificationService } from "../../../../../common/service/notification/notification.service"; import { UserService } from "../../../../../common/service/user/user.service"; import { EntityType } from "../../../../../hub/service/hub.service"; @@ -47,8 +57,13 @@ import { extractErrorMessage } from "../../../../../common/util/error"; import { formatCount } from "src/app/common/util/format.util"; import { formatSize } from "src/app/common/util/size-formatter.util"; import { ModelVersion } from "../../../../../common/type/model"; -import { DatasetFileNode, getFullPathFromDatasetFileNode } from "../../../../../common/type/datasetVersionFileTree"; +import { + DatasetFileNode, + getFullPathFromDatasetFileNode, + getRelativePathFromDatasetFileNode, +} from "../../../../../common/type/datasetVersionFileTree"; import { MarkdownDescriptionComponent } from "../../markdown-description/markdown-description.component"; +import { VersionUploaderComponent } from "../../version-uploader/version-uploader.component"; import { UserDatasetFileRendererComponent } from "../../user-dataset/user-dataset-explorer/user-dataset-file-renderer/user-dataset-file-renderer.component"; import { UserDatasetVersionFiletreeComponent } from "../../user-dataset/user-dataset-explorer/user-dataset-version-filetree/user-dataset-version-filetree.component"; @@ -81,7 +96,10 @@ import { UserDatasetVersionFiletreeComponent } from "../../user-dataset/user-dat NzOptionComponent, NzTabsComponent, NzTabComponent, + NzDividerComponent, + NzInputDirective, MarkdownDescriptionComponent, + VersionUploaderComponent, UserDatasetFileRendererComponent, UserDatasetVersionFiletreeComponent, ], @@ -89,6 +107,7 @@ import { UserDatasetVersionFiletreeComponent } from "../../user-dataset/user-dat export class ModelDetailComponent implements OnInit { public mid: number | undefined; public modelName: string = ""; + public editedModelName: string = ""; public modelDescription: string = ""; public modelCreationTime: string = ""; public modelCreationTimeTooltip: string = ""; @@ -127,6 +146,11 @@ export class ModelDetailComponent implements OnInit { public currentUid: number | undefined = this.userService.getCurrentUser()?.uid; public readonly modelEntityType = EntityType.Model; + public readonly modelEndpoint = MODEL_FILE_RESOURCE_ENDPOINT; + public readonly frameworks = MODEL_FRAMEWORKS; + public readonly formats = MODEL_FORMATS; + + @ViewChild(VersionUploaderComponent) private versionUploader?: VersionUploaderComponent; formatSize = formatSize; formatCount = formatCount; @@ -135,6 +159,7 @@ export class ModelDetailComponent implements OnInit { private route: ActivatedRoute, private modelService: ModelService, private downloadService: DownloadService, + private stagedFileService: StagedFileService, private notificationService: NotificationService, private userService: UserService ) { @@ -193,6 +218,7 @@ export class ModelDetailComponent implements OnInit { next: dashboardModel => { const model = dashboardModel.model; this.modelName = model.name; + this.editedModelName = model.name; this.modelDescription = model.description; this.modelIsPublic = model.isPublic; this.modelIsDownloadable = model.isDownloadable; @@ -350,4 +376,121 @@ export class ModelDetailComponent implements OnInit { } return this.modelIsDownloadable && (this.modelIsPublic || this.userModelAccessLevel !== "NONE"); } + + userHasWriteAccess(): boolean { + return this.userModelAccessLevel === "WRITE"; + } + + onPreviouslyUploadedFileDeleted(node: DatasetFileNode): void { + if (!this.mid) { + return; + } + const relativePath = getRelativePathFromDatasetFileNode(node); + this.stagedFileService + .deleteFile(this.modelEndpoint, this.mid, relativePath) + .pipe(untilDestroyed(this)) + .subscribe({ + next: () => { + this.notificationService.success( + `File ${node.name} is successfully deleted. You may finalize it or revert it at the "Create Version" panel` + ); + this.versionUploader?.notePathStaged(relativePath); + }, + error: () => this.notificationService.error("Failed to delete the file"), + }); + } + + /** Commits the staged files; the panel owns the rest of the version flow. */ + createModelVersion = (versionName: string): Observable => + this.modelService.createModelVersion(this.mid!, versionName); + + onVersionCreated(): void { + this.retrieveModelVersionList(); + } + + // =========================================================================== + // Settings + // =========================================================================== + + onSaveModelName(): void { + if (!this.mid) { + return; + } + const name = this.editedModelName; + const nameError = validateModelName(name); + if (nameError) { + this.notificationService.error(nameError); + return; + } + + this.modelService + .updateModelName(this.mid, name) + .pipe(untilDestroyed(this)) + .subscribe({ + next: () => { + this.modelName = name; + this.editedModelName = name; + this.notificationService.success(`Model name updated to '${name}'`); + }, + error: (err: unknown) => this.notificationService.error(extractErrorMessage(err)), + }); + } + + onModelDescriptionChange(description: string): void { + const updatedDescription = description ?? ""; + const previousDescription = this.modelDescription; + + if (!this.mid || previousDescription === updatedDescription) { + return; + } + this.modelDescription = updatedDescription; + + this.modelService + .updateModelDescription(this.mid, updatedDescription) + .pipe(untilDestroyed(this)) + .subscribe({ + error: () => { + this.modelDescription = previousDescription; + this.notificationService.error("Failed to update model description"); + }, + }); + } + + onFrameworkChange(framework: string): void { + const previous = this.modelFramework; + if (!this.mid || previous === framework) { + return; + } + this.modelFramework = framework; + + this.modelService + .updateModelFramework(this.mid, framework) + .pipe(untilDestroyed(this)) + .subscribe({ + next: () => this.notificationService.success(`Framework set to '${framework}'`), + error: (err: unknown) => { + this.modelFramework = previous; + this.notificationService.error(extractErrorMessage(err)); + }, + }); + } + + onFormatChange(modelFormat: string): void { + const previous = this.modelFormat; + if (!this.mid || previous === modelFormat) { + return; + } + this.modelFormat = modelFormat; + + this.modelService + .updateModelFormat(this.mid, modelFormat) + .pipe(untilDestroyed(this)) + .subscribe({ + next: () => this.notificationService.success(`Format set to '${modelFormat}'`), + error: (err: unknown) => { + this.modelFormat = previous; + this.notificationService.error(extractErrorMessage(err)); + }, + }); + } } diff --git a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.html b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.html new file mode 100644 index 00000000000..384d706f600 --- /dev/null +++ b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.html @@ -0,0 +1,176 @@ + + + + + + + + + + + +
+ {{ fileName }} + +
+ + + + + + +
+
+
+ {{ task.status }}: {{ task.filePath }} + +
+ +
+ + + {{ formatSpeed(task.uploadSpeed) }} - + {{ formatTime(task.totalTime ?? 0) }} elapsed, + {{ formatTime(task.estimatedTimeRemaining ?? 0) }} left + + + + Upload time: {{ formatTime(task.totalTime ?? 0) }} + +
+
+
+ + + + + + + + + + + + + +
+
+ + +
+
+ +
+
+ + diff --git a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.scss b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.scss new file mode 100644 index 00000000000..35e4a14490e --- /dev/null +++ b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.scss @@ -0,0 +1,114 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +.upload-progress-wrapper { + max-height: 25vh; + overflow-y: auto; + padding-right: 4px; +} + +// Rows must stay exactly PENDING_ROW_HEIGHT_PX tall for the fixed-size scroll strategy. The gutter +// lives on the rows: padding on the viewport never reaches its absolutely positioned content wrapper. +.upload-progress-wrapper-pending { + .pending-file-row { + height: 32px; + display: flex; + align-items: center; + justify-content: space-between; + padding-right: 4px; + + .pending-file-name { + overflow: hidden; + white-space: nowrap; + text-overflow: ellipsis; + } + } +} + +.progress-header { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; +} + +.upload-stats { + font-size: 13px; + margin-bottom: 20px; + + nz-progress { + width: 97%; + display: inline-block; + } +} + +:host ::ng-deep .upload-stats .ant-tag { + border: none; +} + +.fixed-width-speed { + display: inline-block; + min-width: 5ch; + text-align: right; +} + +.fixed-width-time { + display: inline-block; + min-width: 2ch; + text-align: right; +} + +.section-divider { + margin: 8px 0; +} + +.version-creator { + margin-top: 20px; + padding: 20px; +} + +.version-input-container { + display: flex; + align-items: center; + gap: 10px; + + label { + font-size: 15px; + } +} + +.version-input { + padding: 6px; +} + +.create-version-button { + display: flex; /* Use flexbox for centering */ + align-items: center; /* Center vertically */ + justify-content: center; /* Center horizontally */ + color: white; + border: none; + padding: 12px 40px; /* Increase padding for a wider button */ + border-radius: 25px; + cursor: pointer; + transition: background-color 0.3s; + margin: 18px auto 0 auto; /* Auto margins for horizontal centering */ + width: 200px; /* Adjust width as needed */ + font-size: 18px; /* Make text slightly bigger */ + font-weight: bold; /* Optional: Make text bold */ +} diff --git a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts new file mode 100644 index 00000000000..1f77c58b81d --- /dev/null +++ b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts @@ -0,0 +1,1121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ComponentFixture, TestBed } from "@angular/core/testing"; +import { HttpErrorResponse, HttpStatusCode } from "@angular/common/http"; +import { By } from "@angular/platform-browser"; +import { NzModalService } from "ng-zorro-antd/modal"; +import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; +import { concat, Observable, of, Subject, throwError } from "rxjs"; +import { + ABORT_RETRY_BACKOFF_BASE_MS, + ABORT_RETRY_MAX_ATTEMPTS, + VersionUploaderComponent, +} from "./version-uploader.component"; +import { + MultipartUploadProgress, + MultipartUploadService, +} from "../../../service/user/file-resource/multipart-upload.service"; +import { StagedFileService } from "../../../service/user/file-resource/staged-file.service"; +import { + DATASET_FILE_RESOURCE_ENDPOINT, + MODEL_FILE_RESOURCE_ENDPOINT, +} from "../../../service/user/file-resource/file-resource-endpoint"; +import { AdminSettingsService } from "../../../service/admin/settings/admin-settings.service"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; +import { DatasetStagedObject } from "../../../../common/type/dataset-staged-object"; +import { FileUploadItem } from "../../../type/dashboard-file.interface"; +import { commonTestImports, commonTestProviders } from "../../../../common/testing/test-utils"; + +describe("VersionUploaderComponent", () => { + let fixture: ComponentFixture; + let component: VersionUploaderComponent; + let uploadSubjects: Subject[]; + let uploadedPaths: string[]; + let multipartUploadSpy: ReturnType; + let createVersionSpy: ReturnType; + const asCreateVersion = (spy: ReturnType) => spy as unknown as (name: string) => Observable; + + const makeFileItem = (name: string): FileUploadItem => ({ + file: new File(["x"], name), + name, + description: "", + uploadProgress: 0, + isUploadingFlag: false, + restart: false, + }); + + const dropFiles = (...names: string[]) => component.onNewUploadFilesChanged(names.map(makeFileItem)); + + const finishUpload = (index: number, filePath: string, totalTime = 1) => + uploadSubjects[index].next({ filePath, percentage: 100, status: "finished", totalTime }); + + beforeEach(() => { + uploadSubjects = []; + uploadedPaths = []; + multipartUploadSpy = vi.fn((_endpoint: unknown, _ownerEmail: string, _resourceName: string, filePath: string) => { + const progress = new Subject(); + uploadSubjects.push(progress); + uploadedPaths.push(filePath); + return progress.asObservable(); + }); + createVersionSpy = vi.fn(() => of({})); + + TestBed.configureTestingModule({ + imports: [VersionUploaderComponent, ...commonTestImports], + providers: [ + { provide: NzModalService, useValue: {} }, + { + provide: MultipartUploadService, + useValue: { + multipartUpload: multipartUploadSpy, + finalizeMultipartUpload: vi.fn(() => of({})), + listMultipartUploads: vi.fn(() => of([])), + findExistingUploadFiles: vi.fn(() => of([])), + }, + }, + { provide: StagedFileService, useValue: { getDiff: vi.fn(() => of([])), resetFileDiff: vi.fn(() => of({})) } }, + { provide: NotificationService, useValue: { success: vi.fn(), error: vi.fn(), info: vi.fn() } }, + // maxConcurrentFiles becomes 3, the cap the queue tests below rely on. + { provide: AdminSettingsService, useValue: { getPublicSetting: vi.fn(() => of("3")) } }, + ...commonTestProviders, + ], + }); + + fixture = TestBed.createComponent(VersionUploaderComponent); + component = fixture.componentInstance; + component.resourceId = 1; + component.ownerEmail = "owner@texera.com"; + component.resourceName = "test-dataset"; + component.endpoint = DATASET_FILE_RESOURCE_ENDPOINT; + component.createVersion = asCreateVersion(createVersionSpy); + fixture.detectChanges(); + }); + + describe("a failed upload", () => { + const notification = () => TestBed.inject(NotificationService) as unknown as { error: ReturnType }; + + /** Fails the in-flight upload of `name` with the given HTTP status. */ + const failUpload = (index: number, status: number) => + uploadSubjects[index].error(new HttpErrorResponse({ status })); + + it("names the 409 conflict so the user knows to retry", () => { + dropFiles("a.csv"); + + failUpload(0, HttpStatusCode.Conflict); + + expect(notification().error).toHaveBeenCalledWith(expect.stringContaining("Upload blocked (409)")); + }); + + it("falls back to a generic message for any other failure", () => { + dropFiles("a.csv"); + + failUpload(0, HttpStatusCode.InternalServerError); + + expect(notification().error).toHaveBeenCalledWith("Upload failed. Please retry."); + }); + + it("marks the task failed and keeps its progress rather than showing it complete", () => { + dropFiles("a.csv"); + // a partially-uploaded file: the bar must not jump to 100 when it fails + uploadSubjects[0].next({ filePath: "a.csv", percentage: 42, status: "uploading" }); + + failUpload(0, HttpStatusCode.InternalServerError); + + const task = component.uploadTasks.find(t => t.filePath === "a.csv"); + expect(task?.status).toBe("failed"); + expect(task?.percentage).toBe(42); + }); + + it("frees the concurrency slot so a queued upload can start", () => { + // maxConcurrentFiles is 3, so a fourth file waits for a slot + dropFiles("a.csv", "b.csv", "c.csv", "d.csv"); + expect(uploadedPaths).toEqual(["a.csv", "b.csv", "c.csv"]); + + failUpload(0, HttpStatusCode.InternalServerError); + + expect(uploadedPaths).toContain("d.csv"); + }); + + it("still reports the failure when the task is no longer in the list", () => { + dropFiles("a.csv"); + component.uploadTasks = []; // the taskIndex === -1 arm + + expect(() => failUpload(0, HttpStatusCode.InternalServerError)).not.toThrow(); + expect(notification().error).toHaveBeenCalled(); + }); + }); + + /** + * A progress event and the five-second hide timer both address a row by its index in + * `uploadTasks`, and that row can already be gone — dismissed by the user — by the time + * either arrives, so both lookups have to survive the miss. The completion path also has + * to pick a key for `uploadTimeMap` out of a name that may carry directories. + */ + describe("progress bookkeeping", () => { + beforeEach(() => { + // The completion path arms a 5s row-hide timer; keep it off the real clock. + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it("ignores progress for a row that is no longer listed", () => { + dropFiles("a.csv"); + component.uploadTasks = []; // dismissed while a chunk was still in flight + + expect(() => uploadSubjects[0].next({ filePath: "a.csv", percentage: 50, status: "uploading" })).not.toThrow(); + // The late event must not resurrect the row or write a phantom index into the list. + expect(component.uploadTasks).toEqual([]); + expect(Object.keys(component.uploadTasks)).toHaveLength(0); + }); + + it("keys the upload time by the last path segment, falling back to the whole name", () => { + // Taking the segment after the last "/" yields "" for a name that ends in one, and + // keying the map under "" would collide every such upload onto one entry; the + // fallback keeps the name the caller gave instead. + dropFiles("nested/dir/"); + + finishUpload(0, "nested/dir/", 7); + + expect(component.uploadTimeMap.get("nested/dir/")).toBe(7); + expect(component.uploadTimeMap.has("")).toBe(false); + + // The reader of this map (staged-objects-list) looks a row up by + // `filePath.split("/").pop() || filePath`, so an ordinary nested name has to be + // keyed by its last segment here or the per-file time silently stops rendering. + dropFiles("dir/sub/a.csv"); + + finishUpload(1, "dir/sub/a.csv", 9); + + expect(component.uploadTimeMap.get("a.csv")).toBe(9); + expect(component.uploadTimeMap.has("dir/sub/a.csv")).toBe(false); + }); + + it("ignores a hide request for a row that is gone", () => { + // Every one of scheduleHide's call sites already checks the index, so the -1 arm + // pins a defensive no-op rather than a reachable scenario: without the guard the + // lookup would read `filePath` off undefined and throw. The valid-index call that + // follows keeps a scheduleHide which does nothing at all from passing this test. + dropFiles("a.csv"); + const before = [...component.uploadTasks]; + + expect(() => (component as any).scheduleHide(-1)).not.toThrow(); + expect(component.uploadTasks).toEqual(before); + + (component as any).scheduleHide(0); + vi.advanceTimersByTime(5000); + + expect(component.uploadTasks).toEqual([]); + }); + }); + + /** + * Aborting an in-flight upload has to survive the backend still finalizing the previous attempt: + * the abort call is retried on 409 up to ABORT_RETRY_MAX_ATTEMPTS, a 404 means it is already gone, + * and the caller's callback must fire exactly once down every one of those paths. + */ + describe("aborting an upload", () => { + let finalize: ReturnType; + + beforeEach(() => { + vi.useFakeTimers(); + finalize = TestBed.inject(MultipartUploadService).finalizeMultipartUpload as unknown as ReturnType; + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + /** Starts an upload and reports progress, leaving one task in flight. */ + function inFlight(name = "a.txt") { + dropFiles(name); + uploadSubjects[0].next({ filePath: name, percentage: 10, status: "uploading", totalTime: 0 }); + return component.uploadTasks.find(t => t.filePath === name)!; + } + + const conflict = () => throwError(() => ({ status: 409 }) as any); + const gone = () => throwError(() => ({ status: 404 }) as any); + + it("marks the task aborted and tells the caller once", () => { + const task = inFlight(); + const onAborted = vi.fn(); + + component.onClickAbortUploadProgress(task as any, onAborted); + + expect(finalize).toHaveBeenCalledWith( + DATASET_FILE_RESOURCE_ENDPOINT, + "owner@texera.com", + "test-dataset", + "a.txt", + true + ); + expect(component.uploadTasks.find(t => t.filePath === "a.txt")!.status).toBe("aborted"); + expect(onAborted).toHaveBeenCalledTimes(1); + + // The aborted row goes on the same five-second hide timer a finished one does, so + // it clears itself out of the list instead of sitting there for the rest of the session. + vi.advanceTimersByTime(5000); + + expect(component.uploadTasks.find(t => t.filePath === "a.txt")).toBeUndefined(); + }); + + it("stops listening to the upload it aborted", () => { + const task = inFlight(); + + component.onClickAbortUploadProgress(task as any); + + // The progress stream is unsubscribed, so a late event cannot resurrect the task. + uploadSubjects[0].next({ filePath: "a.txt", percentage: 100, status: "finished", totalTime: 1 }); + expect(component.uploadTasks.find(t => t.filePath === "a.txt")!.status).toBe("aborted"); + }); + + it("treats a 404 as already aborted rather than an error", () => { + finalize.mockReturnValueOnce(gone()); + const task = inFlight(); + const onAborted = vi.fn(); + + component.onClickAbortUploadProgress(task as any, onAborted); + + expect(onAborted).toHaveBeenCalledTimes(1); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("retries a 409 after a backoff and finishes once the server catches up", () => { + // The server is still finalizing the previous attempt; the abort has to wait it out. + finalize.mockReturnValueOnce(conflict()); + const task = inFlight(); + const onAborted = vi.fn(); + + component.onClickAbortUploadProgress(task as any, onAborted); + expect(onAborted).not.toHaveBeenCalled(); + + vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS); + + expect(finalize).toHaveBeenCalledTimes(2); + expect(onAborted).toHaveBeenCalledTimes(1); + }); + + it("backs off further on each successive conflict", () => { + finalize.mockReturnValue(conflict()); + const task = inFlight(); + + component.onClickAbortUploadProgress(task as any); + expect(finalize).toHaveBeenCalledTimes(1); + + // First wait is BASE * 1, the second BASE * 2, so BASE alone is not enough for the third call. + vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS); + expect(finalize).toHaveBeenCalledTimes(2); + + vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS); + expect(finalize).toHaveBeenCalledTimes(2); + + vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS); + expect(finalize).toHaveBeenCalledTimes(3); + }); + + it("gives up after the attempt limit but still reports the abort", () => { + // Without the bound this would retry forever against a permanently conflicted server. + finalize.mockReturnValue(conflict()); + const task = inFlight(); + const onAborted = vi.fn(); + + component.onClickAbortUploadProgress(task as any, onAborted); + vi.advanceTimersByTime(ABORT_RETRY_BACKOFF_BASE_MS * ABORT_RETRY_MAX_ATTEMPTS * (ABORT_RETRY_MAX_ATTEMPTS + 1)); + + expect(finalize).toHaveBeenCalledTimes(ABORT_RETRY_MAX_ATTEMPTS + 1); + expect(onAborted).toHaveBeenCalledTimes(1); + }); + + it("reports the abort once even on an error the retry does not cover", () => { + finalize.mockReturnValueOnce(throwError(() => ({ status: 500 }) as any)); + const task = inFlight(); + const onAborted = vi.fn(); + + component.onClickAbortUploadProgress(task as any, onAborted); + + expect(onAborted).toHaveBeenCalledTimes(1); + expect(finalize).toHaveBeenCalledTimes(1); + }); + + it("frees the concurrency slot so a queued upload can start", () => { + // Aborting has to release the slot as an ordinary completion would; otherwise the queue + // stalls behind an upload that is no longer running. + dropFiles("a.txt", "b.txt", "c.txt", "d.txt"); + expect(uploadedPaths).toEqual(["a.txt", "b.txt", "c.txt"]); + uploadSubjects[0].next({ filePath: "a.txt", percentage: 10, status: "uploading", totalTime: 0 }); + const task = component.uploadTasks.find(t => t.filePath === "a.txt")!; + + component.onClickAbortUploadProgress(task as any); + + expect(uploadedPaths).toContain("d.txt"); + }); + + it("cancelExistingUpload aborts an upload that is still running", () => { + inFlight("b.txt"); + const onCanceled = vi.fn(); + + component.cancelExistingUpload("b.txt", onCanceled); + + expect(finalize).toHaveBeenCalledWith( + DATASET_FILE_RESOURCE_ENDPOINT, + "owner@texera.com", + "test-dataset", + "b.txt", + true + ); + expect(onCanceled).toHaveBeenCalledTimes(1); + }); + + it("frees the slot of an upload aborted before its first part went out", () => { + // A task sits at "initializing" until the service reports its first progress. + // Cancelling in that window still has to hand the slot to whatever is queued + // behind it, or the queue stalls on an upload that never started. + dropFiles("a.txt", "b.txt", "c.txt", "d.txt"); + expect(uploadedPaths).toEqual(["a.txt", "b.txt", "c.txt"]); + const initializing = component.uploadTasks.find(t => t.filePath === "a.txt")!; + expect(initializing.status).toBe("initializing"); + + component.onClickAbortUploadProgress(initializing as any); + + expect(uploadedPaths).toContain("d.txt"); + expect(component.activeCount).toBe(3); + }); + + it("does not free a second slot when a finished upload's row is dismissed", () => { + // The row's button becomes "Close" once the upload is done, and the slot was + // already released by the completion; releasing it a second time would let a + // fourth upload run past the concurrency cap. + dropFiles("a.txt", "b.txt", "c.txt", "d.txt"); + finishUpload(0, "a.txt"); + expect(uploadedPaths).toContain("d.txt"); + const finished = component.uploadTasks.find(t => t.filePath === "a.txt")!; + expect(finished.status).toBe("finished"); + + component.onClickAbortUploadProgress(finished as any); + + expect(component.activeCount).toBe(3); + const dismissed = component.uploadTasks.find(t => t.filePath === "a.txt")!; + expect(dismissed.status).toBe("aborted"); + + // The row lingers for five seconds after being dismissed, so the same X is + // still there to be clicked again — and that click must not release either. + component.onClickAbortUploadProgress(dismissed as any); + + expect(component.activeCount).toBe(3); + }); + + it("does not free a second slot when a failed upload's row is dismissed", () => { + // The failure handler already released this upload's slot and let the queued + // fourth file start; dismissing the row it left behind must not release a + // second slot, or a fifth upload would run past the cap of three. + dropFiles("a.txt", "b.txt", "c.txt", "d.txt"); + uploadSubjects[0].error(new HttpErrorResponse({ status: 500 })); + expect(uploadedPaths).toContain("d.txt"); + const failed = component.uploadTasks.find(t => t.filePath === "a.txt")!; + expect(failed.status).toBe("failed"); + + component.onClickAbortUploadProgress(failed as any); + + expect(component.activeCount).toBe(3); + expect(multipartUploadSpy).toHaveBeenCalledTimes(4); + }); + + it("cancelExistingUpload aborts an upload whose first part has not gone out", () => { + // Until the service reports a first chunk the task sits at "initializing". + // A re-drop in that window has to abort that attempt rather than fall + // through and race a second multipart upload against it for the same path, + // which is exactly the 409 the upload error handler warns about. + dropFiles("b.txt"); + expect(component.uploadTasks.find(t => t.filePath === "b.txt")!.status).toBe("initializing"); + const onCanceled = vi.fn(); + + component.cancelExistingUpload("b.txt", onCanceled); + + expect(finalize).toHaveBeenCalledWith( + DATASET_FILE_RESOURCE_ENDPOINT, + "owner@texera.com", + "test-dataset", + "b.txt", + true + ); + expect(component.uploadTasks.find(t => t.filePath === "b.txt")!.status).toBe("aborted"); + // The slot goes back to the queue instead of being held by an attempt that + // is no longer running. + expect(component.activeCount).toBe(0); + expect(onCanceled).toHaveBeenCalledTimes(1); + }); + + it("tells the caller once even when the abort call reports more than once", () => { + // The callback is latched so that it fires exactly once no matter how many of the + // subscription's handlers reach it. HttpClient itself delivers a single response, + // so this drives the latch directly: a response followed by a stream failure runs + // the next handler and then the error handler, and both of them report done. + finalize.mockReturnValueOnce( + concat( + of({}), + throwError(() => ({ status: 500 }) as any) + ) + ); + const task = inFlight(); + const onAborted = vi.fn(); + + component.onClickAbortUploadProgress(task as any, onAborted); + + expect(onAborted).toHaveBeenCalledTimes(1); + }); + + it("aborts a task whose row was already dropped without resurrecting it", () => { + const task = inFlight(); + component.uploadTasks = []; // the row was dismissed before the abort was clicked + + component.onClickAbortUploadProgress(task as any); + + expect(finalize).toHaveBeenCalledWith( + DATASET_FILE_RESOURCE_ENDPOINT, + "owner@texera.com", + "test-dataset", + "a.txt", + true + ); + // Writing "aborted" back at a missing index would leave a phantom "-1" property on + // the array, which neither a throw nor `.length` would reveal. + expect(component.uploadTasks).toEqual([]); + expect(Object.keys(component.uploadTasks)).toHaveLength(0); + }); + }); + + /** The panel is the only thing that knows which resource family it is addressing. */ + describe("resource addressing", () => { + it("passes its endpoint to the upload engine, whichever family it serves", () => { + component.endpoint = MODEL_FILE_RESOURCE_ENDPOINT; + component.resourceName = "resnet-50"; + + dropFiles("weights/model.pt"); + + expect(multipartUploadSpy.mock.calls[0][0]).toBe(MODEL_FILE_RESOURCE_ENDPOINT); + expect(multipartUploadSpy.mock.calls[0][2]).toBe("resnet-50"); + }); + + it("reads its upload tuning from its own family's settings keys", () => { + const settings = TestBed.inject(AdminSettingsService) as unknown as { + getPublicSetting: ReturnType; + }; + + // The last is the file picker's own per-file ceiling, which it reads off the same endpoint. + expect(settings.getPublicSetting.mock.calls.map(call => call[0])).toEqual([ + DATASET_FILE_RESOURCE_ENDPOINT.chunkSizeSettingKey, + DATASET_FILE_RESOURCE_ENDPOINT.maxConcurrentChunksSettingKey, + DATASET_FILE_RESOURCE_ENDPOINT.maxConcurrentFilesSettingKey, + DATASET_FILE_RESOURCE_ENDPOINT.maxFileSizeSettingKey, + ]); + expect(component.maxConcurrentFiles).toBe(3); + }); + }); + + describe("staged changes", () => { + it("tracks the pending-change count from the diff response", () => { + const staged: DatasetStagedObject[] = [ + { path: "a", pathType: "file", diffType: "added", sizeBytes: 1 }, + { path: "b", pathType: "file", diffType: "added", sizeBytes: 1 }, + ]; + + component.onStagedObjectsUpdated(staged); + expect(component.pendingChangesCount).toBe(2); + expect(component.userHasPendingChanges).toBe(true); + + component.onStagedObjectsUpdated([]); + expect(component.pendingChangesCount).toBe(0); + expect(component.userHasPendingChanges).toBe(false); + }); + + it("counts a change the host staged elsewhere, such as a file-tree deletion", () => { + component.notePathStaged("nested/a.txt"); + + expect(component.pendingChangesCount).toBe(1); + expect(component.userHasPendingChanges).toBe(true); + }); + }); + + describe("creating a version", () => { + it("commits through the host's callback, trimming the name", () => { + component.versionName = " v2 "; + + component.onClickCreateVersion(); + + expect(createVersionSpy).toHaveBeenCalledWith("v2"); + }); + + it("lets the backend name the version when the box is empty", () => { + component.versionName = " "; + + component.onClickCreateVersion(); + + expect(createVersionSpy).toHaveBeenCalledWith(""); + }); + + it("clears the staged state, tells the host to reload, and re-enables the button", () => { + const versionCreated = vi.fn(); + component.versionCreated.subscribe(versionCreated); + dropFiles("a.csv"); + finishUpload(0, "a.csv"); + expect(component.pendingChangesCount).toBe(1); + + component.onClickCreateVersion(); + + expect(component.versionName).toBe(""); + expect(component.pendingChangesCount).toBe(0); + expect(component.isCreatingVersion).toBe(false); + expect(versionCreated).toHaveBeenCalledTimes(1); + }); + + it("keeps the staged changes and re-enables the button when creation is rejected", () => { + createVersionSpy.mockReturnValue(throwError(() => new HttpErrorResponse({ status: 400 }))); + const versionCreated = vi.fn(); + component.versionCreated.subscribe(versionCreated); + dropFiles("a.csv"); + finishUpload(0, "a.csv"); + + component.onClickCreateVersion(); + + expect(component.isCreatingVersion).toBe(false); + // The user can retry, so what they staged must survive the failure. + expect(component.pendingChangesCount).toBe(1); + expect(versionCreated).not.toHaveBeenCalled(); + }); + + it("ignores a second submit while one is in flight", () => { + createVersionSpy.mockReturnValue(new Subject()); + + component.onClickCreateVersion(); + component.onClickCreateVersion(); + + expect(createVersionSpy).toHaveBeenCalledTimes(1); + }); + + it("commits nothing before the host supplies a resource id", () => { + component.resourceId = undefined; + + component.onClickCreateVersion(); + + expect(createVersionSpy).not.toHaveBeenCalled(); + }); + }); + + describe("progress state for the template", () => { + it("maps the upload status to a progress state", () => { + expect(component.getUploadStatus("uploading")).toBe("active"); + expect(component.getUploadStatus("initializing")).toBe("active"); + expect(component.getUploadStatus("aborted")).toBe("exception"); + expect(component.getUploadStatus("failed")).toBe("exception"); + expect(component.getUploadStatus("finished")).toBe("success"); + }); + + it("tracks a task by its file path", () => { + const task = { filePath: "owner/data/file.csv" } as unknown as Parameters[1]; + expect(component.trackByTask(0, task)).toBe("owner/data/file.csv"); + }); + }); + + it("starts at most maxConcurrentFiles uploads immediately and queues the rest", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); + + expect(multipartUploadSpy).toHaveBeenCalledTimes(3); + expect(uploadedPaths).toEqual(["f1.txt", "f2.txt", "f3.txt"]); + expect(component.activeCount).toBe(3); + expect(component.queuedCount).toBe(2); + expect(component.queuedFileNames).toEqual(["f4.txt", "f5.txt"]); + }); + + it("does nothing when an empty file list is dropped", () => { + dropFiles(); + + expect(multipartUploadSpy).not.toHaveBeenCalled(); + expect(component.activeCount).toBe(0); + expect(component.queuedCount).toBe(0); + expect(component.queuedFileNames).toEqual([]); + }); + + it("starts no upload at all before the host supplies a resource id", () => { + // Every multipart call is addressed to one resource, so without an id there is + // nowhere to upload into: the drop is refused outright rather than leaving + // rows on the panel for uploads that were never started. + component.resourceId = undefined; + + dropFiles("f1.txt", "f2.txt"); + + expect(multipartUploadSpy).not.toHaveBeenCalled(); + expect(component.uploadTasks).toEqual([]); + expect(component.activeCount).toBe(0); + }); + + it("starts the next queued upload when an active upload finishes", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); + + finishUpload(0, "f1.txt"); + + expect(multipartUploadSpy).toHaveBeenCalledTimes(4); + expect(uploadedPaths[3]).toBe("f4.txt"); + expect(component.activeCount).toBe(3); + expect(component.queuedCount).toBe(1); + expect(component.queuedFileNames).toEqual(["f5.txt"]); + }); + + it("removes a cancelled file from the pending queue without starting it", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); + + component.cancelExistingUpload("f4.txt"); + + expect(multipartUploadSpy).toHaveBeenCalledTimes(3); + expect(component.queuedCount).toBe(1); + expect(component.queuedFileNames).toEqual(["f5.txt"]); + }); + + it("ignores cancellation of a file that is neither active nor queued", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); + + component.cancelExistingUpload("missing.txt"); + + expect(component.activeCount).toBe(3); + expect(component.queuedCount).toBe(1); + expect(component.queuedFileNames).toEqual(["f4.txt"]); + }); + + // #5586: the template reads queuedFileNames on every change-detection pass, + // so it must not allocate a new array unless the queue changed. + it("keeps the same queuedFileNames array reference while the queue is unchanged", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); + + const firstRead = component.queuedFileNames; + + expect(component.queuedFileNames).toBe(firstRead); + }); + + it("exposes a new queuedFileNames array after the queue changes", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); + const beforeCancel = component.queuedFileNames; + + component.cancelExistingUpload("f4.txt"); + + expect(component.queuedFileNames).not.toBe(beforeCancel); + expect(component.queuedFileNames).toEqual(["f5.txt"]); + }); + + it("identifies pending queue entries by file name in trackByPendingFile", () => { + expect(component.trackByPendingFile(0, "dir/a.txt")).toBe("dir/a.txt"); + }); + + // A resumed upload with no missing parts finishes with totalTime exactly 0; + // the slot must still be released. + it("releases the concurrency slot when a finished upload reports totalTime 0", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); + + finishUpload(0, "f1.txt", 0); + + expect(multipartUploadSpy).toHaveBeenCalledTimes(4); + expect(uploadedPaths[3]).toBe("f4.txt"); + expect(component.activeCount).toBe(3); + expect(component.queuedCount).toBe(0); + }); + + // The Pending header updates per file, so the Finished header must too — it + // cannot wait for the throttled staged-objects refetch. + it("updates the Finished count immediately when uploads finish", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); + expect(component.pendingChangesCount).toBe(0); + + finishUpload(0, "f1.txt"); + expect(component.pendingChangesCount).toBe(1); + + finishUpload(1, "f2.txt"); + expect(component.pendingChangesCount).toBe(2); + }); + + it("reconciles the optimistic Finished count with a diff response", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt"); + finishUpload(0, "f1.txt"); + finishUpload(1, "f2.txt"); + + const diff: DatasetStagedObject[] = [{ path: "f1.txt", pathType: "file", diffType: "added", sizeBytes: 1 }]; + component.onStagedObjectsUpdated(diff); + + // f1 is confirmed by the response; f2 stays counted until a response includes it. + expect(component.pendingChangesCount).toBe(2); + + component.onStagedObjectsUpdated([...diff, { path: "f2.txt", pathType: "file", diffType: "added", sizeBytes: 1 }]); + expect(component.pendingChangesCount).toBe(2); + }); + + it("keeps an in-progress upload's slot while progress events stream in", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); + + uploadSubjects[0].next({ filePath: "f1.txt", percentage: 50, status: "uploading" }); + + expect(component.uploadTasks.find(t => t.filePath === "f1.txt")?.percentage).toBe(50); + expect(component.activeCount).toBe(3); + expect(component.queuedCount).toBe(1); + }); + + it("does not double-count a finished upload already confirmed by a diff response", () => { + dropFiles("f1.txt"); + finishUpload(0, "f1.txt"); + component.onStagedObjectsUpdated([{ path: "f1.txt", pathType: "file", diffType: "added", sizeBytes: 1 }]); + expect(component.pendingChangesCount).toBe(1); + + dropFiles("f1.txt"); // re-upload the already-staged file + finishUpload(1, "f1.txt"); + + expect(component.pendingChangesCount).toBe(1); + }); + + it("does not start queued uploads beyond a lowered concurrency limit", () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt"); + component.maxConcurrentFiles = 1; + + finishUpload(0, "f1.txt"); + + expect(component.activeCount).toBe(2); + expect(component.queuedCount).toBe(1); + expect(multipartUploadSpy).toHaveBeenCalledTimes(3); + }); + + it("clears the Finished count when a version is created", () => { + dropFiles("f1.txt"); + finishUpload(0, "f1.txt"); + expect(component.pendingChangesCount).toBe(1); + + component.versionName = "v1"; + component.onClickCreateVersion(); + + expect(component.pendingChangesCount).toBe(0); + }); + + it("does not remove a re-uploaded file's active task when hiding its finished predecessor", () => { + vi.useFakeTimers(); + try { + dropFiles("a.txt"); + finishUpload(0, "a.txt"); // schedules the finished row to hide in 5s + + dropFiles("a.txt"); // re-upload the same name within the 5s window + vi.advanceTimersByTime(5000); + + expect(component.uploadTasks).toHaveLength(1); + expect(component.uploadTasks[0].status).not.toBe("finished"); + expect(component.activeCount).toBe(1); + + finishUpload(1, "a.txt"); + expect(component.activeCount).toBe(0); + } finally { + vi.useRealTimers(); + } + }); + + it("renders the virtualized pending list and re-measures viewports on panel expand", async () => { + dropFiles("f1.txt", "f2.txt", "f3.txt", "f4.txt", "f5.txt"); + + // Flush the viewport's init microtask, then render the rows. + fixture.detectChanges(); + await Promise.resolve(); + fixture.detectChanges(); + + expect(component.pendingListHeightPx).toBe(2 * component.PENDING_ROW_HEIGHT_PX); + const rows = fixture.nativeElement.querySelectorAll(".pending-file-row"); + expect(rows.length).toBe(2); + + // Expand the Pending / Uploading / Finished panels. + const headers: NodeListOf = fixture.nativeElement.querySelectorAll( + ".upload-status-panels .ant-collapse-header" + ); + expect(headers.length).toBe(3); + headers.forEach(header => header.click()); + fixture.detectChanges(); + // Flush the checkViewportSize timers. + await new Promise(resolve => setTimeout(resolve)); + + // Collapsing again must be a no-op for the re-measure handler. + headers.forEach(header => header.click()); + fixture.detectChanges(); + + // Cancel a queued file from its row. + const cancelButton = fixture.nativeElement.querySelector(".pending-file-row button") as HTMLButtonElement; + cancelButton.click(); + expect(component.queuedCount).toBe(1); + expect(component.queuedFileNames).toEqual(["f5.txt"]); + }); + + describe("rendered progress", () => { + /** + * Puts one task on the panel in the given state and opens it. The panel is gated on the + * separate activeUploads counter rather than on uploadTasks, and ng-zorro collapses it by + * default, so both have to be arranged before its body exists. + */ + function withTask(over: Record): HTMLElement { + (component as any).activeUploads = 1; + component.uploadTasks = [ + { + filePath: "big.csv", + percentage: 40, + status: "uploading", + uploadSpeed: 1024, + totalTime: 12, + estimatedTimeRemaining: 30, + ...over, + } as any, + ]; + fixture.detectChanges(); + const el = fixture.nativeElement as HTMLElement; + const header = Array.from(el.querySelectorAll(".ant-collapse-header")).find(h => + (h.textContent || "").includes("Uploading:") + ); + header!.click(); + fixture.detectChanges(); + return el; + } + + it("shows no statistics while an upload is still initializing", () => { + // There is nothing to report yet; showing a 0 B/s row reads as a stalled upload. + const el = withTask({ status: "initializing" }); + + expect(el.querySelector(".upload-stats")).toBeNull(); + }); + + it("reports speed and both timings while an upload runs", () => { + const el = withTask({ status: "uploading" }); + + const stats = el.querySelector(".upload-stats")!; + expect(stats.textContent).toContain("elapsed"); + expect(stats.textContent).toContain("left"); + expect(stats.querySelector(".fixed-width-speed")).not.toBeNull(); + }); + + it("replaces the live figures with a total once the upload finishes", () => { + const el = withTask({ status: "finished" }); + + const stats = el.querySelector(".upload-stats")!; + expect(stats.textContent).toContain("Upload time:"); + expect(stats.textContent).not.toContain("left"); + }); + + it("reports a total for an aborted upload too", () => { + const el = withTask({ status: "aborted" }); + + expect(el.querySelector(".upload-stats")!.textContent).toContain("Upload time:"); + }); + }); + + describe("rendered panel", () => { + const host = (): HTMLElement => fixture.nativeElement as HTMLElement; + + const q = (root: ParentNode, selector: string): E => { + const el = root.querySelector(selector); + expect(el, `expected to find "${selector}"`).not.toBeNull(); + return el as unknown as E; + }; + + const text = (el: Element | null | undefined): string => (el?.textContent ?? "").replace(/\s+/g, " ").trim(); + + /** ng-zorro collapses every status panel by default, so its body has to be opened first. */ + const openPanel = (header: string): void => { + const found = Array.from(host().querySelectorAll(".ant-collapse-header")).find(h => + (h.textContent || "").includes(header) + ); + expect(found, `expected a collapse panel headed "${header}"`).toBeDefined(); + found!.click(); + fixture.detectChanges(); + }; + + describe("upload panel", () => { + it("starts an upload for a file the uploader hands over", () => { + const el = host(); + const uploader = fixture.debugElement.query(By.css("texera-user-files-uploader")); + // Distinct tuning values, so the two cannot stand in for each other below. + component.chunkSizeMiB = 50; + component.maxConcurrentChunks = 10; + + uploader.triggerEventHandler("uploadedFiles", [makeFileItem("new.csv")]); + fixture.detectChanges(); + + // Both are plain numbers, so asserting their exact values is the only way to notice + // them exchanged: 10-byte chunks, or 52 million parallel requests, would look + // identical to expect.any(Number). + expect(multipartUploadSpy).toHaveBeenCalledWith( + DATASET_FILE_RESOURCE_ENDPOINT, + "owner@texera.com", + "test-dataset", + "new.csv", + expect.anything(), + 50 * 1024 * 1024, + 10, + false + ); + expect(text(el)).toContain("Uploading: 1 file(s)"); + }); + + /** Renders the given in-flight tasks and expands the "Uploading" panel. */ + const withTasks = (...tasks: Array>): HTMLElement => { + component.uploadTasks = tasks.map(t => ({ + percentage: 40, + status: "uploading", + uploadSpeed: 1024, + totalTime: 12, + estimatedTimeRemaining: 30, + ...t, + })) as never; + (component as unknown as { activeUploads: number }).activeUploads = tasks.length; + fixture.detectChanges(); + openPanel("Uploading:"); + return host(); + }; + + it("aborts the upload whose own row button was clicked", () => { + withTasks({ filePath: "first.csv" }, { filePath: "second.csv" }); + + const rows = fixture.debugElement.queryAll(By.css(".upload-progress-wrapper > div")); + expect(rows.length).toBe(2); + // Each row has to name its own task: identifying the row by position alone + // would not notice every row rendering the first task's name and status. + expect(rows.map(row => text(row.query(By.css(".progress-header")).nativeElement))).toEqual([ + "uploading: first.csv", + "uploading: second.csv", + ]); + + const abort = rows[1].query(By.css(".progress-header button")); + // A live upload is cancelled, not dismissed; the finished row below says "Close". + expect(abort.injector.get(NzTooltipDirective).directiveTitle).toBe("Cancel the upload"); + + abort.nativeElement.click(); + fixture.detectChanges(); + + const finalize = TestBed.inject(MultipartUploadService).finalizeMultipartUpload as unknown as ReturnType< + typeof vi.fn + >; + expect(finalize).toHaveBeenCalledTimes(1); + expect(finalize).toHaveBeenCalledWith( + DATASET_FILE_RESOURCE_ENDPOINT, + "owner@texera.com", + "test-dataset", + "second.csv", + true + ); + }); + + it("reports the elapsed time, the time remaining and the speed in their own slots", () => { + // Distinguishable timings, so the two spans cannot stand in for each other: + // showing 90s elapsed on a 12s-old upload is the defect this guards. + const el = withTasks({ + filePath: "big.csv", + totalTime: 12, + estimatedTimeRemaining: 90, + uploadSpeed: 5 * 1024 * 1024, + }); + const stats = q(el, ".upload-stats"); + + expect(Array.from(stats.querySelectorAll(".fixed-width-time")).map(text)).toEqual(["12s", "1m30s left"]); + expect(text(q(stats, ".fixed-width-speed"))).toBe("5.0 MB/s"); + }); + + it("floors both live timings at one second while an upload reports none", () => { + const el = withTasks({ filePath: "big.csv", totalTime: undefined, estimatedTimeRemaining: undefined }); + + const times = Array.from(q(el, ".upload-stats").querySelectorAll(".fixed-width-time")).map(text); + expect(times).toEqual(["1s", "1s left"]); + }); + + it("reports the total time of a finished upload", () => { + const el = withTasks({ filePath: "big.csv", status: "finished", totalTime: 75 }); + + expect(text(q(el, ".upload-stats"))).toContain("Upload time: 1m15s"); + // A finished row is dismissed rather than cancelled. + const button = fixture.debugElement.query(By.css(".upload-progress-wrapper > div .progress-header button")); + expect(button.injector.get(NzTooltipDirective).directiveTitle).toBe("Close"); + }); + + it("floors the total of a finished upload that timed nothing", () => { + const el = withTasks({ filePath: "big.csv", status: "finished", totalTime: undefined }); + + expect(text(q(el, ".upload-stats"))).toContain("Upload time: 1s"); + }); + }); + + describe("version creator", () => { + /** Renders the creator, which only appears with staged changes to commit. */ + const withPendingChanges = (): HTMLElement => { + component.userHasPendingChanges = true; + fixture.detectChanges(); + return host(); + }; + + const typeName = (el: HTMLElement, value: string): HTMLInputElement => { + const input = q(el, ".version-input"); + input.value = value; + input.dispatchEvent(new Event("input")); + fixture.detectChanges(); + return input; + }; + + it("offers the creator only once there is something to commit", () => { + const el = host(); + expect(el.querySelector(".version-creator")).toBeNull(); + + component.userHasPendingChanges = true; + fixture.detectChanges(); + + expect(el.querySelector(".version-creator")).not.toBeNull(); + expect(text(q(el, ".create-version-button"))).toBe("Submit"); + }); + + it("creates a version named by the creator's own input", () => { + const el = withPendingChanges(); + + typeName(el, "second cut"); + q(el, ".create-version-button").click(); + + expect(createVersionSpy).toHaveBeenCalledWith("second cut"); + }); + + it("submits the version straight from the name field with Enter", () => { + const el = withPendingChanges(); + + typeName(el, "from the keyboard").dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true })); + + expect(createVersionSpy).toHaveBeenCalledWith("from the keyboard"); + }); + + it("spins the submit button and locks the name field while a version is being created", async () => { + const el = withPendingChanges(); + expect(q(el, ".create-version-button").classList).not.toContain("ant-btn-loading"); + expect(q(el, ".version-input").disabled).toBe(false); + + component.isCreatingVersion = true; + fixture.detectChanges(); + // NgModel routes the input's `disabled` binding through control.disable(), + // which it defers to a microtask, so the DOM lags the render by one turn. + await Promise.resolve(); + fixture.detectChanges(); + + expect(q(el, ".create-version-button").classList).toContain("ant-btn-loading"); + // Renaming a version mid-creation would be applied to nothing, so the + // field is locked for as long as the request is in flight. + expect(q(el, ".version-input").disabled).toBe(true); + }); + }); + }); +}); diff --git a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.ts b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.ts new file mode 100644 index 00000000000..8f709a1f6c8 --- /dev/null +++ b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.ts @@ -0,0 +1,469 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Component, EventEmitter, Input, OnInit, Output, ViewChild } from "@angular/core"; +import { UntilDestroy, untilDestroyed } from "@ngneat/until-destroy"; +import { Observable, Subscription } from "rxjs"; +import { HttpErrorResponse, HttpStatusCode } from "@angular/common/http"; +import { NgFor, NgIf } from "@angular/common"; +import { FormsModule } from "@angular/forms"; +import { CdkFixedSizeVirtualScroll, CdkVirtualForOf, CdkVirtualScrollViewport } from "@angular/cdk/scrolling"; +import { NzButtonComponent } from "ng-zorro-antd/button"; +import { NzCollapseComponent, NzCollapsePanelComponent } from "ng-zorro-antd/collapse"; +import { NzDividerComponent } from "ng-zorro-antd/divider"; +import { NzIconDirective } from "ng-zorro-antd/icon"; +import { NzInputDirective } from "ng-zorro-antd/input"; +import { NzProgressComponent } from "ng-zorro-antd/progress"; +import { NzTagComponent } from "ng-zorro-antd/tag"; +import { NzTooltipDirective } from "ng-zorro-antd/tooltip"; +import { ɵNzTransitionPatchDirective } from "ng-zorro-antd/core/transition-patch"; + +import { AdminSettingsService } from "../../../service/admin/settings/admin-settings.service"; +import { + MultipartUploadProgress, + MultipartUploadService, +} from "../../../service/user/file-resource/multipart-upload.service"; +import { + DATASET_FILE_RESOURCE_ENDPOINT, + FileResourceEndpoint, +} from "../../../service/user/file-resource/file-resource-endpoint"; +import { NotificationService } from "../../../../common/service/notification/notification.service"; +import { extractErrorMessage } from "../../../../common/util/error"; +import { formatSpeed, formatTime, parseIntOrDefault } from "src/app/common/util/format.util"; +import { DatasetStagedObject } from "../../../../common/type/dataset-staged-object"; +import { FileUploadItem } from "../../../type/dashboard-file.interface"; +import { FilesUploaderComponent } from "../files-uploader/files-uploader.component"; +import { StagedObjectsListComponent } from "../staged-objects-list/staged-objects-list.component"; + +export const ABORT_RETRY_MAX_ATTEMPTS = 10; +export const ABORT_RETRY_BACKOFF_BASE_MS = 100; +export const FINISHED_TASK_HIDE_DELAY_MS = 5000; + +/** + * The "Create New Version" panel of a versioned resource's detail page: pick files, watch them + * upload, review what is staged, and commit it as a version. Resource-agnostic — addressing comes + * from `endpoint`, and only the version-creation call is passed in, because each resource kind + * types its own response payload. + */ +@UntilDestroy() +@Component({ + selector: "texera-version-uploader", + templateUrl: "./version-uploader.component.html", + styleUrls: ["./version-uploader.component.scss"], + imports: [ + NgIf, + NgFor, + FormsModule, + NzButtonComponent, + NzCollapseComponent, + NzCollapsePanelComponent, + NzDividerComponent, + NzIconDirective, + NzInputDirective, + NzProgressComponent, + NzTagComponent, + NzTooltipDirective, + ɵNzTransitionPatchDirective, + CdkVirtualScrollViewport, + CdkFixedSizeVirtualScroll, + CdkVirtualForOf, + FilesUploaderComponent, + StagedObjectsListComponent, + ], +}) +export class VersionUploaderComponent implements OnInit { + @Input() resourceId: number | undefined; + @Input() resourceName: string = ""; + @Input() ownerEmail: string = ""; + /** Which resource family the ids above belong to. */ + @Input() endpoint: FileResourceEndpoint = DATASET_FILE_RESOURCE_ENDPOINT; + /** Commits the staged files as a new version. */ + @Input() createVersion!: (versionName: string) => Observable; + + /** The host reloads its version list off this. */ + @Output() versionCreated = new EventEmitter(); + + userHasPendingChanges: boolean = false; + pendingChangesCount: number = 0; + // Staged paths from the last diff response, plus locally staged paths not yet in one: counted + // together so the Finished header keeps pace with the real-time Pending header. + private confirmedStagedPaths = new Set(); + private unconfirmedStagedPaths = new Set(); + + // Upload tuning, overridden by this resource family's settings in Admin -> Settings. + chunkSizeMiB: number = 50; + maxConcurrentChunks: number = 10; + maxConcurrentFiles: number = 3; + private uploadSubscriptions = new Map(); + uploadTimeMap = new Map(); + + private activeUploads: number = 0; + // FIFO queue of uploads waiting for a concurrency slot, keyed by file name. + private pendingQueue = new Map void>(); + private pendingQueueDirty = false; + private queuedFileNamesSnapshot: string[] = []; + + // Row height must match .pending-file-row in the SCSS. + readonly PENDING_ROW_HEIGHT_PX = 32; + readonly PENDING_LIST_MAX_HEIGHT_PX = 160; + + @ViewChild(CdkVirtualScrollViewport) private pendingViewport?: CdkVirtualScrollViewport; + + versionName: string = ""; + isCreatingVersion: boolean = false; + + // One row per in-flight or recently finished upload, keyed by file path. + uploadTasks: Array = []; + + // Coalesced by the staged list, which refetches the diff at most once per window. + userMakeChanges = new EventEmitter(); + + formatTime = formatTime; + formatSpeed = formatSpeed; + + constructor( + private multipartUploadService: MultipartUploadService, + private adminSettingsService: AdminSettingsService, + private notificationService: NotificationService + ) {} + + ngOnInit(): void { + this.loadUploadSettings(); + } + + // A missing key or failed fetch keeps the field defaults; NaN here would silently stall the + // upload queue (`activeUploads < NaN` is always false). + private loadUploadSettings(): void { + this.adminSettingsService + .getPublicSetting(this.endpoint.chunkSizeSettingKey) + .pipe(untilDestroyed(this)) + .subscribe({ + next: value => (this.chunkSizeMiB = parseIntOrDefault(value, this.chunkSizeMiB)), + error: () => {}, + }); + this.adminSettingsService + .getPublicSetting(this.endpoint.maxConcurrentChunksSettingKey) + .pipe(untilDestroyed(this)) + .subscribe({ + next: value => (this.maxConcurrentChunks = parseIntOrDefault(value, this.maxConcurrentChunks)), + error: () => {}, + }); + this.adminSettingsService + .getPublicSetting(this.endpoint.maxConcurrentFilesSettingKey) + .pipe(untilDestroyed(this)) + .subscribe({ + next: value => (this.maxConcurrentFiles = parseIntOrDefault(value, this.maxConcurrentFiles)), + error: () => {}, + }); + } + + onNewUploadFilesChanged(files: FileUploadItem[]): void { + if (!this.resourceId) { + return; + } + files.forEach(file => { + const continueWithUpload = () => { + const startUpload = () => { + this.removeFromPendingQueue(file.name); + this.uploadTasks.unshift({ filePath: file.name, percentage: 0, status: "initializing" }); + + const subscription = this.multipartUploadService + .multipartUpload( + this.endpoint, + this.ownerEmail, + this.resourceName, + file.name, + file.file, + this.chunkSizeMiB * 1024 * 1024, + this.maxConcurrentChunks, + file.restart + ) + .pipe(untilDestroyed(this)) + .subscribe({ + next: progress => { + const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); + if (taskIndex === -1) { + return; + } + this.uploadTasks[taskIndex] = { + ...this.uploadTasks[taskIndex], + ...progress, + percentage: progress.percentage ?? this.uploadTasks[taskIndex].percentage ?? 0, + }; + // totalTime may be exactly 0 (resumed upload with no missing parts); a truthiness + // check would leak the concurrency slot. + if (progress.status === "finished" && progress.totalTime !== undefined) { + const filename = file.name.split("/").pop() || file.name; + this.uploadTimeMap.set(filename, progress.totalTime); + this.notePathStaged(file.name); + this.scheduleHide(taskIndex); + this.onUploadComplete(); + } + }, + error: (res: unknown) => { + const err = res as HttpErrorResponse; + if (err?.status === HttpStatusCode.Conflict) { + this.notificationService.error( + "Upload blocked (409). Another upload is likely in progress for this file (another tab/browser), or the server is finalizing a previous upload. Please retry in a moment." + ); + } else { + this.notificationService.error("Upload failed. Please retry."); + } + const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); + if (taskIndex !== -1) { + this.uploadTasks[taskIndex] = { + ...this.uploadTasks[taskIndex], + percentage: this.uploadTasks[taskIndex].percentage ?? 0, + status: "failed", + }; + this.scheduleHide(taskIndex); + } + this.onUploadComplete(); + }, + complete: () => { + const taskIndex = this.uploadTasks.findIndex(t => t.filePath === file.name); + if (taskIndex !== -1 && this.uploadTasks[taskIndex].status !== "finished") { + this.uploadTasks[taskIndex].status = "finished"; + this.notePathStaged(file.name); + this.scheduleHide(taskIndex); + this.onUploadComplete(); + } + }, + }); + this.uploadSubscriptions.set(file.name, subscription); + }; + + if (this.activeUploads < this.maxConcurrentFiles) { + this.activeUploads++; + startUpload(); + } else { + this.pendingQueue.set(file.name, startUpload); + this.pendingQueueDirty = true; + } + }; + + this.cancelExistingUpload(file.name, continueWithUpload); + }); + } + + cancelExistingUpload(fileName: string, onCanceled?: () => void): void { + const task = this.uploadTasks.find(t => t.filePath === fileName); + if (task && (task.status === "uploading" || task.status === "initializing")) { + this.onClickAbortUploadProgress(task, onCanceled); + return; + } + this.removeFromPendingQueue(fileName); + onCanceled?.(); + } + + private processNextQueuedUpload(): void { + if (this.activeUploads >= this.maxConcurrentFiles) { + return; + } + const next = this.pendingQueue.entries().next(); + if (!next.done) { + const [fileName, startUpload] = next.value; + this.pendingQueue.delete(fileName); + this.pendingQueueDirty = true; + this.activeUploads++; + startUpload(); + } + } + + private onUploadComplete(): void { + this.activeUploads--; + this.processNextQueuedUpload(); + } + + private removeFromPendingQueue(fileName: string): void { + if (this.pendingQueue.delete(fileName)) { + this.pendingQueueDirty = true; + } + } + + // Stable array for the template: rebuilt at most once per queue change so change detection does + // not allocate a new array per pass (#5586). + get queuedFileNames(): string[] { + if (this.pendingQueueDirty) { + this.queuedFileNamesSnapshot = Array.from(this.pendingQueue.keys()); + this.pendingQueueDirty = false; + } + return this.queuedFileNamesSnapshot; + } + + get queuedCount(): number { + return this.pendingQueue.size; + } + + get activeCount(): number { + return this.activeUploads; + } + + get pendingListHeightPx(): number { + return Math.min(this.queuedCount * this.PENDING_ROW_HEIGHT_PX, this.PENDING_LIST_MAX_HEIGHT_PX); + } + + get hasAnyActivity(): boolean { + return this.pendingChangesCount > 0 || this.activeCount > 0 || this.queuedCount > 0; + } + + // The viewport initializes inside the collapsed (display: none) panel and measures height 0; the + // CDK only re-measures on window resize. + onPendingPanelActiveChange(active: boolean): void { + if (active) { + setTimeout(() => this.pendingViewport?.checkViewportSize()); + } + } + + // Hide a finished, failed or aborted row after a short delay. + private scheduleHide(idx: number): void { + if (idx === -1) { + return; + } + const task = this.uploadTasks[idx]; + this.uploadSubscriptions.delete(task.filePath); + // Remove by identity, not filePath: a same-named re-upload within the window has its own row, + // which must survive this timer. + setTimeout(() => { + this.uploadTasks = this.uploadTasks.filter(t => t !== task); + }, FINISHED_TASK_HIDE_DELAY_MS); + } + + onClickAbortUploadProgress(task: MultipartUploadProgress & { filePath: string }, onAborted?: () => void): void { + const subscription = this.uploadSubscriptions.get(task.filePath); + if (subscription) { + subscription.unsubscribe(); + this.uploadSubscriptions.delete(task.filePath); + } + + if (task.status === "uploading" || task.status === "initializing") { + this.onUploadComplete(); + } + + let doneCalled = false; + const done = () => { + if (doneCalled) { + return; + } + doneCalled = true; + onAborted?.(); + }; + + const abortWithRetry = (attempt: number) => { + this.multipartUploadService + .finalizeMultipartUpload(this.endpoint, this.ownerEmail, this.resourceName, task.filePath, true) + .pipe(untilDestroyed(this)) + .subscribe({ + next: () => { + this.notificationService.info(`${task.filePath} uploading has been terminated`); + done(); + }, + error: (res: unknown) => { + const err = res as HttpErrorResponse; + // Already gone, treat as done. + if (err.status === HttpStatusCode.NotFound) { + done(); + return; + } + // Backend is still finalizing/aborting; retry with a tiny backoff. + if (err.status === HttpStatusCode.Conflict && attempt < ABORT_RETRY_MAX_ATTEMPTS) { + setTimeout(() => abortWithRetry(attempt + 1), ABORT_RETRY_BACKOFF_BASE_MS * (attempt + 1)); + return; + } + done(); + }, + }); + }; + + abortWithRetry(0); + + const idx = this.uploadTasks.findIndex(t => t.filePath === task.filePath); + if (idx !== -1) { + this.uploadTasks[idx] = { ...this.uploadTasks[idx], status: "aborted" }; + this.scheduleHide(idx); + } + } + + getUploadStatus(status: MultipartUploadProgress["status"]): "active" | "exception" | "success" { + return status === "uploading" || status === "initializing" + ? "active" + : status === "aborted" || status === "failed" + ? "exception" + : "success"; + } + + trackByTask(_: number, task: MultipartUploadProgress & { filePath: string }): string { + return task.filePath; + } + + trackByPendingFile(_: number, fileName: string): string { + return fileName; + } + + onStagedObjectsUpdated(stagedObjects: DatasetStagedObject[]): void { + this.confirmedStagedPaths = new Set(stagedObjects.map(obj => obj.path)); + for (const path of this.confirmedStagedPaths) { + this.unconfirmedStagedPaths.delete(path); + } + this.refreshPendingChanges(); + } + + /** + * Counts a change staged outside this panel — the host stages a deletion from the file tree — + * in the Finished header immediately, ahead of the next diff response. + */ + notePathStaged(path: string): void { + if (!this.confirmedStagedPaths.has(path)) { + this.unconfirmedStagedPaths.add(path); + } + this.refreshPendingChanges(); + this.userMakeChanges.emit(); + } + + private refreshPendingChanges(): void { + this.pendingChangesCount = this.confirmedStagedPaths.size + this.unconfirmedStagedPaths.size; + this.userHasPendingChanges = this.pendingChangesCount > 0; + } + + onClickCreateVersion(): void { + if (!this.resourceId || this.isCreatingVersion) { + return; + } + this.isCreatingVersion = true; + this.createVersion(this.versionName?.trim() || "") + .pipe(untilDestroyed(this)) + .subscribe({ + next: () => { + this.notificationService.success("Version Created"); + this.isCreatingVersion = false; + this.versionName = ""; + // A new version consumes all staged changes. + this.confirmedStagedPaths.clear(); + this.unconfirmedStagedPaths.clear(); + this.refreshPendingChanges(); + this.versionCreated.emit(); + this.userMakeChanges.emit(); + }, + error: (err: unknown) => { + this.notificationService.error(`Version creation failed: ${extractErrorMessage(err)}`); + this.isCreatingVersion = false; + }, + }); + } +} diff --git a/frontend/src/app/dashboard/service/user/dataset/dataset.service.spec.ts b/frontend/src/app/dashboard/service/user/dataset/dataset.service.spec.ts index 1af4773e6da..181bd3b79b0 100644 --- a/frontend/src/app/dashboard/service/user/dataset/dataset.service.spec.ts +++ b/frontend/src/app/dashboard/service/user/dataset/dataset.service.spec.ts @@ -22,6 +22,7 @@ import { HttpClientTestingModule, HttpTestingController } from "@angular/common/ import { firstValueFrom } from "rxjs"; import { DATASET_BASE_URL, DatasetService, MultipartUploadProgress, validateDatasetName } from "./dataset.service"; +import { FakeXMLHttpRequest } from "../file-resource/testing/fake-xml-http-request"; import { AppSettings } from "../../../../common/app-setting"; import { AuthService } from "../../../../common/service/user/auth.service"; import { commonTestProviders } from "../../../../common/testing/test-utils"; @@ -75,63 +76,6 @@ const SAMPLE_FILE_NODES: DatasetFileNode[] = [ { name: "root", type: "directory", parentDir: "", children: [] as DatasetFileNode[] } as DatasetFileNode, ]; -class FakeXMLHttpRequest { - static instances: FakeXMLHttpRequest[] = []; - - // Capturing upload target so tests can drive `upload.progress` events. - readonly upload = { - listeners: new Map(), - addEventListener(type: string, listener: EventListener): void { - this.listeners.set(type, [...(this.listeners.get(type) ?? []), listener]); - }, - }; - status = 0; - url = ""; - readonly requestHeaders = new Map(); - private listeners = new Map(); - - open(_method: string, url: string): void { - this.url = url; - } - - setRequestHeader(name: string, value: string): void { - this.requestHeaders.set(name, value); - } - - send(): void { - FakeXMLHttpRequest.instances.push(this); - } - - abort(): void {} - - addEventListener(type: string, listener: EventListener): void { - this.listeners.set(type, [...(this.listeners.get(type) ?? []), listener]); - } - - /** Drives the `upload.progress` listener registered by the service. */ - emitProgress(loaded: number, lengthComputable = true): void { - const event = { lengthComputable, loaded } as unknown as Event; - for (const listener of this.upload.listeners.get("progress") ?? []) { - listener(event); - } - } - - respond(status: number): void { - this.status = status; - this.emit("load"); - } - - fail(): void { - this.emit("error"); - } - - private emit(type: string): void { - for (const listener of this.listeners.get(type) ?? []) { - listener(new Event(type)); - } - } -} - describe("validateDatasetName", () => { it("returns null for a valid name", () => { expect(validateDatasetName("my-dataset_1")).toBeNull(); diff --git a/frontend/src/app/dashboard/service/user/dataset/dataset.service.ts b/frontend/src/app/dashboard/service/user/dataset/dataset.service.ts index 9d21e41977a..a370158369c 100644 --- a/frontend/src/app/dashboard/service/user/dataset/dataset.service.ts +++ b/frontend/src/app/dashboard/service/user/dataset/dataset.service.ts @@ -19,15 +19,17 @@ import { Injectable } from "@angular/core"; import { HttpClient, HttpErrorResponse, HttpParams } from "@angular/common/http"; -import { catchError, map, mergeMap, switchMap, tap, toArray } from "rxjs/operators"; +import { map, switchMap } from "rxjs/operators"; import { Contributor, Dataset, DatasetVersion } from "../../../../common/type/dataset"; import { AppSettings } from "../../../../common/app-setting"; -import { EMPTY, from, Observable, throwError } from "rxjs"; +import { EMPTY, Observable } from "rxjs"; import { DashboardDataset } from "../../../type/dashboard-dataset.interface"; import { DatasetFileNode } from "../../../../common/type/datasetVersionFileTree"; import { DatasetStagedObject } from "../../../../common/type/dataset-staged-object"; import { GuiConfigService } from "../../../../common/service/gui-config.service"; -import { AuthService } from "src/app/common/service/user/auth.service"; +import { MultipartUploadProgress, MultipartUploadService } from "../file-resource/multipart-upload.service"; +import { StagedFileService } from "../file-resource/staged-file.service"; +import { DATASET_FILE_RESOURCE_ENDPOINT } from "../file-resource/file-resource-endpoint"; export const DATASET_BASE_URL = "dataset"; export const DATASET_CREATE_URL = DATASET_BASE_URL + "/create"; @@ -70,14 +72,8 @@ export const DATASET_PUBLIC_VERSION_BASE_URL = "publicVersion"; export const DATASET_PUBLIC_VERSION_RETRIEVE_LIST_URL = DATASET_PUBLIC_VERSION_BASE_URL + "/list"; export const DATASET_GET_OWNERS_URL = DATASET_BASE_URL + "/user-dataset-owners"; -export interface MultipartUploadProgress { - filePath: string; - percentage: number; - status: "initializing" | "uploading" | "finished" | "aborted" | "failed"; - uploadSpeed?: number; // bytes per second - estimatedTimeRemaining?: number; // seconds - totalTime?: number; // total seconds taken -} +// Re-exported so existing importers keep resolving it from here. +export type { MultipartUploadProgress }; @Injectable({ providedIn: "root", @@ -85,7 +81,9 @@ export interface MultipartUploadProgress { export class DatasetService { constructor( private http: HttpClient, - private config: GuiConfigService + private config: GuiConfigService, + private multipartUploadService: MultipartUploadService, + private stagedFileService: StagedFileService ) {} public createDataset(dataset: Dataset, contributors: Contributor[] = []): Observable { @@ -162,14 +160,8 @@ export class DatasetService { } /** - * Handles multipart upload for large files using RxJS, - * with a concurrency limit on how many parts we process in parallel. - * - * Backend flow: - * POST /dataset/multipart-upload?type=init&ownerEmail=...&datasetName=...&filePath=...&numParts=N - * POST /dataset/multipart-upload/part?ownerEmail=...&datasetName=...&filePath=...&partNumber= (body: raw chunk) - * POST /dataset/multipart-upload?type=finish&ownerEmail=...&datasetName=...&filePath=... - * POST /dataset/multipart-upload?type=abort&ownerEmail=...&datasetName=...&filePath=... + * Multipart upload for large files. The engine lives in MultipartUploadService; this keeps the + * dataset-facing signature so existing callers are untouched. */ public multipartUpload( ownerEmail: string, @@ -180,278 +172,24 @@ export class DatasetService { concurrencyLimit: number, restart: boolean ): Observable { - const partCount = Math.ceil(file.size / partSize); - - return new Observable(observer => { - // Track upload progress (bytes) for each part independently - const partProgress = new Map(); - - let baselineUploaded = 0; - // Progress tracking state - let startTime: number | null = null; - const speedSamples: number[] = []; - let lastETA = 0; - let lastUpdateTime = 0; - - const lastStats = { - uploadSpeed: 0, - estimatedTimeRemaining: 0, - totalTime: 0, - }; - - const getTotalTime = () => (startTime ? (Date.now() - startTime) / 1000 : 0); - - // Calculate stats with smoothing and simple throttling (~1s) - const calculateStats = (totalUploaded: number) => { - if (startTime === null) { - startTime = Date.now(); - } - - const now = Date.now(); - const elapsed = getTotalTime(); - - const shouldUpdate = now - lastUpdateTime >= 1000; - if (!shouldUpdate) { - // keep totalTime fresh even when throttled - lastStats.totalTime = elapsed; - return lastStats; - } - lastUpdateTime = now; - - const sessionUploaded = Math.max(0, totalUploaded - baselineUploaded); - const currentSpeed = elapsed > 0 ? sessionUploaded / elapsed : 0; - speedSamples.push(currentSpeed); - if (speedSamples.length > 5) { - speedSamples.shift(); - } - const avgSpeed = speedSamples.length > 0 ? speedSamples.reduce((a, b) => a + b, 0) / speedSamples.length : 0; - - const remaining = file.size - totalUploaded; - let eta = avgSpeed > 0 ? remaining / avgSpeed : 0; - eta = Math.min(eta, 24 * 60 * 60); // cap ETA at 24h - - if (lastETA > 0 && eta > 0) { - const maxChange = lastETA * 0.3; - const diff = Math.abs(eta - lastETA); - if (diff > maxChange) { - eta = lastETA + (eta > lastETA ? maxChange : -maxChange); - } - } - lastETA = eta; - - const percentComplete = (totalUploaded / file.size) * 100; - if (percentComplete > 95) { - eta = Math.min(eta, 10); - } - - lastStats.uploadSpeed = avgSpeed; - lastStats.estimatedTimeRemaining = Math.max(0, Math.round(eta)); - lastStats.totalTime = elapsed; - - return lastStats; - }; - - // 1. INIT: ask backend to create a LakeFS multipart upload session - const initParams = new HttpParams() - .set("type", "init") - .set("ownerEmail", ownerEmail) - .set("datasetName", datasetName) - .set("filePath", encodeURIComponent(filePath)) - .set("fileSizeBytes", file.size.toString()) - .set("partSizeBytes", partSize.toString()) - .set("restart", restart); - - const init$ = this.http.post<{ missingParts: number[]; completedPartsCount: number }>( - `${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/multipart-upload`, - {}, - { params: initParams } - ); - - const subscription = init$ - .pipe( - switchMap(initResp => { - const missingParts = (initResp?.missingParts ?? []).slice(); - const completedPartsCount = initResp?.completedPartsCount ?? 0; - - const missingBytes = missingParts.reduce((sum, partNumber) => { - const start = (partNumber - 1) * partSize; - const end = Math.min(start + partSize, file.size); - return sum + (end - start); - }, 0); - - baselineUploaded = file.size - missingBytes; - const baselinePct = partCount > 0 ? Math.round((completedPartsCount / partCount) * 100) : 0; - - observer.next({ - filePath, - percentage: baselinePct, - status: "initializing", - uploadSpeed: 0, - estimatedTimeRemaining: 0, - totalTime: 0, - }); - // 2. Upload each part to /multipart-upload/part using XMLHttpRequest - return from(missingParts).pipe( - mergeMap(partNumber => { - const start = (partNumber - 1) * partSize; - const end = Math.min(start + partSize, file.size); - const chunk = file.slice(start, end); - - return new Observable(partObserver => { - const xhr = new XMLHttpRequest(); - - xhr.upload.addEventListener("progress", event => { - if (event.lengthComputable) { - partProgress.set(partNumber, event.loaded); - - let totalUploaded = baselineUploaded; // CHANGED - partProgress.forEach(bytes => { - totalUploaded += bytes; - }); - - const percentage = Math.round((totalUploaded / file.size) * 100); - const stats = calculateStats(totalUploaded); - - observer.next({ - filePath, - percentage: Math.min(percentage, 99), - status: "uploading", - ...stats, - }); - } - }); - - xhr.addEventListener("load", () => { - if (xhr.status === 200 || xhr.status === 204) { - // Mark part as fully uploaded - partProgress.set(partNumber, chunk.size); - - let totalUploaded = baselineUploaded; - partProgress.forEach(bytes => { - totalUploaded += bytes; - }); - - // Force stats recompute on completion - lastUpdateTime = 0; - const percentage = Math.round((totalUploaded / file.size) * 100); - const stats = calculateStats(totalUploaded); - - observer.next({ - filePath, - percentage: Math.min(percentage, 99), - status: "uploading", - ...stats, - }); - - partObserver.complete(); - } else { - partObserver.error(new Error(`Failed to upload part ${partNumber} (HTTP ${xhr.status})`)); - } - }); - - xhr.addEventListener("error", () => { - // Remove failed part from progress - partProgress.delete(partNumber); - partObserver.error(new Error(`Failed to upload part ${partNumber}`)); - }); - - const partUrl = - `${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/multipart-upload/part` + - `?ownerEmail=${encodeURIComponent(ownerEmail)}` + - `&datasetName=${encodeURIComponent(datasetName)}` + - `&filePath=${encodeURIComponent(filePath)}` + - `&partNumber=${partNumber}`; - - xhr.open("POST", partUrl); - xhr.setRequestHeader("Content-Type", "application/octet-stream"); - const token = AuthService.getAccessToken(); - if (token) { - xhr.setRequestHeader("Authorization", `Bearer ${token}`); - } - xhr.send(chunk); - return () => { - try { - xhr.abort(); - } catch {} - }; - }); - }, concurrencyLimit), - toArray(), // wait for all parts - // 3. FINISH: notify backend that all parts are done - switchMap(() => { - const finishParams = new HttpParams() - .set("type", "finish") - .set("ownerEmail", ownerEmail) - .set("datasetName", datasetName) - .set("filePath", encodeURIComponent(filePath)); - - return this.http.post( - `${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/multipart-upload`, - {}, - { params: finishParams } - ); - }), - tap(() => { - const totalTime = getTotalTime(); - observer.next({ - filePath, - percentage: 100, - status: "finished", - uploadSpeed: 0, - estimatedTimeRemaining: 0, - totalTime, - }); - observer.complete(); - }), - catchError((error: unknown) => { - // On error, compute best-effort percentage from bytes we've seen - let totalUploaded = baselineUploaded; - partProgress.forEach(bytes => { - totalUploaded += bytes; - }); - const percentage = file.size > 0 ? Math.round((totalUploaded / file.size) * 100) : 0; - - observer.next({ - filePath, - percentage, - status: "failed", - uploadSpeed: 0, - estimatedTimeRemaining: 0, - totalTime: getTotalTime(), - }); - - return throwError(() => error); - }) - ); - }) - ) - .subscribe({ - error: (err: unknown) => observer.error(err), - }); - - return () => subscription.unsubscribe(); - }); + return this.multipartUploadService.multipartUpload( + DATASET_FILE_RESOURCE_ENDPOINT, + ownerEmail, + datasetName, + filePath, + file, + partSize, + concurrencyLimit, + restart + ); } public listMultipartUploads(ownerEmail: string, datasetName: string): Observable { - const params = new HttpParams().set("type", "list").set("ownerEmail", ownerEmail).set("datasetName", datasetName); - - return this.http - .post<{ - filePaths: string[]; - }>(`${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/multipart-upload`, {}, { params }) - .pipe(map(res => res?.filePaths ?? [])); + return this.multipartUploadService.listMultipartUploads(DATASET_FILE_RESOURCE_ENDPOINT, ownerEmail, datasetName); } public findExistingUploadFiles(did: number, files: { path: string; sizeBytes: number }[]): Observable { - return this.http - .post<{ filePaths: string[] }>( - `${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/${did}/existing-upload-files`, - { - files, - } - ) - .pipe(map(res => res?.filePaths ?? [])); + return this.multipartUploadService.findExistingUploadFiles(DATASET_FILE_RESOURCE_ENDPOINT, did, files); } public finalizeMultipartUpload( @@ -460,16 +198,12 @@ export class DatasetService { filePath: string, isAbort: boolean ): Observable { - const params = new HttpParams() - .set("type", isAbort ? "abort" : "finish") - .set("ownerEmail", ownerEmail) - .set("datasetName", datasetName) - .set("filePath", encodeURIComponent(filePath)); - - return this.http.post( - `${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/multipart-upload`, - {}, - { params } + return this.multipartUploadService.finalizeMultipartUpload( + DATASET_FILE_RESOURCE_ENDPOINT, + ownerEmail, + datasetName, + filePath, + isAbort ); } @@ -479,9 +213,7 @@ export class DatasetService { * @param filePath File path to reset */ public resetDatasetFileDiff(did: number, filePath: string): Observable { - const params = new HttpParams().set("filePath", encodeURIComponent(filePath)); - - return this.http.put(`${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/${did}/diff`, {}, { params }); + return this.stagedFileService.resetFileDiff(DATASET_FILE_RESOURCE_ENDPOINT, did, filePath); } /** @@ -490,9 +222,7 @@ export class DatasetService { * @param filePath File path to delete */ public deleteDatasetFile(did: number, filePath: string): Observable { - const params = new HttpParams().set("filePath", encodeURIComponent(filePath)); - - return this.http.delete(`${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/${did}/file`, { params }); + return this.stagedFileService.deleteFile(DATASET_FILE_RESOURCE_ENDPOINT, did, filePath); } /** @@ -500,7 +230,7 @@ export class DatasetService { * @param did Dataset ID */ public getDatasetDiff(did: number): Observable { - return this.http.get(`${AppSettings.getApiEndpoint()}/${DATASET_BASE_URL}/${did}/diff`); + return this.stagedFileService.getDiff(DATASET_FILE_RESOURCE_ENDPOINT, did); } /** diff --git a/frontend/src/app/dashboard/service/user/file-resource/file-resource-endpoint.ts b/frontend/src/app/dashboard/service/user/file-resource/file-resource-endpoint.ts new file mode 100644 index 00000000000..4a89c129a81 --- /dev/null +++ b/frontend/src/app/dashboard/service/user/file-resource/file-resource-endpoint.ts @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Addresses the file-upload endpoints of one versioned resource kind. Resource families differ only + * in their base path and in the query-param name carrying the resource name, so the upload engine is + * parameterized by this rather than duplicated per family. + */ +export interface FileResourceEndpoint { + /** Path segment under the API root, e.g. "dataset". */ + readonly baseUrl: string; + /** Query-param name carrying the resource name, e.g. "datasetName". */ + readonly nameParamKey: string; + /** site_settings key holding this resource's per-file upload ceiling, in MiB. */ + readonly maxFileSizeSettingKey: string; + /** Fallback ceiling in MiB when that setting is absent or unparsable. */ + readonly defaultMaxFileSizeMiB: number; + /** site_settings keys tuning this resource's multipart upload. */ + readonly chunkSizeSettingKey: string; + readonly maxConcurrentChunksSettingKey: string; + readonly maxConcurrentFilesSettingKey: string; +} + +export const DATASET_FILE_RESOURCE_ENDPOINT: FileResourceEndpoint = { + baseUrl: "dataset", + nameParamKey: "datasetName", + maxFileSizeSettingKey: "dataset_single_file_upload_max_size_mib", + defaultMaxFileSizeMiB: 20, + chunkSizeSettingKey: "dataset_multipart_upload_chunk_size_mib", + maxConcurrentChunksSettingKey: "dataset_max_number_of_concurrent_uploading_file_chunks", + maxConcurrentFilesSettingKey: "dataset_max_number_of_concurrent_uploading_file", +}; + +export const MODEL_FILE_RESOURCE_ENDPOINT: FileResourceEndpoint = { + baseUrl: "model", + nameParamKey: "modelName", + maxFileSizeSettingKey: "model_single_file_upload_max_size_mib", + defaultMaxFileSizeMiB: 2048, + chunkSizeSettingKey: "model_multipart_upload_chunk_size_mib", + maxConcurrentChunksSettingKey: "model_max_number_of_concurrent_uploading_file_chunks", + maxConcurrentFilesSettingKey: "model_max_number_of_concurrent_uploading_file", +}; diff --git a/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.spec.ts b/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.spec.ts new file mode 100644 index 00000000000..94bdca97398 --- /dev/null +++ b/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.spec.ts @@ -0,0 +1,313 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { TestBed } from "@angular/core/testing"; +import { HttpClientTestingModule, HttpTestingController } from "@angular/common/http/testing"; +import { firstValueFrom } from "rxjs"; + +import { MultipartUploadProgress, MultipartUploadService } from "./multipart-upload.service"; +import { DATASET_FILE_RESOURCE_ENDPOINT, FileResourceEndpoint } from "./file-resource-endpoint"; +import { FakeXMLHttpRequest } from "./testing/fake-xml-http-request"; +import { commonTestProviders } from "../../../../common/testing/test-utils"; +import { AuthService } from "src/app/common/service/user/auth.service"; + +const API = "api"; + +/** + * A second endpoint that is neither dataset nor model. Using a synthetic one proves the engine is + * genuinely parameterized rather than accidentally dataset-shaped. + */ +const WIDGET_ENDPOINT: FileResourceEndpoint = { + baseUrl: "widget", + nameParamKey: "widgetName", + maxFileSizeSettingKey: "widget_single_file_upload_max_size_mib", + defaultMaxFileSizeMiB: 64, + chunkSizeSettingKey: "widget_multipart_upload_chunk_size_mib", + maxConcurrentChunksSettingKey: "widget_max_number_of_concurrent_uploading_file_chunks", + maxConcurrentFilesSettingKey: "widget_max_number_of_concurrent_uploading_file", +}; + +describe("MultipartUploadService", () => { + let service: MultipartUploadService; + let http: HttpTestingController; + + const isType = + (endpoint: FileResourceEndpoint, type: string) => (r: { url: string; params: { get(k: string): string | null } }) => + r.url === `${API}/${endpoint.baseUrl}/multipart-upload` && r.params.get("type") === type; + + beforeEach(() => { + TestBed.configureTestingModule({ + imports: [HttpClientTestingModule], + providers: [MultipartUploadService, ...commonTestProviders], + }); + service = TestBed.inject(MultipartUploadService); + http = TestBed.inject(HttpTestingController); + FakeXMLHttpRequest.reset(); + }); + + afterEach(() => { + http.verify(); + }); + + // ─── the endpoint seam ──────────────────────────────────────────────────── + + it("addresses init, part and finish through the supplied endpoint", async () => { + vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest); + const file = new File([new Uint8Array(4)], "w.bin"); + const done = new Promise((resolve, reject) => { + service + .multipartUpload(WIDGET_ENDPOINT, "o@e.com", "my-widget", "w.bin", file, 4, 1, false) + .subscribe({ error: reject, complete: resolve }); + }); + + const init = http.expectOne(isType(WIDGET_ENDPOINT, "init")); + expect(init.request.params.get("widgetName")).toBe("my-widget"); + expect(init.request.params.get("datasetName")).toBeNull(); + init.flush({ missingParts: [1], completedPartsCount: 0 }); + + // The part URL is built by hand rather than through HttpParams, so it needs asserting + // separately — a hardcoded name key here would only break for non-dataset resources. + const xhr = FakeXMLHttpRequest.instances[0]; + expect(xhr.url.startsWith(`${API}/widget/multipart-upload/part`)).toBe(true); + expect(xhr.params().get("widgetName")).toBe("my-widget"); + expect(xhr.params().get("datasetName")).toBeNull(); + xhr.respond(204); + + const finish = http.expectOne(isType(WIDGET_ENDPOINT, "finish")); + expect(finish.request.params.get("widgetName")).toBe("my-widget"); + finish.flush({}); + await done; + }); + + it("uses the endpoint for listMultipartUploads", async () => { + const pending = firstValueFrom(service.listMultipartUploads(WIDGET_ENDPOINT, "o@e.com", "my-widget")); + const req = http.expectOne(isType(WIDGET_ENDPOINT, "list")); + expect(req.request.params.get("widgetName")).toBe("my-widget"); + req.flush({ filePaths: ["a"] }); + expect(await pending).toEqual(["a"]); + }); + + it("uses the endpoint for findExistingUploadFiles", async () => { + const pending = firstValueFrom( + service.findExistingUploadFiles(WIDGET_ENDPOINT, 7, [{ path: "a.csv", sizeBytes: 1 }]) + ); + const req = http.expectOne(`${API}/widget/7/existing-upload-files`); + expect(req.request.body).toEqual({ files: [{ path: "a.csv", sizeBytes: 1 }] }); + req.flush({ filePaths: ["a.csv"] }); + expect(await pending).toEqual(["a.csv"]); + }); + + it("uses the endpoint for finalizeMultipartUpload and distinguishes abort from finish", () => { + service.finalizeMultipartUpload(WIDGET_ENDPOINT, "o@e.com", "my-widget", "f", true).subscribe(); + const abort = http.expectOne(isType(WIDGET_ENDPOINT, "abort")); + expect(abort.request.params.get("widgetName")).toBe("my-widget"); + abort.flush({}); + + service.finalizeMultipartUpload(WIDGET_ENDPOINT, "o@e.com", "my-widget", "f", false).subscribe(); + http.expectOne(isType(WIDGET_ENDPOINT, "finish")).flush({}); + }); + + // ─── upload flow ────────────────────────────────────────────────────────── + + it("emits progress, attaches the auth header, and finishes at 100%", async () => { + const tokenSpy = vi.spyOn(AuthService, "getAccessToken").mockReturnValue("tok123"); + vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest); + try { + const file = new File([new Uint8Array(8)], "d.bin"); + const emissions: MultipartUploadProgress[] = []; + const done = new Promise((resolve, reject) => { + service + .multipartUpload(DATASET_FILE_RESOURCE_ENDPOINT, "o@e.com", "ds", "d.bin", file, 8, 1, false) + .subscribe({ next: p => emissions.push(p), error: reject, complete: resolve }); + }); + + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "init")).flush({ + missingParts: [1], + completedPartsCount: 0, + }); + + const xhr = FakeXMLHttpRequest.instances[0]; + expect(xhr.requestHeaders.get("Content-Type")).toBe("application/octet-stream"); + expect(xhr.requestHeaders.get("Authorization")).toBe("Bearer tok123"); + + xhr.emitProgress(4); + xhr.respond(200); + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "finish")).flush({}); + await done; + + expect(emissions[0]).toMatchObject({ status: "initializing" }); + expect(emissions.some(e => e.status === "uploading" && e.percentage > 0 && e.percentage <= 99)).toBe(true); + expect(emissions.at(-1)).toMatchObject({ status: "finished", percentage: 100 }); + } finally { + tokenSpy.mockRestore(); + } + }); + + it("omits the auth header when there is no access token", () => { + const tokenSpy = vi.spyOn(AuthService, "getAccessToken").mockReturnValue(null); + vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest); + try { + const file = new File([new Uint8Array(4)], "anon.bin"); + const subscription = service + .multipartUpload(DATASET_FILE_RESOURCE_ENDPOINT, "o@e.com", "ds", "anon.bin", file, 4, 1, false) + .subscribe(); + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "init")).flush({ + missingParts: [1], + completedPartsCount: 0, + }); + + expect(FakeXMLHttpRequest.instances[0].requestHeaders.has("Authorization")).toBe(false); + subscription.unsubscribe(); + } finally { + tokenSpy.mockRestore(); + } + }); + + it("resumes by uploading only the parts the backend reports missing", async () => { + vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest); + const file = new File(["abcdefgh"], "resume.txt"); + const emissions: MultipartUploadProgress[] = []; + const done = new Promise((resolve, reject) => { + service + .multipartUpload(DATASET_FILE_RESOURCE_ENDPOINT, "o@e.com", "ds", "resume.txt", file, 4, 1, false) + .subscribe({ next: p => emissions.push(p), error: reject, complete: resolve }); + }); + + // One of two parts already landed, so the baseline starts at 50%. + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "init")).flush({ + missingParts: [2], + completedPartsCount: 1, + }); + + expect(emissions[0]).toMatchObject({ percentage: 50, status: "initializing" }); + expect(FakeXMLHttpRequest.instances.map(x => x.params().get("partNumber"))).toEqual(["2"]); + FakeXMLHttpRequest.instances[0].respond(204); + + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "finish")).flush({}); + await done; + expect(emissions.at(-1)).toMatchObject({ percentage: 100, status: "finished" }); + }); + + it("finishes without uploading anything when no parts are missing", async () => { + vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest); + const file = new File([new Uint8Array(4)], "c.bin"); + const done = new Promise((resolve, reject) => { + service + .multipartUpload(DATASET_FILE_RESOURCE_ENDPOINT, "o@e.com", "ds", "c.bin", file, 4, 1, false) + .subscribe({ error: reject, complete: resolve }); + }); + + // A sparse payload also exercises the nullish-coalescing defaults. + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "init")).flush({}); + expect(FakeXMLHttpRequest.instances.length).toBe(0); + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "finish")).flush({}); + await done; + }); + + // ─── failure and teardown ───────────────────────────────────────────────── + + it("fails the upload when a part returns a non-2xx status", async () => { + vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest); + const file = new File([new Uint8Array(4)], "e.bin"); + const emissions: MultipartUploadProgress[] = []; + const outcome = new Promise(resolve => { + service + .multipartUpload(DATASET_FILE_RESOURCE_ENDPOINT, "o@e.com", "ds", "e.bin", file, 4, 1, false) + .subscribe({ next: p => emissions.push(p), error: resolve, complete: () => resolve(null) }); + }); + + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "init")).flush({ + missingParts: [1], + completedPartsCount: 0, + }); + FakeXMLHttpRequest.instances[0].respond(500); + + const err = await outcome; + expect((err as Error).message).toContain("HTTP 500"); + expect(emissions.at(-1)).toMatchObject({ status: "failed" }); + }); + + it("fails the upload when a part errors at the transport level", async () => { + vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest); + const file = new File([new Uint8Array(4)], "x.bin"); + const outcome = new Promise(resolve => { + service + .multipartUpload(DATASET_FILE_RESOURCE_ENDPOINT, "o@e.com", "ds", "x.bin", file, 4, 1, false) + .subscribe({ error: resolve, complete: () => resolve(null) }); + }); + + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "init")).flush({ + missingParts: [1], + completedPartsCount: 0, + }); + FakeXMLHttpRequest.instances[0].fail(); + + expect(await outcome).toBeInstanceOf(Error); + }); + + it("aborts the in-flight part request when the caller unsubscribes", () => { + vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest); + const file = new File([new Uint8Array(4)], "t.bin"); + const subscription = service + .multipartUpload(DATASET_FILE_RESOURCE_ENDPOINT, "o@e.com", "ds", "t.bin", file, 4, 1, false) + .subscribe(); + + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "init")).flush({ + missingParts: [1], + completedPartsCount: 0, + }); + + const xhr = FakeXMLHttpRequest.instances[0]; + expect(xhr.aborted).toBe(false); + + // Teardown must reach the inner per-part closure, not just the outer subscription. + subscription.unsubscribe(); + + expect(xhr.aborted).toBe(true); + }); + + // ─── payload tolerance ──────────────────────────────────────────────────── + + it("tolerates null payloads from the list and existing-files endpoints", async () => { + const listPending = firstValueFrom(service.listMultipartUploads(DATASET_FILE_RESOURCE_ENDPOINT, "o@e.com", "ds")); + http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "list")).flush(null); + expect(await listPending).toEqual([]); + + const existingPending = firstValueFrom(service.findExistingUploadFiles(DATASET_FILE_RESOURCE_ENDPOINT, 7, [])); + http.expectOne(`${API}/dataset/7/existing-upload-files`).flush(null); + expect(await existingPending).toEqual([]); + }); + + it("percent-encodes the file path on both the init params and the part URL", async () => { + vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest); + const nested = "folder sub/a+b.csv"; + const file = new File([new Uint8Array(4)], "a.csv"); + const subscription = service + .multipartUpload(DATASET_FILE_RESOURCE_ENDPOINT, "o@e.com", "ds", nested, file, 4, 1, false) + .subscribe(); + + const init = http.expectOne(isType(DATASET_FILE_RESOURCE_ENDPOINT, "init")); + // HttpParams decodes on read, so this asserts the pre-encoding the backend expects. + expect(init.request.params.get("filePath")).toBe(encodeURIComponent(nested)); + init.flush({ missingParts: [1], completedPartsCount: 0 }); + + // The hand-built part URL encodes once, so reading it back yields the raw path. + expect(FakeXMLHttpRequest.instances[0].params().get("filePath")).toBe(nested); + subscription.unsubscribe(); + }); +}); diff --git a/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.ts b/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.ts new file mode 100644 index 00000000000..cfae34b82e7 --- /dev/null +++ b/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.ts @@ -0,0 +1,371 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Injectable } from "@angular/core"; +import { HttpClient, HttpParams } from "@angular/common/http"; +import { catchError, map, mergeMap, switchMap, tap, toArray } from "rxjs/operators"; +import { from, Observable, throwError } from "rxjs"; +import { AppSettings } from "../../../../common/app-setting"; +import { AuthService } from "src/app/common/service/user/auth.service"; +import { FileResourceEndpoint } from "./file-resource-endpoint"; + +export interface MultipartUploadProgress { + filePath: string; + percentage: number; + status: "initializing" | "uploading" | "finished" | "aborted" | "failed"; + uploadSpeed?: number; // bytes per second + estimatedTimeRemaining?: number; // seconds + totalTime?: number; // total seconds taken +} + +/** + * The multipart upload engine, shared by every versioned resource kind. Addressing comes from the + * FileResourceEndpoint the caller supplies; everything else here is resource-agnostic. + */ +@Injectable({ + providedIn: "root", +}) +export class MultipartUploadService { + constructor(private http: HttpClient) {} + + /** + * Handles multipart upload for large files using RxJS, + * with a concurrency limit on how many parts we process in parallel. + * + * Backend flow, where {base} and {nameKey} come from the endpoint: + * POST /{base}/multipart-upload?type=init&ownerEmail=...&{nameKey}=...&filePath=...&numParts=N + * POST /{base}/multipart-upload/part?ownerEmail=...&{nameKey}=...&filePath=...&partNumber= (body: raw chunk) + * POST /{base}/multipart-upload?type=finish&ownerEmail=...&{nameKey}=...&filePath=... + * POST /{base}/multipart-upload?type=abort&ownerEmail=...&{nameKey}=...&filePath=... + */ + public multipartUpload( + endpoint: FileResourceEndpoint, + ownerEmail: string, + resourceName: string, + filePath: string, + file: File, + partSize: number, + concurrencyLimit: number, + restart: boolean + ): Observable { + const partCount = Math.ceil(file.size / partSize); + + return new Observable(observer => { + // Track upload progress (bytes) for each part independently + const partProgress = new Map(); + + let baselineUploaded = 0; + // Progress tracking state + let startTime: number | null = null; + const speedSamples: number[] = []; + let lastETA = 0; + let lastUpdateTime = 0; + + const lastStats = { + uploadSpeed: 0, + estimatedTimeRemaining: 0, + totalTime: 0, + }; + + const getTotalTime = () => (startTime ? (Date.now() - startTime) / 1000 : 0); + + // Calculate stats with smoothing and simple throttling (~1s) + const calculateStats = (totalUploaded: number) => { + if (startTime === null) { + startTime = Date.now(); + } + + const now = Date.now(); + const elapsed = getTotalTime(); + + const shouldUpdate = now - lastUpdateTime >= 1000; + if (!shouldUpdate) { + // keep totalTime fresh even when throttled + lastStats.totalTime = elapsed; + return lastStats; + } + lastUpdateTime = now; + + const sessionUploaded = Math.max(0, totalUploaded - baselineUploaded); + const currentSpeed = elapsed > 0 ? sessionUploaded / elapsed : 0; + speedSamples.push(currentSpeed); + if (speedSamples.length > 5) { + speedSamples.shift(); + } + const avgSpeed = speedSamples.length > 0 ? speedSamples.reduce((a, b) => a + b, 0) / speedSamples.length : 0; + + const remaining = file.size - totalUploaded; + let eta = avgSpeed > 0 ? remaining / avgSpeed : 0; + eta = Math.min(eta, 24 * 60 * 60); // cap ETA at 24h + + if (lastETA > 0 && eta > 0) { + const maxChange = lastETA * 0.3; + const diff = Math.abs(eta - lastETA); + if (diff > maxChange) { + eta = lastETA + (eta > lastETA ? maxChange : -maxChange); + } + } + lastETA = eta; + + const percentComplete = (totalUploaded / file.size) * 100; + if (percentComplete > 95) { + eta = Math.min(eta, 10); + } + + lastStats.uploadSpeed = avgSpeed; + lastStats.estimatedTimeRemaining = Math.max(0, Math.round(eta)); + lastStats.totalTime = elapsed; + + return lastStats; + }; + + // 1. INIT: ask backend to create a LakeFS multipart upload session + const initParams = new HttpParams() + .set("type", "init") + .set("ownerEmail", ownerEmail) + .set(endpoint.nameParamKey, resourceName) + .set("filePath", encodeURIComponent(filePath)) + .set("fileSizeBytes", file.size.toString()) + .set("partSizeBytes", partSize.toString()) + .set("restart", restart); + + const init$ = this.http.post<{ missingParts: number[]; completedPartsCount: number }>( + `${AppSettings.getApiEndpoint()}/${endpoint.baseUrl}/multipart-upload`, + {}, + { params: initParams } + ); + + const subscription = init$ + .pipe( + switchMap(initResp => { + const missingParts = (initResp?.missingParts ?? []).slice(); + const completedPartsCount = initResp?.completedPartsCount ?? 0; + + const missingBytes = missingParts.reduce((sum, partNumber) => { + const start = (partNumber - 1) * partSize; + const end = Math.min(start + partSize, file.size); + return sum + (end - start); + }, 0); + + baselineUploaded = file.size - missingBytes; + const baselinePct = partCount > 0 ? Math.round((completedPartsCount / partCount) * 100) : 0; + + observer.next({ + filePath, + percentage: baselinePct, + status: "initializing", + uploadSpeed: 0, + estimatedTimeRemaining: 0, + totalTime: 0, + }); + // 2. Upload each part to /multipart-upload/part using XMLHttpRequest + return from(missingParts).pipe( + mergeMap(partNumber => { + const start = (partNumber - 1) * partSize; + const end = Math.min(start + partSize, file.size); + const chunk = file.slice(start, end); + + return new Observable(partObserver => { + const xhr = new XMLHttpRequest(); + + xhr.upload.addEventListener("progress", event => { + if (event.lengthComputable) { + partProgress.set(partNumber, event.loaded); + + let totalUploaded = baselineUploaded; // CHANGED + partProgress.forEach(bytes => { + totalUploaded += bytes; + }); + + const percentage = Math.round((totalUploaded / file.size) * 100); + const stats = calculateStats(totalUploaded); + + observer.next({ + filePath, + percentage: Math.min(percentage, 99), + status: "uploading", + ...stats, + }); + } + }); + + xhr.addEventListener("load", () => { + if (xhr.status === 200 || xhr.status === 204) { + // Mark part as fully uploaded + partProgress.set(partNumber, chunk.size); + + let totalUploaded = baselineUploaded; + partProgress.forEach(bytes => { + totalUploaded += bytes; + }); + + // Force stats recompute on completion + lastUpdateTime = 0; + const percentage = Math.round((totalUploaded / file.size) * 100); + const stats = calculateStats(totalUploaded); + + observer.next({ + filePath, + percentage: Math.min(percentage, 99), + status: "uploading", + ...stats, + }); + + partObserver.complete(); + } else { + partObserver.error(new Error(`Failed to upload part ${partNumber} (HTTP ${xhr.status})`)); + } + }); + + xhr.addEventListener("error", () => { + // Remove failed part from progress + partProgress.delete(partNumber); + partObserver.error(new Error(`Failed to upload part ${partNumber}`)); + }); + + const partUrl = + `${AppSettings.getApiEndpoint()}/${endpoint.baseUrl}/multipart-upload/part` + + `?ownerEmail=${encodeURIComponent(ownerEmail)}` + + `&${endpoint.nameParamKey}=${encodeURIComponent(resourceName)}` + + `&filePath=${encodeURIComponent(filePath)}` + + `&partNumber=${partNumber}`; + + xhr.open("POST", partUrl); + xhr.setRequestHeader("Content-Type", "application/octet-stream"); + const token = AuthService.getAccessToken(); + if (token) { + xhr.setRequestHeader("Authorization", `Bearer ${token}`); + } + xhr.send(chunk); + return () => { + try { + xhr.abort(); + } catch {} + }; + }); + }, concurrencyLimit), + toArray(), // wait for all parts + // 3. FINISH: notify backend that all parts are done + switchMap(() => { + const finishParams = new HttpParams() + .set("type", "finish") + .set("ownerEmail", ownerEmail) + .set(endpoint.nameParamKey, resourceName) + .set("filePath", encodeURIComponent(filePath)); + + return this.http.post( + `${AppSettings.getApiEndpoint()}/${endpoint.baseUrl}/multipart-upload`, + {}, + { params: finishParams } + ); + }), + tap(() => { + const totalTime = getTotalTime(); + observer.next({ + filePath, + percentage: 100, + status: "finished", + uploadSpeed: 0, + estimatedTimeRemaining: 0, + totalTime, + }); + observer.complete(); + }), + catchError((error: unknown) => { + // On error, compute best-effort percentage from bytes we've seen + let totalUploaded = baselineUploaded; + partProgress.forEach(bytes => { + totalUploaded += bytes; + }); + const percentage = file.size > 0 ? Math.round((totalUploaded / file.size) * 100) : 0; + + observer.next({ + filePath, + percentage, + status: "failed", + uploadSpeed: 0, + estimatedTimeRemaining: 0, + totalTime: getTotalTime(), + }); + + return throwError(() => error); + }) + ); + }) + ) + .subscribe({ + error: (err: unknown) => observer.error(err), + }); + + return () => subscription.unsubscribe(); + }); + } + + public listMultipartUploads( + endpoint: FileResourceEndpoint, + ownerEmail: string, + resourceName: string + ): Observable { + const params = new HttpParams() + .set("type", "list") + .set("ownerEmail", ownerEmail) + .set(endpoint.nameParamKey, resourceName); + + return this.http + .post<{ + filePaths: string[]; + }>(`${AppSettings.getApiEndpoint()}/${endpoint.baseUrl}/multipart-upload`, {}, { params }) + .pipe(map(res => res?.filePaths ?? [])); + } + + public findExistingUploadFiles( + endpoint: FileResourceEndpoint, + resourceId: number, + files: { path: string; sizeBytes: number }[] + ): Observable { + return this.http + .post<{ filePaths: string[] }>( + `${AppSettings.getApiEndpoint()}/${endpoint.baseUrl}/${resourceId}/existing-upload-files`, + { + files, + } + ) + .pipe(map(res => res?.filePaths ?? [])); + } + + public finalizeMultipartUpload( + endpoint: FileResourceEndpoint, + ownerEmail: string, + resourceName: string, + filePath: string, + isAbort: boolean + ): Observable { + const params = new HttpParams() + .set("type", isAbort ? "abort" : "finish") + .set("ownerEmail", ownerEmail) + .set(endpoint.nameParamKey, resourceName) + .set("filePath", encodeURIComponent(filePath)); + + return this.http.post( + `${AppSettings.getApiEndpoint()}/${endpoint.baseUrl}/multipart-upload`, + {}, + { params } + ); + } +} diff --git a/frontend/src/app/dashboard/service/user/file-resource/staged-file.service.spec.ts b/frontend/src/app/dashboard/service/user/file-resource/staged-file.service.spec.ts new file mode 100644 index 00000000000..d7ac2d2d8f3 --- /dev/null +++ b/frontend/src/app/dashboard/service/user/file-resource/staged-file.service.spec.ts @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { TestBed } from "@angular/core/testing"; +import { HttpClientTestingModule, HttpTestingController } from "@angular/common/http/testing"; +import { firstValueFrom } from "rxjs"; +import { StagedFileService } from "./staged-file.service"; +import { + DATASET_FILE_RESOURCE_ENDPOINT, + FileResourceEndpoint, + MODEL_FILE_RESOURCE_ENDPOINT, +} from "./file-resource-endpoint"; +import { DatasetStagedObject } from "../../../../common/type/dataset-staged-object"; +import { commonTestProviders } from "../../../../common/testing/test-utils"; + +const API = "api"; + +describe("StagedFileService", () => { + let service: StagedFileService; + let http: HttpTestingController; + + beforeEach(() => { + TestBed.configureTestingModule({ + imports: [HttpClientTestingModule], + providers: [StagedFileService, ...commonTestProviders], + }); + service = TestBed.inject(StagedFileService); + http = TestBed.inject(HttpTestingController); + }); + + afterEach(() => http.verify()); + + const endpoints: Array<[string, FileResourceEndpoint]> = [ + ["dataset", DATASET_FILE_RESOURCE_ENDPOINT], + ["model", MODEL_FILE_RESOURCE_ENDPOINT], + ]; + + for (const [name, endpoint] of endpoints) { + it(`lists the staged objects of a ${name}`, async () => { + const staged: DatasetStagedObject[] = [{ path: "a.txt", pathType: "file", diffType: "added", sizeBytes: 1 }]; + const pending = firstValueFrom(service.getDiff(endpoint, 7)); + + http.expectOne(`${API}/${endpoint.baseUrl}/7/diff`).flush(staged); + + expect(await pending).toEqual(staged); + }); + + it(`reverts one staged ${name} path`, () => { + service.resetFileDiff(endpoint, 7, "dir/a b.txt").subscribe(); + + // The path is encoded into the query param, so a space or slash survives the round trip. + const req = http.expectOne( + `${API}/${endpoint.baseUrl}/7/diff?filePath=${encodeURIComponent(encodeURIComponent("dir/a b.txt"))}` + ); + expect(req.request.method).toBe("PUT"); + req.flush({}); + }); + + it(`stages a deletion of a committed ${name} file`, () => { + service.deleteFile(endpoint, 7, "dir/a.txt").subscribe(); + + const req = http.expectOne( + `${API}/${endpoint.baseUrl}/7/file?filePath=${encodeURIComponent(encodeURIComponent("dir/a.txt"))}` + ); + expect(req.request.method).toBe("DELETE"); + req.flush({}); + }); + } + + it("surfaces a server error rather than swallowing it", async () => { + const outcome = firstValueFrom(service.getDiff(MODEL_FILE_RESOURCE_ENDPOINT, 7)).catch((err: unknown) => err); + http.expectOne(`${API}/model/7/diff`).flush({ message: "nope" }, { status: 500, statusText: "Server Error" }); + expect(await outcome).toMatchObject({ status: 500 }); + }); +}); diff --git a/frontend/src/app/dashboard/service/user/file-resource/staged-file.service.ts b/frontend/src/app/dashboard/service/user/file-resource/staged-file.service.ts new file mode 100644 index 00000000000..47ea46c878b --- /dev/null +++ b/frontend/src/app/dashboard/service/user/file-resource/staged-file.service.ts @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { Injectable } from "@angular/core"; +import { HttpClient, HttpParams } from "@angular/common/http"; +import { Observable } from "rxjs"; +import { AppSettings } from "../../../../common/app-setting"; +import { DatasetStagedObject } from "../../../../common/type/dataset-staged-object"; +import { FileResourceEndpoint } from "./file-resource-endpoint"; + +/** + * The uncommitted-change operations every versioned resource shares: list what is staged, revert one + * staged path, and stage a deletion. Addressing comes from the caller's FileResourceEndpoint. + */ +@Injectable({ + providedIn: "root", +}) +export class StagedFileService { + constructor(private http: HttpClient) {} + + /** Uncommitted changes, i.e. what a new version would consume. */ + public getDiff(endpoint: FileResourceEndpoint, resourceId: number): Observable { + return this.http.get( + `${AppSettings.getApiEndpoint()}/${endpoint.baseUrl}/${resourceId}/diff` + ); + } + + public resetFileDiff(endpoint: FileResourceEndpoint, resourceId: number, filePath: string): Observable { + const params = new HttpParams().set("filePath", encodeURIComponent(filePath)); + + return this.http.put( + `${AppSettings.getApiEndpoint()}/${endpoint.baseUrl}/${resourceId}/diff`, + {}, + { params } + ); + } + + /** Stages a deletion of an already-committed file; the next version applies it. */ + public deleteFile(endpoint: FileResourceEndpoint, resourceId: number, filePath: string): Observable { + const params = new HttpParams().set("filePath", encodeURIComponent(filePath)); + + return this.http.delete(`${AppSettings.getApiEndpoint()}/${endpoint.baseUrl}/${resourceId}/file`, { + params, + }); + } +} diff --git a/frontend/src/app/dashboard/service/user/file-resource/testing/fake-xml-http-request.ts b/frontend/src/app/dashboard/service/user/file-resource/testing/fake-xml-http-request.ts new file mode 100644 index 00000000000..9d7edb5a861 --- /dev/null +++ b/frontend/src/app/dashboard/service/user/file-resource/testing/fake-xml-http-request.ts @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +/** + * Stand-in for XMLHttpRequest, which the multipart engine uses directly for per-part upload + * progress. Install with `vi.stubGlobal("XMLHttpRequest", FakeXMLHttpRequest)`. + */ +export class FakeXMLHttpRequest { + static instances: FakeXMLHttpRequest[] = []; + + /** Capturing upload target so tests can drive `upload.progress` events. */ + readonly upload = { + listeners: new Map(), + addEventListener(type: string, listener: EventListener): void { + this.listeners.set(type, [...(this.listeners.get(type) ?? []), listener]); + }, + }; + status = 0; + url = ""; + aborted = false; + readonly requestHeaders = new Map(); + private listeners = new Map(); + + static reset(): void { + FakeXMLHttpRequest.instances = []; + } + + open(_method: string, url: string): void { + this.url = url; + } + + setRequestHeader(name: string, value: string): void { + this.requestHeaders.set(name, value); + } + + send(): void { + FakeXMLHttpRequest.instances.push(this); + } + + abort(): void { + this.aborted = true; + } + + addEventListener(type: string, listener: EventListener): void { + this.listeners.set(type, [...(this.listeners.get(type) ?? []), listener]); + } + + /** Drives the `upload.progress` listener registered by the engine. */ + emitProgress(loaded: number, lengthComputable = true): void { + const event = { lengthComputable, loaded } as unknown as Event; + for (const listener of this.upload.listeners.get("progress") ?? []) { + listener(event); + } + } + + respond(status: number): void { + this.status = status; + this.emit("load"); + } + + fail(): void { + this.emit("error"); + } + + /** Query params of the URL this request was opened with. */ + params(): URLSearchParams { + return new URL(this.url, "http://localhost").searchParams; + } + + private emit(type: string): void { + for (const listener of this.listeners.get(type) ?? []) { + listener(new Event(type)); + } + } +} diff --git a/frontend/src/app/dashboard/service/user/model/model.service.spec.ts b/frontend/src/app/dashboard/service/user/model/model.service.spec.ts index 903455a702a..418052af201 100644 --- a/frontend/src/app/dashboard/service/user/model/model.service.spec.ts +++ b/frontend/src/app/dashboard/service/user/model/model.service.spec.ts @@ -226,6 +226,37 @@ describe("ModelService", () => { expect(await pending).toEqual({ url: null }); }); + it("posts a version name as text/plain and folds the file nodes into the version", async () => { + const pending = firstValueFrom(service.createModelVersion(7, "v2")); + const req = http.expectOne(`${API}/model/7/version/create`); + + expect(req.request.method).toBe("POST"); + expect(req.request.body).toBe("v2"); + expect(req.request.headers.get("Content-Type")).toBe("text/plain"); + + const fileNodes = [{ name: "model.pt", type: "file", parentDir: "/model/a/m/v2", size: 4 }]; + req.flush({ modelVersion: { mvid: 2, mid: 7, creatorUid: 1, name: "v2" }, fileNodes }); + + expect(await pending).toMatchObject({ mvid: 2, name: "v2", fileNodes }); + }); + + it("lets the backend name the version when none is given", () => { + service.createModelVersion(7, "").subscribe(); + expect(http.expectOne(`${API}/model/7/version/create`).request.body).toBe(""); + }); + + it("updates the framework and the format through their own endpoints", () => { + service.updateModelFramework(7, "onnx").subscribe(); + const framework = http.expectOne(`${API}/model/update/framework`); + expect(framework.request.body).toEqual({ mid: 7, framework: "onnx" }); + framework.flush({}); + + service.updateModelFormat(7, "safetensors").subscribe(); + const format = http.expectOne(`${API}/model/update/format`); + expect(format.request.body).toEqual({ mid: 7, format: "safetensors" }); + format.flush({}); + }); + it("surfaces a server error rather than swallowing it", async () => { const outcome = firstValueFrom(service.retrieveAccessibleModels()).catch((err: unknown) => err); http.expectOne(`${API}/model/list`).flush({ message: "nope" }, { status: 500, statusText: "Server Error" }); diff --git a/frontend/src/app/dashboard/service/user/model/model.service.ts b/frontend/src/app/dashboard/service/user/model/model.service.ts index c5d26b5ecd4..b4095139002 100644 --- a/frontend/src/app/dashboard/service/user/model/model.service.ts +++ b/frontend/src/app/dashboard/service/user/model/model.service.ts @@ -20,7 +20,7 @@ import { Injectable } from "@angular/core"; import { HttpClient, HttpParams } from "@angular/common/http"; import { Observable } from "rxjs"; -import { switchMap } from "rxjs/operators"; +import { map, switchMap } from "rxjs/operators"; import { AppSettings } from "../../../../common/app-setting"; import { Model, ModelVersion } from "../../../../common/type/model"; import { DashboardModel } from "../../../type/dashboard-model.interface"; @@ -31,6 +31,8 @@ export const MODEL_CREATE_URL = MODEL_BASE_URL + "/create"; export const MODEL_UPDATE_BASE_URL = MODEL_BASE_URL + "/update"; export const MODEL_UPDATE_NAME_URL = MODEL_UPDATE_BASE_URL + "/name"; export const MODEL_UPDATE_DESCRIPTION_URL = MODEL_UPDATE_BASE_URL + "/description"; +export const MODEL_UPDATE_FRAMEWORK_URL = MODEL_UPDATE_BASE_URL + "/framework"; +export const MODEL_UPDATE_FORMAT_URL = MODEL_UPDATE_BASE_URL + "/format"; export const MODEL_LIST_URL = MODEL_BASE_URL + "/list"; export const MODEL_VERSION_BASE_URL = "version"; @@ -109,6 +111,40 @@ export class ModelService { }); } + public updateModelFramework(mid: number, framework: string): Observable { + return this.http.post(`${AppSettings.getApiEndpoint()}/${MODEL_UPDATE_FRAMEWORK_URL}`, { + mid: mid, + framework: framework, + }); + } + + public updateModelFormat(mid: number, format: string): Observable { + return this.http.post(`${AppSettings.getApiEndpoint()}/${MODEL_UPDATE_FORMAT_URL}`, { + mid: mid, + format: format, + }); + } + + /** + * Commits the model's staged changes as a new version. The backend rejects this with a 400 when + * nothing is staged, and names the version itself when `newVersion` is blank. + */ + public createModelVersion(mid: number, newVersion: string): Observable { + return this.http + .post<{ + modelVersion: ModelVersion; + fileNodes: DatasetFileNode[]; + }>(`${AppSettings.getApiEndpoint()}/${MODEL_BASE_URL}/${mid}/${MODEL_VERSION_BASE_URL}/create`, newVersion, { + headers: { "Content-Type": "text/plain" }, + }) + .pipe( + map(response => { + response.modelVersion.fileNodes = response.fileNodes; + return response.modelVersion; + }) + ); + } + /** A model's versions, newest first; an anonymous caller gets the public-only endpoint. */ public retrieveModelVersionList(mid: number, isLogin: boolean = true): Observable { const listUrl = isLogin ? MODEL_VERSION_RETRIEVE_LIST_URL : MODEL_PUBLIC_VERSION_RETRIEVE_LIST_URL; From fb2ad191a9b955fcf641e8483a12ea553e1cad3a Mon Sep 17 00:00:00 2001 From: Tanishq Gandhi Date: Fri, 28 Aug 2026 15:05:01 -0700 Subject: [PATCH 2/6] feat(frontend): add model file upload and version creation --- .../model-detail.component.spec.ts | 34 +++++++++++++++++++ .../model-detail.component.ts | 5 +++ 2 files changed, 39 insertions(+) diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts index 71f7963743b..abea30e305a 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts @@ -600,6 +600,40 @@ describe("ModelDetailComponent", () => { expect(component.modelName).toBe("resnet-101"); }); + it("refreshes the file tree after a rename, so paths carry the new name", () => { + modelService["updateModelName"] = vi.fn(() => of({})); + modelService["retrieveModelVersionList"] = vi.fn(() => of([aVersion(1, "v1")])); + modelService["retrieveModelVersionFileTree"] = vi.fn(() => + of({ fileNodes: [aFile("model.pt", `/model/${OWNER}/resnet-50/v1`)], size: 4 }) + ); + create(); + modelService["retrieveModelVersionFileTree"].mockClear(); + // The renamed model's tree comes back under the new path. + modelService["retrieveModelVersionFileTree"] = vi.fn(() => + of({ fileNodes: [aFile("model.pt", `/model/${OWNER}/resnet-101/v1`)], size: 4 }) + ); + component.editedModelName = "resnet-101"; + + component.onSaveModelName(); + + // Preview and single-file download resolve a model by (owner, name), so a tree still + // holding the old name would 404 on every file until the page was reloaded. + expect(modelService["retrieveModelVersionFileTree"]).toHaveBeenCalledWith(MID, 1, true); + expect(component.currentDisplayedFileName).toBe(`/model/${OWNER}/resnet-101/v1/model.pt`); + }); + + it("skips the tree refresh for a model with no versions", () => { + modelService["updateModelName"] = vi.fn(() => of({})); + create(); + modelService["retrieveModelVersionFileTree"].mockClear(); + component.editedModelName = "resnet-101"; + + component.onSaveModelName(); + + expect(modelService["retrieveModelVersionFileTree"]).not.toHaveBeenCalled(); + expect(component.modelName).toBe("resnet-101"); + }); + it("rejects an invalid name without calling the server", () => { modelService["updateModelName"] = vi.fn(() => of({})); create(); diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts index 4faa098a497..955d5fb6594 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts @@ -394,6 +394,8 @@ export class ModelDetailComponent implements OnInit { this.notificationService.success( `File ${node.name} is successfully deleted. You may finalize it or revert it at the "Create Version" panel` ); + // Undefined only when the panel is not rendered, which is the same write-access + // condition that gates the tree's delete control. this.versionUploader?.notePathStaged(relativePath); }, error: () => this.notificationService.error("Failed to delete the file"), @@ -430,6 +432,9 @@ export class ModelDetailComponent implements OnInit { next: () => { this.modelName = name; this.editedModelName = name; + // File nodes carry the model name inside their paths, and preview and single-file + // download resolve a model by (owner, name) — a stale tree 404s until reload. + this.onVersionSelected(this.selectedVersion); this.notificationService.success(`Model name updated to '${name}'`); }, error: (err: unknown) => this.notificationService.error(extractErrorMessage(err)), From 85e2284c5e792a99c72097d789bb3a7aa6b57968 Mon Sep 17 00:00:00 2001 From: Tanishq Gandhi Date: Fri, 28 Aug 2026 16:31:43 -0700 Subject: [PATCH 3/6] fix(frontend): keep the model card's latest-version facts off the selection The card describes the newest version whatever the version picker shows, but its file name and size were written inside onVersionSelected, guarded on the selected version happening to be versions[0]. Renaming while browsing an older version therefore left "Latest version file" on a path built from the old name. Fetch those facts in their own right instead, mirroring the dataset page's retrieveLatestVersionFile, and refresh them alongside the tree after a rename. Using retrieveModelVersionList would also have fixed the display, but it resets the picker to the newest version, which is not something a rename should do. Co-Authored-By: Claude Opus 5 --- .../model-detail.component.spec.ts | 24 +++++++++++ .../model-detail.component.ts | 40 ++++++++++++++----- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts index abea30e305a..3e0fde26dee 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts @@ -622,6 +622,30 @@ describe("ModelDetailComponent", () => { expect(component.currentDisplayedFileName).toBe(`/model/${OWNER}/resnet-101/v1/model.pt`); }); + it("refreshes the Model Card too when an older version is on screen, without moving the picker", () => { + const versions = [aVersion(2, "v2"), aVersion(1, "v1")]; + modelService["updateModelName"] = vi.fn(() => of({})); + modelService["retrieveModelVersionList"] = vi.fn(() => of(versions)); + let name = "resnet-50"; + modelService["retrieveModelVersionFileTree"] = vi.fn((_mid: number, mvid: number) => + of({ fileNodes: [aFile(`v${mvid}.pt`, `/model/${OWNER}/${name}/v${mvid}`)], size: mvid * 100 }) + ); + create(); + component.onVersionSelected(versions[1]); + expect(component.latestVersionFileName).toBe(`/model/${OWNER}/resnet-50/v2/v2.pt`); + + name = "resnet-101"; + component.editedModelName = name; + component.onSaveModelName(); + + // The Model Card describes the newest version, which is not the one being browsed — deriving + // its facts from the selection would have left this path on the old name. + expect(component.latestVersionFileName).toBe(`/model/${OWNER}/resnet-101/v2/v2.pt`); + expect(component.currentDisplayedFileName).toBe(`/model/${OWNER}/resnet-101/v1/v1.pt`); + // Renaming is not a reason to move the user off the version they opened. + expect(component.selectedVersion).toBe(versions[1]); + }); + it("skips the tree refresh for a model with no versions", () => { modelService["updateModelName"] = vi.fn(() => of({})); create(); diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts index 955d5fb6594..7baaeb4e535 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts @@ -267,9 +267,8 @@ export class ModelDetailComponent implements OnInit { if (versions.length === 0) { return; } - const latest = versions[0]; - this.latestVersionCreationTime = this.formatCreationTime(latest); - this.onVersionSelected(latest); + this.retrieveLatestVersionFacts(); + this.onVersionSelected(versions[0]); }, error: (err: unknown) => this.notificationService.error(extractErrorMessage(err)), }); @@ -290,10 +289,6 @@ export class ModelDetailComponent implements OnInit { this.selectedVersionCreationTime = this.formatCreationTime(version); const firstFile = this.getFirstFileNode(this.fileTreeNodeList); - if (version === this.versions[0]) { - this.latestVersionFileName = firstFile ? getFullPathFromDatasetFileNode(firstFile) : ""; - this.latestVersionSize = data.size; - } if (!firstFile) { this.currentDisplayedFileName = ""; this.currentFileSize = undefined; @@ -305,6 +300,32 @@ export class ModelDetailComponent implements OnInit { }); } + /** + * The Model Card describes the newest version whatever the picker is showing, so its facts are + * fetched in their own right rather than derived from the current selection. + */ + private retrieveLatestVersionFacts(): void { + const latest = this.versions[0]; + if (!this.mid || !latest?.mvid) { + this.latestVersionCreationTime = ""; + this.latestVersionFileName = ""; + this.latestVersionSize = undefined; + return; + } + this.latestVersionCreationTime = this.formatCreationTime(latest); + this.modelService + .retrieveModelVersionFileTree(this.mid, latest.mvid, this.isLogin) + .pipe(untilDestroyed(this)) + .subscribe({ + next: data => { + const firstFile = this.getFirstFileNode(data.fileNodes); + this.latestVersionFileName = firstFile ? getFullPathFromDatasetFileNode(firstFile) : ""; + this.latestVersionSize = data.size; + }, + error: (err: unknown) => this.notificationService.error(extractErrorMessage(err)), + }); + } + onVersionFileTreeNodeSelected(node: DatasetFileNode): void { this.loadFileContent(node); } @@ -432,9 +453,10 @@ export class ModelDetailComponent implements OnInit { next: () => { this.modelName = name; this.editedModelName = name; - // File nodes carry the model name inside their paths, and preview and single-file - // download resolve a model by (owner, name) — a stale tree 404s until reload. + // Every file path embeds the model name, and preview and single-file download resolve + // a model by (owner, name) — a stale tree 404s until reload. this.onVersionSelected(this.selectedVersion); + this.retrieveLatestVersionFacts(); this.notificationService.success(`Model name updated to '${name}'`); }, error: (err: unknown) => this.notificationService.error(extractErrorMessage(err)), From e0b19f16f725c7bf3079286c22646022616891c0 Mon Sep 17 00:00:00 2001 From: Tanishq Gandhi Date: Sat, 29 Aug 2026 11:04:56 -0700 Subject: [PATCH 4/6] fix(frontend): address review findings on the model upload PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Drop the invented .progress-header flex rule. The class is unstyled on main, so adding it silently changed the dataset upload row's layout, against this PR's claim of leaving dataset behavior alone. - Stop listing the newest version's objects twice on every model page load. onVersionSelected already fetches that tree whenever the newest version is the one on screen, so the card now reuses that response and the extra fetch runs only when the two differ — after a rename while browsing an older one. - Reword the conflicting-file modal, which said "in this dataset" and is now shown to model owners too. - Remove a duplicate VersionUploaderComponent import and a comment orphaned when its rule moved to the shared panel. Co-Authored-By: Claude Opus 5 --- .../files-uploader.component.spec.ts | 1 + .../files-uploader.component.ts | 2 +- .../dataset-detail.component.scss | 4 --- .../dataset-detail.component.ts | 1 - .../model-detail.component.spec.ts | 13 +++++++++ .../model-detail.component.ts | 29 +++++++++++++------ .../version-uploader.component.scss | 7 ----- .../file-resource/file-resource-endpoint.ts | 4 +++ .../multipart-upload.service.spec.ts | 1 + 9 files changed, 40 insertions(+), 22 deletions(-) diff --git a/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.spec.ts b/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.spec.ts index c72bfa6feb6..f895435475c 100644 --- a/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.spec.ts @@ -68,6 +68,7 @@ const droppedFile = (relativePath: string, file: File): NgxFileDropEntry => /** A resource family that is neither dataset nor model, to prove the uploader is parameterized. */ const WIDGET_ENDPOINT: FileResourceEndpoint = { baseUrl: "widget", + label: "widget", nameParamKey: "widgetName", maxFileSizeSettingKey: "widget_single_file_upload_max_size_mib", defaultMaxFileSizeMiB: 64, diff --git a/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts b/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts index 806c9e3123f..f7d81b30a87 100644 --- a/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts +++ b/frontend/src/app/dashboard/component/user/files-uploader/files-uploader.component.ts @@ -187,7 +187,7 @@ export class FilesUploaderComponent implements OnInit { fileName, path: item.name, size: formatSize(item.file.size), - hint: "A file with the same path and size exists in this dataset. Skip only if you expect it is the same file.", + hint: `A file with the same path and size exists in this ${this.endpoint.label}. Skip only if you expect it is the same file.`, }, nzFooter: [ ...(showForAll ? [button("Upload For All", "uploadAll"), button("Skip For All", "skipAll")] : []), diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss index 1cb00a8cf7c..0bd67b60672 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss +++ b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.scss @@ -127,10 +127,6 @@ nz-select { margin-top: 15%; } -// Rows must stay exactly PENDING_ROW_HEIGHT_PX tall for the fixed-size scroll -// strategy. The gutter lives on the rows: padding on the viewport never -// reaches its absolutely positioned content wrapper. - .status-tag-row { margin-top: 16px; display: flex; diff --git a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts index 01dd8b3d84f..535156b9541 100644 --- a/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts +++ b/frontend/src/app/dashboard/component/user/user-dataset/user-dataset-explorer/dataset-detail.component.ts @@ -111,7 +111,6 @@ export const THROTTLE_TIME_MS = 1000; NzDividerComponent, VersionUploaderComponent, NzInputDirective, - VersionUploaderComponent, NzDropdownDirective, NzDropdownMenuComponent, NzMenuDirective, diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts index 3e0fde26dee..cb9f97da7b6 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts @@ -622,6 +622,19 @@ describe("ModelDetailComponent", () => { expect(component.currentDisplayedFileName).toBe(`/model/${OWNER}/resnet-101/v1/model.pt`); }); + it("lists the newest version's objects once on load, not once per consumer", () => { + modelService["retrieveModelVersionList"] = vi.fn(() => of([aVersion(2, "v2"), aVersion(1, "v1")])); + modelService["retrieveModelVersionFileTree"] = vi.fn(() => + of({ fileNodes: [aFile("model.pt", `/model/${OWNER}/resnet-50/v2`)], size: 4 }) + ); + create(); + + // The file tree and the Model Card both describe v2 here, so one response serves both. + expect(modelService["retrieveModelVersionFileTree"]).toHaveBeenCalledTimes(1); + expect(component.latestVersionFileName).toBe(`/model/${OWNER}/resnet-50/v2/model.pt`); + expect(component.latestVersionSize).toBe(4); + }); + it("refreshes the Model Card too when an older version is on screen, without moving the picker", () => { const versions = [aVersion(2, "v2"), aVersion(1, "v1")]; modelService["updateModelName"] = vi.fn(() => of({})); diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts index 7baaeb4e535..eebd436616a 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts @@ -267,7 +267,7 @@ export class ModelDetailComponent implements OnInit { if (versions.length === 0) { return; } - this.retrieveLatestVersionFacts(); + this.latestVersionCreationTime = this.formatCreationTime(versions[0]); this.onVersionSelected(versions[0]); }, error: (err: unknown) => this.notificationService.error(extractErrorMessage(err)), @@ -288,6 +288,12 @@ export class ModelDetailComponent implements OnInit { this.currentModelVersionSize = data.size; this.selectedVersionCreationTime = this.formatCreationTime(version); + // The Model Card describes the newest version, so when that is the one just fetched its + // facts come from this response rather than a second identical request. + if (version === this.versions[0]) { + this.applyLatestVersionFacts(data); + } + const firstFile = this.getFirstFileNode(this.fileTreeNodeList); if (!firstFile) { this.currentDisplayedFileName = ""; @@ -300,9 +306,15 @@ export class ModelDetailComponent implements OnInit { }); } + private applyLatestVersionFacts(data: { fileNodes: DatasetFileNode[]; size: number }): void { + const firstFile = this.getFirstFileNode(data.fileNodes); + this.latestVersionFileName = firstFile ? getFullPathFromDatasetFileNode(firstFile) : ""; + this.latestVersionSize = data.size; + } + /** - * The Model Card describes the newest version whatever the picker is showing, so its facts are - * fetched in their own right rather than derived from the current selection. + * Refreshes the Model Card when the newest version is *not* the one on screen. Whenever they + * coincide, onVersionSelected fills it in from the tree it already fetched. */ private retrieveLatestVersionFacts(): void { const latest = this.versions[0]; @@ -317,11 +329,7 @@ export class ModelDetailComponent implements OnInit { .retrieveModelVersionFileTree(this.mid, latest.mvid, this.isLogin) .pipe(untilDestroyed(this)) .subscribe({ - next: data => { - const firstFile = this.getFirstFileNode(data.fileNodes); - this.latestVersionFileName = firstFile ? getFullPathFromDatasetFileNode(firstFile) : ""; - this.latestVersionSize = data.size; - }, + next: data => this.applyLatestVersionFacts(data), error: (err: unknown) => this.notificationService.error(extractErrorMessage(err)), }); } @@ -456,7 +464,10 @@ export class ModelDetailComponent implements OnInit { // Every file path embeds the model name, and preview and single-file download resolve // a model by (owner, name) — a stale tree 404s until reload. this.onVersionSelected(this.selectedVersion); - this.retrieveLatestVersionFacts(); + // That call covers the card only when the newest version is the one on screen. + if (this.selectedVersion !== this.versions[0]) { + this.retrieveLatestVersionFacts(); + } this.notificationService.success(`Model name updated to '${name}'`); }, error: (err: unknown) => this.notificationService.error(extractErrorMessage(err)), diff --git a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.scss b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.scss index 35e4a14490e..788b226a408 100644 --- a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.scss +++ b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.scss @@ -41,13 +41,6 @@ } } -.progress-header { - display: flex; - align-items: center; - justify-content: space-between; - gap: 8px; -} - .upload-stats { font-size: 13px; margin-bottom: 20px; diff --git a/frontend/src/app/dashboard/service/user/file-resource/file-resource-endpoint.ts b/frontend/src/app/dashboard/service/user/file-resource/file-resource-endpoint.ts index 4a89c129a81..9aa89ffbdfa 100644 --- a/frontend/src/app/dashboard/service/user/file-resource/file-resource-endpoint.ts +++ b/frontend/src/app/dashboard/service/user/file-resource/file-resource-endpoint.ts @@ -25,6 +25,8 @@ export interface FileResourceEndpoint { /** Path segment under the API root, e.g. "dataset". */ readonly baseUrl: string; + /** How to name this resource kind in user-facing copy, e.g. "dataset". */ + readonly label: string; /** Query-param name carrying the resource name, e.g. "datasetName". */ readonly nameParamKey: string; /** site_settings key holding this resource's per-file upload ceiling, in MiB. */ @@ -39,6 +41,7 @@ export interface FileResourceEndpoint { export const DATASET_FILE_RESOURCE_ENDPOINT: FileResourceEndpoint = { baseUrl: "dataset", + label: "dataset", nameParamKey: "datasetName", maxFileSizeSettingKey: "dataset_single_file_upload_max_size_mib", defaultMaxFileSizeMiB: 20, @@ -49,6 +52,7 @@ export const DATASET_FILE_RESOURCE_ENDPOINT: FileResourceEndpoint = { export const MODEL_FILE_RESOURCE_ENDPOINT: FileResourceEndpoint = { baseUrl: "model", + label: "model", nameParamKey: "modelName", maxFileSizeSettingKey: "model_single_file_upload_max_size_mib", defaultMaxFileSizeMiB: 2048, diff --git a/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.spec.ts b/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.spec.ts index 94bdca97398..05795a092e0 100644 --- a/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.spec.ts +++ b/frontend/src/app/dashboard/service/user/file-resource/multipart-upload.service.spec.ts @@ -34,6 +34,7 @@ const API = "api"; */ const WIDGET_ENDPOINT: FileResourceEndpoint = { baseUrl: "widget", + label: "widget", nameParamKey: "widgetName", maxFileSizeSettingKey: "widget_single_file_upload_max_size_mib", defaultMaxFileSizeMiB: 64, From 5db68f63760bfc8cb45a27209cb950929196f886 Mon Sep 17 00:00:00 2001 From: Tanishq Gandhi Date: Sat, 29 Aug 2026 18:28:28 -0700 Subject: [PATCH 5/6] fix(frontend): restore the upload-tuning tests and keep your place on rename - Port the three loadUploadSettings fallback tests that were dropped when the panel moved out of DatasetDetailComponent. They pin the invariant the code comment warns about: a failed or unparsable settings fetch must leave the fields alone, because a NaN concurrency limit stalls the queue outright (`activeUploads < NaN` is never true). - Renaming a model reopens the file that was on screen instead of resetting to the version's first. Only the paths change, so the file is still there. - Derive the admin upload card's title from FileResourceEndpoint.label, so a resource family is named once rather than in two places that could drift. Co-Authored-By: Claude Opus 5 --- .../settings/admin-settings.component.ts | 13 ++-- .../model-detail.component.spec.ts | 44 ++++++++++++++ .../model-detail.component.ts | 35 +++++++++-- .../version-uploader.component.spec.ts | 59 +++++++++++++++++++ 4 files changed, 140 insertions(+), 11 deletions(-) diff --git a/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.ts b/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.ts index 104a70578a9..744aa7ab266 100644 --- a/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.ts +++ b/frontend/src/app/dashboard/component/admin/settings/admin-settings.component.ts @@ -52,6 +52,7 @@ export interface UploadSettingsForm { /** One upload card: a resource family, the site_settings keys it owns, and the edited values. */ export interface UploadSettingsGroup { + /** Title case of the endpoint's own label, so a family is never named twice. */ readonly label: string; readonly endpoint: FileResourceEndpoint; readonly form: UploadSettingsForm; @@ -63,8 +64,8 @@ const DEFAULT_MAX_CONCURRENT_FILES = 3; const DEFAULT_MAX_CONCURRENT_CHUNKS = 10; const DEFAULT_CHUNK_SIZE_MIB = 50; -const uploadGroup = (label: string, endpoint: FileResourceEndpoint): UploadSettingsGroup => ({ - label, +const uploadGroup = (endpoint: FileResourceEndpoint): UploadSettingsGroup => ({ + label: endpoint.label.charAt(0).toUpperCase() + endpoint.label.slice(1), endpoint, form: { maxConcurrentFiles: DEFAULT_MAX_CONCURRENT_FILES, @@ -117,8 +118,8 @@ export class AdminSettingsComponent implements OnInit { // One card per resource family; adding a family here adds its card, with no new save/reset code. readonly uploadGroups: UploadSettingsGroup[] = [ - uploadGroup("Dataset", DATASET_FILE_RESOURCE_ENDPOINT), - uploadGroup("Model", MODEL_FILE_RESOURCE_ENDPOINT), + uploadGroup(DATASET_FILE_RESOURCE_ENDPOINT), + uploadGroup(MODEL_FILE_RESOURCE_ENDPOINT), ]; csvMaxColumns: number = 512; @@ -318,7 +319,7 @@ export class AdminSettingsComponent implements OnInit { .pipe(untilDestroyed(this)) .subscribe({ next: () => this.message.success(`${group.label} upload settings saved successfully.`), - error: () => this.message.error(`Failed to save ${group.label.toLowerCase()} settings.`), + error: () => this.message.error(`Failed to save ${group.endpoint.label} settings.`), }); } @@ -327,7 +328,7 @@ export class AdminSettingsComponent implements OnInit { this.adminSettingsService.resetSetting(key).pipe(untilDestroyed(this)).subscribe({}) ); - this.message.info(`Resetting ${group.label.toLowerCase()} settings...`); + this.message.info(`Resetting ${group.endpoint.label} settings...`); setTimeout(() => window.location.reload(), this.RELOAD_DELAY); } diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts index cb9f97da7b6..d6434908816 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts @@ -659,6 +659,50 @@ describe("ModelDetailComponent", () => { expect(component.selectedVersion).toBe(versions[1]); }); + it("reopens the file you were reading after a rename, not the version's first", () => { + modelService["updateModelName"] = vi.fn(() => of({})); + modelService["retrieveModelVersionList"] = vi.fn(() => of([aVersion(1, "v1")])); + let name = "resnet-50"; + const tree = () => ({ + fileNodes: [ + aFile("first.txt", `/model/${OWNER}/${name}/v1`), + { + name: "weights", + type: "directory" as const, + parentDir: `/model/${OWNER}/${name}/v1`, + children: [aFile("model.pt", `/model/${OWNER}/${name}/v1/weights`)], + }, + ], + size: 8, + }); + modelService["retrieveModelVersionFileTree"] = vi.fn(() => of(tree())); + create(); + // Open a nested file that is not the one the tree opens by default. + component.onVersionFileTreeNodeSelected(aFile("model.pt", `/model/${OWNER}/resnet-50/v1/weights`)); + expect(component.currentDisplayedFileName).toBe(`/model/${OWNER}/resnet-50/v1/weights/model.pt`); + + name = "resnet-101"; + component.editedModelName = name; + component.onSaveModelName(); + + // Same file, new path — renaming should not lose the reader's place. + expect(component.currentDisplayedFileName).toBe(`/model/${OWNER}/resnet-101/v1/weights/model.pt`); + }); + + it("falls back to the first file when a rename outlives the file that was open", () => { + modelService["updateModelName"] = vi.fn(() => of({})); + modelService["retrieveModelVersionList"] = vi.fn(() => of([aVersion(1, "v1")])); + modelService["retrieveModelVersionFileTree"] = vi.fn(() => + of({ fileNodes: [aFile("only.txt", `/model/${OWNER}/resnet-101/v1`)], size: 4 }) + ); + create(); + component.editedModelName = "resnet-101"; + + component.onSaveModelName(); + + expect(component.currentDisplayedFileName).toBe(`/model/${OWNER}/resnet-101/v1/only.txt`); + }); + it("skips the tree refresh for a model with no versions", () => { modelService["updateModelName"] = vi.fn(() => of({})); create(); diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts index eebd436616a..b67af0adf06 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts @@ -133,6 +133,8 @@ export class ModelDetailComponent implements OnInit { public currentDisplayedFileName: string = ""; public currentFileSize: number | undefined; + // Path within the version, which survives a rename — unlike currentDisplayedFileName. + private openFileRelativePath: string = ""; // Placeholders until models reach the hub. The hub backend has no model entity type // (`hub/EntityType.scala` is Workflow and Dataset only), so nothing can populate these yet. @@ -274,7 +276,11 @@ export class ModelDetailComponent implements OnInit { }); } - onVersionSelected(version: ModelVersion | undefined): void { + /** + * @param preferredRelativePath reopens this file rather than the version's first, when the + * refetched tree still holds it. Used after a rename, which invalidates every path. + */ + onVersionSelected(version: ModelVersion | undefined, preferredRelativePath?: string): void { this.selectedVersion = version; if (!this.mid || !version?.mvid) { return; @@ -294,13 +300,17 @@ export class ModelDetailComponent implements OnInit { this.applyLatestVersionFacts(data); } - const firstFile = this.getFirstFileNode(this.fileTreeNodeList); - if (!firstFile) { + const preferred = preferredRelativePath + ? this.findFileByRelativePath(this.fileTreeNodeList, preferredRelativePath) + : undefined; + const target = preferred ?? this.getFirstFileNode(this.fileTreeNodeList); + if (!target) { this.currentDisplayedFileName = ""; this.currentFileSize = undefined; + this.openFileRelativePath = ""; return; } - this.loadFileContent(firstFile); + this.loadFileContent(target); }, error: (err: unknown) => this.notificationService.error(extractErrorMessage(err)), }); @@ -341,6 +351,20 @@ export class ModelDetailComponent implements OnInit { loadFileContent(node: DatasetFileNode): void { this.currentDisplayedFileName = getFullPathFromDatasetFileNode(node); this.currentFileSize = node.size; + this.openFileRelativePath = getRelativePathFromDatasetFileNode(node); + } + + private findFileByRelativePath(nodes: DatasetFileNode[], relativePath: string): DatasetFileNode | undefined { + for (const node of nodes) { + if (node.type === "file" && getRelativePathFromDatasetFileNode(node) === relativePath) { + return node; + } + const inChildren = node.children && this.findFileByRelativePath(node.children, relativePath); + if (inChildren) { + return inChildren; + } + } + return undefined; } // Walk from the first node into directories until reaching a file. @@ -463,7 +487,8 @@ export class ModelDetailComponent implements OnInit { this.editedModelName = name; // Every file path embeds the model name, and preview and single-file download resolve // a model by (owner, name) — a stale tree 404s until reload. - this.onVersionSelected(this.selectedVersion); + // Reopen whatever was on screen: only the paths changed, not the files. + this.onVersionSelected(this.selectedVersion, this.openFileRelativePath); // That call covers the card only when the newest version is the one on screen. if (this.selectedVersion !== this.versions[0]) { this.retrieveLatestVersionFacts(); diff --git a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts index 1f77c58b81d..72d46d9da25 100644 --- a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts @@ -503,6 +503,23 @@ describe("VersionUploaderComponent", () => { }); }); + const settingsStub = () => + TestBed.inject(AdminSettingsService) as unknown as { getPublicSetting: ReturnType }; + + /** Rebuilds the fixture so a per-test settings stub is in place before ngOnInit runs. */ + const rebuild = (seed?: (c: VersionUploaderComponent) => void): void => { + fixture.destroy(); + fixture = TestBed.createComponent(VersionUploaderComponent); + component = fixture.componentInstance; + component.resourceId = 1; + component.ownerEmail = "owner@texera.com"; + component.resourceName = "test-dataset"; + component.endpoint = DATASET_FILE_RESOURCE_ENDPOINT; + component.createVersion = asCreateVersion(createVersionSpy); + seed?.(component); + fixture.detectChanges(); + }; + /** The panel is the only thing that knows which resource family it is addressing. */ describe("resource addressing", () => { it("passes its endpoint to the upload engine, whichever family it serves", () => { @@ -529,6 +546,48 @@ describe("VersionUploaderComponent", () => { ]); expect(component.maxConcurrentFiles).toBe(3); }); + + it("keeps the default upload settings when the public settings are missing", () => { + settingsStub().getPublicSetting.mockReturnValue(of(null)); + rebuild(); + + expect(component.chunkSizeMiB).toBe(50); + expect(component.maxConcurrentChunks).toBe(10); + expect(component.maxConcurrentFiles).toBe(3); + }); + + it("leaves the chunk size untouched when only that setting fails to load", () => { + // A distinct value per key, so a setting that lands in the wrong field is visible: + // 7 chunks and 2 files cannot stand in for one another. + settingsStub().getPublicSetting.mockImplementation((key: string) => + key === DATASET_FILE_RESOURCE_ENDPOINT.chunkSizeSettingKey + ? throwError(() => new Error("boom")) + : of(key === DATASET_FILE_RESOURCE_ENDPOINT.maxConcurrentChunksSettingKey ? "7" : "2") + ); + // A sentinel the class default cannot supply, so "the failed fetch wrote nothing" is + // distinguishable from "it wrote the default back". + rebuild(c => (c.chunkSizeMiB = 42)); + + expect(component.chunkSizeMiB).toBe(42); + expect(component.maxConcurrentChunks).toBe(7); + expect(component.maxConcurrentFiles).toBe(2); + }); + + it("leaves both concurrency limits untouched when their settings fail to load", () => { + settingsStub().getPublicSetting.mockImplementation((key: string) => + key === DATASET_FILE_RESOURCE_ENDPOINT.chunkSizeSettingKey ? of("128") : throwError(() => new Error("boom")) + ); + rebuild(c => { + c.maxConcurrentChunks = 41; + c.maxConcurrentFiles = 40; + }); + + // A failed fetch that wrote anything here — a reset, or a NaN — would stall the queue + // outright, since `activeUploads < NaN` is never true. + expect(component.chunkSizeMiB).toBe(128); + expect(component.maxConcurrentChunks).toBe(41); + expect(component.maxConcurrentFiles).toBe(40); + }); }); describe("staged changes", () => { From 22ac69fa6372ef56a4036d22e2b2023e5fbe25ce Mon Sep 17 00:00:00 2001 From: Tanishq Gandhi Date: Sun, 30 Aug 2026 20:16:37 -0700 Subject: [PATCH 6/6] fix(frontend): block a model rename while an upload is in flight The upload engine captures the resource name when an upload starts, so renaming mid-flight leaves the remaining part and finish calls addressed to a model that no longer answers to that name. Aborting cannot rescue it either: the abort reads the name at click time, i.e. the new one, so it 404s too and the multipart session is left for the retry loop to grind through. The panel now reports whether anything is in flight and the Settings tab disables the name field until it is idle, with onSaveModelName guarding the same invariant. The dataset page has the same bug and is left alone here, queued with the other pre-existing dataset fixes. Co-Authored-By: Claude Opus 5 --- .../model-detail.component.html | 11 +++++++- .../model-detail.component.spec.ts | 15 +++++++++++ .../model-detail.component.ts | 8 ++++++ .../version-uploader.component.spec.ts | 25 +++++++++++++++++++ .../version-uploader.component.ts | 17 ++++++++++--- 5 files changed, 72 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.html b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.html index f6387ae0f5b..94be936b129 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.html +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.html @@ -360,7 +360,8 @@
Choose a Version:
[resourceName]="modelName" [endpoint]="modelEndpoint" [createVersion]="createModelVersion" - (versionCreated)="onVersionCreated()"> + (versionCreated)="onVersionCreated()" + (uploadsInFlightChange)="uploadsInFlight = $event"> @@ -378,14 +379,22 @@

General

Letters, numbers, underscores and hyphens.

+

+ An upload is in progress — finish or cancel it in Versions & Files first, or it will be left + incomplete. +

diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts index d6434908816..93a89bfd0ca 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.spec.ts @@ -715,6 +715,21 @@ describe("ModelDetailComponent", () => { expect(component.modelName).toBe("resnet-101"); }); + it("refuses to rename while an upload is in flight", () => { + modelService["updateModelName"] = vi.fn(() => of({})); + create(); + component.uploadsInFlight = true; + component.editedModelName = "resnet-101"; + + component.onSaveModelName(); + + // The engine captured the old name when the upload started; renaming now would strand its + // remaining parts, and the abort — which reads the new name — could not clean them up. + expect(modelService["updateModelName"]).not.toHaveBeenCalled(); + expect(notificationService["error"]).toHaveBeenCalled(); + expect(component.modelName).toBe("resnet-50"); + }); + it("rejects an invalid name without calling the server", () => { modelService["updateModelName"] = vi.fn(() => of({})); create(); diff --git a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts index b67af0adf06..cd5fa68a860 100644 --- a/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts +++ b/frontend/src/app/dashboard/component/user/user-model/user-model-explorer/model-detail.component.ts @@ -154,6 +154,10 @@ export class ModelDetailComponent implements OnInit { @ViewChild(VersionUploaderComponent) private versionUploader?: VersionUploaderComponent; + // Renaming mid-upload strands the in-flight parts under the old name, so the Settings tab + // blocks it until the panel is idle. + public uploadsInFlight = false; + formatSize = formatSize; formatCount = formatCount; @@ -471,6 +475,10 @@ export class ModelDetailComponent implements OnInit { if (!this.mid) { return; } + if (this.uploadsInFlight) { + this.notificationService.error("Finish or cancel the upload in progress before renaming this model"); + return; + } const name = this.editedModelName; const nameError = validateModelName(name); if (nameError) { diff --git a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts index 72d46d9da25..754bf14eebd 100644 --- a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts +++ b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.spec.ts @@ -590,6 +590,31 @@ describe("VersionUploaderComponent", () => { }); }); + describe("in-flight signalling", () => { + it("tells the host while an upload is running, so a rename cannot strand it", () => { + const inFlight: boolean[] = []; + component.uploadsInFlightChange.subscribe((v: boolean) => inFlight.push(v)); + + dropFiles("a.csv"); + expect(inFlight.at(-1)).toBe(true); + + component.onClickAbortUploadProgress(component.uploadTasks[0]); + expect(inFlight.at(-1)).toBe(false); + }); + + it("stays flagged until the last of several uploads finishes", () => { + const inFlight: boolean[] = []; + component.uploadsInFlightChange.subscribe((v: boolean) => inFlight.push(v)); + + dropFiles("a.csv", "b.csv"); + finishUpload(0, "a.csv"); + expect(inFlight.at(-1)).toBe(true); + + finishUpload(1, "b.csv"); + expect(inFlight.at(-1)).toBe(false); + }); + }); + describe("staged changes", () => { it("tracks the pending-change count from the diff response", () => { const staged: DatasetStagedObject[] = [ diff --git a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.ts b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.ts index 8f709a1f6c8..cfdaca02e6f 100644 --- a/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.ts +++ b/frontend/src/app/dashboard/component/user/version-uploader/version-uploader.component.ts @@ -98,6 +98,12 @@ export class VersionUploaderComponent implements OnInit { /** The host reloads its version list off this. */ @Output() versionCreated = new EventEmitter(); + /** + * True while any upload is mid-flight. The engine captured the resource name when the upload + * started, so a rename in that window strands the remaining part/finish calls under the old + * name — and the abort, which reads the new one, cannot clean them up either. + */ + @Output() uploadsInFlightChange = new EventEmitter(); userHasPendingChanges: boolean = false; pendingChangesCount: number = 0; @@ -250,7 +256,7 @@ export class VersionUploaderComponent implements OnInit { }; if (this.activeUploads < this.maxConcurrentFiles) { - this.activeUploads++; + this.setActiveUploads(this.activeUploads + 1); startUpload(); } else { this.pendingQueue.set(file.name, startUpload); @@ -281,16 +287,21 @@ export class VersionUploaderComponent implements OnInit { const [fileName, startUpload] = next.value; this.pendingQueue.delete(fileName); this.pendingQueueDirty = true; - this.activeUploads++; + this.setActiveUploads(this.activeUploads + 1); startUpload(); } } private onUploadComplete(): void { - this.activeUploads--; + this.setActiveUploads(this.activeUploads - 1); this.processNextQueuedUpload(); } + private setActiveUploads(count: number): void { + this.activeUploads = count; + this.uploadsInFlightChange.emit(count > 0); + } + private removeFromPendingQueue(fileName: string): void { if (this.pendingQueue.delete(fileName)) { this.pendingQueueDirty = true;