refactor: file loading

src/components/App.vue

@@ -207,6 +207,14 @@ import { UrlParams } from '@vueuse/core';
 import vtkURLExtract from '@kitware/vtk.js/Common/Core/URLExtract';
 import { URL } from 'whatwg-url';
 
+import { basename } from '@/src/utils/path';
+import { useDatasetStore } from '@/src/store/datasets';
+import { logError } from '@/src/utils/loggers';
+import {
+  importDataSources,
+  ImportDataSourcesResult,
+  convertSuccessResultToDataSelection,
+} from '@/src/io/import/importDataSources';
 import ResizableNavDrawer from './ResizableNavDrawer.vue';
 import ToolButton from './ToolButton.vue';
 import LayoutGrid from './LayoutGrid.vue';
@@ -220,77 +228,57 @@ import Settings from './Settings.vue';
 import VolViewFullLogo from './icons/VolViewFullLogo.vue';
 import VolViewLogo from './icons/VolViewLogo.vue';
 import {
-  useDatasetStore,
-  convertSuccessResultToDataSelection,
-  LoadResult,
-  FileLoadSuccess,
-  DICOMLoadSuccess,
-  FileLoadFailure,
-  DICOMLoadFailure,
-} from '../store/datasets';
+  DataSource,
+  fileToDataSource,
+  getDataSourceName,
+  uriToDataSource,
+} from '../io/import/dataSource';
 import { useImageStore } from '../store/datasets-images';
-import { makeLocal, DatasetFile, makeRemote } from '../store/datasets-files';
 import { useViewStore } from '../store/views';
 import { MessageType, useMessageStore } from '../store/messages';
 import { Layouts } from '../config';
-import { isStateFile, loadState, serialize } from '../io/state-file';
+import { serialize } from '../io/state-file';
 import SaveSession from './SaveSession.vue';
 import { useGlobalErrorHook } from '../composables/useGlobalErrorHook';
 import { useWebGLWatchdog } from '../composables/useWebGLWatchdog';
 import { useAppLoadingNotifications } from '../composables/useAppLoadingNotifications';
-import { isFulfilled, partition, pluck, wrapInArray } from '../utils';
-import { canFetchUrl, fetchFile } from '../utils/fetch';
-import { basename } from '../utils/path';
+import { partition, wrapInArray } from '../utils';
 
-async function loadFiles(files: DatasetFile[], setError: (err: Error) => void) {
+async function loadFiles(
+  sources: DataSource[],
+  setError: (err: Error) => void
+) {
   const dataStore = useDatasetStore();
 
-  const loadFirstDataset = !dataStore.primarySelection;
-
-  let statuses: LoadResult[] = [];
-  let stateFile = false;
+  let results: ImportDataSourcesResult[];
   try {
-    // For now only support restoring from a single state files.
-    stateFile = files.length === 1 && (await isStateFile(files[0].file));
-    if (stateFile) {
-      statuses = await loadState(files[0].file);
-    } else {
-      statuses = await dataStore.loadFiles(files);
-    }
+    results = await importDataSources(sources);
   } catch (error) {
     setError(error as Error);
     return;
   }
 
-  const loaded = statuses.filter((s) => s.loaded) as (
-    | FileLoadSuccess
-    | DICOMLoadSuccess
-  )[];
-  const errored = statuses.filter((s) => !s.loaded) as (
-    | FileLoadFailure
-    | DICOMLoadFailure
-  )[];
-
-  if (
-    loaded.length &&
-    !stateFile &&
-    (loadFirstDataset || loaded.length === 1)
-  ) {
-    const selection = convertSuccessResultToDataSelection(loaded[0]);
+  const [succeeded, errored] = partition((result) => result.ok, results);
+
+  if (!dataStore.primarySelection && succeeded.length) {
+    const selection = convertSuccessResultToDataSelection(succeeded[0]);
     if (selection) {
       dataStore.setPrimarySelection(selection);
     }
   }
 
   if (errored.length) {
-    const failedFilenames = errored.map((result) => {
-      if (result.type === 'file') {
-        return result.filename;
-      }
-      return 'DICOM files';
+    const errorMessages = errored.map((errResult) => {
+      // pick first error
+      const [firstError] = errResult.errors;
+      // pick innermost dataset that errored
+      const name = getDataSourceName(firstError.inputDataStackTrace[0]);
+      // log error for debugging
+      logError(firstError.cause);
+      return `- ${name}: ${firstError.message}`;
     });
     const failedError = new Error(
-      `These files failed to load:\n${failedFilenames.join('\n')}`
+      `These files failed to load:\n${errorMessages.join('\n')}`
     );
 
     setError(failedError);
@@ -303,49 +291,14 @@ async function loadRemoteFilesFromURLParams(
 ) {
   const urls = wrapInArray(params.urls);
   const names = wrapInArray(params.names ?? []); // optional names should resolve to [] if params.names === undefined
-  const datasets = urls.map((url, idx) => ({
-    url,
-    remoteFilename:
-      names[idx] ||
-      basename(new URL(url, window.location.href).pathname) ||
+  const sources = urls.map((url, idx) =>
+    uriToDataSource(
       url,
-    // loadFiles will treat empty files as URLs to download
-    file: new File([], ''),
-  }));
-
-  const [downloadNow, downloadLater] = partition(
-    (dataset) => canFetchUrl(dataset.url),
-    datasets
-  );
-
-  const fetchResults = await Promise.allSettled(
-    downloadNow.map(({ url, remoteFilename }) => fetchFile(url, remoteFilename))
-  );
-
-  const withParams = fetchResults.map((result, idx) => ({
-    result,
-    ...downloadNow[idx],
-  }));
-
-  const [downloaded, rejected] = partition(
-    ({ result }) => isFulfilled(result) && result.value.size !== 0,
-    withParams
-  );
-
-  if (rejected.length) {
-    setError(
-      new Error(
-        `Failed to download URLs:\n${rejected.map(pluck('url')).join('\n')}`
-      )
-    );
-  }
-
-  const downloadedDatasetFiles = downloaded.map(({ result, url }) =>
-    makeRemote(url, (result as PromiseFulfilledResult<File>).value)
+      names[idx] || basename(new URL(url, window.location.href).pathname) || url
+    )
   );
 
-  // must await for setError to work
-  await loadFiles([...downloadedDatasetFiles, ...downloadLater], setError);
+  await loadFiles(sources, setError);
 }
 
 export default defineComponent({
@@ -409,8 +362,8 @@ export default defineComponent({
         return;
       }
 
-      const datasetFiles = Array.from(files).map(makeLocal);
-      runAsLoading((setError) => loadFiles(datasetFiles, setError));
+      const dataSources = Array.from(files).map(fileToDataSource);
+      runAsLoading((setError) => loadFiles(dataSources, setError));
     }
 
     const fileEl = document.createElement('input');

src/components/SampleDataBrowser.vue

@@ -7,17 +7,18 @@ import {
   computed,
 } from '@vue/composition-api';
 import ImageListCard from '@/src/components/ImageListCard.vue';
-import { SAMPLE_DATA } from '../config';
+import { useDatasetStore } from '@/src/store/datasets';
 import {
   convertSuccessResultToDataSelection,
-  useDatasetStore,
-} from '../store/datasets';
+  importDataSources,
+} from '@/src/io/import/importDataSources';
+import { remoteFileToDataSource } from '@/src/io/import/dataSource';
+import { SAMPLE_DATA } from '../config';
 import { useMessageStore } from '../store/messages';
 import { SampleDataset } from '../types';
 import { useImageStore } from '../store/datasets-images';
 import { useDICOMStore } from '../store/datasets-dicom';
 import { fetchFile } from '../utils/fetch';
-import { makeRemote } from '../store/datasets-files';
 
 enum ProgressState {
   Pending,
@@ -94,23 +95,25 @@ export default defineComponent({
         });
         status.progress[sample.name].state = ProgressState.Done;
 
-        if (sampleFile) {
-          const [loadResult] = await datasetStore.loadFiles([
-            makeRemote(sample.url, sampleFile),
-          ]);
-          if (loadResult?.loaded) {
-            const selection = convertSuccessResultToDataSelection(loadResult);
-            if (selection) {
-              const id =
-                selection.type === 'image'
-                  ? selection.dataID
-                  : selection.volumeKey;
-              set(loaded.idToURL, id, sample.url);
-              set(loaded.urlToID, sample.url, id);
-            }
-            datasetStore.setPrimarySelection(selection);
-          }
+        const [loadResult] = await importDataSources([
+          remoteFileToDataSource(sampleFile, sample.url),
+        ]);
+
+        if (!loadResult) {
+          throw new Error('Did not receive a load result');
+        }
+        if (!loadResult.ok) {
+          throw loadResult.errors[0].cause;
+        }
+
+        const selection = convertSuccessResultToDataSelection(loadResult);
+        if (selection) {
+          const id =
+            selection.type === 'image' ? selection.dataID : selection.volumeKey;
+          set(loaded.idToURL, id, sample.url);
+          set(loaded.urlToID, sample.url, id);
         }
+        datasetStore.setPrimarySelection(selection);
       } catch (error) {
         status.progress[sample.name].state = ProgressState.Error;
         const messageStore = useMessageStore();

src/core/__tests__/pipeline.spec.ts

@@ -175,6 +175,52 @@ describe('Pipeline', () => {
     expect(result.errors[0].message).to.equal(error.message);
   });
 
+  it('should handle nested executions', async () => {
+    // handlers encode fibonacci
+    const handlers: Array<Handler<number, number>> = [
+      async (idx, { done }) => {
+        if (idx === 0 || idx === 1) {
+          return done(1);
+        }
+        return idx;
+      },
+      async (idx, { execute, done }) => {
+        let fnum = (await execute(idx - 1)).data[0];
+        if (idx > 1) {
+          fnum += (await execute(idx - 2)).data[0];
+        }
+        return done(fnum);
+      },
+    ];
+
+    const pipeline = new Pipeline(handlers);
+    const N = 5;
+    const result = await pipeline.execute(N);
+
+    expect(result.ok).to.be.true;
+    // pick first result data, which is the top-level pipeline result
+    expect(result.data[0]).to.equal(8);
+  });
+
+  it('should handle allow extra context overriding', async () => {
+    type Extra = number;
+    const handlers: Array<Handler<number, number, Extra>> = [
+      (val, { done, execute, extra }) => {
+        if (extra === 42) {
+          return done(extra);
+        }
+        execute(val, 42);
+        return val;
+      },
+    ];
+
+    const pipeline = new Pipeline(handlers);
+    const result = await pipeline.execute(0, 21);
+
+    expect(result.ok).to.be.true;
+    expect(result.data).to.deep.equal([42]);
+  });
+
   it('should handle nested async errors', async () => {
     const error = new Error('Some failure');
     const handlers: Array<Handler<number>> = [

src/core/dicom-web-api.ts

@@ -1,3 +1,4 @@
+import { cleanUndefined } from '@/src/utils';
 import { api } from 'dicomweb-client-typed';
 
 export interface FetchStudyOptions {
@@ -47,15 +48,6 @@ function parseTag(value: any) {
   return v;
 }
 
-// remove undefined properties
-function cleanUndefined(obj: Object) {
-  return Object.entries(obj).reduce(
-    (cleaned, [key, value]) =>
-      value === undefined ? cleaned : { ...cleaned, [key]: value },
-    {}
-  );
-}
-
 function parseInstance(instance: any) {
   const withNamedTags = Object.entries(tags).reduce(
     (info, [key, tag]) => ({ ...info, [key]: parseTag(instance[tag]) }),