Upload exactly the server-validated package

client/platform/web-girder/api/dataset.service.ts

@@ -5,7 +5,7 @@ import {
   registrationValuesSummary, filterRegistrationValues, mergeRegistrationValues,
 } from 'vue-media-annotator/alignedView/cameraRegistrationFiles';
 import {
-  DatasetMetaMutable, FrameImage, SaveAttributeArgs, SaveAttributeTrackFilterArgs,
+  DatasetMetaMutable, DatasetType, FrameImage, SaveAttributeArgs, SaveAttributeTrackFilterArgs,
 } from 'dive-common/apispec';
 import { calibrationFileMarker, jsonCalibrationFileMarker, metadataFileMarker } from 'dive-common/constants';
 import { attachFrameTimestamps } from 'dive-common/frameTimestamp';
@@ -247,14 +247,33 @@ async function importCameraRegistration(
   return summary;
 }
 
-interface ValidationResponse {
-  ok: boolean;
-  type: 'video' | 'image-sequence' | 'large-image';
-  media: string[];
-  annotations: string[];
+export type UploadRole = 'media' | 'annotations' | 'datasetConfig' | 'ignored';
+
+/** Every validated filename under exactly one role. `ignored` is the files not accepted. */
+export type ValidatedUploadRoleMap = Record<UploadRole, string[]>;
+
+/** A filename with the reason it is not uploaded. Display shape; the wire carries `reasons`. */
+export interface IgnoredUploadFile {
+  name: string;
+  reason: string;
+}
+
+interface ValidationBase {
   message: string;
+  roles: ValidatedUploadRoleMap;
+  /** Why a file landed in its role, keyed by filename. Populated for the `ignored` role. */
+  reasons: Record<string, string>;
 }
 
+/**
+ * Server classification of one upload selection. `roles` is authoritative for what each file
+ * is for, so the set to upload is the selection minus `roles.ignored`. Only an accepted
+ * selection has a media type, so `ok` narrows `type` to a real DatasetType.
+ */
+export type ValidationResponse =
+  | (ValidationBase & { ok: true; type: DatasetType })
+  | (ValidationBase & { ok: false; type?: undefined });
+
 function validateUploadGroup(names: string[]) {
   return girderRest.post<ValidationResponse>('dive_dataset/validate_files', names);
 }

client/platform/web-girder/multicamFileRegistry.spec.ts

@@ -6,11 +6,25 @@ import {
 import {
   clearMulticamFileRegistry,
   getCalibrationFile,
+  getCameraPackageFiles,
   getLastCalibration,
+  mediaFileNamesForImport,
   saveCalibration,
+  stashAnnotationFile,
   stashCalibrationFile,
 } from './multicamFileRegistry';
 
+/** Fixed lastModified so "did the user pick this very file?" is deterministic in tests. */
+function file(name: string, content = name): File {
+  return new File([content], name, { type: 'application/octet-stream', lastModified: 0 });
+}
+
+function fileWithPath(name: string, relPath: string, content = name): File {
+  const f = file(name, content);
+  Object.defineProperty(f, 'webkitRelativePath', { value: relPath, configurable: true });
+  return f;
+}
+
 describe('multicamFileRegistry calibration', () => {
   const storage = new Map<string, string>();
 
@@ -30,10 +44,10 @@ describe('multicamFileRegistry calibration', () => {
   });
 
   it('resolves calibration by basename when stashed with a path-like key', () => {
-    const file = new File(['{}'], 'stereo-cal.json', { type: 'application/json' });
-    stashCalibrationFile('folder/stereo-cal.json', file);
-    expect(getCalibrationFile('stereo-cal.json')).toBe(file);
-    expect(getCalibrationFile('folder/stereo-cal.json')).toBe(file);
+    const cal = new File(['{}'], 'stereo-cal.json', { type: 'application/json' });
+    stashCalibrationFile('folder/stereo-cal.json', cal);
+    expect(getCalibrationFile('stereo-cal.json')).toBe(cal);
+    expect(getCalibrationFile('folder/stereo-cal.json')).toBe(cal);
   });
 
   it('does not restore last calibration from localStorage without a session File', async () => {
@@ -43,9 +57,94 @@ describe('multicamFileRegistry calibration', () => {
   });
 
   it('restores last calibration when the File is still in the registry', async () => {
-    const file = new File(['{}'], 'cal.json', { type: 'application/json' });
-    stashCalibrationFile('cal.json', file);
+    const cal = new File(['{}'], 'cal.json', { type: 'application/json' });
+    stashCalibrationFile('cal.json', cal);
     await saveCalibration('cal.json');
     await expect(getLastCalibration()).resolves.toBe('cal.json');
   });
 });
+
+describe('multicam camera package construction', () => {
+  beforeEach(() => {
+    clearMulticamFileRegistry();
+  });
+
+  it('flattens folder files before validation so names match uploaded names', () => {
+    const folderFiles = [
+      fileWithPath('img001.png', 'cam1/img001.png'),
+      fileWithPath('img002.png', 'cam1/img002.png'),
+    ];
+    const { files: cameraFiles } = getCameraPackageFiles(folderFiles);
+    // The server validates and Girder uploads flat basenames, not folder paths.
+    expect(cameraFiles.map((f) => f.name)).toEqual(['img001.png', 'img002.png']);
+    expect(
+      cameraFiles.every((f) => !f.webkitRelativePath || f.webkitRelativePath === f.name),
+    ).toBe(true);
+  });
+
+  it('reports only media filenames for multicam pre-import validation', () => {
+    const files = [
+      fileWithPath('img001.png', 'cam1/img001.png'),
+      fileWithPath('nav.csv', 'cam1/nav.csv'),
+      fileWithPath('tracks.csv', 'cam1/tracks.csv'),
+    ];
+    expect(mediaFileNamesForImport(files, 'image-sequence')).toEqual(['img001.png']);
+  });
+
+  it('reports TIFF camera files as media so Begin Import stays enabled', () => {
+    // Subfolder discovery registers a camera folder of .tif images (the SealTK IR layout) and
+    // imports it as large-image; reporting zero files here disables Begin Import permanently.
+    const files = [
+      fileWithPath('ir_0001.tif', 'ir/ir_0001.tif'),
+      fileWithPath('ir_0002.TIFF', 'ir/ir_0002.TIFF'),
+      fileWithPath('scan.nitf', 'ir/scan.nitf'),
+      fileWithPath('notes.txt', 'ir/notes.txt'),
+    ];
+    expect(mediaFileNamesForImport(files, 'image-sequence')).toEqual([
+      'ir_0001.tif', 'ir_0002.TIFF', 'scan.nitf',
+    ]);
+  });
+
+  it('appends the explicit track file to the validation input', () => {
+    const folderFiles = [file('img001.png')];
+    const track = file('tracks.csv');
+    stashAnnotationFile('cam1/tracks.csv', track);
+    const { files: cameraFiles, replaced } = getCameraPackageFiles(folderFiles, 'cam1/tracks.csv');
+    expect(cameraFiles.map((f) => f.name)).toEqual(['img001.png', 'tracks.csv']);
+    expect(cameraFiles).toContain(track);
+    expect(replaced).toEqual([]);
+  });
+
+  it('deduplicates the explicit track file by name', () => {
+    const track = file('tracks.csv');
+    stashAnnotationFile('cam1/tracks.csv', track);
+    // The same File object is also part of the camera folder selection.
+    const folderFiles = [file('img001.png'), track];
+    const { files: cameraFiles, replaced } = getCameraPackageFiles(folderFiles, 'cam1/tracks.csv');
+    expect(cameraFiles.map((f) => f.name)).toEqual(['img001.png', 'tracks.csv']);
+    expect(cameraFiles.filter((f) => f.name === 'tracks.csv')).toHaveLength(1);
+    // Re-picking the folder's own file is not a file leaving the upload.
+    expect(replaced).toEqual([]);
+  });
+
+  it('uploads the explicitly picked track file, and reports the folder copy it displaced', () => {
+    const explicitTrack = file('tracks.csv', 'chosen elsewhere');
+    stashAnnotationFile('cam1/tracks.csv', explicitTrack);
+    const folderTrack = file('tracks.csv', 'the camera folder copy');
+    const folderFiles = [file('img001.png'), folderTrack];
+    const { files: cameraFiles, replaced } = getCameraPackageFiles(folderFiles, 'cam1/tracks.csv');
+    expect(cameraFiles.map((f) => f.name)).toEqual(['img001.png', 'tracks.csv']);
+    expect(cameraFiles).toContain(explicitTrack);
+    expect(cameraFiles).not.toContain(folderTrack);
+    expect(replaced).toEqual([folderTrack]);
+  });
+
+  it('keeps camera-folder annotations and config in the package, not just media', () => {
+    // tracks.csv and config.json are auto-detected in the camera folder, and both belong to
+    // the camera: the package is everything the server validates, not only the media.
+    const folderFiles = [file('img001.png'), file('tracks.csv'), file('config.json')];
+    expect(getCameraPackageFiles(folderFiles).files.map((f) => f.name)).toEqual([
+      'img001.png', 'tracks.csv', 'config.json',
+    ]);
+  });
+});

client/platform/web-girder/multicamFileRegistry.ts

@@ -2,6 +2,13 @@
 
 import { Location } from '@girder/components/src';
 import { parentDatasetId } from 'dive-common/compositeDatasetId';
+import {
+  ImageSequenceType,
+  VideoType,
+  fileVideoTypes,
+  largeImageFileExtensions,
+} from 'dive-common/constants';
+import { fileImageTypes } from 'dive-common/components/ImportMultiCamDialog/multicamSubfolderLayout';
 import { openFromDisk, GirderUploadManager } from './utils';
 
 const LAST_CALIBRATION_STORAGE_KEY = 'dive_web_last_calibration';
@@ -89,8 +96,26 @@ export function flattenUploadFiles(files: File[]): File[] {
   });
 }
 
-export function mediaFileNamesForImport(files: File[]): string[] {
-  return flattenUploadFiles(files).map((file) => file.name);
+function fileExtension(fileName: string): string {
+  return fileName.split('.').pop()?.toLowerCase() ?? '';
+}
+
+/**
+ * Extensions a camera folder can contribute as importable images: the set multicam subfolder
+ * discovery accepts, plus the large-image formats an all-TIFF camera folder imports as. A
+ * camera that discovery registered must never report zero media files here, or Begin Import
+ * is disabled with no way to correct it.
+ */
+const importableImageExtensions = [...fileImageTypes, ...largeImageFileExtensions];
+
+export function mediaFileNamesForImport(
+  files: File[],
+  mediaType: typeof ImageSequenceType | typeof VideoType = ImageSequenceType,
+): string[] {
+  const allowedExtensions = mediaType === VideoType ? fileVideoTypes : importableImageExtensions;
+  return files
+    .map((file) => file.name)
+    .filter((name) => allowedExtensions.includes(fileExtension(name)));
 }
 
 export function stashAnnotationFile(key: string, file: File): void {
@@ -101,6 +126,52 @@ export function getAnnotationFile(key: string): File | undefined {
   return annotationFilesByKey.get(key);
 }
 
+export interface CameraPackage {
+  /** Files to validate and upload with the camera. */
+  files: File[];
+  /**
+   * Camera-folder files left out only because an explicit pick claimed their name. They are
+   * a different file than the one the user chose, so the caller must report them.
+   */
+  replaced: File[];
+}
+
+/** Two selections of the same file on disk are distinct File objects; treat them as one. */
+function isSameSelection(a: File, b: File): boolean {
+  return a === b
+    || (a.name === b.name && a.size === b.size && a.lastModified === b.lastModified);
+}
+
+/**
+ * Assemble the complete file package for one multicam camera: the camera folder's files plus
+ * the annotation file explicitly chosen for that camera.
+ *
+ * An explicit pick always wins over a folder file of the same name — the folder copy is
+ * dropped, so the user uploads the file they chose and Girder never sees a duplicate name.
+ *
+ * A dropped folder copy that is not the picked file is a real file leaving the upload, so it
+ * is returned in `replaced` rather than vanishing.
+ *
+ * `flattenUploadFiles` is applied here, before validation, so the names sent to
+ * the server for validation match the names uploaded to Girder.
+ */
+export function getCameraPackageFiles(
+  folderFiles: File[],
+  annotationKey?: string,
+): CameraPackage {
+  const annotationFile = annotationKey ? getAnnotationFile(annotationKey) : undefined;
+  const explicitFiles = annotationFile ? [annotationFile] : [];
+  const explicitNames = new Set(explicitFiles.map((file) => file.name));
+  const folderPackage = flattenUploadFiles(
+    folderFiles.filter((file) => !explicitNames.has(file.name)),
+  );
+  return {
+    files: annotationFile ? [...folderPackage, annotationFile] : folderPackage,
+    replaced: folderFiles.filter((file) => explicitNames.has(file.name)
+      && !explicitFiles.some((pick) => isSameSelection(pick, file))),
+  };
+}
+
 function calibrationLookupKeys(key: string): string[] {
   const keys = new Set<string>();
   if (key) {

client/platform/web-girder/uploadSlots.spec.ts

@@ -0,0 +1,72 @@
+// eslint-disable-next-line import/no-extraneous-dependencies -- Vitest is only used in tests
+import { describe, expect, it } from 'vitest';
+
+import suggestUploadSlots from './uploadSlots';
+
+function file(name: string): File {
+  return new File([name], name, { type: 'application/octet-stream' });
+}
+
+function accountedNames(slots: ReturnType<typeof suggestUploadSlots>): string[] {
+  return [
+    ...slots.mediaList.map((f) => f.name),
+    ...(slots.annotationFile ? [slots.annotationFile.name] : []),
+    ...(slots.configFile ? [slots.configFile.name] : []),
+    ...slots.unslotted.map((entry) => entry.name),
+  ].sort();
+}
+
+describe('suggestUploadSlots', () => {
+  it('suggests a single annotation CSV and keeps it out of the media slot', () => {
+    const slots = suggestUploadSlots([file('img001.png'), file('tracks.csv')]);
+    expect(slots.mediaList.map((f) => f.name)).toEqual(['img001.png']);
+    expect(slots.annotationFile?.name).toBe('tracks.csv');
+  });
+
+  it('detects a config JSON alongside an annotation CSV', () => {
+    const slots = suggestUploadSlots([file('img001.png'), file('tracks.csv'), file('dataset.meta.json')]);
+    expect(slots.configFile?.name).toBe('dataset.meta.json');
+    expect(slots.annotationFile?.name).toBe('tracks.csv');
+    expect(slots.mediaList.map((f) => f.name)).toEqual(['img001.png']);
+  });
+
+  it('leaves a second CSV unslotted rather than in the media slot the server rejects', () => {
+    // A second CSV in the media slot makes server validation fail outright ("Can only upload a
+    // single CSV Annotation per import"), which blocks the whole selection. Report it instead.
+    const slots = suggestUploadSlots([file('img001.png'), file('tracks.csv'), file('nav_2024.csv')]);
+    expect(slots.mediaList.map((f) => f.name)).toEqual(['img001.png']);
+    expect(slots.annotationFile?.name).toBe('tracks.csv');
+    expect(slots.unslotted).toEqual([
+      { name: 'nav_2024.csv', reason: 'Only one annotation file can be uploaded per dataset' },
+    ]);
+  });
+
+  it('reports a second configuration JSON rather than sending both to validation', () => {
+    const slots = suggestUploadSlots([
+      file('img001.png'), file('a.meta.json'), file('b.config.json'),
+    ]);
+    expect(slots.configFile?.name).toBe('a.meta.json');
+    expect(slots.annotationFile).toBeNull();
+    expect(slots.mediaList.map((f) => f.name)).toEqual(['img001.png']);
+    expect(slots.unslotted).toEqual([
+      { name: 'b.config.json', reason: 'Only one configuration file can be uploaded per dataset' },
+    ]);
+  });
+
+  it('slots a lone YAML annotation instead of leaving it to look like media', () => {
+    const slots = suggestUploadSlots([file('video.mp4'), file('tracks.yml')]);
+    expect(slots.mediaList.map((f) => f.name)).toEqual(['video.mp4']);
+    expect(slots.annotationFile?.name).toBe('tracks.yml');
+  });
+
+  it('never silently drops any picked file (every input is slotted or reported)', () => {
+    const picked = [
+      file('img001.png'), file('img002.png'),
+      file('tracks.csv'), file('extra.csv'),
+      file('dataset.meta.json'), file('other.json'),
+      file('nav.unknown'),
+    ];
+    const slots = suggestUploadSlots(picked);
+    expect(accountedNames(slots)).toEqual(picked.map((f) => f.name).sort());
+  });
+});