Introduce ReferenceImageWithMedia decorator to simplify the code

src/trackers/image-tracker/image-tracker-utils.ts

@@ -29,7 +29,7 @@ import { SpeedyPoint2 } from 'speedy-vision/types/core/speedy-point';
 import { SpeedyVector2 } from 'speedy-vision/types/core/speedy-vector';
 import { SpeedyKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
 import { ImageTracker } from './image-tracker';
-import { ReferenceImage } from './reference-image';
+import { ReferenceImageWithMedia } from './reference-image';
 import { Utils } from '../../utils/utils';
 import { IllegalOperationError, IllegalArgumentError, NumericalError } from '../../utils/errors';
 import { NIS_SIZE, TRACK_GRID_GRANULARITY } from './settings';
@@ -166,12 +166,10 @@ export class ImageTrackerUtils
      * @param referenceImage
      * @returns a best-fit aspect ratio
      */
-    static bestFitAspectRatioNDC(imageTracker: ImageTracker, referenceImage: ReferenceImage): number
+    static bestFitAspectRatioNDC(imageTracker: ImageTracker, referenceImage: ReferenceImageWithMedia): number
     {
         const screenSize = imageTracker.screenSize;
         const screenAspectRatio = screenSize.width / screenSize.height;
-        const referenceImageMedia = imageTracker.database._findMedia(referenceImage.name);
-        const referenceImageAspectRatio = referenceImageMedia.size.width / referenceImageMedia.size.height;
 
         /*
         
@@ -194,7 +192,7 @@ export class ImageTrackerUtils
 
         */
 
-        return referenceImageAspectRatio / screenAspectRatio;
+        return referenceImage.aspectRatio / screenAspectRatio;
     }
 
     /**

src/trackers/image-tracker/image-tracker.ts

@@ -37,7 +37,7 @@ import { Tracker, TrackerOutput, TrackerResult, Trackable } from '../tracker';
 import { Session } from '../../core/session';
 import { IllegalOperationError, IllegalArgumentError } from '../../utils/errors';
 import { Resolution } from '../../utils/resolution';
-import { ReferenceImage } from './reference-image';
+import { ReferenceImage, ReferenceImageWithMedia } from './reference-image';
 import { ReferenceImageDatabase } from './reference-image-database';
 import { ImageTrackerState } from './states/state';
 import { ImageTrackerInitialState } from './states/initial';
@@ -320,7 +320,7 @@ export class ImageTracker extends AREventTarget<ImageTrackerEventType> implement
      * @returns reference image
      * @internal
      */
-    _referenceImageOfKeypoint(keypointIndex: number): Nullable<ReferenceImage>
+    _referenceImageOfKeypoint(keypointIndex: number): Nullable<ReferenceImageWithMedia>
     {
         const training = this._state.training as ImageTrackerTrainingState;
         return training.referenceImageOfKeypoint(keypointIndex);

src/trackers/image-tracker/reference-image-database.ts

@@ -23,7 +23,7 @@
 import Speedy from 'speedy-vision';
 import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
 import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
-import { ReferenceImage } from './reference-image';
+import { ReferenceImage, ReferenceImageWithMedia } from './reference-image';
 import { Utils } from '../../utils/utils';
 import { IllegalArgumentError, IllegalOperationError } from '../../utils/errors';
 
@@ -33,28 +33,13 @@ const DEFAULT_CAPACITY = 100; // this number should exceed normal usage
                               // further testing is needed to verify the appropriateness of this number;
                               // it depends on the images, on the keypoint descriptors, and even on the target devices
 
-/** Generate a unique name for a reference image */
-const generateUniqueName = () => 'target-' + Math.random().toString(16).substr(2);
-
-/**
- * An entry of a Reference Image Database
- */
-interface ReferenceImageDatabaseEntry
-{
-    /** reference image */
-    readonly referenceImage: ReferenceImage;
-
-    /** previously loaded media */
-    readonly media: SpeedyMedia;
-}
-
 /**
  * A collection of Reference Images
  */
 export class ReferenceImageDatabase implements Iterable<ReferenceImage>
 {
     /** Entries */
-    private _entries: Map<string, ReferenceImageDatabaseEntry>;
+    private _entries: Map<string, ReferenceImageWithMedia>;
 
     /** Maximum number of entries */
     private _capacity: number;
@@ -62,8 +47,6 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
     /** Is the database locked? */
     private _locked: boolean;
 
-    /** Are we busy loading an image? */
-    private _busy: boolean;
 
 
 
@@ -75,7 +58,6 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
         this._capacity = DEFAULT_CAPACITY;
         this._entries = new Map();
         this._locked = false;
-        this._busy = false;
     }
 
     /**
@@ -111,11 +93,9 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
     /**
      * Iterates over the collection
      */
-    *[Symbol.iterator](): Iterator<ReferenceImage>
+    [Symbol.iterator](): Iterator<ReferenceImageWithMedia>
     {
-        const arr = Array.from(this._entries.values());
-        const ref = arr.map(entry => entry.referenceImage);
-        yield* ref;
+        return this._entries.values();
     }
 
     /**
@@ -128,56 +108,40 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
      */
     add(referenceImages: ReferenceImage[]): SpeedyPromise<void>
     {
-        const n = referenceImages.length;
-
-        // handle no input
-        if(n == 0)
-            return Speedy.Promise.resolve();
-
-        // handle multiple images as input
-        if(n > 1) {
-            Utils.log(`Loading ${n} reference image${n != 1 ? 's' : ''}...`);
-            const preloadMedias = referenceImages.map(referenceImage => Speedy.load(referenceImage.image));
-            return Speedy.Promise.all(preloadMedias).then(() => {
-                const promises = referenceImages.map(referenceImage => this.add([ referenceImage ]));
-                return Utils.runInSequence(promises);
+        return this._preloadMany(referenceImages).then(referenceImagesWithMedia => {
+            referenceImagesWithMedia.forEach(referenceImageWithMedia => {
+                this._addOne(referenceImageWithMedia);
             });
-        }
+        });
+    }
 
-        // handle a single image as input
-        const referenceImage = referenceImages[0];
+    /**
+     * Add a single preloaded reference image to the database
+     * @param referenceImage
+     */
+    _addOne(referenceImage: ReferenceImageWithMedia): void
+    {
+        const name = referenceImage.name;
 
         // locked database?
         if(this._locked)
-            throw new IllegalOperationError(`Can't add reference image "${referenceImage.name}" to the database: it's locked`);
-
-        // busy loading another image?
-        if(this._busy)
-            return Utils.wait(4).then(() => this.add(referenceImages)); // try again later
+            throw new IllegalOperationError(`Can't add reference image "${name}" to the database: it's locked`);
 
         // reached full capacity?
         if(this.count >= this.capacity)
-            throw new IllegalOperationError(`Can't add reference image "${referenceImage.name}" to the database: the capacity of ${this.capacity} images has been exceeded.`);
+            throw new IllegalOperationError(`Can't add reference image "${name}" to the database: the capacity of ${this.capacity} images has been exceeded.`);
 
         // check if the image is valid
         if(!(referenceImage.image instanceof HTMLImageElement) && !(referenceImage.image instanceof HTMLCanvasElement) && !(referenceImage.image instanceof ImageBitmap))
-            throw new IllegalArgumentError(`Can't add reference image "${referenceImage.name}" to the database: invalid image`);
+            throw new IllegalArgumentError(`Can't add reference image "${name}" to the database: invalid image`);
 
         // check for duplicate names
-        if(this._entries.has(referenceImage.name))
-            throw new IllegalArgumentError(`Can't add reference image "${referenceImage.name}" to the database: found duplicated name`);
+        if(this._entries.has(name))
+            throw new IllegalArgumentError(`Can't add reference image "${name}" to the database: found duplicated name`);
 
         // add the reference image to the database
-        Utils.log(`Adding reference image "${referenceImage.name}" to the database...`);
-        this._busy = true;
-        return Speedy.load(referenceImage.image).then(media => {
-            const name = referenceImage.name || generateUniqueName();
-            this._busy = false;
-            this._entries.set(name, {
-                referenceImage: Object.freeze(Object.assign({ }, referenceImage, { name })),
-                media: media
-            });
-        });
+        Utils.log(`Adding reference image "${name}" to the database...`);
+        this._entries.set(name, referenceImage);
     }
 
     /**
@@ -186,25 +150,51 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
      */
     _lock(): void
     {
-        if(this._busy)
-            throw new IllegalOperationError(`Can't lock the reference image database: we're busy loading an image`);
-
         this._locked = true;
     }
 
     /**
-     * Get the media object associated to a reference image
-     * @param name reference image name
-     * @returns media
+     * Get reference image by name
+     * @param name
+     * @returns the reference image with the given name, or null if there isn't any
      * @internal
      */
-    _findMedia(name: string): SpeedyMedia
+    _find(name: string): ReferenceImageWithMedia | null
+    {
+        return this._entries.get(name) || null;
+    }
+
+    /**
+     * Load a reference image
+     * @param referenceImage
+     * @returns a promise that resolves to a corresponding ReferenceImageWithMedia
+     */
+    private _preloadOne(referenceImage: ReferenceImage): SpeedyPromise<ReferenceImageWithMedia>
     {
-        const entry = this._entries.get(name);
+        if(referenceImage.name !== undefined)
+            Utils.log(`Loading reference image \"${referenceImage.name}\"...`);
+        else
+            Utils.log(`Loading reference image...`);
+
+        if(!referenceImage.image)
+            return Speedy.Promise.reject(new IllegalArgumentError('The reference image was not provided!'));
 
-        if(!entry)
-            throw new IllegalArgumentError(`Can't find reference image "${name}"`);
+        return Speedy.load(referenceImage.image).then(media => {
+            return new ReferenceImageWithMedia(referenceImage, media);
+        });
+    }
+
+    /**
+     * Load multiple reference images
+     * @param referenceImages
+     * @returns a promise that resolves to corresponding ReferenceImageWithMedia objects
+     */
+    private _preloadMany(referenceImages: ReferenceImage[]): SpeedyPromise<ReferenceImageWithMedia[]>
+    {
+        const n = referenceImages.length;
+        Utils.log(`Loading ${n} reference image${n != 1 ? 's' : ''}...`);
 
-        return entry.media;
+        const promises = referenceImages.map(referenceImage => this._preloadOne(referenceImage));
+        return Speedy.Promise.all<ReferenceImageWithMedia>(promises);
     }
 }

src/trackers/image-tracker/reference-image.ts

@@ -20,14 +20,104 @@
  * Reference Image for tracking
  */
 
+import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
+
+type ReferenceImageType = HTMLImageElement | HTMLCanvasElement | ImageBitmap;
+
+
+
 /**
  * Reference Image for tracking
  */
 export interface ReferenceImage
 {
     /** Reference Images should have unique names given by the user */
-    readonly name: string;
+    name?: string;
 
     /** Image data */
-    readonly image: HTMLImageElement | HTMLCanvasElement | ImageBitmap;
+    readonly image: ReferenceImageType;
+}
+
+/**
+ * A ReferenceImage decorated with a SpeedyMedia
+ */
+export class ReferenceImageWithMedia implements ReferenceImage
+{
+    /** The decorated reference image */
+    private _referenceImage: ReferenceImage;
+
+    /** A SpeedyMedia corresponding to the reference image */
+    private readonly _media: SpeedyMedia;
+
+    /** The aspect ratio of the reference image */
+    private readonly _aspectRatio: number;
+
+
+
+    /**
+     * Constructor
+     * @param referenceImage
+     * @param media
+     */
+    constructor(referenceImage: ReferenceImage, media: SpeedyMedia)
+    {
+        this._referenceImage = Object.assign({}, referenceImage);
+        this._media = media;
+
+        // generate a unique name if none is given
+        if(this._referenceImage.name === undefined)
+            this._referenceImage.name = this._generateUniqueName();
+
+        // store the aspect ratio
+        this._aspectRatio = media.width / media.height;
+    }
+
+    /**
+     * Getter of the name of the reference image
+     */
+    get name(): string
+    {
+        return this._referenceImage.name!;
+    }
+
+    /**
+     * Setter of the name of the reference image
+     */
+    set name(name: string)
+    {
+        this._referenceImage.name = name;
+    }
+
+    /**
+     * The image data
+     */
+    get image(): ReferenceImageType
+    {
+        return this._referenceImage.image;
+    }
+
+    /**
+     * A SpeedyMedia corresponding to the reference media
+     */
+    get media(): SpeedyMedia
+    {
+        return this._media;
+    }
+
+    /**
+     * The aspect ratio of the reference image
+     */
+    get aspectRatio(): number
+    {
+        return this._aspectRatio;
+    }
+
+    /**
+     * Generate a unique name for a reference image
+     * @returns a unique name
+     */
+    private _generateUniqueName(): string
+    {
+        return 'target-' + Math.random().toString(16).substr(2);
+    }
 }

src/trackers/image-tracker/states/pre-tracking-a.ts

@@ -38,7 +38,7 @@ import { SpeedyKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/
 import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
 import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
 import { ImageTrackerState, ImageTrackerStateOutput } from './state';
-import { ReferenceImage } from '../reference-image';
+import { ReferenceImage, ReferenceImageWithMedia } from '../reference-image';
 import { Nullable, Utils } from '../../../utils/utils';
 import { TrackingError } from '../../../utils/errors';
 import {
@@ -62,7 +62,7 @@ import {
 export class ImageTrackerPreTrackingAState extends ImageTrackerState
 {
     /** reference image */
-    private _referenceImage: Nullable<ReferenceImage>;
+    private _referenceImage: Nullable<ReferenceImageWithMedia>;
 
     /** a snapshot of the video from the scanning state and corresponding to the initial homography */
     private _snapshot: Nullable<SpeedyPipelineNodeImagePortalSink>;
@@ -92,7 +92,7 @@ export class ImageTrackerPreTrackingAState extends ImageTrackerState
     onEnterState(settings: Record<string,any>)
     {
         const homography = settings.homography as SpeedyMatrix;
-        const referenceImage = settings.referenceImage as ReferenceImage;
+        const referenceImage = settings.referenceImage as ReferenceImageWithMedia;
         const snapshot = settings.snapshot as SpeedyPipelineNodeImagePortalSink;
 
         // set attributes
@@ -114,7 +114,7 @@ export class ImageTrackerPreTrackingAState extends ImageTrackerState
         const borderClipper = this._pipeline.node('borderClipper') as SpeedyPipelineNodeKeypointBorderClipper;
 
         // set the reference image as the source image
-        source.media = this._imageTracker.database._findMedia(this._referenceImage!.name);
+        source.media = this._referenceImage!.media;
 
         // clip keypoints from the borders of the target image
         borderClipper.imageSize = screenSize;

src/trackers/image-tracker/states/tracking.ts

@@ -44,7 +44,7 @@ import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-ut
 import { ImageTrackerEvent } from '../image-tracker-event';
 import { ImageTrackerState, ImageTrackerStateOutput } from './state';
 import { Nullable, Utils } from '../../../utils/utils';
-import { ReferenceImage } from '../reference-image';
+import { ReferenceImage, ReferenceImageWithMedia } from '../reference-image';
 import { CameraModel } from '../../../geometry/camera-model';
 import { Viewer } from '../../../geometry/viewer';
 import { Pose } from '../../../geometry/pose';
@@ -77,7 +77,7 @@ const NUMBER_OF_PBOS = 2;
 export class ImageTrackerTrackingState extends ImageTrackerState
 {
     /** tracked image */
-    private _referenceImage: Nullable<ReferenceImage>;
+    private _referenceImage: Nullable<ReferenceImageWithMedia>;
 
     /** current homography (for warping) */
     private _warpHomography: SpeedyMatrix;
@@ -139,7 +139,7 @@ export class ImageTrackerTrackingState extends ImageTrackerState
     onEnterState(settings: Record<string,any>)
     {
         const homography = settings.homography as SpeedyMatrix; // NDC, from reference image to video
-        const referenceImage = settings.referenceImage as Nullable<ReferenceImage>;
+        const referenceImage = settings.referenceImage as Nullable<ReferenceImageWithMedia>;
         const templateKeypoints = settings.templateKeypoints as SpeedyKeypoint[];
         const templateKeypointPortalSink = settings.templateKeypointPortalSink as SpeedyPipelineNodeKeypointPortalSink;
         const initialScreenSize = settings.initialScreenSize as SpeedySize; // this.screenSize is not yet set
@@ -325,10 +325,8 @@ export class ImageTrackerTrackingState extends ImageTrackerState
             // We transform the keypoints of the reference image to NDC as a
             // convenience. However, doing so distorts the aspect ratio. Here
             // we undo the distortion.
-            const referenceImageMedia = this._imageTracker.database._findMedia(this._referenceImage!.name);
-            const referenceImageAspectRatio = referenceImageMedia.size.width / referenceImageMedia.size.height;
-            //const scale = ImageTrackerUtils.inverseBestFitScaleNDC(referenceImageAspectRatio); // not preferred; extrapolates the bounds of NDC
-            const scale = ImageTrackerUtils.bestFitScaleNDC(1 / referenceImageAspectRatio); // preferred
+            //const scale = ImageTrackerUtils.inverseBestFitScaleNDC(referenceImage.aspectRatio); // not preferred; extrapolates the bounds of NDC
+            const scale = ImageTrackerUtils.bestFitScaleNDC(1 / referenceImage.aspectRatio); // preferred
             const homography = Speedy.Matrix(this._poseHomography.times(scale));
             //this._poseHomography = homography; // visualize the polyline becoming a square
 

src/trackers/image-tracker/states/training.ts

@@ -33,8 +33,7 @@ import { Resolution } from '../../../utils/resolution';
 import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
 import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
 import { ImageTrackerState, ImageTrackerStateOutput } from './state';
-import { ReferenceImage } from '../reference-image';
-import { ReferenceImageDatabase } from '../reference-image-database';
+import { ReferenceImage, ReferenceImageWithMedia } from '../reference-image';
 import { Nullable, Utils } from '../../../utils/utils';
 import { IllegalOperationError, TrainingError } from '../../../utils/errors';
 import {
@@ -60,7 +59,7 @@ interface TrainingMap
     readonly referenceImageIndex: number[];
 
     /** reference images */
-    readonly referenceImages: ReferenceImage[];
+    readonly referenceImages: ReferenceImageWithMedia[];
 }
 
 
@@ -142,15 +141,13 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
 
         // set the appropriate training media
-        const database = this._imageTracker.database;
         const referenceImage = this._trainingMap.referenceImages[this._currentImageIndex];
-        const media = database._findMedia(referenceImage.name);
-        source.media = media;
+        source.media = referenceImage.media;
 
         // compute the appropriate size of the training image space
         const resolution = this._imageTracker.resolution;
         const scale = TRAIN_IMAGE_SCALE; // ORB is not scale-invariant
-        const aspectRatioOfTrainingImage = media.width / media.height;
+        const aspectRatioOfTrainingImage = referenceImage.aspectRatio;
 
         screen.size = Utils.resolution(resolution, aspectRatioOfTrainingImage);
         screen.size.width = Math.round(screen.size.width * scale);
@@ -305,7 +302,7 @@ export class ImageTrackerTrainingState extends ImageTrackerState
      * @param keypointIndex -1 if not found
      * @returns reference image
      */
-    referenceImageOfKeypoint(keypointIndex: number): Nullable<ReferenceImage>
+    referenceImageOfKeypoint(keypointIndex: number): Nullable<ReferenceImageWithMedia>
     {
         const imageIndex = this.referenceImageIndexOfKeypoint(keypointIndex);
         if(imageIndex < 0)