Merge branch 'ndc'

demos/hello-aframe/index.html

@@ -86,7 +86,7 @@
             <ar-root reference-image="mage">
 
                 <!-- Switch from top view to front view -->
-                <a-entity rotation="-90 0 0" position="0 -0.5 0">
+                <a-entity rotation="-90 0 0" position="0 -0.8 0">
 
                     <!-- Light -->
                     <a-light type="ambient" intensity="1.5"></a-light>
@@ -109,7 +109,7 @@
 
             <!-- The cat -->
             <ar-root reference-image="cat">
-                <a-entity rotation="-90 0 0" position="0 -0.5 0">
+                <a-entity rotation="-90 0 0" position="0 -0.8 0">
                     <a-light type="ambient" intensity="1.5"></a-light>
 
                     <a-entity

demos/hello-aframe/video.html

@@ -86,7 +86,7 @@
             <ar-root reference-image="mage">
 
                 <!-- Switch from top view to front view -->
-                <a-entity rotation="-90 0 0" position="0 -0.5 0">
+                <a-entity rotation="-90 0 0" position="0 -0.8 0">
 
                     <!-- Light -->
                     <a-light type="ambient" intensity="1.5"></a-light>
@@ -109,7 +109,7 @@
 
             <!-- The cat -->
             <ar-root reference-image="cat">
-                <a-entity rotation="-90 0 0" position="0 -0.5 0">
+                <a-entity rotation="-90 0 0" position="0 -0.8 0">
                     <a-light type="ambient" intensity="1.5"></a-light>
 
                     <a-entity

demos/hello-babylon/demo.js

@@ -18,6 +18,7 @@ class EnchantedDemo extends ARDemo
         super();
 
         this._objects = { };
+        this._initialized = false;
     }
 
     /**
@@ -99,7 +100,7 @@ class EnchantedDemo extends ARDemo
         });
 
         // Change the point of view - slightly
-        ar.root.position.y = -0.5;
+        ar.root.position.y = -0.8;
 
         // Initialize objects
         this._initLight(ar);
@@ -110,6 +111,9 @@ class EnchantedDemo extends ARDemo
             this._initMage(ar),
             this._initCat(ar),
         ]);
+
+        // done!
+        this._initialized = true;
     }
 
     /**
@@ -130,7 +134,7 @@ class EnchantedDemo extends ARDemo
 
     _initLight(ar)
     {
-        const light = new BABYLON.HemisphericLight('light', BABYLON.Vector3.Down());
+        const light = new BABYLON.HemisphericLight('light', BABYLON.Vector3.Up());
         light.intensity = 1.0;
         light.diffuse.set(1, 1, 0.9);
         light.specular.set(0, 0, 0);
@@ -229,6 +233,12 @@ class EnchantedDemo extends ARDemo
 
     _onTargetFound(referenceImage)
     {
+        // make sure that the scene is initialized
+        if(!this._initialized) {
+            alert(`Target \"${referenceImage.name}\" was found, but the 3D scene is not yet initialized!`);
+            return;
+        }
+
         // change the scene based on the tracked image
         switch(referenceImage.name) {
             case 'mage':

demos/hello-three/demo.js

@@ -92,6 +92,7 @@ class EnchantedDemo extends ARDemo
         super();
 
         this._objects = { };
+        this._initialized = false;
     }
 
     /**
@@ -169,9 +170,9 @@ class EnchantedDemo extends ARDemo
         // ar.root, a node that is automatically aligned to the physical scene.
         // Adjusting ar.root will adjust all virtual objects.
         Utils.switchToFrontView(ar);
-        ar.root.position.set(0, -0.5, 0);
+        ar.root.position.set(0, -0.8, 0);
 
-        // initialize objects
+        // Initialize objects
         this._initLight(ar);
         this._initText(ar);
         this._initMagicCircle(ar);
@@ -180,6 +181,9 @@ class EnchantedDemo extends ARDemo
             this._initMage(ar),
             this._initCat(ar),
         ]);
+
+        // done!
+        this._initialized = true;
     }
 
     /**
@@ -298,6 +302,12 @@ class EnchantedDemo extends ARDemo
 
     _onTargetFound(referenceImage)
     {
+        // make sure that the scene is initialized
+        if(!this._initialized) {
+            alert(`Target \"${referenceImage.name}\" was found, but the 3D scene is not yet initialized!`);
+            return;
+        }
+
         // change the scene based on the tracked image
         switch(referenceImage.name) {
             case 'mage':

docs/api/reference-image.md

@@ -12,6 +12,8 @@ A name used to identify this reference image in a [database](reference-image-dat
 
 ### image
 
-`referenceImage.image: HTMLImageElement | HTMLCanvasElement | ImageBitmap, read-only`
+`referenceImage.image: HTMLImageElement | ImageBitmap | ImageData, read-only`
 
-Image template with pixel data.
+Image template with pixel data.
+
+*Note:* `ImageData` is acceptable since version 0.4.0.

src/core/session.ts

@@ -541,14 +541,14 @@ export class Session extends AREventTarget<SessionEventType>
         // render user media
         if(this._primarySource !== null) {
             const media = this._primarySource._internalMedia;
-            this._renderMedia(ctx, media);
+            this._renderMedia(ctx, media, true);
         }
 
-        // render output image(s)
+        // render output image(s) for debugging
         for(let i = 0; i < this._trackers.length; i++) {
             const media = this._trackers[i]._output.image;
             if(media !== undefined)
-                this._renderMedia(ctx, media);
+                this._renderMedia(ctx, media, false);
         }
 
         // render gizmos
@@ -559,18 +559,21 @@ export class Session extends AREventTarget<SessionEventType>
      * Render a SpeedyMedia
      * @param ctx rendering context
      * @param media
+     * @param stretch
      */
-    private _renderMedia(ctx: CanvasRenderingContext2D, media: SpeedyMedia): void
+    private _renderMedia(ctx: CanvasRenderingContext2D, media: SpeedyMedia, stretch: boolean): void
     {
         const canvas = ctx.canvas;
+        const width = stretch ? canvas.width : media.width;
+        const height = stretch ? canvas.height : media.height;
 
         if(media.type != 'data') {
             const image = media.source as Exclude<SpeedyMediaSourceNativeElement, ImageData>;
-            ctx.drawImage(image, 0, 0, canvas.width, canvas.height);
+            ctx.drawImage(image, 0, 0, width, height);
         }
         else {
             const image = media.source as ImageData;
-            ctx.putImageData(image, 0, 0, 0, 0, canvas.width, canvas.height);
+            ctx.putImageData(image, 0, 0, 0, 0, width, height);
         }
     }
 

src/geometry/camera-model.ts

@@ -29,13 +29,13 @@ import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
 import { Nullable, Utils } from '../utils/utils';
 import { Settings } from '../core/settings';
 import { PoseFilter } from './pose-filter';
-import { IllegalOperationError, IllegalArgumentError } from '../utils/errors';
+import { NumericalError } from '../utils/errors';
 
 /** A guess of the horizontal field-of-view of a typical camera, in degrees */
 const HFOV_GUESS = 60; // https://developer.apple.com/library/archive/documentation/DeviceInformation/Reference/iOSDeviceCompatibility/Cameras/Cameras.html
 
-/** Number of iterations used to refine the estimated pose */
-const POSE_ITERATIONS = 30;
+/** The default scale of the image plane. The scale affects the focal length */
+const DEFAULT_SCALE = 2; // the length of the [-1,+1] interval
 
 /** Convert degrees to radians */
 const DEG2RAD = 0.017453292519943295; // pi / 180
@@ -47,17 +47,28 @@ const RAD2DEG = 57.29577951308232; // 180 / pi
 const EPSILON = 1e-6;
 
 /** Index of the horizontal focal length in the camera intrinsics matrix (column-major format) */
-export const FX = 0;
+const FX = 0;
 
 /** Index of the vertical focal length in the camera intrinsics matrix */
-export const FY = 4;
+const FY = 4;
 
 /** Index of the horizontal position of the principal point in the camera intrinsics matrix */
-export const U0 = 6;
+const U0 = 6;
 
 /** Index of the vertical position of the principal point in the camera intrinsics matrix */
-export const V0 = 7;
+const V0 = 7;
+
+/** Number of iterations used to refine the estimated pose */
+const POSE_REFINEMENT_ITERATIONS = 30;
 
+/** Maximum number of iterations used when refining the translation vector */
+const TRANSLATION_REFINEMENT_ITERATIONS = 15;
+
+/** Tolerance used to exit early when refining the translation vector */
+const TRANSLATION_REFINEMENT_TOLERANCE = DEFAULT_SCALE * 0.01;
+
+/** Size of the grid used to refine the translation vector */
+const TRANSLATION_REFINEMENT_GRIDSIZE = 5; //3;
 
 
 
@@ -66,16 +77,19 @@ export const V0 = 7;
  */
 export class CameraModel
 {
-    /** size of the image sensor, in pixels */
-    private _screenSize: SpeedySize;
+    /** size of the image plane */
+    private _imageSize: SpeedySize;
 
     /** 3x4 camera matrix */
     private _matrix: SpeedyMatrix;
 
-    /** intrinsics matrix, in column-major format */
+    /** a helper to switch the handedness of a coordinate system */
+    private _flipZ: SpeedyMatrix;
+
+    /** entries of the intrinsics matrix in column-major format */
     private _intrinsics: number[];
 
-    /** extrinsics matrix, in column-major format */
+    /** entries of the extrinsics matrix in column-major format */
     private _extrinsics: number[];
 
     /** smoothing filter */
@@ -88,32 +102,43 @@ export class CameraModel
      */
     constructor()
     {
-        this._screenSize = Speedy.Size(0, 0);
+        this._imageSize = Speedy.Size(0, 0);
         this._matrix = Speedy.Matrix.Eye(3, 4);
         this._intrinsics = [1,0,0,0,1,0,0,0,1]; // 3x3 identity matrix
         this._extrinsics = [1,0,0,0,1,0,0,0,1,0,0,0]; // 3x4 matrix [ R | t ] = [ I | 0 ] no rotation & no translation
         this._filter = new PoseFilter();
+        this._flipZ = Speedy.Matrix(4, 4, [
+            1, 0, 0, 0,
+            0, 1, 0, 0,
+            0, 0,-1, 0,
+            0, 0, 0, 1
+        ]);
     }
 
     /**
      * Initialize the model
-     * @param screenSize
+     * @param aspectRatio aspect ratio of the image plane
+     * @param scale optional scale factor of the image plane
      */
-    init(screenSize: SpeedySize): void
+    init(aspectRatio: number, scale: number = DEFAULT_SCALE): void
     {
-        // validate
-        if(screenSize.area() == 0)
-            throw new IllegalArgumentError(`Can't initialize the camera model with screenSize = ${screenSize.toString()}`);
+        // log
+        Utils.log(`Initializing the camera model...`);
+        Utils.assert(aspectRatio > 0 && scale > 1e-5);
 
-        // set the screen size
-        this._screenSize.width = screenSize.width;
-        this._screenSize.height = screenSize.height;
+        // set the size of the image plane
+        // this rule is conceived so that min(w,h) = s and w/h = a
+        if(aspectRatio >= 1) {
+            this._imageSize.width = aspectRatio * scale;
+            this._imageSize.height = scale;
+        }
+        else {
+            this._imageSize.width = scale;
+            this._imageSize.height = scale / aspectRatio;
+        }
 
         // reset the model
         this.reset();
-
-        // log
-        Utils.log(`Initializing the camera model...`);
     }
 
     /**
@@ -127,31 +152,15 @@ export class CameraModel
 
     /**
      * Update the camera model
-     * @param homography 3x3 perspective transform
-     * @param screenSize may change over time (e.g., when going from portrait to landscape or vice-versa)
-     * @returns promise that resolves to a camera matrix
+     * @param homographyNDC 3x3 perspective transform
+     * @returns a promise that resolves to a camera matrix
      */
-    update(homography: SpeedyMatrix, screenSize: SpeedySize): SpeedyPromise<SpeedyMatrix>
+    update(homographyNDC: SpeedyMatrix): SpeedyPromise<SpeedyMatrix>
     {
-        // validate the shape of the homography
-        if(homography.rows != 3 || homography.columns != 3)
-            throw new IllegalArgumentError(`Camera model: provide a homography matrix`);
+        Utils.assert(homographyNDC.rows == 3 && homographyNDC.columns == 3);
 
-        // validate screenSize
-        if(screenSize.area() == 0)
-            throw new IllegalArgumentError(`Camera model: invalid screenSize = ${screenSize.toString()}`);
-
-        // changed screen size?
-        if(!this._screenSize.equals(screenSize)) {
-            Utils.log(`Camera model: detected a change in screen size...`);
-
-            // update the screen size
-            this._screenSize.width = screenSize.width;
-            this._screenSize.height = screenSize.height;
-
-            // reset camera
-            this.reset();
-        }
+        // convert to image space
+        const homography = this._convertToImageSpace(homographyNDC);
 
         // read the entries of the homography
         const h = homography.read();
@@ -161,10 +170,8 @@ export class CameraModel
 
         // validate the homography (homography matrices aren't singular)
         const det = h13 * (h21 * h32 - h22 * h31) - h23 * (h11 * h32 - h12 * h31) + h33 * (h11 * h22 - h12 * h21);
-        if(Math.abs(det) < EPSILON) {
-            Utils.warning(`Can't update the camera model using an invalid homography matrix`);
-            return Speedy.Promise.resolve(this._matrix);
-        }
+        if(Math.abs(det) < EPSILON || Number.isNaN(det))
+            return Speedy.Promise.reject(new NumericalError(`Can't update the camera model using an invalid homography matrix`));
 
         // estimate the pose
         const pose = this._estimatePose(homography);
@@ -172,12 +179,22 @@ export class CameraModel
             this._extrinsics = this._filter.output().read();
 
         // compute the camera matrix
-        const C = this.denormalizer();
+        const Z = this._flipZ; // switch to a right handed system
         const K = Speedy.Matrix(3, 3, this._intrinsics);
         const E = Speedy.Matrix(3, 4, this._extrinsics);
-        this._matrix.setToSync(K.times(E).times(C));
-        //console.log("intrinsics -----------", K.toString());
-        //console.log("matrix ----------------",this._matrix.toString());
+        this._matrix.setToSync(K.times(E).times(Z));
+
+        /*
+        // test
+        console.log("homography ------------", homography.toString());
+        console.log("intrinsics ------------", K.toString());
+        console.log("extrinsics ------------", E.toString());
+        console.log("extrinsicsINV ---------", Speedy.Matrix(this.computeViewMatrix().inverse()).toString());
+        console.log("matrix ----------------", this._matrix.toString());
+        console.log("projectionMatrix ----- ", this.computeProjectionMatrix(0.1,100).toString());
+        */
+
+        // done!
         return Speedy.Promise.resolve(this._matrix);
     }
 
@@ -191,9 +208,7 @@ export class CameraModel
     }
 
     /**
-     * The camera matrix that maps the 3D normalized space [-1,1]^3 to the
-     * 2D AR screen space (measured in pixels)
-     * @returns 3x4 camera matrix
+     * The 3x4 camera matrix
      */
     get matrix(): SpeedyMatrix
     {
@@ -201,106 +216,99 @@ export class CameraModel
     }
 
     /**
-     * Camera intrinsics matrix
-     * @returns 3x3 intrinsics matrix in column-major format
+     * The size of the image plane
      */
-    get intrinsics(): number[]
+    get imageSize(): SpeedySize
     {
-        return this._intrinsics;
+        return this._imageSize;
     }
 
     /**
-     * Camera extrinsics matrix
-     * @returns 3x4 extrinsics matrix [ R | t ] in column-major format
+     * The aspect ratio of the image
      */
-    get extrinsics(): number[]
+    get aspectRatio(): number
     {
-        return this._extrinsics;
+        return this._imageSize.width / this._imageSize.height;
     }
 
     /**
-     * Convert coordinates from normalized space [-1,1]^3 to a
-     * "3D pixel space" based on the dimensions of the AR screen.
-     *
-     * We perform a 180-degrees rotation around the x-axis so that
-     * it looks nicer (the y-axis grows downwards in image space).
-     *
-     * The final camera matrix is P = K * [ R | t ] * C, where
-     * C is this conversion matrix. The intent behind this is to
-     * make tracking independent of target and screen sizes.
-     *
-     * Reminder: we use a right-handed coordinate system in 3D!
-     * In 2D image space the coordinate system is left-handed.
-     *
-     * @returns 4x4 conversion matrix C
+     * Focal length in "pixels" (projection distance in the pinhole camera model)
+     * same as (focal length in mm) * (number of "pixels" per world unit in "pixels"/mm)
+     * "pixels" means image plane units
      */
-    denormalizer(): SpeedyMatrix
+    get focalLength(): number
     {
-        const w = this._screenSize.width / 2; // half width, in pixels
-        const h = this._screenSize.height / 2; // half height, in pixels
-        const d = Math.min(w, h); // virtual unit length, in pixels
-
-        /*
-        return Speedy.Matrix(4, 4, [
-            1, 0, 0, 0,
-            0,-1, 0, 0,
-            0, 0,-1, 0,
-            w/d, h/d, 0, 1/d
-        ]);
-        */
-
-        return Speedy.Matrix(4, 4, [
-            d, 0, 0, 0,
-            0,-d, 0, 0,
-            0, 0,-d, 0,
-            w, h, 0, 1,
-        ]);
+        return this._intrinsics[FX]; // fx == fy
     }
 
     /**
-     * Size of the AR screen space, in pixels
-     * @returns size in pixels
+     * Horizontal field-of-view, given in radians
      */
-    get screenSize(): SpeedySize
+    get fovx(): number
     {
-        return this._screenSize;
+        const halfWidth = this._imageSize.width / 2;
+        return 2 * Math.atan(halfWidth / this._intrinsics[FX]);
     }
 
     /**
-     * Focal length in pixel units (projection distance in the pinhole camera model)
-     * same as (focal length in mm) * (number of pixels per world unit in pixels/mm)
-     * @returns focal length
+     * Vertical field-of-view, given in radians
      */
-    get focalLength(): number
+    get fovy(): number
     {
-        return this._intrinsics[FY]; // fx == fy
+        const halfHeight = this._imageSize.height / 2;
+        return 2 * Math.atan(halfHeight / this._intrinsics[FY]);
     }
 
     /**
-     * Horizontal field-of-view, given in radians
-     * @returns vertical field-of-view
+     * Compute the view matrix. This 4x4 matrix moves 3D points from
+     * world space to view space. We want the camera looking in the
+     * direction of the negative z-axis (WebGL-friendly)
+     * @returns a view matrix
      */
-    get fovx(): number
+    computeViewMatrix(): SpeedyMatrix
     {
-        return 2 * Math.atan(this._intrinsics[U0] / this._intrinsics[FX]);
-    }
+        const E = this._extrinsics;
 
-    /**
-     * Vertical field-of-view, given in radians
-     * @returns vertical field-of-view
-     */
-    get fovy(): number
-    {
-        return 2 * Math.atan(this._intrinsics[V0] / this._intrinsics[FY]);
+        // We augment the 3x4 extrinsics matrix E with the [ 0  0  0  1 ] row
+        // and get E+. Let Z be 4x4 flipZ, the identity matrix with the third
+        // column negated. The following matrix is View = Z * E+ * Z. We get
+        // the camera looking in the direction of the negative z-axis in a
+        // right handed system!
+        return Speedy.Matrix(4, 4, [
+            E[0], E[1],-E[2], 0, // r1
+            E[3], E[4],-E[5], 0, // r2
+           -E[6],-E[7],+E[8], 0, // r3
+            E[9], E[10],-E[11], 1 // t
+        ]);
     }
 
     /**
-     * Principal point
-     * @returns principal point, in pixel coordinates
+     * Compute a perspective projection matrix for WebGL
+     * @param near distance of the near plane
+     * @param far distance of the far plane
      */
-    principalPoint(): SpeedyPoint2
+    computeProjectionMatrix(near: number, far: number): SpeedyMatrix
     {
-        return Speedy.Point2(this._intrinsics[U0], this._intrinsics[V0]);
+        const fx = this._intrinsics[FX];
+        const fy = this._intrinsics[FY];
+        const halfWidth = this._imageSize.width / 2;
+        const halfHeight = this._imageSize.height / 2;
+
+        // we assume that the principal point is at the center of the image plane
+        const right = near * (halfWidth / fx);
+        const top = near * (halfHeight / fy);
+        //const top = right * (halfHeight / halfWidth); // same thing
+        const bottom = -top, left = -right; // symmetric frustum
+
+        // a derivation of this projection matrix can be found at
+        // https://www.songho.ca/opengl/gl_projectionmatrix.html
+        // http://learnwebgl.brown37.net/08_projections/projections_perspective.html
+        return Speedy.Matrix(4, 4, [
+            2 * near / (right - left), 0, 0, 0,
+            0, 2 * near / (top - bottom), 0, 0,
+            (right + left) / (right - left), (top + bottom) / (top - bottom), -(far + near) / (far - near), -1,
+            0, 0, -2 * far * near / (far - near), 0
+        ]);
     }
 
     /**
@@ -321,10 +329,10 @@ export class CameraModel
      */
     private _resetIntrinsics(): void
     {
-        const cameraWidth = Math.max(this._screenSize.width, this._screenSize.height); // portrait or landscape?
+        const cameraWidth = Math.max(this._imageSize.width, this._imageSize.height); // portrait or landscape?
 
-        const u0 = this._screenSize.width / 2;
-        const v0 = this._screenSize.height / 2;
+        const u0 = 0; // principal point at the center of the image plane
+        const v0 = 0;
         const fx = (cameraWidth / 2) / Math.tan(DEG2RAD * HFOV_GUESS / 2);
         const fy = fx;
 
@@ -335,6 +343,34 @@ export class CameraModel
     }
 
     /**
+     * Convert a homography from NDC to image space
+     * @param homographyNDC
+     * @returns a new homography
+     */
+    private _convertToImageSpace(homographyNDC: SpeedyMatrix): SpeedyMatrix
+    {
+        const w = this._imageSize.width / 2;
+        const h = this._imageSize.height / 2;
+
+        // fromNDC converts points from NDC to image space
+        const fromNDC = Speedy.Matrix(3, 3, [
+            w, 0, 0,
+            0, h, 0,
+            0, 0, 1
+        ]);
+
+        /*
+        // make h33 = 1 (wanted?)
+        const data = homographyNDC.read();
+        const h33 = data[8];
+        const hom = homographyNDC.times(1/h33);
+        */
+
+        // convert homography
+        return Speedy.Matrix(fromNDC.times(homographyNDC));
+    }
+
+    /**
      * Compute a normalized homography H^ = K^(-1) * H for an
      * ideal pinhole with f = 1 and principal point = (0,0)
      * @param homography homography H to be normalized
@@ -342,7 +378,6 @@ export class CameraModel
      */
     private _normalizeHomography(homography: SpeedyMatrix): SpeedyMatrix
     {
-        const h = homography.read();
         const u0 = this._intrinsics[U0];
         const v0 = this._intrinsics[V0];
         const fx = this._intrinsics[FX];
@@ -350,6 +385,7 @@ export class CameraModel
         const u0fx = u0 / fx;
         const v0fy = v0 / fy;
 
+        const h = homography.read();
         const h11 = h[0] / fx - u0fx * h[2], h12 = h[3] / fx - u0fx * h[5], h13 = h[6] / fx - u0fx * h[8];
         const h21 = h[1] / fy - v0fy * h[2], h22 = h[4] / fy - v0fy * h[5], h23 = h[7] / fy - v0fy * h[8];
         const h31 = h[2], h32 = h[5], h33 = h[8];
@@ -401,10 +437,10 @@ export class CameraModel
 
         // sanity check
         if(Number.isNaN(scale))
-            return Speedy.Matrix(3, 3, (new Array(9)).fill(Number.NaN));
+            return Speedy.Matrix(3, 3, (new Array<number>(9)).fill(Number.NaN));
 
         // recover the rotation
-        let r = new Array(6) as number[];
+        let r = new Array<number>(6);
         r[0] = scale * h11;
         r[1] = scale * h21;
         r[2] = scale * h31;
@@ -412,8 +448,8 @@ export class CameraModel
         r[4] = scale * h22;
         r[5] = scale * h32;
 
-        // refine the rotation
-        r = this._refineRotation(r); // r is initially noisy
+        // refine the rotation (r is initially noisy)
+        r = this._refineRotation(r);
 
         /*
 
@@ -438,7 +474,7 @@ export class CameraModel
         scale /= h1norm2 + h2norm2;
 
         // recover the translation
-        let t = new Array(3) as number[];
+        let t = new Array<number>(3);
         t[0] = scale * h13;
         t[1] = scale * h23;
         t[2] = scale * h33;
@@ -539,6 +575,8 @@ export class CameraModel
         // compute the Cholesky decomposition LL' of the diagonal matrix D
         // whose entries are the two eigenvalues of R'R and then invert L
         const s1 = Math.sqrt(eigval1), s2 = Math.sqrt(eigval2); // singular values of R (pick s1 >= s2)
+
+        /*
         const Linv = Speedy.Matrix(2, 2, [1/s1, 0, 0, 1/s2]); // L inverse
 
         // compute the correction matrix C = Q * Linv * Q', where Q = [q1|q2]
@@ -550,6 +588,25 @@ export class CameraModel
         // correct the rotation vectors r1 and r2 using C
         const R = Speedy.Matrix(3, 2, [r11, r21, r31, r12, r22, r32]);
         return Speedy.Matrix(R.times(C)).read();
+        */
+
+        // find C = Q * Linv * Q' manually
+        // [ a  b ] is symmetric
+        // [ b  c ]
+        const a = x1*x1/s1 + x2*x2/s2;
+        const b = x1*y1/s1 + x2*y2/s2;
+        const c = y1*y1/s1 + y2*y2/s2;
+
+        // find RC manually
+        return [
+            a*r11 + b*r12,
+            a*r21 + b*r22,
+            a*r31 + b*r32,
+
+            b*r11 + c*r12,
+            b*r21 + c*r22,
+            b*r31 + c*r32
+        ];
     }
 
     /**
@@ -587,25 +644,33 @@ export class CameraModel
         const r21 = rot[1], r22 = rot[4];
         const r31 = rot[2], r32 = rot[5];
 
-        // sample points [ xi  yi ]' in AR screen space
-        //const x = [ 0.5, 0.0, 1.0, 1.0, 0.0, 0.5, 1.0, 0.5, 0.0 ];
-        //const y = [ 0.5, 0.0, 0.0, 1.0, 1.0, 0.0, 0.5, 1.0, 0.5 ];
-        const x = [ 0.5, 0.0, 1.0, 1.0, 0.0 ];
-        const y = [ 0.5, 0.0, 0.0, 1.0, 1.0 ];
-        const n = x.length;
-        const n3 = 3*n;
-
-        const width = this._screenSize.width;
-        const height = this._screenSize.height;
-        for(let i = 0; i < n; i++) {
-            x[i] *= width;
-            y[i] *= height;
+        // generate a grid of sample points [ xi  yi ]' in the image
+        //const x = [ 0, -1, +1, +1, -1 ];
+        //const y = [ 0, -1, -1, +1, +1 ];
+        const g = TRANSLATION_REFINEMENT_GRIDSIZE;
+        const x = new Array<number>(g*g);
+        const y = new Array<number>(g*g);
+        const halfWidth = this._imageSize.width / 2;
+        const halfHeight = this._imageSize.height / 2;
+
+        for(let k = 0, i = 0; i < g; i++) {
+            for(let j = 0; j < g; j++, k++) {
+                // in [-1,+1]
+                x[k] = (i/(g-1)) * 2 - 1;
+                y[k] = (j/(g-1)) * 2 - 1;
+
+                // in [-s/2,+s/2], where s = w,h
+                x[k] *= halfWidth;
+                y[k] *= halfHeight;
+            }
         }
+        //console.log(x.toString(), y.toString());
 
         // set auxiliary values: ai = H [ xi  yi  1 ]'
-        const a1 = new Array(n) as number[];
-        const a2 = new Array(n) as number[];
-        const a3 = new Array(n) as number[];
+        const n = x.length;
+        const a1 = new Array<number>(n);
+        const a2 = new Array<number>(n);
+        const a3 = new Array<number>(n);
         for(let i = 0; i < n; i++) {
             a1[i] = x[i] * h11 + y[i] * h12 + h13;
             a2[i] = x[i] * h21 + y[i] * h22 + h23;
@@ -614,8 +679,9 @@ export class CameraModel
 
         // we'll solve M t = v for t with linear least squares
         // M: 3n x 3, v: 3n x 1, t: 3 x 1
-        const m = new Array(3*n * 3) as number[];
-        const v = new Array(3*n) as number[];
+        const n3 = 3*n;
+        const m = new Array<number>(n3 * 3);
+        const v = new Array<number>(n3);
         for(let i = 0, k = 0; k < n; i += 3, k++) {
             m[i] = m[i+n3+1] = m[i+n3+n3+2] = 0;
             m[i+n3] = -(m[i+1] = a3[k]);
@@ -676,20 +742,18 @@ export class CameraModel
         */
 
         // gradient descent: super lightweight implementation
-        const r = new Array(3*n) as number[];
-        const c = new Array(3) as number[];
-        const Mc = new Array(3*n) as number[];
+        const r = new Array<number>(3*n);
+        const c = new Array<number>(3);
+        const Mc = new Array<number>(3*n);
 
         // initial guess
-        const t = new Array(3) as number[];
+        const t = new Array<number>(3);
         t[0] = t0[0];
         t[1] = t0[1];
         t[2] = t0[2];
 
         // iterate
-        const MAX_ITERATIONS = 15;
-        const TOLERANCE = 1;
-        for(let it = 0; it < MAX_ITERATIONS; it++) {
+        for(let it = 0; it < TRANSLATION_REFINEMENT_ITERATIONS; it++) {
             //console.log("it",it+1);
 
             // compute residual r = Mt - v
@@ -718,8 +782,8 @@ export class CameraModel
             let num = 0;
             for(let i = 0; i < 3; i++)
                 num += c[i] * c[i];
-            //console.log("c'c=",num);
-            if(num < TOLERANCE)
+            //console.log("c'c=",num," at #",it+1);
+            if(num < TRANSLATION_REFINEMENT_TOLERANCE)
                 break;
 
             // compute (Mc)'(Mc)
@@ -778,7 +842,7 @@ export class CameraModel
     }
 
     /**
-     * Estimate the pose [ R | t ] given a homography in AR screen space
+     * Estimate the pose [ R | t ] given a homography in sensor space
      * @param homography must be valid
      * @returns 3x4 matrix
      */
@@ -793,7 +857,7 @@ export class CameraModel
         // it won't be a perfect equality due to noise in the homography.
         // remark: composition of homographies
         const residual = Speedy.Matrix(normalizedHomography);
-        for(let k = 0; k < POSE_ITERATIONS; k++) {
+        for(let k = 0; k < POSE_REFINEMENT_ITERATIONS; k++) {
             // incrementally improve the partial pose
             const rt = this._estimatePartialPose(residual); // rt should converge to the identity matrix
             partialPose.setToSync(rt.times(partialPose));

src/geometry/view.ts

@@ -23,16 +23,15 @@
 
 import Speedy from 'speedy-vision';
 import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix';
-import { CameraModel, FX, FY, U0, V0 } from './camera-model';
+import { CameraModel } from './camera-model';
 import { IllegalArgumentError } from '../utils/errors';
 import { Nullable } from '../utils/utils';
 
-/** Default distance in pixels of the near plane to the optical center of the camera */
-const DEFAULT_NEAR = 1;
-
-/** Default distance in pixels of the far plane to the optical center of the camera */
-const DEFAULT_FAR = 20000;
+/** Default distance of the near plane to the optical center of the camera */
+const DEFAULT_NEAR = 0.1;
 
+/** Default distance of the far plane to the optical center of the camera */
+const DEFAULT_FAR = 10000 * DEFAULT_NEAR;
 
 
 /**
@@ -48,33 +47,27 @@ export interface View
     readonly _projectionMatrixInverse: SpeedyMatrix;
 }
 
+
 /**
  * A PerspectiveView is a View defining a symmetric frustum around the z-axis
  * (perspective projection)
  */
 export class PerspectiveView implements View
 {
-    /** A 4x4 matrix that projects the viewer space into the clip space, i.e., [-1,1]^3 */
-    private readonly _projectionMatrix: SpeedyMatrix;
-
-    /** The inverse of the projection matrix, computed lazily */
-    private _inverseProjection: Nullable<SpeedyMatrix>;
-
-    /** Tangent of the half of the horizontal field-of-view */
-    private readonly _tanOfHalfFovx: number;
-
-    /** Tangent of the half of the vertical field-of-view */
-    private readonly _tanOfHalfFovy: number;
+    /** Camera model */
+    private readonly _camera: CameraModel;
 
-    /** Aspect ratio of the frustum */
-    private readonly _aspect: number;
-
-    /** Distance of the near plane to the Z = 0 plane in viewer space */
+    /** Distance of the near plane to the optical center of the camera */
     private readonly _near: number;
 
-    /** Distance of the far plane to the Z = 0 plane in viewer space */
+    /** Distance of the far plane to the optical center of the camera*/
     private readonly _far: number;
 
+    /** A 4x4 matrix that projects viewer space into clip space, i.e., [-1,1]^3 */
+    private readonly _projectionMatrix: SpeedyMatrix;
+
+    /** The inverse of the projection matrix, computed lazily */
+    private _inverseProjection: Nullable<SpeedyMatrix>;
 
 
 
@@ -86,19 +79,16 @@ export class PerspectiveView implements View
      */
     constructor(camera: CameraModel, near: number = DEFAULT_NEAR, far: number = DEFAULT_FAR)
     {
-        const intrinsics = camera.intrinsics;
-        const screenSize = camera.screenSize;
-
-        this._near = Math.max(0, +near);
-        this._far = Math.max(0, +far);
+        this._near = +near;
+        this._far = +far;
 
         if(this._near >= this._far)
             throw new IllegalArgumentError(`View expects near < far (found near = ${this._near} and far = ${this._far})`);
+        else if(this._near <= 0)
+            throw new IllegalArgumentError(`View expects a positive near (found ${this._near})`);
 
-        this._aspect = screenSize.width / screenSize.height;
-        this._tanOfHalfFovx = intrinsics[U0] / intrinsics[FX];
-        this._tanOfHalfFovy = intrinsics[V0] / intrinsics[FY];
-        this._projectionMatrix = PerspectiveView._computeProjectionMatrix(intrinsics, this._near, this._far);
+        this._camera = camera;
+        this._projectionMatrix = camera.computeProjectionMatrix(this._near, this._far);
         this._inverseProjection = null;
     }
 
@@ -111,11 +101,23 @@ export class PerspectiveView implements View
     }
 
     /**
+     * The inverse of the projection matrix
+     * @internal
+     */
+    get _projectionMatrixInverse(): SpeedyMatrix
+    {
+        if(this._inverseProjection === null)
+            this._inverseProjection = Speedy.Matrix(this._projectionMatrix.inverse());
+
+        return this._inverseProjection;
+    }
+
+    /**
      * Aspect ratio of the frustum
      */
     get aspect(): number
     {
-        return this._aspect;
+        return this._camera.aspectRatio;
     }
 
     /**
@@ -123,7 +125,7 @@ export class PerspectiveView implements View
      */
     get fovx(): number
     {
-        return 2 * Math.atan(this._tanOfHalfFovx);
+        return this._camera.fovx;
     }
 
     /**
@@ -131,7 +133,7 @@ export class PerspectiveView implements View
      */
     get fovy(): number
     {
-        return 2 * Math.atan(this._tanOfHalfFovy);
+        return this._camera.fovy;
     }
 
     /**
@@ -149,40 +151,4 @@ export class PerspectiveView implements View
     {
         return this._far;
     }
-
-    /**
-     * The inverse of the projection matrix
-     * @internal
-     */
-    get _projectionMatrixInverse(): SpeedyMatrix
-    {
-        if(this._inverseProjection === null)
-            this._inverseProjection = Speedy.Matrix(this._projectionMatrix.inverse());
-
-        return this._inverseProjection;
-    }
-
-    /**
-     * Compute a perspective projection matrix for WebGL
-     * @param K camera intrinsics
-     * @param near distance of the near plane
-     * @param far distance of the far plane
-     */
-    private static _computeProjectionMatrix(K: number[], near: number, far: number): SpeedyMatrix
-    {
-        // we assume that the principal point is at the center of the image
-        const top = near * (K[V0] / K[FY]);
-        const right = near * (K[U0] / K[FX]);
-        const bottom = -top, left = -right; // symmetric frustum
-
-        // a derivation of this projection matrix can be found at
-        // https://www.songho.ca/opengl/gl_projectionmatrix.html
-        // http://learnwebgl.brown37.net/08_projections/projections_perspective.html
-        return Speedy.Matrix(4, 4, [
-            2 * near / (right - left), 0, 0, 0,
-            0, 2 * near / (top - bottom), 0, 0,
-            (right + left) / (right - left), (top + bottom) / (top - bottom), -(far + near) / (far - near), -1,
-            0, 0, -2 * far * near / (far - near), 0
-        ]);
-    }
 }

src/geometry/viewer-pose.ts

@@ -44,64 +44,22 @@ export class ViewerPose extends Pose
      */
     constructor(camera: CameraModel)
     {
-        // compute the view matrix and its inverse in AR screen space
-        const viewMatrix = ViewerPose._computeViewMatrix(camera);
-        const inverseTransform = new Transform(viewMatrix); // from world space to view space
-        const transform = inverseTransform.inverse; // from view space to world space
+        const viewMatrix = camera.computeViewMatrix();
+        const modelMatrix = Speedy.Matrix(viewMatrix.inverse());
 
+        const transform = new Transform(modelMatrix);
         super(transform);
+
         this._viewMatrix = viewMatrix;
     }
 
     /**
-     * This 4x4 matrix moves 3D points from world space to view space. We
-     * assume that the camera is looking in the direction of the negative
-     * z-axis (WebGL-friendly)
+     * This 4x4 matrix moves 3D points from world space to view space.
+     * We assume that the camera is looking in the direction of the
+     * negative z-axis (WebGL-friendly)
      */
     get viewMatrix(): SpeedyMatrix
     {
         return this._viewMatrix;
     }
-
-    /**
-     * Compute the view matrix in AR screen space, measured in pixels
-     * @param camera
-     * @returns a 4x4 matrix describing a rotation and a translation
-     */
-    private static _computeViewMatrix(camera: CameraModel): SpeedyMatrix
-    {
-        /*
-
-        // this is the view matrix in AR screen space, measured in pixels
-        // we augment the extrinsics matrix, making it 4x4 by adding a
-        // [ 0  0  0  1 ] row. Below, E is a 3x4 extrinsics matrix
-        const V = Speedy.Matrix(4, 4, [
-            E[0], E[1], E[2], 0,
-            E[3], E[4], E[5], 0,
-            E[6], E[7], E[8], 0,
-            E[9], E[10], E[11], 1
-        ]);
-
-        // we premultiply V by F, which performs a rotation around the
-        // x-axis by 180 degrees, so that we get the 3D objects in front
-        // of the camera pointing in the direction of the negative z-axis
-        const F = Speedy.Matrix(4, 4, [
-            1, 0, 0, 0,
-            0,-1, 0, 0,
-            0, 0,-1, 0,
-            0, 0, 0, 1
-        ]);
-
-        Matrix F * V is matrix V with the second and third rows negated
-
-        */
-
-        const E = camera.extrinsics;
-        return Speedy.Matrix(4, 4, [
-            E[0],-E[1],-E[2], 0,
-            E[3],-E[4],-E[5], 0,
-            E[6],-E[7],-E[8], 0,
-            E[9],-E[10],-E[11], 1
-        ]);
-    }
 }

src/trackers/image-tracker/image-tracker-utils.ts

@@ -0,0 +1,456 @@
+/*
+ * encantar.js
+ * GPU-accelerated Augmented Reality for the web
+ * Copyright (C) 2022-2024 Alexandre Martins <alemartf(at)gmail.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * image-tracker-utils.ts
+ * Image Tracker: Utilities
+ */
+
+import Speedy from 'speedy-vision';
+import { SpeedySize } from 'speedy-vision/types/core/speedy-size';
+import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
+import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
+import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix';
+import { SpeedyPoint2 } from 'speedy-vision/types/core/speedy-point';
+import { SpeedyVector2 } from 'speedy-vision/types/core/speedy-vector';
+import { SpeedyKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
+import { ReferenceImageWithMedia } from './reference-image';
+import { Utils } from '../../utils/utils';
+import { IllegalOperationError, IllegalArgumentError, NumericalError } from '../../utils/errors';
+import { NIS_SIZE, TRACK_GRID_GRANULARITY } from './settings';
+
+/*
+
+Definitions:
+------------
+
+1. Raster space:
+   an image space whose top-left coordinate is (0,0) and whose bottom-right
+   coordinate is (w-1,h-1), where (w,h) is its size. The y-axis points down.
+
+2. AR screen size:
+   size in pixels used for image processing operations. It's determined by the
+   resolution of the tracker and by the aspect ratio of the input media.
+
+3. AR screen space (screen):
+   a raster space whose size is the AR screen size.
+
+4. Normalized Image Space (NIS):
+   a raster space whose size is N x N, where N = NIS_SIZE.
+
+5. Normalized Device Coordinates (NDC):
+   the normalized 2D space [-1,1]x[-1,1]. The origin is at the center and the
+   y-axis points up.
+
+*/
+
+/** An ordered pair [src, dest] of keypoints */
+export type ImageTrackerKeypointPair = [ Readonly<SpeedyKeypoint>, Readonly<SpeedyKeypoint> ];
+
+/**
+ * Utilities for the Image Tracker
+ */
+export class ImageTrackerUtils
+{
+    /**
+     * Find a transformation that converts a raster space to NIS
+     * @param size size of the raster space
+     * @returns a 3x3 matrix
+     */
+    static rasterToNIS(size: SpeedySize): SpeedyMatrix
+    {
+        const sx = NIS_SIZE / size.width;
+        const sy = NIS_SIZE / size.height;
+
+        return Speedy.Matrix(3, 3, [
+            sx, 0,  0,
+            0,  sy, 0,
+            0,  0,  1
+        ]);
+    }
+
+    /**
+     * Find a transformation that converts a raster space to NDC
+     * @param size size of the raster space
+     * @returns a 3x3 matrix
+     */
+    static rasterToNDC(size: SpeedySize): SpeedyMatrix
+    {
+        const w = size.width, h = size.height;
+
+        return Speedy.Matrix(3, 3, [
+            2/w, 0,   0,
+            0,  -2/h, 0,
+           -1,   1,   1
+        ]);
+    }
+
+    /**
+     * Find a transformation that converts NDC to a raster space
+     * @param size size of the raster space
+     * @returns a 3x3 matrix
+     */
+    static NDCToRaster(size: SpeedySize): SpeedyMatrix
+    {
+        const w = size.width, h = size.height;
+
+        return Speedy.Matrix(3, 3, [
+            w/2, 0,   0,
+            0,  -h/2, 0,
+            w/2, h/2, 1
+        ]);
+    }
+
+    /**
+     * Find a transformation that scales points in NDC
+     * @param sx horizontal scale factor
+     * @param sy vertical scale factor
+     * @returns a 3x3 matrix
+     */
+    static scaleNDC(sx: number, sy: number = sx): SpeedyMatrix
+    {
+        // In NDC, the origin is at the center of the space!
+        return Speedy.Matrix(3, 3, [
+            sx, 0,  0,
+            0,  sy, 0,
+            0,  0,  1
+        ]);
+    }
+
+    /**
+     * Find a scale transformation in NDC such that the output has a desired aspect ratio
+     * @param aspectRatio desired aspect ratio
+     * @param scale optional scale factor in both axes
+     * @returns a 3x3 matrix
+     */
+    static bestFitScaleNDC(aspectRatio: number, scale: number = 1): SpeedyMatrix
+    {
+        if(aspectRatio >= 1)
+            return this.scaleNDC(scale, scale / aspectRatio); // s/(s/a) = a, sx >= sy
+        else
+            return this.scaleNDC(scale * aspectRatio, scale); // (s*a)/s = a, sx < sy
+    }
+
+    /**
+     * Find the inverse matrix of bestFitScaleNDC()
+     * @param aspectRatio as given to bestFitScaleNDC()
+     * @param scale optional, as given to bestFitScaleNDC()
+     * @returns a 3x3 matrix
+     */
+    static inverseBestFitScaleNDC(aspectRatio: number, scale: number = 1): SpeedyMatrix
+    {
+        if(aspectRatio >= 1)
+            return this.scaleNDC(1 / scale, aspectRatio / scale);
+        else
+            return this.scaleNDC(1 / (scale * aspectRatio), 1 / scale);
+    }
+
+    /**
+     * Find the best-fit aspect ratio for the rectification of the reference image in NDC
+     * @param screenSize
+     * @param referenceImage
+     * @returns a best-fit aspect ratio
+     */
+    static bestFitAspectRatioNDC(screenSize: SpeedySize, referenceImage: ReferenceImageWithMedia): number
+    {
+        /*
+        
+        The best-fit aspectRatio (a) is constructed as follows:
+
+        1) a fully stretched(*) and distorted reference image in NDC:
+           a = 1
+
+        2) a square in NDC:
+           a = 1 / screenAspectRatio
+
+        3) an image with the aspect ratio of the reference image in NDC:
+           a = referenceImageAspectRatio * (1 / screenAspectRatio)
+
+        (*) AR screen space
+
+        By transforming the reference image twice, first by converting it to AR
+        screen space, and then by rectifying it, we lose a little bit of quality.
+        Nothing to be too concerned about, though?
+
+        */
+
+        const screenAspectRatio = screenSize.width / screenSize.height;
+        return referenceImage.aspectRatio / screenAspectRatio;
+    }
+
+    /**
+     * Given n > 0 pairs (src_i, dest_i) of keypoints in NIS,
+     * convert them to NDC and output a 2 x 2n matrix of the form:
+     * [ src_0.x  src_1.x  ... | dest_0.x  dest_1.x  ... ]
+     * [ src_0.y  src_1.y  ... | dest_0.y  dest_1.y  ... ]
+     * @param pairs pairs of keypoints in NIS
+     * @returns 2 x 2n matrix with two 2 x n blocks: [ src | dest ]
+     * @throws
+     */
+    static compilePairsOfKeypointsNDC(pairs: ImageTrackerKeypointPair[]): SpeedyMatrix
+    {
+        const n = pairs.length;
+
+        if(n == 0)
+            throw new IllegalArgumentError();
+
+        const scale = 2 / NIS_SIZE;
+        const data = new Array<number>(2 * 2*n);
+
+        for(let i = 0, j = 0, k = 2*n; i < n; i++, j += 2, k += 2) {
+            const src = pairs[i][0];
+            const dest = pairs[i][1];
+
+            data[j] = src.x * scale - 1; // convert from NIS to NDC
+            data[j+1] = 1 - src.y * scale; // flip y-axis
+
+            data[k] = dest.x * scale - 1;
+            data[k+1] = 1 - dest.y * scale;
+        }
+
+        return Speedy.Matrix(2, 2*n, data);
+    }
+
+    /**
+     * Given n > 0 pairs of keypoints in NDC as a 2 x 2n [ src | dest ] matrix,
+     * find a perspective warp (homography) from src to dest in NDC
+     * @param points compiled pairs of keypoints in NDC
+     * @param options to be passed to speedy-vision
+     * @returns a pair [ 3x3 transformation matrix, quality score ]
+     */
+    static findPerspectiveWarpNDC(points: SpeedyMatrix, options: object): SpeedyPromise<[SpeedyMatrix,number]>
+    {
+        // too few data points?
+        const n = points.columns / 2;
+        if(n < 4) {
+            return Speedy.Promise.reject(
+                new IllegalArgumentError(`Too few data points to compute a perspective warp`)
+            );
+        }
+
+        // compute a homography
+        const src = points.block(0, 1, 0, n-1);
+        const dest = points.block(0, 1, n, 2*n-1);
+        const mask = Speedy.Matrix.Zeros(1, n);
+
+        return Speedy.Matrix.findHomography(
+            Speedy.Matrix.Zeros(3),
+            src,
+            dest,
+            Object.assign({ mask }, options)
+        ).then(homography => {
+
+            // check if this is a valid warp
+            const a00 = homography.at(0,0);
+            if(Number.isNaN(a00))
+                throw new NumericalError(`Can't compute a perspective warp: bad keypoints`);
+
+            // count the number of inliers
+            const inliers = mask.read();
+            let inlierCount = 0;
+            for(let i = inliers.length - 1; i >= 0; i--)
+                inlierCount += inliers[i];
+            const score = inlierCount / inliers.length;
+
+            // done!
+            return [ homography, score ];
+
+        });
+    }
+
+    /**
+     * Given n > 0 pairs of keypoints in NDC as a 2 x 2n [ src | dest ] matrix,
+     * find an affine warp from src to dest in NDC. The affine warp is given as
+     * a 3x3 matrix whose last row is [0 0 1]
+     * @param points compiled pairs of keypoints in NDC
+     * @param options to be passed to speedy-vision
+     * @returns a pair [ 3x3 transformation matrix, quality score ]
+     */
+    static findAffineWarpNDC(points: SpeedyMatrix, options: object): SpeedyPromise<[SpeedyMatrix,number]>
+    {
+        // too few data points?
+        const n = points.columns / 2;
+        if(n < 3) {
+            return Speedy.Promise.reject(
+                new IllegalArgumentError(`Too few data points to compute an affine warp`)
+            );
+        }
+
+        // compute an affine transformation
+        const model = Speedy.Matrix.Eye(3);
+        const src = points.block(0, 1, 0, n-1);
+        const dest = points.block(0, 1, n, 2*n-1);
+        const mask = Speedy.Matrix.Zeros(1, n);
+
+        return Speedy.Matrix.findAffineTransform(
+            model.block(0, 1, 0, 2), // 2x3 submatrix
+            src,
+            dest,
+            Object.assign({ mask }, options)
+        ).then(_ => {
+
+            // check if this is a valid warp
+            const a00 = model.at(0,0);
+            if(Number.isNaN(a00))
+                throw new NumericalError(`Can't compute an affine warp: bad keypoints`);
+
+            // count the number of inliers
+            const inliers = mask.read();
+            let inlierCount = 0;
+            for(let i = inliers.length - 1; i >= 0; i--)
+                inlierCount += inliers[i];
+            const score = inlierCount / inliers.length;
+
+            // done!
+            return [ model, score ];
+
+        });
+    }
+
+    /**
+     * Find a polyline in Normalized Device Coordinates (NDC)
+     * @param homography maps the corners of NDC to a quadrilateral in NDC
+     * @returns 4 points in NDC
+     */
+    static findPolylineNDC(homography: SpeedyMatrix): SpeedyPoint2[]
+    {
+        const h = homography.read();
+        const uv = [ -1, +1,    -1, -1,    +1, -1,    +1, +1 ]; // the corners of a reference image in NDC
+        const polyline = new Array<SpeedyPoint2>(4);
+
+        for(let i = 0, j = 0; i < 4; i++, j += 2) {
+            const u = uv[j], v = uv[j+1];
+
+            const x = h[0]*u + h[3]*v + h[6];
+            const y = h[1]*u + h[4]*v + h[7];
+            const w = h[2]*u + h[5]*v + h[8];
+
+            polyline[i] = Speedy.Point2(x/w, y/w);
+        }
+
+        return polyline;
+    }
+
+    /**
+     * Find a better spatial distribution of the input matches
+     * @param pairs in the [src, dest] format
+     * @returns refined pairs of quality matches
+     */
+    static refineMatchingPairs(pairs: ImageTrackerKeypointPair[]): ImageTrackerKeypointPair[]
+    {
+        // collect all keypoints obtained in this frame
+        const m = pairs.length;
+        const destKeypoints = new Array<SpeedyKeypoint>(m);
+
+        for(let j = 0; j < m; j++)
+            destKeypoints[j] = pairs[j][1];
+
+        // find a better spatial distribution of the keypoints
+        const indices = this._distributeKeypoints(destKeypoints);
+
+        // assemble output
+        const n = indices.length; // number of refined matches
+        const result = new Array<ImageTrackerKeypointPair>(n);
+
+        for(let i = 0; i < n; i++)
+            result[i] = pairs[indices[i]];
+
+        // done!
+        return result;
+    }
+
+    /**
+     * Spatially distribute keypoints over a grid
+     * @param keypoints keypoints to be distributed
+     * @returns a list of indices of keypoints[]
+     */
+    private static _distributeKeypoints(keypoints: SpeedyKeypoint[]): number[]
+    {
+        // create a grid
+        const gridCells = TRACK_GRID_GRANULARITY; // number of grid elements in each axis
+        const numberOfCells = gridCells * gridCells;
+        const n = keypoints.length;
+
+        // get the coordinates of the keypoints
+        const points: number[] = new Array(2 * n);
+        for(let i = 0, j = 0; i < n; i++, j += 2) {
+            points[j] = keypoints[i].x;
+            points[j+1] = keypoints[i].y;
+        }
+
+        // normalize the coordinates to [0,1) x [0,1)
+        this._normalizePoints(points);
+
+        // distribute the keypoints over the grid
+        const grid = new Array<number>(numberOfCells).fill(-1);
+        for(let i = 0, j = 0; i < n; i++, j += 2) {
+            // find the grid location of the i-th point
+            const xg = Math.floor(points[j] * gridCells); // 0 <= xg,yg < gridCells
+            const yg = Math.floor(points[j+1] * gridCells);
+
+            // store the index of the i-th point in the grid
+            const k = yg * gridCells + xg;
+            if(grid[k] < 0)
+                grid[k] = i;
+        }
+
+        // retrieve points of the grid
+        let m = 0;
+        const indices = new Array<number>(numberOfCells);
+        for(let g = 0; g < numberOfCells; g++) {
+            if(grid[g] >= 0)
+                indices[m++] = grid[g];
+        }
+        indices.length = m;
+
+        // done!
+        return indices;
+    }
+
+    /**
+     * Normalize points to [0,1)^2
+     * @param points 2 x n matrix of points in column-major format
+     * @returns points
+     */
+    private static _normalizePoints(points: number[]): number[]
+    {
+        Utils.assert(points.length % 2 == 0);
+
+        const n = points.length / 2;
+        if(n == 0)
+            return points;
+
+        let xmin = Number.POSITIVE_INFINITY, xmax = Number.NEGATIVE_INFINITY;
+        let ymin = Number.POSITIVE_INFINITY, ymax = Number.NEGATIVE_INFINITY;
+        for(let i = 0, j = 0; i < n; i++, j += 2) {
+            const x = points[j], y = points[j+1];
+            xmin = x < xmin ? x : xmin;
+            ymin = y < ymin ? y : ymin;
+            xmax = x > xmax ? x : xmax;
+            ymax = y > ymax ? y : ymax;
+        }
+
+        const xlen = xmax - xmin + 1; // +1 is a correction factor, so that 0 <= x,y < 1
+        const ylen = ymax - ymin + 1;
+        for(let i = 0, j = 0; i < n; i++, j += 2) {
+            points[j] = (points[j] - xmin) / xlen;
+            points[j+1] = (points[j+1] - ymin) / ylen;
+        }
+
+        return points;
+    }
+}

src/trackers/image-tracker/image-tracker.ts

@@ -37,13 +37,14 @@ import { Tracker, TrackerOutput, TrackerResult, Trackable } from '../tracker';
 import { Session } from '../../core/session';
 import { IllegalOperationError, IllegalArgumentError } from '../../utils/errors';
 import { Resolution } from '../../utils/resolution';
-import { ReferenceImage } from './reference-image';
+import { ReferenceImage, ReferenceImageWithMedia } from './reference-image';
 import { ReferenceImageDatabase } from './reference-image-database';
 import { ImageTrackerState } from './states/state';
 import { ImageTrackerInitialState } from './states/initial';
 import { ImageTrackerTrainingState } from './states/training';
 import { ImageTrackerScanningState } from './states/scanning';
-import { ImageTrackerPreTrackingState } from './states/pre-tracking';
+import { ImageTrackerPreTrackingAState } from './states/pre-tracking-a';
+import { ImageTrackerPreTrackingBState } from './states/pre-tracking-b';
 import { ImageTrackerTrackingState } from './states/tracking';
 import { Nullable, Utils } from '../../utils/utils';
 import { AREventTarget } from '../../utils/ar-events';
@@ -52,20 +53,7 @@ import { ImageTrackerEvent, ImageTrackerEventType } from './image-tracker-event'
 import { SpeedyPoint2 } from 'speedy-vision/types/core/speedy-point';
 import { Viewer } from '../../geometry/viewer';
 import { Pose } from '../../geometry/pose';
-
-/*
-
-A few definitions:
-
-1. Viewport size:
-    size of the drawing buffer of the background canvas = size of the input
-    media, in pixels
-
-2. AR screen size:
-    size for image processing operations, determined by the resolution of the
-    tracker and by the aspect ratio of the input media
-
-*/
+import { CameraModel } from '../../geometry/camera-model';
 
 /** A trackable target */
 export interface TrackableImage extends Trackable
@@ -96,24 +84,21 @@ export interface ImageTrackerOutput extends TrackerOutput
     /** tracker result to be consumed by the user */
     readonly exports?: ImageTrackerResult;
 
-    /** size of the AR screen space, in pixels */
-    readonly screenSize?: SpeedySize;
-
-    /** optional keypoints */
+    /** keypoints found in this framestep */
     readonly keypoints?: SpeedyKeypoint[];
 
-    /** optional polyline for testing */
-    readonly polyline?: SpeedyPoint2[];
+    /** optional keypoints for visualizing & testing */
+    readonly keypointsNIS?: SpeedyKeypoint[];
 
-    /** optional 3x4 camera matrix in AR screen space */
-    readonly cameraMatrix?: SpeedyMatrix;
+    /** optional polyline for visualizing & testing */
+    readonly polylineNDC?: SpeedyPoint2[];
 
-    /** 3x3 homography in AR screen space */
-    homography?: SpeedyMatrix;
+    /** optional camera model for visualizing & testing */
+    readonly camera?: CameraModel;
 }
 
 /** All possible states of an Image Tracker */
-export type ImageTrackerStateName = 'initial' | 'training' | 'scanning' | 'pre-tracking' | 'tracking';
+export type ImageTrackerStateName = 'initial' | 'training' | 'scanning' | 'pre-tracking-a' | 'pre-tracking-b' | 'tracking';
 
 /** A helper */
 const formatSize = (size: SpeedySize) => `${size.width}x${size.height}`;
@@ -163,7 +148,8 @@ export class ImageTracker extends AREventTarget<ImageTrackerEventType> implement
             'initial': new ImageTrackerInitialState(this),
             'training': new ImageTrackerTrainingState(this),
             'scanning': new ImageTrackerScanningState(this),
-            'pre-tracking': new ImageTrackerPreTrackingState(this),
+            'pre-tracking-a': new ImageTrackerPreTrackingAState(this),
+            'pre-tracking-b': new ImageTrackerPreTrackingBState(this),
             'tracking': new ImageTrackerTrackingState(this),
         };
 
@@ -311,8 +297,7 @@ export class ImageTracker extends AREventTarget<ImageTrackerEventType> implement
         // compute the screen size for image processing purposes
         // note: this may change over time...!
         const media = this._source!._internalMedia;
-        const aspectRatio = media.width / media.height;
-        const screenSize = Utils.resolution(this._resolution, aspectRatio);
+        const screenSize = this._computeScreenSize();
 
         // run the active state
         const activeState = this._state[this._activeStateName];
@@ -335,7 +320,7 @@ export class ImageTracker extends AREventTarget<ImageTrackerEventType> implement
      * @returns reference image
      * @internal
      */
-    _referenceImageOfKeypoint(keypointIndex: number): Nullable<ReferenceImage>
+    _referenceImageOfKeypoint(keypointIndex: number): Nullable<ReferenceImageWithMedia>
     {
         const training = this._state.training as ImageTrackerTrainingState;
         return training.referenceImageOfKeypoint(keypointIndex);
@@ -364,4 +349,18 @@ export class ImageTracker extends AREventTarget<ImageTrackerEventType> implement
         const training = this._state.training as ImageTrackerTrainingState;
         return training.referenceKeypoint(keypointIndex);
     }
+
+    /**
+     * Compute the current size of the AR screen space
+     * Note that this may change over time
+     * @returns size
+     */
+    private _computeScreenSize(): SpeedySize
+    {
+        const media = this._source!._internalMedia;
+        const aspectRatio = media.width / media.height;
+        const screenSize = Utils.resolution(this._resolution, aspectRatio);
+
+        return screenSize;
+    }
 }

src/trackers/image-tracker/reference-image-database.ts

@@ -23,7 +23,7 @@
 import Speedy from 'speedy-vision';
 import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
 import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
-import { ReferenceImage } from './reference-image';
+import { ReferenceImage, ReferenceImageWithMedia } from './reference-image';
 import { Utils } from '../../utils/utils';
 import { IllegalArgumentError, IllegalOperationError } from '../../utils/errors';
 
@@ -33,28 +33,13 @@ const DEFAULT_CAPACITY = 100; // this number should exceed normal usage
                               // further testing is needed to verify the appropriateness of this number;
                               // it depends on the images, on the keypoint descriptors, and even on the target devices
 
-/** Generate a unique name for a reference image */
-const generateUniqueName = () => 'target-' + Math.random().toString(16).substr(2);
-
-/**
- * An entry of a Reference Image Database
- */
-interface ReferenceImageDatabaseEntry
-{
-    /** reference image */
-    readonly referenceImage: ReferenceImage;
-
-    /** previously loaded media */
-    readonly media: SpeedyMedia;
-}
-
 /**
  * A collection of Reference Images
  */
 export class ReferenceImageDatabase implements Iterable<ReferenceImage>
 {
-    /** Image database */
-    private _database: ReferenceImageDatabaseEntry[];
+    /** Entries */
+    private _entries: Map<string, ReferenceImageWithMedia>;
 
     /** Maximum number of entries */
     private _capacity: number;
@@ -62,8 +47,6 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
     /** Is the database locked? */
     private _locked: boolean;
 
-    /** Are we busy loading an image? */
-    private _busy: boolean;
 
 
 
@@ -73,9 +56,8 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
     constructor()
     {
         this._capacity = DEFAULT_CAPACITY;
-        this._database = [];
+        this._entries = new Map();
         this._locked = false;
-        this._busy = false;
     }
 
     /**
@@ -83,7 +65,7 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
      */
     get count(): number
     {
-        return this._database.length;
+        return this._entries.size;
     }
 
     /**
@@ -111,10 +93,9 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
     /**
      * Iterates over the collection
      */
-    *[Symbol.iterator](): Iterator<ReferenceImage>
+    [Symbol.iterator](): Iterator<ReferenceImageWithMedia>
     {
-        const ref = this._database.map(entry => entry.referenceImage);
-        yield* ref;
+        return this._entries.values();
     }
 
     /**
@@ -127,51 +108,44 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
      */
     add(referenceImages: ReferenceImage[]): SpeedyPromise<void>
     {
-        // handle no input
-        if(referenceImages.length == 0)
-            return Speedy.Promise.resolve();
-
-        // handle multiple images as input
-        if(referenceImages.length > 1) {
-            const promises = referenceImages.map(image => this.add([ image ]));
-            return Utils.runInSequence(promises);
-        }
+        return this._preloadMany(referenceImages).then(referenceImagesWithMedia => {
+            referenceImagesWithMedia.forEach(referenceImageWithMedia => {
+                this._addOne(referenceImageWithMedia);
+            });
+        });
+    }
 
-        // handle a single image as input
-        const referenceImage = referenceImages[0];
+    /**
+     * Add a single preloaded reference image to the database
+     * @param referenceImage
+     */
+    _addOne(referenceImage: ReferenceImageWithMedia): void
+    {
+        const name = referenceImage.name;
 
         // locked database?
         if(this._locked)
-            throw new IllegalOperationError(`Can't add reference image "${referenceImage.name}" to the database: it's locked`);
-
-        // busy loading another image?
-        if(this._busy)
-            return Utils.wait(4).then(() => this.add(referenceImages)); // try again later
+            throw new IllegalOperationError(`Can't add reference image "${name}" to the database: it's locked`);
 
         // reached full capacity?
         if(this.count >= this.capacity)
-            throw new IllegalOperationError(`Can't add reference image "${referenceImage.name}" to the database: the capacity of ${this.capacity} images has been exceeded.`);
+            throw new IllegalOperationError(`Can't add reference image "${name}" to the database: the capacity of ${this.capacity} images has been exceeded.`);
 
         // check if the image is valid
-        if(!(referenceImage.image instanceof HTMLImageElement) && !(referenceImage.image instanceof HTMLCanvasElement) && !(referenceImage.image instanceof ImageBitmap))
-            throw new IllegalArgumentError(`Can't add reference image "${referenceImage.name}" to the database: invalid image`);
+        if(
+            !(referenceImage.image instanceof HTMLImageElement) &&
+            !(referenceImage.image instanceof ImageBitmap) &&
+            !(referenceImage.image instanceof ImageData)
+        )
+            throw new IllegalArgumentError(`Can't add reference image "${name}" to the database: invalid image`);
 
         // check for duplicate names
-        if(this._database.find(entry => entry.referenceImage.name === referenceImage.name) !== undefined)
-            throw new IllegalArgumentError(`Can't add reference image "${referenceImage.name}" to the database: found duplicated name`);
+        if(this._entries.has(name))
+            throw new IllegalArgumentError(`Can't add reference image "${name}" to the database: found duplicated name`);
 
-        // load the media and add the reference image to the database
-        this._busy = true;
-        return Speedy.load(referenceImage.image).then(media => {
-            this._busy = false;
-            this._database.push({
-                referenceImage: Object.freeze({
-                    ...referenceImage,
-                    name: referenceImage.name || generateUniqueName()
-                }),
-                media: media
-            });
-        });
+        // add the reference image to the database
+        Utils.log(`Adding reference image "${name}" to the database...`);
+        this._entries.set(name, referenceImage);
     }
 
     /**
@@ -180,25 +154,51 @@ export class ReferenceImageDatabase implements Iterable<ReferenceImage>
      */
     _lock(): void
     {
-        if(this._busy)
-            throw new IllegalOperationError(`Can't lock the reference image database: we're busy loading an image`);
-
         this._locked = true;
     }
 
     /**
-     * Get the media object associated to a reference image
-     * @param name reference image name
-     * @returns media
+     * Get reference image by name
+     * @param name
+     * @returns the reference image with the given name, or null if there isn't any
      * @internal
      */
-    _findMedia(name: string): SpeedyMedia
+    _find(name: string): ReferenceImageWithMedia | null
+    {
+        return this._entries.get(name) || null;
+    }
+
+    /**
+     * Load a reference image
+     * @param referenceImage
+     * @returns a promise that resolves to a corresponding ReferenceImageWithMedia
+     */
+    private _preloadOne(referenceImage: ReferenceImage): SpeedyPromise<ReferenceImageWithMedia>
+    {
+        if(referenceImage.name !== undefined)
+            Utils.log(`Loading reference image \"${referenceImage.name}\"...`);
+        else
+            Utils.log(`Loading reference image...`);
+
+        if(!referenceImage.image)
+            return Speedy.Promise.reject(new IllegalArgumentError('The reference image was not provided!'));
+
+        return Speedy.load(referenceImage.image).then(media => {
+            return new ReferenceImageWithMedia(referenceImage, media);
+        });
+    }
+
+    /**
+     * Load multiple reference images
+     * @param referenceImages
+     * @returns a promise that resolves to corresponding ReferenceImageWithMedia objects
+     */
+    private _preloadMany(referenceImages: ReferenceImage[]): SpeedyPromise<ReferenceImageWithMedia[]>
     {
-        for(let i = 0; i < this._database.length; i++) {
-            if(this._database[i].referenceImage.name === name)
-                return this._database[i].media;
-        }
+        const n = referenceImages.length;
+        Utils.log(`Loading ${n} reference image${n != 1 ? 's' : ''}...`);
 
-        throw new IllegalArgumentError(`Can't find reference image "${name}"`);
+        const promises = referenceImages.map(referenceImage => this._preloadOne(referenceImage));
+        return Speedy.Promise.all<ReferenceImageWithMedia>(promises);
     }
 }

src/trackers/image-tracker/reference-image.ts

@@ -20,14 +20,104 @@
  * Reference Image for tracking
  */
 
+import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
+
+type ReferenceImageType = HTMLImageElement | ImageBitmap | ImageData;
+
+
+
 /**
  * Reference Image for tracking
  */
 export interface ReferenceImage
 {
     /** Reference Images should have unique names given by the user */
-    readonly name: string;
+    name?: string;
 
     /** Image data */
-    readonly image: HTMLImageElement | HTMLCanvasElement | ImageBitmap;
+    readonly image: ReferenceImageType;
+}
+
+/**
+ * A ReferenceImage decorated with a SpeedyMedia
+ */
+export class ReferenceImageWithMedia implements ReferenceImage
+{
+    /** The decorated reference image */
+    private readonly _referenceImage: ReferenceImage;
+
+    /** A SpeedyMedia corresponding to the reference image */
+    private readonly _media: SpeedyMedia;
+
+    /** The aspect ratio of the reference image */
+    private readonly _aspectRatio: number;
+
+
+
+    /**
+     * Constructor
+     * @param referenceImage
+     * @param media
+     */
+    constructor(referenceImage: ReferenceImage, media: SpeedyMedia)
+    {
+        this._referenceImage = Object.assign({}, referenceImage);
+        this._media = media;
+
+        // generate a unique name if none is given
+        if(this._referenceImage.name === undefined)
+            this._referenceImage.name = this._generateUniqueName();
+
+        // store the aspect ratio
+        this._aspectRatio = media.width / media.height;
+    }
+
+    /**
+     * Getter of the name of the reference image
+     */
+    get name(): string
+    {
+        return this._referenceImage.name!;
+    }
+
+    /**
+     * Setter of the name of the reference image
+     */
+    set name(name: string)
+    {
+        this._referenceImage.name = name;
+    }
+
+    /**
+     * Image data
+     */
+    get image(): ReferenceImageType
+    {
+        return this._referenceImage.image;
+    }
+
+    /**
+     * A SpeedyMedia corresponding to the reference media
+     */
+    get media(): SpeedyMedia
+    {
+        return this._media;
+    }
+
+    /**
+     * The aspect ratio of the reference image
+     */
+    get aspectRatio(): number
+    {
+        return this._aspectRatio;
+    }
+
+    /**
+     * Generate a unique name for a reference image
+     * @returns a unique name
+     */
+    private _generateUniqueName(): string
+    {
+        return 'target-' + Math.random().toString(16).substr(2);
+    }
 }

src/trackers/image-tracker/settings.ts

@@ -29,8 +29,8 @@ export const TRAIN_MAX_KEYPOINTS = 1024; //512;
 /** Percentage relative to the screen size adjusted to the aspect ratio of the reference image */
 export const TRAIN_IMAGE_SCALE = 0.8; // ORB is not scale invariant
 
-/** Normalized width & height of an image target, in pixels */
-export const TRAIN_TARGET_NORMALIZED_SIZE = 1024; // keypoint positions are stored as fixed point
+/** Width and height of the Normalized Image Space (NIS) */
+export const NIS_SIZE = 1024; // keypoint positions are stored as fixed point
 
 /** Used to identify the best maches */
 export const SCAN_MATCH_RATIO = 0.7; // usually a value in [0.6, 0.8]
@@ -47,14 +47,17 @@ export const SCAN_PYRAMID_SCALEFACTOR = 1.19; // 2 ^ 0.25
 /** Threshold of the FAST corner detector used in the scanning/training states */
 export const SCAN_FAST_THRESHOLD = 60;
 
-/** Minimum number of accepted matches for us to move out from the scanning state */
+/** Minimum number of accepted matches for us to move out of the scanning state */
 export const SCAN_MIN_MATCHES = 20; //30;
 
 /** When in the scanning state, we require the image to be matched during a few consecutive frames before accepting it */
 export const SCAN_CONSECUTIVE_FRAMES = 30;//15;//45;
 
-/** Reprojection error, in pixels, used when estimating a motion model (scanning state) */
-export const SCAN_RANSAC_REPROJECTIONERROR = 5;
+/** Reprojection error, in NIS pixels, used when estimating a motion model (scanning state) */
+export const SCAN_RANSAC_REPROJECTIONERROR_NIS = (NIS_SIZE * 0.02) | 0;
+
+/** Reprojection error, in NDC, used when estimating a motion model (scanning state) */
+export const SCAN_RANSAC_REPROJECTIONERROR_NDC = SCAN_RANSAC_REPROJECTIONERROR_NIS / (NIS_SIZE / 2);
 
 /** Number of tables used in the LSH-based keypoint matching */
 export const SCAN_LSH_TABLES = 8; // up to 32
@@ -92,6 +95,9 @@ export const SUBPIXEL_GAUSSIAN_SIGMA = 1.0;
 /** Subpixel refinement method */
 export const SUBPIXEL_METHOD = 'bilinear-upsample'; // 'quadratic1d';
 
+/** Minimum acceptable number of matched keypoints when in a pre-tracking state */
+export const PRE_TRACK_MIN_MATCHES = 4;
+
 /** Minimum acceptable number of matched keypoints when in the tracking state */
 export const TRACK_MIN_MATCHES = 4;//10; //20;
 
@@ -113,11 +119,14 @@ export const TRACK_RECTIFIED_BORDER = 0.15; //0.20;
 /** Relative size (%) used to clip keypoints from the borders of the rectified image */
 export const TRACK_CLIPPING_BORDER = TRACK_RECTIFIED_BORDER * 1.20; //1.25; //1.15;
 
-/** Number of iterations used to refine the target image before tracking */
-export const TRACK_REFINEMENT_ITERATIONS = 3;
+/** Scale of the rectified image in NDC, without taking the aspect ratio into consideration */
+export const TRACK_RECTIFIED_SCALE = 1 - 2 * TRACK_RECTIFIED_BORDER;
+
+/** Reprojection error, in NIS pixels, used when estimating a motion model (tracking state) */
+export const TRACK_RANSAC_REPROJECTIONERROR_NIS = (NIS_SIZE * 0.0125) | 0;
 
-/** Reprojection error, in pixels, used when estimating a motion model (tracking state) */
-export const TRACK_RANSAC_REPROJECTIONERROR = 3; //2.5;
+/** Reprojection error, in NDC, used when estimating a motion model (tracking state) */
+export const TRACK_RANSAC_REPROJECTIONERROR_NDC = TRACK_RANSAC_REPROJECTIONERROR_NIS / (NIS_SIZE / 2);
 
 /** We use a N x N grid to spatially distribute the keypoints in order to compute a better homography */
 export const TRACK_GRID_GRANULARITY = 10; //20; // the value of N
@@ -126,4 +135,4 @@ export const TRACK_GRID_GRANULARITY = 10; //20; // the value of N
 export const TRACK_MATCH_RATIO = 0.75; // usually a value in [0.6, 0.8] - low values => strict tracking
 
 /** Number of consecutive frames in which we tolerate a  "target lost" situation */
-export const TRACK_LOST_TOLERANCE = 10;
+export const TRACK_LOST_TOLERANCE = 15;

src/trackers/image-tracker/states/initial.ts

@@ -78,6 +78,16 @@ export class ImageTrackerInitialState extends ImageTrackerState
     }
 
     /**
+     * Called when leaving the state, after update()
+     */
+    onLeaveState(): void
+    {
+        // we don't return to this state, so we can release the pipeline early
+        this._pipeline.release();
+        this._pipelineReleased = true;
+    }
+
+    /**
      * Create & setup the pipeline
      * @returns pipeline
      */

src/trackers/image-tracker/states/pre-tracking-a.ts

@@ -0,0 +1,273 @@
+/*
+ * encantar.js
+ * GPU-accelerated Augmented Reality for the web
+ * Copyright (C) 2022-2024 Alexandre Martins <alemartf(at)gmail.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * pre-tracking-a.ts
+ * Image tracker: Pre-Tracking A state
+ */
+
+import Speedy from 'speedy-vision';
+import { SpeedySize } from 'speedy-vision/types/core/speedy-size';
+import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
+import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix';
+import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
+import { SpeedyPipeline, SpeedyPipelineOutput } from 'speedy-vision/types/core/pipeline/pipeline';
+import { SpeedyPipelineNodeImageSource } from 'speedy-vision/types/core/pipeline/nodes/images/source';
+import { SpeedyPipelineNodeImageMultiplexer } from 'speedy-vision/types/core/pipeline/nodes/images/multiplexer';
+import { SpeedyPipelineNodeImagePortalSource, SpeedyPipelineNodeImagePortalSink } from 'speedy-vision/types/core/pipeline/nodes/images/portal';
+import { SpeedyPipelineNodeKeypointPortalSource, SpeedyPipelineNodeKeypointPortalSink } from 'speedy-vision/types/core/pipeline/nodes/keypoints/portal';
+import { SpeedyPipelineNodeResize } from 'speedy-vision/types/core/pipeline/nodes/transforms/resize';
+import { SpeedyPipelineNodePerspectiveWarp } from 'speedy-vision/types/core/pipeline/nodes/transforms/perspective-warp';
+import { SpeedyPipelineNodeKeypointBorderClipper } from 'speedy-vision/types/core/pipeline/nodes/keypoints/border-clipper';
+import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/transformer';
+import { SpeedyKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
+import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
+import { ImageTrackerState, ImageTrackerStateOutput } from './state';
+import { ReferenceImage, ReferenceImageWithMedia } from '../reference-image';
+import { Nullable, Utils } from '../../../utils/utils';
+import { TrackingError } from '../../../utils/errors';
+import {
+    TRACK_RECTIFIED_SCALE, TRACK_CLIPPING_BORDER,
+    NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY, TRACK_WITH_NIGHTVISION,
+    ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_SIGMA,
+    TRACK_HARRIS_QUALITY, TRACK_DETECTOR_CAPACITY, TRACK_MAX_KEYPOINTS,
+    SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_SIGMA,
+    PRE_TRACK_MIN_MATCHES,
+    NIGHTVISION_QUALITY,
+    SUBPIXEL_METHOD,
+} from '../settings';
+
+
+
+/**
+ * Pre-Tracking A is a new training phase. The reference image that was found
+ * in the scanning state is transported to AR screen space, and a new training
+ * takes place there, with new keypoints and in a suitable warp.
+ */
+export class ImageTrackerPreTrackingAState extends ImageTrackerState
+{
+    /** reference image */
+    private _referenceImage: Nullable<ReferenceImageWithMedia>;
+
+    /** a snapshot of the video from the scanning state and corresponding to the initial homography */
+    private _snapshot: Nullable<SpeedyPipelineNodeImagePortalSink>;
+
+    /** initial homography, from reference image to scanned image, NDC */
+    private _homography: SpeedyMatrix;
+
+
+
+    /**
+     * Constructor
+     * @param imageTracker
+     */
+    constructor(imageTracker: ImageTracker)
+    {
+        super('pre-tracking-a', imageTracker);
+
+        this._homography = Speedy.Matrix.Eye(3);
+        this._referenceImage = null;
+        this._snapshot = null;
+    }
+
+    /**
+     * Called as soon as this becomes the active state, just before update() runs for the first time
+     * @param settings
+     */
+    onEnterState(settings: Record<string,any>)
+    {
+        const homography = settings.homography as SpeedyMatrix;
+        const referenceImage = settings.referenceImage as ReferenceImageWithMedia;
+        const snapshot = settings.snapshot as SpeedyPipelineNodeImagePortalSink;
+
+        // set attributes
+        this._homography = homography;
+        this._referenceImage = referenceImage;
+        this._snapshot = snapshot;
+    }
+
+    /**
+     * Called just before the GPU processing
+     * @returns promise
+     */
+    protected _beforeUpdate(): SpeedyPromise<void>
+    {
+        const screenSize = this.screenSize;
+        const source = this._pipeline.node('source') as SpeedyPipelineNodeImageSource;
+        const imageRectifier = this._pipeline.node('imageRectifier') as SpeedyPipelineNodePerspectiveWarp;
+        const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
+        const borderClipper = this._pipeline.node('borderClipper') as SpeedyPipelineNodeKeypointBorderClipper;
+
+        // set the reference image as the source image
+        source.media = this._referenceImage!.media;
+
+        // clip keypoints from the borders of the target image
+        borderClipper.imageSize = screenSize;
+        borderClipper.borderSize = Speedy.Vector2(
+            screenSize.width * TRACK_CLIPPING_BORDER,
+            screenSize.height * TRACK_CLIPPING_BORDER
+        );
+
+        // convert keypoints to NIS
+        keypointScaler.transform = ImageTrackerUtils.rasterToNIS(screenSize);
+
+        // rectify the image
+        const scale = TRACK_RECTIFIED_SCALE;
+        const aspectRatio = ImageTrackerUtils.bestFitAspectRatioNDC(screenSize, this._referenceImage!);
+        const shrink = ImageTrackerUtils.bestFitScaleNDC(aspectRatio, scale);
+        const toScreen = ImageTrackerUtils.NDCToRaster(screenSize);
+        const toNDC = ImageTrackerUtils.rasterToNDC(screenSize);
+
+        return imageRectifier.transform.setTo(
+            toScreen.times(shrink).times(toNDC)
+        ).then(() => void 0);
+    }
+
+    /**
+     * Post processing that takes place just after the GPU processing
+     * @param result pipeline results
+     * @returns state output
+     */
+    protected _afterUpdate(result: SpeedyPipelineOutput): SpeedyPromise<ImageTrackerStateOutput>
+    {
+        const referenceImage = this._referenceImage!;
+        const keypointPortalSink = this._pipeline.node('keypointPortalSink') as SpeedyPipelineNodeKeypointPortalSink;
+        const keypoints = result.keypoints as SpeedyKeypoint[];
+        const image = result.image as SpeedyMedia | undefined;
+
+        // tracker output
+        const trackerOutput: ImageTrackerOutput = {
+            keypointsNIS: image !== undefined ? keypoints : undefined, // debug only
+            image: image,
+        };
+
+        // not enough keypoints? something went wrong!
+        if(keypoints.length < PRE_TRACK_MIN_MATCHES) {
+            Utils.warning(`Can't pre-track "${referenceImage.name}" in ${this.name}!`);
+            return Speedy.Promise.resolve({
+                nextState: 'scanning',
+                trackerOutput: trackerOutput,
+            });
+        }
+
+        // done!
+        return Speedy.Promise.resolve({
+            nextState: 'pre-tracking-b',
+            trackerOutput: trackerOutput,
+            nextStateSettings: {
+                referenceKeypointPortalSink: keypointPortalSink,
+                referenceImage: this._referenceImage,
+                snapshot: this._snapshot,
+                homography: this._homography,
+            }
+        });
+    }
+
+    /**
+     * Create & setup the pipeline
+     * @returns pipeline
+     */
+    protected _createPipeline(): SpeedyPipeline
+    {
+        const pipeline = Speedy.Pipeline();
+
+        const source = Speedy.Image.Source('source');
+        const screen = Speedy.Transform.Resize('screen');
+        const greyscale = Speedy.Filter.Greyscale();
+        const imageRectifier = Speedy.Transform.PerspectiveWarp('imageRectifier');
+        const nightvision = Speedy.Filter.Nightvision();
+        const nightvisionMux = Speedy.Image.Multiplexer();
+        const detector = Speedy.Keypoint.Detector.Harris();
+        const descriptor = Speedy.Keypoint.Descriptor.ORB();
+        const blur = Speedy.Filter.GaussianBlur();
+        const clipper = Speedy.Keypoint.Clipper();
+        const borderClipper = Speedy.Keypoint.BorderClipper('borderClipper');
+        const denoiser = Speedy.Filter.GaussianBlur();
+        const subpixel = Speedy.Keypoint.SubpixelRefiner();
+        const keypointScaler = Speedy.Keypoint.Transformer('keypointScaler');
+        const keypointPortalSink = Speedy.Keypoint.Portal.Sink('keypointPortalSink');
+        const keypointSink = Speedy.Keypoint.Sink('keypoints');
+        //const imageSink = Speedy.Image.Sink('image');
+
+        source.media = null;
+        imageRectifier.transform = Speedy.Matrix.Eye(3);
+        screen.size = Speedy.Size(0,0);
+        nightvision.gain = NIGHTVISION_GAIN;
+        nightvision.offset = NIGHTVISION_OFFSET;
+        nightvision.decay = NIGHTVISION_DECAY;
+        nightvision.quality = NIGHTVISION_QUALITY;
+        nightvisionMux.port = TRACK_WITH_NIGHTVISION ? 1 : 0; // 1 = enable nightvision
+        blur.kernelSize = Speedy.Size(ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_KSIZE);
+        blur.sigma = Speedy.Vector2(ORB_GAUSSIAN_SIGMA, ORB_GAUSSIAN_SIGMA);
+        denoiser.kernelSize = Speedy.Size(SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_KSIZE);
+        denoiser.sigma = Speedy.Vector2(SUBPIXEL_GAUSSIAN_SIGMA, SUBPIXEL_GAUSSIAN_SIGMA);
+        detector.quality = TRACK_HARRIS_QUALITY;
+        detector.capacity = TRACK_DETECTOR_CAPACITY;
+        subpixel.method = SUBPIXEL_METHOD;
+        clipper.size = TRACK_MAX_KEYPOINTS;
+        borderClipper.imageSize = screen.size;
+        borderClipper.borderSize = Speedy.Vector2(0,0);
+        keypointScaler.transform = Speedy.Matrix.Eye(3);
+        keypointSink.turbo = false;
+
+        // prepare input
+        source.output().connectTo(screen.input());
+        screen.output().connectTo(greyscale.input());
+
+        // preprocess images
+        greyscale.output().connectTo(imageRectifier.input());
+        imageRectifier.output().connectTo(nightvisionMux.input('in0'));
+        imageRectifier.output().connectTo(nightvision.input());
+        nightvision.output().connectTo(nightvisionMux.input('in1'));
+
+        // keypoint detection & clipping
+        nightvisionMux.output().connectTo(detector.input());
+        detector.output().connectTo(borderClipper.input());
+        borderClipper.output().connectTo(clipper.input());
+
+        // keypoint refinement
+        imageRectifier.output().connectTo(denoiser.input());
+        denoiser.output().connectTo(subpixel.input('image'));
+        clipper.output().connectTo(subpixel.input('keypoints'));
+
+        // keypoint description
+        nightvisionMux.output().connectTo(blur.input());
+        blur.output().connectTo(descriptor.input('image'));
+        subpixel.output().connectTo(descriptor.input('keypoints'));
+
+        // prepare output
+        descriptor.output().connectTo(keypointScaler.input());
+        keypointScaler.output().connectTo(keypointSink.input());
+        keypointScaler.output().connectTo(keypointPortalSink.input());
+        //imageRectifier.output().connectTo(imageSink.input());
+
+        // done!
+        pipeline.init(
+            source, screen,
+            greyscale, imageRectifier,
+            nightvision, nightvisionMux,
+            detector, borderClipper, clipper,
+            denoiser, subpixel,
+            blur, descriptor,
+            keypointScaler, keypointSink, keypointPortalSink,
+            //imageSink
+        );
+
+        return pipeline;
+    }
+}

src/trackers/image-tracker/states/pre-tracking-b.ts

@@ -0,0 +1,391 @@
+/*
+ * encantar.js
+ * GPU-accelerated Augmented Reality for the web
+ * Copyright (C) 2022-2024 Alexandre Martins <alemartf(at)gmail.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * pre-tracking-b.ts
+ * Image tracker: Pre-Tracking B state
+ */
+
+import Speedy from 'speedy-vision';
+import { SpeedySize } from 'speedy-vision/types/core/speedy-size';
+import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
+import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix';
+import { SpeedyMatrixExpr } from 'speedy-vision/types/core/speedy-matrix-expr';
+import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
+import { SpeedyPipeline, SpeedyPipelineOutput } from 'speedy-vision/types/core/pipeline/pipeline';
+import { SpeedyPipelineNodeImageSource } from 'speedy-vision/types/core/pipeline/nodes/images/source';
+import { SpeedyPipelineNodeImageMultiplexer } from 'speedy-vision/types/core/pipeline/nodes/images/multiplexer';
+import { SpeedyPipelineNodeImagePortalSource, SpeedyPipelineNodeImagePortalSink } from 'speedy-vision/types/core/pipeline/nodes/images/portal';
+import { SpeedyPipelineNodeKeypointPortalSource, SpeedyPipelineNodeKeypointPortalSink } from 'speedy-vision/types/core/pipeline/nodes/keypoints/portal';
+import { SpeedyPipelineNodeResize } from 'speedy-vision/types/core/pipeline/nodes/transforms/resize';
+import { SpeedyPipelineNodePerspectiveWarp } from 'speedy-vision/types/core/pipeline/nodes/transforms/perspective-warp';
+import { SpeedyPipelineNodeKeypointBorderClipper } from 'speedy-vision/types/core/pipeline/nodes/keypoints/border-clipper';
+import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/transformer';
+import { SpeedyKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
+import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
+import { ImageTrackerState, ImageTrackerStateOutput } from './state';
+import { ReferenceImageWithMedia } from '../reference-image';
+import { Nullable, Utils } from '../../../utils/utils';
+import { TrackingError } from '../../../utils/errors';
+import {
+    TRACK_RECTIFIED_SCALE, TRACK_CLIPPING_BORDER,
+    NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY, TRACK_WITH_NIGHTVISION,
+    ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_SIGMA,
+    TRACK_HARRIS_QUALITY, TRACK_DETECTOR_CAPACITY, TRACK_MAX_KEYPOINTS,
+    SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_SIGMA,
+    PRE_TRACK_MIN_MATCHES, TRACK_MATCH_RATIO, TRACK_RANSAC_REPROJECTIONERROR_NDC,
+    NIGHTVISION_QUALITY,
+    SUBPIXEL_METHOD,
+} from '../settings';
+
+
+
+
+/**
+ * In Pre-Tracking B, we refine the homography obtained at the scanning state.
+ * We find a transformation that warps the snapshot obtained from the scanning
+ * state to an image that closely resembles the output of Pre-Tracking A.
+ */
+export class ImageTrackerPreTrackingBState extends ImageTrackerState
+{
+    /** reference image */
+    private _referenceImage: Nullable<ReferenceImageWithMedia>;
+
+    /** a snapshot of the video from the scanning state and corresponding to the initial homography */
+    private _snapshot: Nullable<SpeedyPipelineNodeImagePortalSink>;
+
+    /** initial homography, from reference image to scanned image, NDC */
+    private _homography: SpeedyMatrix;
+
+    /** portal with keypoints from Pre-Tracking A */
+    private _referenceKeypointPortalSink: Nullable<SpeedyPipelineNodeKeypointPortalSink>;
+
+
+
+
+
+
+    /**
+     * Constructor
+     * @param imageTracker
+     */
+    constructor(imageTracker: ImageTracker)
+    {
+        super('pre-tracking-b', imageTracker);
+
+        this._homography = Speedy.Matrix.Eye(3);
+        this._referenceImage = null;
+        this._snapshot = null;
+        this._referenceKeypointPortalSink = null;
+    }
+
+    /**
+     * Called as soon as this becomes the active state, just before update() runs for the first time
+     * @param settings
+     */
+    onEnterState(settings: Record<string,any>)
+    {
+        const homography = settings.homography as SpeedyMatrix;
+        const referenceImage = settings.referenceImage as ReferenceImageWithMedia;
+        const snapshot = settings.snapshot as SpeedyPipelineNodeImagePortalSink;
+        const referenceKeypointPortalSink = settings.referenceKeypointPortalSink as SpeedyPipelineNodeKeypointPortalSink;
+
+        // set attributes
+        this._homography = homography;
+        this._referenceImage = referenceImage;
+        this._snapshot = snapshot;
+        this._referenceKeypointPortalSink = referenceKeypointPortalSink;
+    }
+
+    /**
+     * Called just before the GPU processing
+     * @returns promise
+     */
+    protected _beforeUpdate(): SpeedyPromise<void>
+    {
+        const screenSize = this.screenSize;
+        const imageRectifier = this._pipeline.node('imageRectifier') as SpeedyPipelineNodePerspectiveWarp;
+        const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
+        const borderClipper = this._pipeline.node('borderClipper') as SpeedyPipelineNodeKeypointBorderClipper;
+        const imagePortalSource = this._pipeline.node('imagePortalSource') as SpeedyPipelineNodeImagePortalSource;
+        const referenceKeypointPortalSource = this._pipeline.node('referenceKeypointPortalSource') as SpeedyPipelineNodeKeypointPortalSource;
+
+        // get the snapshot from the scanning state
+        imagePortalSource.source = this._snapshot!;
+
+        // get the reference keypoints from Pre-Tracking A
+        referenceKeypointPortalSource.source = this._referenceKeypointPortalSink!;
+
+        // clip keypoints from the borders of the target image
+        borderClipper.imageSize = screenSize;
+        borderClipper.borderSize = Speedy.Vector2(
+            screenSize.width * TRACK_CLIPPING_BORDER,
+            screenSize.height * TRACK_CLIPPING_BORDER
+        );
+
+        // convert keypoints to NIS
+        keypointScaler.transform = ImageTrackerUtils.rasterToNIS(screenSize);
+
+        // rectify the image
+        const scale = TRACK_RECTIFIED_SCALE;
+        const aspectRatio = ImageTrackerUtils.bestFitAspectRatioNDC(screenSize, this._referenceImage!);
+        const shrink = ImageTrackerUtils.bestFitScaleNDC(aspectRatio, scale);
+        const undistort = this._homography.inverse();
+        const toScreen = ImageTrackerUtils.NDCToRaster(screenSize);
+        const toNDC = ImageTrackerUtils.rasterToNDC(screenSize);
+
+        return imageRectifier.transform.setTo(
+            toScreen.times(shrink.times(undistort)).times(toNDC)
+        ).then(() => void 0);
+    }
+
+    /**
+     * Post processing that takes place just after the GPU processing
+     * @param result pipeline results
+     * @returns state output
+     */
+    protected _afterUpdate(result: SpeedyPipelineOutput): SpeedyPromise<ImageTrackerStateOutput>
+    {
+        const referenceImage = this._referenceImage!;
+        const referenceKeypoints = result.referenceKeypoints as SpeedyKeypoint[]; // from Pre-Tracking A
+        const keypoints = result.keypoints as SpeedyMatchedKeypoint[]; // from Pre-Tracking B
+        const image = result.image as SpeedyMedia | undefined;
+        const keypointPortalSink = this._pipeline.node('keypointPortalSink') as SpeedyPipelineNodeKeypointPortalSink;
+
+        // tracker output
+        const trackerOutput: ImageTrackerOutput = {
+            keypointsNIS: image !== undefined ? keypoints : undefined, // debug only
+            image: image,
+        };
+
+        return Speedy.Promise.resolve()
+        .then(() => {
+
+            // find matching pairs of keypoints
+            const pairs = this._findMatchingPairs(referenceKeypoints, keypoints);
+            //const pairs = ImageTrackerUtils.refineMatchingPairs(allPairs);
+            if(pairs.length < PRE_TRACK_MIN_MATCHES)
+                throw new TrackingError('Not enough data points');
+
+            // find a warp
+            const points = ImageTrackerUtils.compilePairsOfKeypointsNDC(pairs);
+            return this._findAffineMotionNDC(points);
+
+        })
+        .then(warp => {
+
+            // refine the homography
+            return this._homography.setTo(warp.times(this._homography));
+
+        })
+        .then(_ => ({
+            nextState: 'tracking',
+            //nextState: 'pre-tracking-b',
+            trackerOutput: trackerOutput,
+            nextStateSettings: {
+                // we export keypoints obtained in Pre-Tracking B, not in A.
+                // lighting conditions match, but what if the snapshot is too blurry?
+                templateKeypoints: keypoints,
+                templateKeypointPortalSink: keypointPortalSink,
+                referenceImage: this._referenceImage,
+                homography: this._homography,
+                initialScreenSize: this.screenSize,
+            }
+        }))
+        .catch(err => {
+            Utils.warning(`Can't pre-track "${referenceImage.name}" in ${this.name}! ${err.toString()}`);
+            return {
+                nextState: 'scanning',
+                trackerOutput: trackerOutput,
+            };
+        });
+    }
+
+    /**
+     * Find an affine motion model in NDC between pairs of keypoints in NDC
+     * given as a 2 x 2n [ src | dest ] matrix
+     * @param points compiled pairs of keypoints in NDC
+     * @returns a promise that resolves to a 3x3 warp in NDC that maps source to destination
+     */
+    private _findAffineMotionNDC(points: SpeedyMatrix): SpeedyPromise<SpeedyMatrixExpr>
+    {
+        return ImageTrackerUtils.findAffineWarpNDC(points, {
+            method: 'pransac',
+            reprojectionError: TRACK_RANSAC_REPROJECTIONERROR_NDC,
+            numberOfHypotheses: 512*4,
+            bundleSize: 128,
+            mask: undefined // score is not needed
+        }).then(([ warp, score ]) => {
+
+            const scale = TRACK_RECTIFIED_SCALE;
+            const aspectRatio = ImageTrackerUtils.bestFitAspectRatioNDC(this.screenSize, this._referenceImage!);
+            const shrink = ImageTrackerUtils.bestFitScaleNDC(aspectRatio, scale);
+            const grow = ImageTrackerUtils.inverseBestFitScaleNDC(aspectRatio, scale);
+            const scaledWarp = grow.times(warp).times(shrink);
+
+            const distort = this._homography;
+            const undistort = distort.inverse();
+            const correctedWarp = distort.times(scaledWarp).times(undistort);
+
+            //console.log(Speedy.Matrix(warp).toString());
+            //console.log(Speedy.Matrix(scaledWarp).toString());
+            //console.log(Speedy.Matrix(correctedWarp).toString());
+
+            return correctedWarp;
+
+        });
+    }
+
+    /**
+     * Find matching pairs of two sets of keypoints matched via brute force
+     * @param srcKeypoints source (database)
+     * @param destKeypoints destination
+     * @returns an array of matching pairs [src, dest]
+     */
+    private _findMatchingPairs(srcKeypoints: SpeedyKeypoint[], destKeypoints: SpeedyMatchedKeypoint[]): ImageTrackerKeypointPair[]
+    {
+        const pairs: ImageTrackerKeypointPair[] = [];
+
+        for(let i = 0; i < destKeypoints.length; i++) {
+            const destKeypoint = destKeypoints[i];
+
+            if(destKeypoint.matches[0].index >= 0 && destKeypoint.matches[1].index >= 0) {
+                const d1 = destKeypoint.matches[0].distance;
+                const d2 = destKeypoint.matches[1].distance;
+
+                // the best match should be "much better" than the second best match,
+                // which means that they are "distinct enough"
+                if(d1 <= TRACK_MATCH_RATIO * d2) {
+                    const srcKeypoint = srcKeypoints[destKeypoint.matches[0].index];
+                    pairs.push([srcKeypoint, destKeypoint]);
+                }
+            }
+        }
+
+        return pairs;
+    }
+
+    /**
+     * Create & setup the pipeline
+     * @returns pipeline
+     */
+    protected _createPipeline(): SpeedyPipeline
+    {
+        const pipeline = Speedy.Pipeline();
+
+        const source = Speedy.Image.Source('source');
+        const imagePortalSource = Speedy.Image.Portal.Source('imagePortalSource');
+        const referenceKeypointPortalSource = Speedy.Keypoint.Portal.Source('referenceKeypointPortalSource');
+        const screen = Speedy.Transform.Resize('screen');
+        const greyscale = Speedy.Filter.Greyscale();
+        const imageRectifier = Speedy.Transform.PerspectiveWarp('imageRectifier');
+        const nightvision = Speedy.Filter.Nightvision();
+        const nightvisionMux = Speedy.Image.Multiplexer();
+        const detector = Speedy.Keypoint.Detector.Harris();
+        const descriptor = Speedy.Keypoint.Descriptor.ORB();
+        const blur = Speedy.Filter.GaussianBlur();
+        const clipper = Speedy.Keypoint.Clipper();
+        const borderClipper = Speedy.Keypoint.BorderClipper('borderClipper');
+        const denoiser = Speedy.Filter.GaussianBlur();
+        const subpixel = Speedy.Keypoint.SubpixelRefiner();
+        const matcher = Speedy.Keypoint.Matcher.BFKNN();
+        const keypointScaler = Speedy.Keypoint.Transformer('keypointScaler');
+        const keypointSink = Speedy.Keypoint.SinkOfMatchedKeypoints('keypoints');
+        const keypointPortalSink = Speedy.Keypoint.Portal.Sink('keypointPortalSink');
+        const referenceKeypointSink = Speedy.Keypoint.Sink('referenceKeypoints');
+        //const imageSink = Speedy.Image.Sink('image');
+
+        source.media = null;
+        imagePortalSource.source = null;
+        referenceKeypointPortalSource.source = null;
+        imageRectifier.transform = Speedy.Matrix.Eye(3);
+        screen.size = Speedy.Size(0,0);
+        nightvision.gain = NIGHTVISION_GAIN;
+        nightvision.offset = NIGHTVISION_OFFSET;
+        nightvision.decay = NIGHTVISION_DECAY;
+        nightvision.quality = NIGHTVISION_QUALITY;
+        nightvisionMux.port = TRACK_WITH_NIGHTVISION ? 1 : 0; // 1 = enable nightvision
+        blur.kernelSize = Speedy.Size(ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_KSIZE);
+        blur.sigma = Speedy.Vector2(ORB_GAUSSIAN_SIGMA, ORB_GAUSSIAN_SIGMA);
+        denoiser.kernelSize = Speedy.Size(SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_KSIZE);
+        denoiser.sigma = Speedy.Vector2(SUBPIXEL_GAUSSIAN_SIGMA, SUBPIXEL_GAUSSIAN_SIGMA);
+        detector.quality = TRACK_HARRIS_QUALITY;
+        detector.capacity = TRACK_DETECTOR_CAPACITY;
+        subpixel.method = SUBPIXEL_METHOD;
+        clipper.size = TRACK_MAX_KEYPOINTS;
+        borderClipper.imageSize = screen.size;
+        borderClipper.borderSize = Speedy.Vector2(0,0);
+        matcher.k = 2;
+        keypointScaler.transform = Speedy.Matrix.Eye(3);
+        keypointSink.turbo = false;
+
+        // prepare input
+        //source.output(); // ignore, but keep it in the pipeline
+        imagePortalSource.output().connectTo(screen.input());
+        screen.output().connectTo(greyscale.input());
+
+        // preprocess images
+        greyscale.output().connectTo(imageRectifier.input());
+        imageRectifier.output().connectTo(nightvisionMux.input('in0'));
+        imageRectifier.output().connectTo(nightvision.input());
+        nightvision.output().connectTo(nightvisionMux.input('in1'));
+
+        // keypoint detection & clipping
+        nightvisionMux.output().connectTo(detector.input());
+        detector.output().connectTo(borderClipper.input());
+        borderClipper.output().connectTo(clipper.input());
+
+        // keypoint refinement
+        imageRectifier.output().connectTo(denoiser.input());
+        denoiser.output().connectTo(subpixel.input('image'));
+        clipper.output().connectTo(subpixel.input('keypoints'));
+
+        // keypoint description
+        nightvisionMux.output().connectTo(blur.input());
+        blur.output().connectTo(descriptor.input('image'));
+        subpixel.output().connectTo(descriptor.input('keypoints'));
+
+        // keypoint matching
+        descriptor.output().connectTo(matcher.input('keypoints'));
+        referenceKeypointPortalSource.output().connectTo(matcher.input('database'));
+
+        // prepare output
+        descriptor.output().connectTo(keypointScaler.input());
+        keypointScaler.output().connectTo(keypointPortalSink.input());
+        keypointScaler.output().connectTo(keypointSink.input());
+        matcher.output().connectTo(keypointSink.input('matches'));
+        referenceKeypointPortalSource.output().connectTo(referenceKeypointSink.input());
+        //imageRectifier.output().connectTo(imageSink.input());
+
+        // done!
+        pipeline.init(
+            source, screen, imagePortalSource,
+            referenceKeypointPortalSource,
+            greyscale, imageRectifier,
+            nightvision, nightvisionMux,
+            detector, borderClipper, clipper,
+            denoiser, subpixel,
+            blur, descriptor,
+            matcher,
+            keypointScaler, keypointSink, keypointPortalSink, referenceKeypointSink,
+            //imageSink
+        );
+
+        return pipeline;
+    }
+}

src/trackers/image-tracker/states/pre-tracking.ts

@@ -1,517 +0,0 @@
-/*
- * encantar.js
- * GPU-accelerated Augmented Reality for the web
- * Copyright (C) 2022-2024 Alexandre Martins <alemartf(at)gmail.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Lesser General Public License as published
- * by the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Lesser General Public License for more details.
- *
- * You should have received a copy of the GNU Lesser General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- *
- * pre-tracking.ts
- * Pre-tracking state of the Image Tracker
- */
-
-import Speedy from 'speedy-vision';
-import { SpeedySize } from 'speedy-vision/types/core/speedy-size';
-import { SpeedyPoint2 } from 'speedy-vision/types/core/speedy-point';
-import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
-import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix';
-import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
-import { SpeedyPipeline, SpeedyPipelineOutput } from 'speedy-vision/types/core/pipeline/pipeline';
-import { SpeedyPipelineNodeImageSource } from 'speedy-vision/types/core/pipeline/nodes/images/source';
-import { SpeedyPipelineNodeImageMultiplexer } from 'speedy-vision/types/core/pipeline/nodes/images/multiplexer';
-import { SpeedyPipelineNodeImagePortalSource, SpeedyPipelineNodeImagePortalSink } from 'speedy-vision/types/core/pipeline/nodes/images/portal';
-import { SpeedyPipelineNodeKeypointPortalSource, SpeedyPipelineNodeKeypointPortalSink } from 'speedy-vision/types/core/pipeline/nodes/keypoints/portal';
-import { SpeedyPipelineNodeResize } from 'speedy-vision/types/core/pipeline/nodes/transforms/resize';
-import { SpeedyPipelineNodePerspectiveWarp } from 'speedy-vision/types/core/pipeline/nodes/transforms/perspective-warp';
-import { SpeedyPipelineNodeKeypointBorderClipper } from 'speedy-vision/types/core/pipeline/nodes/keypoints/border-clipper';
-import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/transformer';
-import { SpeedyPipelineNodeKeypointMultiplexer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/multiplexer';
-import { SpeedyPipelineNodeKeypointBuffer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/buffer';
-import { SpeedyPipelineNodeStaticLSHTables } from 'speedy-vision/types/core/pipeline/nodes/keypoints/matchers/lsh-static-tables';
-import { SpeedyKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
-import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
-import { ImageTrackerTrackingState } from './tracking';
-import { ImageTrackerState, ImageTrackerStateOutput } from './state';
-import { Nullable, Utils } from '../../../utils/utils';
-import { IllegalOperationError, TrackingError } from '../../../utils/errors';
-import { ReferenceImage } from '../reference-image';
-import {
-    TRACK_RECTIFIED_BORDER, TRACK_CLIPPING_BORDER, TRACK_REFINEMENT_ITERATIONS,
-    NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY, TRACK_WITH_NIGHTVISION,
-    ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_SIGMA,
-    TRACK_HARRIS_QUALITY, TRACK_DETECTOR_CAPACITY, TRACK_MAX_KEYPOINTS,
-    SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_SIGMA,
-    TRACK_RANSAC_REPROJECTIONERROR,
-    TRAIN_TARGET_NORMALIZED_SIZE,
-    TRACK_MATCH_RATIO,
-    NIGHTVISION_QUALITY,
-    SUBPIXEL_METHOD,
-} from '../settings';
-
-
-/** The pre-tracking follows a fixed sequence of steps */
-type PreTrackingStep = 'read-reference-image' | 'warp-camera-image' | 'train-camera-image';
-
-/** Default target space size (used when training) */
-const DEFAULT_TARGET_SPACE_SIZE = Speedy.Size(TRAIN_TARGET_NORMALIZED_SIZE, TRAIN_TARGET_NORMALIZED_SIZE);
-
-/** Use the camera stream as the input of the pipeline */
-const PORT_CAMERA_IMAGE = 1;
-
-/** Use the reference image as the input of the pipeline */
-const PORT_REFERENCE_IMAGE = 0;
-
-
-
-/**
- * The pre-tracking state of the Image Tracker is a new training
- * phase for the particular, actual target we'll be tracking
- */
-export class ImageTrackerPreTrackingState extends ImageTrackerState
-{
-    /** reference image */
-    private _referenceImage: Nullable<ReferenceImage>;
-
-    /** initial homography mapping the target image space to the AR screen space */
-    private _homography: SpeedyMatrix;
-
-    /** current step */
-    private _step: PreTrackingStep;
-
-    /** stored keypoints of the reference image */
-    private _referenceKeypoints: SpeedyKeypoint[];
-
-    /** current number of iterations for warp refinement */
-    private _iterations: number;
-
-
-
-    /**
-     * Constructor
-     * @param imageTracker
-     */
-    constructor(imageTracker: ImageTracker)
-    {
-        super('pre-tracking', imageTracker);
-
-        this._homography = Speedy.Matrix.Eye(3);
-        this._referenceImage = null;
-        this._step = 'read-reference-image';
-        this._referenceKeypoints = [];
-        this._iterations = 0;
-    }
-
-    /**
-     * Called as soon as this becomes the active state, just before update() runs for the first time
-     * @param settings
-     */
-    onEnterState(settings: Record<string,any>)
-    {
-        const imagePortalSource = this._pipeline.node('imagePortalSource') as SpeedyPipelineNodeImagePortalSource;
-        const muxOfReferenceKeypoints = this._pipeline.node('muxOfReferenceKeypoints') as SpeedyPipelineNodeKeypointMultiplexer;
-        const muxOfBufferOfReferenceKeypoints = this._pipeline.node('muxOfBufferOfReferenceKeypoints') as SpeedyPipelineNodeKeypointMultiplexer;
-        const bufferOfReferenceKeypoints = this._pipeline.node('bufferOfReferenceKeypoints') as SpeedyPipelineNodeKeypointBuffer;
-        const homography = settings.homography as SpeedyMatrix;
-        const referenceImage = settings.referenceImage as Nullable<ReferenceImage>;
-        const snapshot = settings.snapshot as SpeedyPipelineNodeImagePortalSink;
-
-        // this shouldn't happen
-        if(!referenceImage)
-            throw new TrackingError(`Can't track a null reference image`);
-
-        // set attributes
-        this._homography = homography;
-        this._referenceImage = referenceImage;
-        this._step = 'read-reference-image';
-        this._referenceKeypoints = [];
-        this._iterations = 0;
-
-        // setup the pipeline
-        imagePortalSource.source = snapshot;
-        muxOfReferenceKeypoints.port = 0;
-        muxOfBufferOfReferenceKeypoints.port = 0;
-        bufferOfReferenceKeypoints.frozen = false;
-    }
-
-    /**
-     * Called just before the GPU processing
-     * @returns promise
-     */
-    protected _beforeUpdate(): SpeedyPromise<void>
-    {
-        const referenceImage = this._referenceImage as ReferenceImage;
-        const source = this._pipeline.node('source') as SpeedyPipelineNodeImageSource;
-        const sourceMux = this._pipeline.node('sourceMux') as SpeedyPipelineNodeImageMultiplexer;
-        const imageRectifier = this._pipeline.node('imageRectifier') as SpeedyPipelineNodePerspectiveWarp;
-        const keypointRectifier = this._pipeline.node('keypointRectifier') as SpeedyPipelineNodeKeypointTransformer;
-        const borderClipper = this._pipeline.node('borderClipper') as SpeedyPipelineNodeKeypointBorderClipper;
-        const screenSize = this.screenSize;
-
-        // set the source media to the reference image we're going to track
-        const targetMedia = this._imageTracker.database._findMedia(referenceImage.name);
-        source.media = targetMedia;
-
-        // setup the source multiplexer
-        if(this._step == 'read-reference-image')
-            sourceMux.port = PORT_REFERENCE_IMAGE;
-        else
-            sourceMux.port = PORT_CAMERA_IMAGE;
-
-        // clip keypoints from the borders of the target image
-        borderClipper.imageSize = screenSize;
-        borderClipper.borderSize = Speedy.Vector2(
-            screenSize.width * TRACK_CLIPPING_BORDER,
-            screenSize.height * TRACK_CLIPPING_BORDER
-        );
-
-        // rectify the image
-        const rectify = (this._step == 'read-reference-image') ?
-            this._findRectificationMatrixOfFullscreenImage(targetMedia, screenSize) :
-            this._findRectificationMatrixOfCameraImage(this._homography, DEFAULT_TARGET_SPACE_SIZE, targetMedia, screenSize);
-
-        return rectify.then(rectificationMatrix => {
-            imageRectifier.transform = rectificationMatrix;
-        });
-    }
-
-    /**
-     * Post processing that takes place just after the GPU processing
-     * @param result pipeline results
-     * @returns state output
-     */
-    protected _afterUpdate(result: SpeedyPipelineOutput): SpeedyPromise<ImageTrackerStateOutput>
-    {
-        const referenceImage = this._referenceImage as ReferenceImage;
-        const imagePortalSink = this._pipeline.node('imagePortal') as SpeedyPipelineNodeImagePortalSink;
-        const keypointPortalSink = this._pipeline.node('keypointPortalSink') as SpeedyPipelineNodeKeypointPortalSink;
-        const muxOfReferenceKeypoints = this._pipeline.node('muxOfReferenceKeypoints') as SpeedyPipelineNodeKeypointMultiplexer;
-        const muxOfBufferOfReferenceKeypoints = this._pipeline.node('muxOfBufferOfReferenceKeypoints') as SpeedyPipelineNodeKeypointMultiplexer;
-        const bufferOfReferenceKeypoints = this._pipeline.node('bufferOfReferenceKeypoints') as SpeedyPipelineNodeKeypointBuffer;
-        const keypoints = result.keypoints as SpeedyMatchedKeypoint[];
-        const image = result.image as SpeedyMedia | undefined;
-
-        // tracker output
-        const trackerOutput: ImageTrackerOutput = {
-            keypoints: image !== undefined ? keypoints : undefined, // debug only
-            image: image,
-            screenSize: this.screenSize,
-        };
-
-        // decide what to do next
-        switch(this._step) {
-            case 'read-reference-image': {
-                // enable matching
-                muxOfReferenceKeypoints.port = 1;
-
-                // store reference keypoints
-                this._referenceKeypoints = keypoints;
-
-                // next step
-                this._step = 'warp-camera-image';
-                return Speedy.Promise.resolve({
-                    nextState: 'pre-tracking',
-                    trackerOutput: trackerOutput,
-                });
-            }
-
-            case 'warp-camera-image': {
-                // freeze reference keypoints
-                bufferOfReferenceKeypoints.frozen = true;
-                muxOfBufferOfReferenceKeypoints.port = 1;
-
-                // refine warp?
-                if(++this._iterations < TRACK_REFINEMENT_ITERATIONS)
-                    this._step = 'warp-camera-image';
-                else
-                    this._step = 'train-camera-image';
-
-                // warp image & go to next step
-                return this._findWarp(keypoints, this._referenceKeypoints).then(warp =>
-                    this._homography.setTo(this._homography.times(warp))
-                ).then(_ => ({
-                    nextState: 'pre-tracking',
-                    trackerOutput: trackerOutput,
-                })).catch(err => {
-                    Utils.warning(`Can't pre-track target image "${referenceImage.name}". ${err.toString()}`);
-                    return {
-                        nextState: 'scanning',
-                        trackerOutput: trackerOutput,
-                    };
-                });
-            }
-
-            case 'train-camera-image': {
-                // log
-                Utils.log(`Took a snapshot of target image "${referenceImage.name}". Found ${keypoints.length} keypoints.`);
-
-                // change the coordinates
-                return this._changeSpace(this._homography, this.screenSize).then(homography => {
-
-                    // we're ready to track the target!
-                    return Speedy.Promise.resolve({
-                        //nextState: 'pre-tracking',
-                        nextState: 'tracking',
-                        trackerOutput: trackerOutput,
-                        nextStateSettings: {
-                            homography: homography,
-                            referenceImage: referenceImage,
-                            templateKeypoints: keypoints,
-                            keypointPortalSink: keypointPortalSink,
-                            imagePortalSink: imagePortalSink,
-                            screenSize: this.screenSize,
-                        },
-                    });
-
-                });
-            }
-        }
-    }
-
-    /**
-     * Find an adjustment warp between the camera image and the reference image
-     * @param dstKeypoints destination
-     * @param srcKeypoints source
-     * @returns a promise that resolves to a 3x3 homography
-     */
-    private _findWarp(dstKeypoints: SpeedyMatchedKeypoint[], srcKeypoints: SpeedyKeypoint[]): SpeedyPromise<SpeedyMatrix>
-    {
-        //return Speedy.Promise.resolve(Speedy.Matrix.Eye(3));
-        const srcCoords: number[] = [];
-        const dstCoords: number[] = [];
-
-        // find matching coordinates of the keypoints
-        for(let i = 0; i < dstKeypoints.length; i++) {
-            const dstKeypoint = dstKeypoints[i];
-            if(dstKeypoint.matches[0].index >= 0 && dstKeypoint.matches[1].index >= 0) {
-                const d1 = dstKeypoint.matches[0].distance, d2 = dstKeypoint.matches[1].distance;
-
-                // the best match should be "much better" than the second best match,
-                // which means that they are "distinct enough"
-                if(d1 <= TRACK_MATCH_RATIO * d2) {
-                    const srcKeypoint = srcKeypoints[dstKeypoint.matches[0].index];
-                    srcCoords.push(srcKeypoint.x);
-                    srcCoords.push(srcKeypoint.y);
-                    dstCoords.push(dstKeypoint.x);
-                    dstCoords.push(dstKeypoint.y);
-                }
-            }
-        }
-
-        // too few points?
-        const n = srcCoords.length / 2;
-        if(n < 4) {
-            return Speedy.Promise.reject(
-                new TrackingError('Too few points to compute a warp')
-            );
-        }
-
-        // compute warp
-        const model = Speedy.Matrix.Eye(3);
-        return this._findKeypointWarp().then(transform =>
-
-            // rectify keypoints
-            Speedy.Matrix.applyAffineTransform(
-                Speedy.Matrix.Zeros(2, 2*n),
-                Speedy.Matrix(2, 2*n, srcCoords.concat(dstCoords)),
-                transform.block(0,1,0,2)
-            )
-
-        ).then(points =>
-
-            // find warp
-            Speedy.Matrix.findAffineTransform(
-                model.block(0,1,0,2),
-                points.block(0,1,0,n-1),
-                points.block(0,1,n,2*n-1), {
-                method: 'pransac',
-                reprojectionError: TRACK_RANSAC_REPROJECTIONERROR,
-                numberOfHypotheses: 512*4,
-                bundleSize: 128,
-            })
-
-        ).then(_ => {
-
-            // validate the model
-            const a00 = model.at(0,0);
-            if(Number.isNaN(a00))
-                throw new TrackingError(`Can't compute warp: bad keypoints`);
-
-            // done!
-            return model;
-
-        });
-    }
-
-    /**
-     * Find a warp to be applied to the keypoints
-     * @returns affine transform
-     */
-    private _findKeypointWarp(): SpeedyPromise<SpeedyMatrix>
-    {
-        const referenceImage = this._referenceImage as ReferenceImage;
-        const media = this._imageTracker.database._findMedia(referenceImage.name);
-        const screenSize = this.screenSize;
-
-        // no rotation is needed
-        if(!this._mustRotateWarpedImage(media, screenSize))
-            return Speedy.Promise.resolve(Speedy.Matrix.Eye(3));
-
-        // rotate by 90 degrees clockwise around the pivot
-        const px = screenSize.width / 2, py = screenSize.height / 2; // pivot
-        return Speedy.Promise.resolve(Speedy.Matrix(3, 3, [
-            0, 1, 0,
-            -1, 0, 0,
-            py+px, py-px, 1,
-        ]));
-    }
-
-    /**
-     * Change the space of the homography in order to improve tracking quality
-     * @param homography mapping coordinates from normalized target space to AR screen space
-     * @param screenSize AR screen size
-     * @returns homography mapping coordinates from AR screen space to AR screen space
-     */
-    private _changeSpace(homography: SpeedyMatrix, screenSize: SpeedySize): SpeedyPromise<SpeedyMatrix>
-    {
-        const sw = screenSize.width, sh = screenSize.height;
-        const screen = Speedy.Matrix(2, 4, [ 0, 0, sw, 0, sw, sh, 0, sh ]);
-
-        const mat = Speedy.Matrix.Zeros(3);
-        return this._findPolylineCoordinates(homography, DEFAULT_TARGET_SPACE_SIZE).then(polyline =>
-            Speedy.Matrix.perspective(mat, screen, polyline)
-        );
-    }
-
-    /**
-     * Create & setup the pipeline
-     * @returns pipeline
-     */
-    protected _createPipeline(): SpeedyPipeline
-    {
-        const pipeline = Speedy.Pipeline();
-
-        const source = Speedy.Image.Source('source');
-        const imagePortalSource = Speedy.Image.Portal.Source('imagePortalSource');
-        const sourceMux = Speedy.Image.Multiplexer('sourceMux');
-        const screen = Speedy.Transform.Resize('screen');
-        const greyscale = Speedy.Filter.Greyscale();
-        const imageRectifier = Speedy.Transform.PerspectiveWarp('imageRectifier');
-        const nightvision = Speedy.Filter.Nightvision();
-        const nightvisionMux = Speedy.Image.Multiplexer();
-        const detector = Speedy.Keypoint.Detector.Harris();
-        const descriptor = Speedy.Keypoint.Descriptor.ORB();
-        const blur = Speedy.Filter.GaussianBlur();
-        const clipper = Speedy.Keypoint.Clipper();
-        const borderClipper = Speedy.Keypoint.BorderClipper('borderClipper');
-        const denoiser = Speedy.Filter.GaussianBlur();
-        const subpixel = Speedy.Keypoint.SubpixelRefiner();
-        const matcher = Speedy.Keypoint.Matcher.BFKNN();
-        const keypointRectifier = Speedy.Keypoint.Transformer('keypointRectifier');
-        const keypointPortalSink = Speedy.Keypoint.Portal.Sink('keypointPortalSink');
-        const keypointPortalSource = Speedy.Keypoint.Portal.Source('keypointPortalSource');
-        const muxOfReferenceKeypoints = Speedy.Keypoint.Multiplexer('muxOfReferenceKeypoints');
-        const bufferOfReferenceKeypoints = Speedy.Keypoint.Buffer('bufferOfReferenceKeypoints');
-        const muxOfBufferOfReferenceKeypoints = Speedy.Keypoint.Multiplexer('muxOfBufferOfReferenceKeypoints');
-        const keypointSink = Speedy.Keypoint.SinkOfMatchedKeypoints('keypoints');
-        const imageSink = Speedy.Image.Sink('image');
-
-        source.media = null;
-        screen.size = Speedy.Size(0,0);
-        imagePortalSource.source = null;
-        imageRectifier.transform = Speedy.Matrix.Eye(3);
-        sourceMux.port = PORT_REFERENCE_IMAGE;
-        nightvision.gain = NIGHTVISION_GAIN;
-        nightvision.offset = NIGHTVISION_OFFSET;
-        nightvision.decay = NIGHTVISION_DECAY;
-        nightvision.quality = NIGHTVISION_QUALITY;
-        nightvisionMux.port = TRACK_WITH_NIGHTVISION ? 1 : 0; // 1 = enable nightvision
-        blur.kernelSize = Speedy.Size(ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_KSIZE);
-        blur.sigma = Speedy.Vector2(ORB_GAUSSIAN_SIGMA, ORB_GAUSSIAN_SIGMA);
-        denoiser.kernelSize = Speedy.Size(SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_KSIZE);
-        denoiser.sigma = Speedy.Vector2(SUBPIXEL_GAUSSIAN_SIGMA, SUBPIXEL_GAUSSIAN_SIGMA);
-        detector.quality = TRACK_HARRIS_QUALITY;
-        detector.capacity = TRACK_DETECTOR_CAPACITY;
-        subpixel.method = SUBPIXEL_METHOD;
-        clipper.size = TRACK_MAX_KEYPOINTS;
-        borderClipper.imageSize = screen.size;
-        borderClipper.borderSize = Speedy.Vector2(0,0);
-        matcher.k = 2;
-        keypointRectifier.transform = Speedy.Matrix.Eye(3);
-        keypointPortalSource.source = keypointPortalSink;
-        muxOfReferenceKeypoints.port = 0;
-        muxOfBufferOfReferenceKeypoints.port = 0;
-        bufferOfReferenceKeypoints.frozen = false;
-        keypointSink.turbo = false;
-
-        // prepare input
-        source.output().connectTo(sourceMux.input('in0')); // port 0: reference image
-        imagePortalSource.output().connectTo(sourceMux.input('in1')); // port 1: camera image (via portal)
-        sourceMux.output().connectTo(screen.input());
-        screen.output().connectTo(greyscale.input());
-
-        // preprocess images
-        greyscale.output().connectTo(imageRectifier.input());
-        imageRectifier.output().connectTo(nightvisionMux.input('in0'));
-        imageRectifier.output().connectTo(nightvision.input());
-        nightvision.output().connectTo(nightvisionMux.input('in1'));
-        nightvisionMux.output().connectTo(blur.input());
-
-        // keypoint detection & clipping
-        nightvisionMux.output().connectTo(detector.input());
-        detector.output().connectTo(borderClipper.input());
-        borderClipper.output().connectTo(clipper.input());
-
-        // keypoint refinement
-        imageRectifier.output().connectTo(denoiser.input());
-        denoiser.output().connectTo(subpixel.input('image'));
-        clipper.output().connectTo(subpixel.input('keypoints'));
-
-        // keypoint description
-        blur.output().connectTo(descriptor.input('image'));
-        subpixel.output().connectTo(descriptor.input('keypoints'));
-
-        // keypoint matching
-        descriptor.output().connectTo(muxOfReferenceKeypoints.input('in0'));
-        muxOfBufferOfReferenceKeypoints.output().connectTo(muxOfReferenceKeypoints.input('in1'));
-        muxOfReferenceKeypoints.output().connectTo(matcher.input('database'));
-        descriptor.output().connectTo(matcher.input('keypoints'));
-
-        // store reference keypoints
-        keypointPortalSource.output().connectTo(muxOfBufferOfReferenceKeypoints.input('in0'));
-        bufferOfReferenceKeypoints.output().connectTo(muxOfBufferOfReferenceKeypoints.input('in1'));
-        keypointPortalSource.output().connectTo(bufferOfReferenceKeypoints.input());
-
-        // portals
-        descriptor.output().connectTo(keypointPortalSink.input());
-
-        // prepare output
-        descriptor.output().connectTo(keypointRectifier.input());
-        keypointRectifier.output().connectTo(keypointSink.input());
-        matcher.output().connectTo(keypointSink.input('matches'));
-        //imageRectifier.output().connectTo(imageSink.input());
-
-        // done!
-        pipeline.init(
-            source, imagePortalSource, sourceMux, screen,
-            greyscale, imageRectifier, nightvision, nightvisionMux, blur,
-            detector, subpixel, clipper, borderClipper, denoiser, descriptor,
-            keypointPortalSource, muxOfReferenceKeypoints, matcher,
-            bufferOfReferenceKeypoints, muxOfBufferOfReferenceKeypoints,
-            keypointRectifier, keypointSink,
-            keypointPortalSink,
-            //imageSink
-        );
-
-        return pipeline;
-    }
-}

src/trackers/image-tracker/states/scanning.ts

@@ -37,25 +37,21 @@ import { SpeedyPipelineNodeImagePortalSource, SpeedyPipelineNodeImagePortalSink
 import { SpeedyPipelineNodeStaticLSHTables } from 'speedy-vision/types/core/pipeline/nodes/keypoints/matchers/lsh-static-tables';
 import { SpeedyKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
 import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
 import { ImageTrackerState, ImageTrackerStateOutput } from './state';
-import { ImageTrackerPreTrackingState } from './pre-tracking';
 import { Nullable, Utils } from '../../../utils/utils';
-import { IllegalOperationError, IllegalArgumentError, DetectionError } from '../../../utils/errors';
+import { DetectionError } from '../../../utils/errors';
 import { 
     SCAN_MATCH_RATIO, SCAN_MIN_MATCHES, SCAN_CONSECUTIVE_FRAMES,
     ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_SIGMA,
     NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY,
     SCAN_WITH_NIGHTVISION, SCAN_PYRAMID_LEVELS, SCAN_PYRAMID_SCALEFACTOR,
     SCAN_FAST_THRESHOLD, SCAN_MAX_KEYPOINTS, SCAN_LSH_TABLES, SCAN_LSH_HASHSIZE,
-    SCAN_RANSAC_REPROJECTIONERROR,
-    TRAIN_TARGET_NORMALIZED_SIZE,
+    SCAN_RANSAC_REPROJECTIONERROR_NDC,
     NIGHTVISION_QUALITY,
 } from '../settings';
 
 
-/** Default target space size (used when training) */
-const DEFAULT_TARGET_SPACE_SIZE = Speedy.Size(TRAIN_TARGET_NORMALIZED_SIZE, TRAIN_TARGET_NORMALIZED_SIZE);
-
 /** Port of the portal multiplexer: get new data from the camera */
 const PORT_CAMERA = 0;
 
@@ -65,7 +61,7 @@ const PORT_MEMORY = 1;
 
 
 /**
- * Scanning state of the Image Tracker
+ * In the scanning state we look for a reference image in the video
  */
 export class ImageTrackerScanningState extends ImageTrackerState
 {
@@ -101,7 +97,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
     {
         const imagePortalMux = this._pipeline.node('imagePortalMux') as SpeedyPipelineNodeImageMultiplexer;
         const lshTables = this._pipeline.node('lshTables') as SpeedyPipelineNodeStaticLSHTables;
-        const keypoints = settings.keypoints as SpeedyKeypoint[] | undefined;
+        const database = settings.database as SpeedyKeypoint[] | undefined;
 
         // set attributes
         this._counter = 0;
@@ -111,8 +107,24 @@ export class ImageTrackerScanningState extends ImageTrackerState
         imagePortalMux.port = PORT_CAMERA;
 
         // prepare the keypoint matcher
-        if(keypoints !== undefined)
-            lshTables.keypoints = keypoints;
+        if(database !== undefined)
+            lshTables.keypoints = database;
+    }
+
+    /**
+     * Called just before the GPU processing
+     * @returns promise
+     */
+    protected _beforeUpdate(): SpeedyPromise<void>
+    {
+        const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
+        const screenSize = this.screenSize;
+
+        // convert keypoints to NIS
+        keypointScaler.transform = ImageTrackerUtils.rasterToNIS(screenSize);
+
+        // done!
+        return Speedy.Promise.resolve();
     }
 
     /**
@@ -124,116 +136,123 @@ export class ImageTrackerScanningState extends ImageTrackerState
     {
         const imagePortalMux = this._pipeline.node('imagePortalMux') as SpeedyPipelineNodeImageMultiplexer;
         const keypoints = result.keypoints as SpeedyMatchedKeypoint[];
-        const matchedKeypoints = this._goodMatches(keypoints);
+        const image = result.image as SpeedyMedia | undefined;
 
         // tracker output
         const trackerOutput: ImageTrackerOutput = {
-            keypoints: keypoints,
-            screenSize: this.screenSize
+            keypointsNIS: keypoints,
+            polylineNDC: [],
+            image: image,
         };
 
         // keep the last memorized image
         imagePortalMux.port = PORT_MEMORY;
 
-        // have we found enough matches...?
-        if(matchedKeypoints.length >= SCAN_MIN_MATCHES) {
-            return this._findHomography(matchedKeypoints).then(([homography, score]) => {
+        // find high quality matches
+        const matchedKeypoints = this._selectGoodMatches(keypoints);
+        if(matchedKeypoints.length < SCAN_MIN_MATCHES) {
+
+            // not enough high quality matches?
+            // we'll continue to scan the scene
+            this._counter = 0;
+            this._bestScore = 0;
 
-                // have we found the best homography so far?
-                if(score >= this._bestScore) {
-                    // store it only if we'll be running the pipeline again
-                    if(this._counter < SCAN_CONSECUTIVE_FRAMES - 1) {
-                        this._bestScore = score;
-                        this._bestHomography = homography;
+            return Speedy.Promise.resolve({
+                nextState: 'scanning',
+                trackerOutput: trackerOutput,
+            });
 
-                        // memorize the last image, corresponding to the best homography(*)
-                        imagePortalMux.port = PORT_CAMERA;
+        }
 
-                        /*
+        // we have enough high quality matches!
+        const pairs = this._findMatchingPairs(matchedKeypoints);
+        const points = ImageTrackerUtils.compilePairsOfKeypointsNDC(pairs);
 
-                        (*) technically speaking, this is not exactly the case. Since we're
-                            using turbo to download the keypoints, there's a slight difference
-                            between the data used to compute the homography and the last image.
-                            Still, assuming continuity of the video stream, this logic is
-                            good enough.
+        // find a homography
+        return this._findHomographyNDC(points).then(([homography, score]) => {
 
-                        */
-                    }
-                }
+            // have we found the best homography so far?
+            if(score >= this._bestScore) {
 
-                // find a polyline surrounding the target
-                return this._findPolyline(homography, DEFAULT_TARGET_SPACE_SIZE);
+                // store it only if we'll be running the pipeline again
+                if(this._counter < SCAN_CONSECUTIVE_FRAMES - 1) {
+                    this._bestScore = score;
+                    this._bestHomography = homography;
 
-            }).then(polyline => {
+                    // memorize the last image, corresponding to the best homography(*)
+                    imagePortalMux.port = PORT_CAMERA;
 
-                // continue a little longer in the scanning state
-                if(++this._counter < SCAN_CONSECUTIVE_FRAMES) {
-                    return {
-                        nextState: this.name,
-                        trackerOutput: {
-                            polyline: polyline,
-                            ...trackerOutput,
-                        },
-                    };
-                }
+                    /*
 
-                // this image should correspond to the best homography
-                const snapshot = this._pipeline.node('imagePortalSink') as SpeedyPipelineNodeImagePortalSink;
+                    (*) technically speaking, this is not exactly the case. Since we're
+                        using turbo to download the keypoints, there's a slight difference
+                        between the data used to compute the homography and the last image.
+                        Still, assuming continuity of the video stream, this logic is
+                        good enough.
 
-                // the reference image that we'll track
-                const referenceImage = this._imageTracker._referenceImageOfKeypoint(
-                    matchedKeypoints[0].matches[0].index
-                );
+                    */
+                }
 
-                // let's track the target!
-                return {
-                    nextState: 'pre-tracking',
-                    nextStateSettings: {
-                        homography: this._bestHomography,
-                        snapshot: snapshot,
-                        referenceImage: referenceImage,
-                    },
-                    trackerOutput: {
-                        polyline: polyline,
-                        ...trackerOutput,
-                    },
-                };
+            }
 
-            }).catch(() => {
+            // find a polyline surrounding the target
+            const polylineNDC = ImageTrackerUtils.findPolylineNDC(homography);
+            trackerOutput.polylineNDC!.push(...polylineNDC);
 
-                // continue in the scanning state
+            // continue a little longer in the scanning state
+            if(++this._counter < SCAN_CONSECUTIVE_FRAMES) {
                 return {
-                    nextState: this.name,
-                    trackerOutput: trackerOutput,
+                    nextState: 'scanning',
+                    trackerOutput: trackerOutput
                 };
+            }
 
-            });
-        }
-        else {
+            // this image should correspond to the best homography
+            const snapshot = this._pipeline.node('imagePortalSink') as SpeedyPipelineNodeImagePortalSink;
 
-            // not enough matches...!
-            this._counter = 0;
-            this._bestScore = 0;
+            // the reference image that we'll track
+            const referenceImage = this._imageTracker._referenceImageOfKeypoint(
+                matchedKeypoints[0].matches[0].index
+            );
 
-        }
+            // this shouldn't happen
+            if(!referenceImage)
+                throw new DetectionError(`Can't track an unknown reference image`);
+
+            // let's track the target!
+            return {
+                nextState: 'pre-tracking-a',
+                nextStateSettings: {
+                    homography: this._bestHomography,
+                    snapshot: snapshot,
+                    referenceImage: referenceImage,
+                },
+                trackerOutput: trackerOutput
+            };
+
+        })
+        .catch(err => {
+
+            // continue in the scanning state
+            Utils.warning(`Error when scanning: ${err.toString()}`)
+            return {
+                nextState: 'scanning',
+                trackerOutput: trackerOutput,
+            };
 
-        // we'll continue to scan the scene
-        return Speedy.Promise.resolve({
-            nextState: this.name,
-            trackerOutput: trackerOutput,
         });
     }
 
     /**
-     * Find "high quality" matches of a single reference image
-     * @param keypoints
-     * @returns high quality matches
+     * Select high quality matches of a single reference image
+     * @param keypoints matched keypoints of any quality, to any reference image
+     * @returns high quality matches of a single reference image
      */
-    private _goodMatches(keypoints: SpeedyMatchedKeypoint[]): SpeedyMatchedKeypoint[]
+    private _selectGoodMatches(keypoints: SpeedyMatchedKeypoint[]): SpeedyMatchedKeypoint[]
     {
         const matchedKeypointsPerImageIndex: Record<number,SpeedyMatchedKeypoint[]> = Object.create(null);
 
-        // filter "good matches"
+        // find high quality matches, regardless of reference image
         for(let j = keypoints.length - 1; j >= 0; j--) {
             const keypoint = keypoints[j];
             if(keypoint.matches[0].index >= 0 && keypoint.matches[1].index >= 0) {
@@ -255,7 +274,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
             }
         }
 
-        // find the image with the most matches
+        // find the reference image with the most high quality matches
         let matchedKeypoints: SpeedyMatchedKeypoint[] = [];
         for(const imageIndex in matchedKeypointsPerImageIndex) {
             if(matchedKeypointsPerImageIndex[imageIndex].length > matchedKeypoints.length)
@@ -267,71 +286,41 @@ export class ImageTrackerScanningState extends ImageTrackerState
     }
 
     /**
-     * Find a homography matrix using matched keypoints
-     * @param matchedKeypoints "good" matches only
-     * @returns homography from reference image space to AR screen space & homography "quality" score
+     * Find a homography matrix using matched keypoints in NDC
+     * @param points compiled pairs of keypoints in NDC
+     * @returns homography (from reference to matched, NDC) & "quality" score
      */
-    private _findHomography(matchedKeypoints: SpeedyMatchedKeypoint[]): SpeedyPromise<[SpeedyMatrix,number]>
+    private _findHomographyNDC(points: SpeedyMatrix): SpeedyPromise<[SpeedyMatrix,number]>
     {
-        const srcCoords: number[] = [];
-        const dstCoords: number[] = [];
-
-        // find matching coordinates of the keypoints
-        for(let i = matchedKeypoints.length - 1; i >= 0; i--) {
-            const matchedKeypoint = matchedKeypoints[i];
-            const referenceKeypoint = this._imageTracker._referenceKeypoint(matchedKeypoint.matches[0].index);
-            if(referenceKeypoint != null) {
-                srcCoords.push(referenceKeypoint.x);
-                srcCoords.push(referenceKeypoint.y);
-                dstCoords.push(matchedKeypoint.x);
-                dstCoords.push(matchedKeypoint.y);
-            }
-            else {
-                // this shouldn't happen
-                return Speedy.Promise.reject(
-                    new DetectionError(`Invalid keypoint match index: ${matchedKeypoint.matches[0].index} from ${matchedKeypoint.toString()}`)
-                );
-            }
-        }
-
-        // too few points?
-        const n = srcCoords.length / 2;
-        if(n < 4) {
-            return Speedy.Promise.reject(
-                new DetectionError(`Too few points to compute a homography`)
-            );
-        }
-
-        // compute a homography
-        const src = Speedy.Matrix(2, n, srcCoords);
-        const dst = Speedy.Matrix(2, n, dstCoords);
-        const mask = Speedy.Matrix.Zeros(1, n);
-
-        const homography = Speedy.Matrix.Zeros(3);
-        return Speedy.Matrix.findHomography(homography, src, dst, {
+        return ImageTrackerUtils.findPerspectiveWarpNDC(points, {
             method: 'pransac',
-            reprojectionError: SCAN_RANSAC_REPROJECTIONERROR,
+            reprojectionError: SCAN_RANSAC_REPROJECTIONERROR_NDC,
             numberOfHypotheses: 512,
             bundleSize: 128,
-            mask: mask,
-        }).then(homography => {
+        });
+    }
 
-            // check if this is a valid homography
-            const a00 = homography.at(0,0);
-            if(Number.isNaN(a00))
-                throw new DetectionError(`Can't compute homography`);
+    /**
+     * Find matching pairs of keypoints from reference image (src) to matched image (dest)
+     * @param matchedKeypoints
+     * @returns an array of matching pairs [src, dest]
+     */
+    private _findMatchingPairs(matchedKeypoints: SpeedyMatchedKeypoint[]): ImageTrackerKeypointPair[]
+    {
+        const pairs = new Array<ImageTrackerKeypointPair>(matchedKeypoints.length);
 
-            // count the number of inliers
-            const inliers = mask.read();
-            let inlierCount = 0;
-            for(let i = inliers.length - 1; i >= 0; i--)
-                inlierCount += inliers[i];
-            const score = inlierCount / inliers.length;
+        for(let i = matchedKeypoints.length - 1; i >= 0; i--) {
+            const matchedKeypoint = matchedKeypoints[i];
+            const referenceKeypoint = this._imageTracker._referenceKeypoint(matchedKeypoint.matches[0].index);
 
-            // done!
-            return [ homography, score ];
+            // this shouldn't happen
+            if(referenceKeypoint == null)
+                throw new DetectionError(`Invalid keypoint match index: ${matchedKeypoint.matches[0].index} from ${matchedKeypoint.toString()}`);
 
-        });
+            pairs[i] = [ referenceKeypoint, matchedKeypoint ];
+        }
+
+        return pairs;
     }
 
     /**
@@ -354,6 +343,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
         const clipper = Speedy.Keypoint.Clipper();
         const lshTables = Speedy.Keypoint.Matcher.StaticLSHTables('lshTables');
         const knn = Speedy.Keypoint.Matcher.LSHKNN();
+        const keypointScaler = Speedy.Keypoint.Transformer('keypointScaler');
         const keypointSink = Speedy.Keypoint.SinkOfMatchedKeypoints('keypoints');
         const imagePortalSink = Speedy.Image.Portal.Sink('imagePortalSink');
         const imagePortalSource = Speedy.Image.Portal.Source('imagePortalSource');
@@ -386,6 +376,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
         imagePortalMux.port = PORT_CAMERA; // 0 = camera stream; 1 = lock image
         imagePortalCopy.size = Speedy.Size(0,0);
         imagePortalCopy.scale = Speedy.Vector2(1,1);
+        keypointScaler.transform = Speedy.Matrix.Eye(3);
         keypointSink.turbo = true;
 
         // prepare input
@@ -412,7 +403,8 @@ export class ImageTrackerScanningState extends ImageTrackerState
         lshTables.output().connectTo(knn.input('lsh'));
 
         // prepare output
-        clipper.output().connectTo(keypointSink.input());
+        clipper.output().connectTo(keypointScaler.input());
+        keypointScaler.output().connectTo(keypointSink.input());
         knn.output().connectTo(keypointSink.input('matches'));
         //pyramid.output().connectTo(imageSink.input());
 
@@ -429,7 +421,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
             greyscale, blur, nightvision, nightvisionMux, pyramid,
             detector, descriptor, clipper,
             lshTables, knn,
-            keypointSink,
+            keypointScaler, keypointSink,
             imagePortalSink, imagePortalSource,
             imagePortalMux, imagePortalBuffer, imagePortalCopy,
             //, imageSink

src/trackers/image-tracker/states/state.ts

@@ -33,10 +33,10 @@ import { SpeedyPipelineNodeResize } from 'speedy-vision/types/core/pipeline/node
 import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/transformer';
 import { SpeedyKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
 import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ReferenceImage } from '../reference-image';
 import { TrackerOutput } from '../../tracker';
 import { Nullable } from '../../../utils/utils';
-import { IllegalOperationError } from '../../../utils/errors';
-import { TRACK_RECTIFIED_BORDER } from '../settings';
+import { IllegalOperationError, IllegalArgumentError } from '../../../utils/errors';
 
 /** State output */
 export interface ImageTrackerStateOutput
@@ -46,7 +46,6 @@ export interface ImageTrackerStateOutput
     readonly nextStateSettings?: Record<string,any>;
 }
 
-
 /**
  * Abstract state of the Image Tracker
  */
@@ -61,6 +60,9 @@ export abstract class ImageTrackerState
     /** pipeline */
     protected _pipeline: SpeedyPipeline;
 
+    /** a flag telling whether or not the pipeline has been released */
+    protected _pipelineReleased: boolean;
+
 
     /**
      * Constructor
@@ -72,6 +74,7 @@ export abstract class ImageTrackerState
         this._name = name;
         this._imageTracker = imageTracker;
         this._pipeline = this._createPipeline();
+        this._pipelineReleased = false;
     }
 
     /**
@@ -84,6 +87,7 @@ export abstract class ImageTrackerState
 
     /**
      * AR screen size
+     * It may change over time, as when flipping a phone
      */
     get screenSize(): SpeedySize
     {
@@ -107,7 +111,12 @@ export abstract class ImageTrackerState
      */
     release(): null
     {
-        return this._pipeline.release();
+        if(!this._pipelineReleased) {
+            this._pipeline.release();
+            this._pipelineReleased = true;
+        }
+
+        return null;
     }
 
     /**
@@ -183,154 +192,4 @@ export abstract class ImageTrackerState
      * @returns pipeline
      */
     protected abstract _createPipeline(): SpeedyPipeline;
-
-
-
-    //
-    // Some utility methods common to various states
-    //
-
-    /**
-     * Find the coordinates of a polyline surrounding the target image
-     * @param homography maps the target image to the AR screen
-     * @param targetSize size of the target space
-     * @returns promise that resolves to 4 points in AR screen space
-     */
-    protected _findPolylineCoordinates(homography: SpeedyMatrix, targetSize: SpeedySize): SpeedyPromise<SpeedyMatrix>
-    {
-        const w = targetSize.width, h = targetSize.height;
-        const referenceImageCoordinates = Speedy.Matrix(2, 4, [
-            0, 0,
-            w, 0,
-            w, h,
-            0, h,
-        ]);
-
-        const polylineCoordinates = Speedy.Matrix.Zeros(2, 4);
-        return Speedy.Matrix.applyPerspectiveTransform(
-            polylineCoordinates,
-            referenceImageCoordinates,
-            homography
-        );
-    }
-
-    /**
-     * Find a polyline surrounding the target image
-     * @param homography maps the target image to the AR screen
-     * @param targetSize size of the target space
-     * @returns promise that resolves to 4 points in AR screen space
-     */
-    protected _findPolyline(homography: SpeedyMatrix, targetSize: SpeedySize): SpeedyPromise<SpeedyPoint2[]>
-    {
-        return this._findPolylineCoordinates(homography, targetSize).then(polylineCoordinates => {
-            const polydata = polylineCoordinates.read();
-            const polyline = Array.from({ length: 4 }, (_, i) => Speedy.Point2(polydata[2*i], polydata[2*i+1]));
-
-            return polyline;
-        });
-    }
-
-    /**
-     * Whether or not to rotate the warped image in order to best fit the AR screen
-     * @param media media associated with the reference image
-     * @param screenSize AR screen
-     * @returns boolean
-     */
-    protected _mustRotateWarpedImage(media: SpeedyMedia, screenSize: SpeedySize): boolean
-    {
-        const screenAspectRatio = screenSize.width / screenSize.height;
-        const mediaAspectRatio = media.width / media.height;
-        const eps = 0.1;
-
-        return (mediaAspectRatio >= 1+eps && screenAspectRatio < 1-eps) || (mediaAspectRatio < 1-eps && screenAspectRatio >= 1+eps);
-    }
-
-    /**
-     * Find a rectification matrix to be applied to an image fitting the entire AR screen
-     * @param media media associated with the reference image
-     * @param screenSize AR screen
-     * @returns promise that resolves to a rectification matrix
-     */
-    protected _findRectificationMatrixOfFullscreenImage(media: SpeedyMedia, screenSize: SpeedySize): SpeedyPromise<SpeedyMatrix>
-    {
-        const b = TRACK_RECTIFIED_BORDER;
-        const sw = screenSize.width, sh = screenSize.height;
-        const mediaAspectRatio = media.width / media.height;
-        const mustRotate = this._mustRotateWarpedImage(media, screenSize);
-
-        // compute the vertices of the target in screen space
-        // we suppose portrait or landscape mode for both screen & media
-        const c = mustRotate ? 1 / mediaAspectRatio : mediaAspectRatio;
-        const top = sw >= sh ? b * sh : (sh - sw * (1-2*b) / c) / 2;
-        const left = sw >= sh ? (sw - sh * (1-2*b) * c) / 2 : b * sw;
-        const right = sw - left;
-        const bottom = sh - top;
-
-        const targetVertices = Speedy.Matrix(2, 4, [
-            left, top,
-            right, top,
-            right, bottom,
-            left, bottom,
-        ]);
-
-        const screenVertices = Speedy.Matrix(2, 4, [
-            0, 0,
-            sw, 0,
-            sw, sh,
-            0, sh
-        ]);
-
-        const preRectificationMatrix = Speedy.Matrix.Eye(3);
-        const alignmentMatrix = Speedy.Matrix.Zeros(3);
-        const rectificationMatrix = Speedy.Matrix.Zeros(3);
-
-        return (mustRotate ? Speedy.Matrix.perspective(
-            // pre-rectifation: rotate by 90 degrees counterclockwise and scale to screenSize
-            preRectificationMatrix,
-            screenVertices,
-            Speedy.Matrix(2, 4, [ 0,sh , 0,0 , sw,0 , sw,sh ])
-        ) : Speedy.Promise.resolve(preRectificationMatrix)).then(_ =>
-            // alignment: align the target to the center of the screen
-            Speedy.Matrix.perspective(
-                alignmentMatrix,
-                screenVertices,
-                targetVertices
-            )
-        ).then(_ =>
-            // pre-rectify and then align
-            rectificationMatrix.setTo(alignmentMatrix.times(preRectificationMatrix))
-        );
-    }
-
-    /**
-     * Find a rectification matrix to be applied to the target image
-     * @param homography maps a reference image to the AR screen
-     * @param targetSize size of the target space
-     * @param media media associated with the reference image
-     * @param screenSize AR screen
-     * @returns promise that resolves to a rectification matrix
-     */
-    protected _findRectificationMatrixOfCameraImage(homography: SpeedyMatrix, targetSize: SpeedySize, media: SpeedyMedia, screenSize: SpeedySize): SpeedyPromise<SpeedyMatrix>
-    {
-        const sw = screenSize.width, sh = screenSize.height;
-        const screen = Speedy.Matrix(2, 4, [ 0, 0, sw, 0, sw, sh, 0, sh ]);
-
-        const rectificationMatrix = Speedy.Matrix.Zeros(3);
-        return this._findPolylineCoordinates(homography, targetSize).then(polyline =>
-
-            // from target space to (full)screen
-            Speedy.Matrix.perspective(rectificationMatrix, polyline, screen)
-
-        ).then(_ =>
-
-            // from (full)screen to rectified coordinates
-            this._findRectificationMatrixOfFullscreenImage(media, screenSize)
-
-        ).then(mat =>
-
-            // function composition
-            rectificationMatrix.setTo(mat.times(rectificationMatrix))
-
-        );
-    }
-}
+}

src/trackers/image-tracker/states/training.ts

@@ -31,9 +31,9 @@ import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/
 import { SpeedyKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
 import { Resolution } from '../../../utils/resolution';
 import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
 import { ImageTrackerState, ImageTrackerStateOutput } from './state';
-import { ReferenceImage } from '../reference-image';
-import { ReferenceImageDatabase } from '../reference-image-database';
+import { ReferenceImage, ReferenceImageWithMedia } from '../reference-image';
 import { Nullable, Utils } from '../../../utils/utils';
 import { IllegalOperationError, TrainingError } from '../../../utils/errors';
 import {
@@ -43,7 +43,6 @@ import {
     SCAN_WITH_NIGHTVISION, NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY,
     SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_SIGMA,
     TRAIN_IMAGE_SCALE,
-    TRAIN_TARGET_NORMALIZED_SIZE,
     NIGHTVISION_QUALITY,
     SUBPIXEL_METHOD,
 } from '../settings';
@@ -53,14 +52,14 @@ import {
 /** The training map maps keypoints to reference images */
 interface TrainingMap
 {
+    /** the collection of all keypoints (of all images) */
+    readonly keypoints: SpeedyKeypoint[];
+
     /** maps a keypoint index to an image index */
     readonly referenceImageIndex: number[];
 
-    /** maps an image index to a reference image */
-    readonly referenceImage: ReferenceImage[];
-
-    /** the collection of all keypoints (of all images) */
-    readonly keypoints: SpeedyKeypoint[];
+    /** reference images */
+    readonly referenceImages: ReferenceImageWithMedia[];
 }
 
 
@@ -70,8 +69,10 @@ interface TrainingMap
  */
 export class ImageTrackerTrainingState extends ImageTrackerState
 {
+    /** index of the image being used to train the tracker */
     private _currentImageIndex = 0;
-    private _image: ReferenceImage[] = [];
+
+    /** training map */
     private _trainingMap: TrainingMap;
 
 
@@ -86,9 +87,9 @@ export class ImageTrackerTrainingState extends ImageTrackerState
 
         // initialize the training map
         this._trainingMap = {
+            keypoints: [],
             referenceImageIndex: [],
-            referenceImage: [],
-            keypoints: []
+            referenceImages: [],
         };
     }
 
@@ -106,10 +107,9 @@ export class ImageTrackerTrainingState extends ImageTrackerState
 
         // prepare to train...
         this._currentImageIndex = 0;
-        this._image.length = 0;
-        this._trainingMap.referenceImageIndex.length = 0;
-        this._trainingMap.referenceImage.length = 0;
         this._trainingMap.keypoints.length = 0;
+        this._trainingMap.referenceImageIndex.length = 0;
+        this._trainingMap.referenceImages.length = 0;
 
         // lock the database
         Utils.log(`Image Tracker: training using ${database.count} reference image${database.count != 1 ? 's' : ''}`);
@@ -117,7 +117,17 @@ export class ImageTrackerTrainingState extends ImageTrackerState
 
         // collect all images
         for(const referenceImage of database)
-            this._image.push(referenceImage);
+            this._trainingMap.referenceImages.push(referenceImage);
+    }
+
+    /**
+     * Called when leaving the state, after update()
+     */
+    onLeaveState(): void
+    {
+        // we don't return to this state, so we can release the pipeline early
+        this._pipeline.release();
+        this._pipelineReleased = true;
     }
 
     /**
@@ -126,66 +136,25 @@ export class ImageTrackerTrainingState extends ImageTrackerState
      */
     protected _beforeUpdate(): SpeedyPromise<void>
     {
-        const arScreenSize = this.screenSize;
         const source = this._pipeline.node('source') as SpeedyPipelineNodeImageSource;
         const screen = this._pipeline.node('screen') as SpeedyPipelineNodeResize;
         const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
 
-        // this shouldn't happen
-        if(this._currentImageIndex >= this._image.length)
-            return Speedy.Promise.reject(new IllegalOperationError());
-
         // set the appropriate training media
-        const database = this._imageTracker.database;
-        const referenceImage = this._image[this._currentImageIndex];
-        const media = database._findMedia(referenceImage.name);
-        source.media = media;
+        const referenceImage = this._trainingMap.referenceImages[this._currentImageIndex];
+        source.media = referenceImage.media;
 
         // compute the appropriate size of the training image space
         const resolution = this._imageTracker.resolution;
         const scale = TRAIN_IMAGE_SCALE; // ORB is not scale-invariant
-        const aspectRatioOfTrainingImage = media.width / media.height;
+        const aspectRatioOfTrainingImage = referenceImage.aspectRatio;
 
-        /*
-        let sin = 0, cos = 1;
-
-        if((aspectRatioOfSourceVideo - 1) * (aspectRatioOfTrainingImage - 1) >= 0) {
-            // training image and source video: both in landscape mode or both in portrait mode
-            screen.size = Utils.resolution(resolution, aspectRatioOfTrainingImage);
-            screen.size.width = Math.round(screen.size.width * scale);
-            screen.size.height = Math.round(screen.size.height * scale);
-        }
-        else if(aspectRatioOfTrainingImage > aspectRatioOfSourceVideo) {
-            // training image: portrait mode; source video: landscape mode
-            screen.size = Utils.resolution(resolution, 1 / aspectRatioOfTrainingImage);
-            screen.size.width = Math.round(screen.size.width * scale);
-            screen.size.height = Math.round(screen.size.height * scale);
-            sin = 1; cos = 0; // rotate 90deg
-        }
-        else {
-            // training image: landscape mode; source video: portrait mode
-        }
-        */
         screen.size = Utils.resolution(resolution, aspectRatioOfTrainingImage);
         screen.size.width = Math.round(screen.size.width * scale);
         screen.size.height = Math.round(screen.size.height * scale);
 
-
-        // convert keypoints from the training image space to AR screen space
-        // let's pretend that trained keypoints belong to the AR screen space,
-        // regardless of the size of the target image. This will make things
-        // easier when computing the homography.
-        /*
-        const sw = arScreenSize.width / screen.size.width;
-        const sh = arScreenSize.height / screen.size.height;
-        */
-        const sw = TRAIN_TARGET_NORMALIZED_SIZE / screen.size.width;
-        const sh = TRAIN_TARGET_NORMALIZED_SIZE / screen.size.height;
-        keypointScaler.transform = Speedy.Matrix(3, 3, [
-            sw, 0,  0,
-            0,  sh, 0,
-            0,  0,  1,
-        ]);
+        // convert keypoints to NIS
+        keypointScaler.transform = ImageTrackerUtils.rasterToNIS(screen.size);
 
         // log
         Utils.log(`Image Tracker: training using reference image "${referenceImage.name}" at ${screen.size.width}x${screen.size.height}...`);
@@ -201,15 +170,20 @@ export class ImageTrackerTrainingState extends ImageTrackerState
      */
     protected _afterUpdate(result: SpeedyPipelineOutput): SpeedyPromise<ImageTrackerStateOutput>
     {
-        const referenceImage = this._image[this._currentImageIndex];
+        const referenceImage = this._trainingMap.referenceImages[this._currentImageIndex];
         const keypoints = result.keypoints as SpeedyKeypoint[];
         const image = result.image as SpeedyMedia | undefined;
 
         // log
         Utils.log(`Image Tracker: found ${keypoints.length} keypoints in reference image "${referenceImage.name}"`);
 
+        // tracker output
+        const trackerOutput: ImageTrackerOutput = {
+            keypointsNIS: image !== undefined ? keypoints : undefined, // debug only
+            image: image,
+        };
+
         // set the training map, so that we can map all keypoints of the current image to the current image
-        this._trainingMap.referenceImage.push(referenceImage);
         for(let i = 0; i < keypoints.length; i++) {
             this._trainingMap.keypoints.push(keypoints[i]);
             this._trainingMap.referenceImageIndex.push(this._currentImageIndex);
@@ -218,31 +192,22 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         // the current image has been processed!
         ++this._currentImageIndex;
 
-        // set output
-        if(this._currentImageIndex >= this._image.length) {
-
-            // finished training!
-            return Speedy.Promise.resolve({
-                //nextState: 'training',
-                nextState: 'scanning',
-                nextStateSettings: {
-                    keypoints: this._trainingMap.keypoints,
-                },
-                trackerOutput: { },
-                //trackerOutput: { image, keypoints, screenSize: this.screenSize },
-            });
-
-        }
-        else {
-
-            // we're not done yet
+        // we're not done yet
+        if(this._currentImageIndex < this._trainingMap.referenceImages.length) {
             return Speedy.Promise.resolve({
                 nextState: 'training',
-                trackerOutput: { },
-                //trackerOutput: { image, keypoints, screenSize: this.screenSize },
+                trackerOutput: trackerOutput
             });
-
         }
+
+        // finished training!
+        return Speedy.Promise.resolve({
+            nextState: 'scanning',
+            trackerOutput: trackerOutput,
+            nextStateSettings: {
+                database: this._trainingMap.keypoints,
+            }
+        });
     }
 
     /**
@@ -268,7 +233,7 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         const clipper = Speedy.Keypoint.Clipper();
         const keypointScaler = Speedy.Keypoint.Transformer('keypointScaler');
         const keypointSink = Speedy.Keypoint.Sink('keypoints');
-        const imageSink = Speedy.Image.Sink('image');
+        //const imageSink = Speedy.Image.Sink('image');
 
         source.media = null;
         screen.size = Speedy.Size(0,0);
@@ -313,12 +278,12 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         // keypoint description
         greyscale.output().connectTo(blur.input());
         blur.output().connectTo(descriptor.input('image'));
-        clipper.output().connectTo(descriptor.input('keypoints'));
+        subpixel.output().connectTo(descriptor.input('keypoints'));
 
         // prepare output
         descriptor.output().connectTo(keypointScaler.input());
         keypointScaler.output().connectTo(keypointSink.input());
-        nightvisionMux.output().connectTo(imageSink.input());
+        //nightvisionMux.output().connectTo(imageSink.input());
 
         // done!
         pipeline.init(
@@ -327,27 +292,27 @@ export class ImageTrackerTrainingState extends ImageTrackerState
             pyramid, detector, blur, descriptor, clipper,
             denoiser, blurredPyramid, subpixel,
             keypointScaler, keypointSink,
-            imageSink
+            //imageSink
         );
         return pipeline;
     }
 
     /**
-     * Get reference image
+     * Get the reference image associated with a keypoint index in the training map
      * @param keypointIndex -1 if not found
      * @returns reference image
      */
-    referenceImageOfKeypoint(keypointIndex: number): Nullable<ReferenceImage>
+    referenceImageOfKeypoint(keypointIndex: number): Nullable<ReferenceImageWithMedia>
     {
         const imageIndex = this.referenceImageIndexOfKeypoint(keypointIndex);
         if(imageIndex < 0)
             return null;
 
-        return this._trainingMap.referenceImage[imageIndex];
+        return this._trainingMap.referenceImages[imageIndex];
     }
 
     /**
-     * Get reference image index
+     * Get the reference image index associated with a keypoint index in the training map
      * @param keypointIndex -1 if not found
      * @returns reference image index, or -1 if not found
      */
@@ -358,14 +323,14 @@ export class ImageTrackerTrainingState extends ImageTrackerState
             return -1;
 
         const imageIndex = this._trainingMap.referenceImageIndex[keypointIndex];
-        if(imageIndex < 0 || imageIndex >= this._trainingMap.referenceImage.length)
+        if(imageIndex < 0 || imageIndex >= this._trainingMap.referenceImages.length)
             return -1;
 
         return imageIndex;
     }
 
     /**
-     * Get keypoint of the trained set
+     * Get a keypoint of the trained set
      * @param keypointIndex -1 if not found
      * @returns a keypoint
      */

src/ui/gizmos.ts

@@ -26,8 +26,10 @@ import { SpeedySize } from 'speedy-vision/types/core/speedy-size';
 import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix';
 import { SpeedyKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
 import { Viewport } from '../core/viewport';
+import { CameraModel } from '../geometry/camera-model';
 import { Tracker, TrackerOutput } from '../trackers/tracker';
 import { ImageTrackerOutput } from '../trackers/image-tracker/image-tracker';
+import { NIS_SIZE } from '../trackers/image-tracker/settings';
 
 
 
@@ -122,10 +124,9 @@ class ImageTrackerGizmos implements GizmosRenderer
             return;
 
         const viewportSize = viewport._realSize;
-        const screenSize = output.screenSize;
-        const keypoints = output.keypoints;
-        const polyline = output.polyline;
-        const cameraMatrix = output.cameraMatrix;
+        const keypointsNIS = output.keypointsNIS;
+        const polylineNDC = output.polylineNDC;
+        const camera = output.camera;
 
         // debug
         //ctx.fillStyle = '#000';
@@ -133,43 +134,48 @@ class ImageTrackerGizmos implements GizmosRenderer
         //ctx.clearRect(0, 0, canvas.width, canvas.height);
 
         // render keypoints
-        if(keypoints !== undefined && screenSize !== undefined)
-            this._splitAndRenderKeypoints(ctx, keypoints, screenSize, viewportSize);
+        if(keypointsNIS !== undefined)
+            this._splitAndRenderKeypointsNIS(ctx, keypointsNIS, viewportSize);
 
         // render polylines
-        if(polyline !== undefined && screenSize !== undefined)
-            this._renderPolyline(ctx, polyline, screenSize, viewportSize);
+        if(polylineNDC !== undefined)
+            this._renderPolylineNDC(ctx, polylineNDC, viewportSize);
 
         // render the axes of the 3D coordinate system
-        if(cameraMatrix !== undefined && screenSize !== undefined)
-            this._renderAxes(ctx, cameraMatrix, screenSize, viewportSize);
+        if(camera !== undefined)
+            this._renderAxes(ctx, camera, viewportSize);
     }
 
     /**
      * Split keypoints in matched/unmatched categories and
      * render them for testing & development purposes
      * @param ctx canvas 2D context
-     * @param keypoints keypoints to render
-     * @param screenSize AR screen size
+     * @param keypoints keypoints in Normalized Image Space (NIS)
      * @param viewportSize viewport size
      * @param size base keypoint rendering size
      */
-    private _splitAndRenderKeypoints(ctx: CanvasRenderingContext2D, keypoints: SpeedyKeypoint[], screenSize: SpeedySize, viewportSize: SpeedySize, size = 1): void
+    private _splitAndRenderKeypointsNIS(ctx: CanvasRenderingContext2D, keypoints: SpeedyKeypoint[], viewportSize: SpeedySize, size = 1): void
     {
         if(keypoints.length == 0)
             return;
 
         if(!Object.prototype.hasOwnProperty.call(keypoints[0], '_matches')) { // hack...
-            this._renderKeypoints(ctx, keypoints, screenSize, viewportSize, '#f00', size);
+            this._renderKeypointsNIS(ctx, keypoints, viewportSize, '#f00', size);
             return;
         }
 
-        const matchedKeypoints = keypoints as SpeedyMatchedKeypoint[];
-        const goodMatches = matchedKeypoints.filter(keypoint => this._isGoodMatch(keypoint));
-        const badMatches = matchedKeypoints.filter(keypoint => !this._isGoodMatch(keypoint));
+        const goodMatches = [], badMatches = [];
+        for(let i = 0; i < keypoints.length; i++) {
+            const keypoint = keypoints[i] as SpeedyMatchedKeypoint;
 
-        this._renderKeypoints(ctx, badMatches, screenSize, viewportSize, '#f00', size);
-        this._renderKeypoints(ctx, goodMatches, screenSize, viewportSize, '#0f0', size);
+            if(this._isGoodMatch(keypoint))
+                goodMatches.push(keypoint);
+            else
+                badMatches.push(keypoint);
+        }
+
+        this._renderKeypointsNIS(ctx, badMatches, viewportSize, '#f00', size);
+        this._renderKeypointsNIS(ctx, goodMatches, viewportSize, '#0f0', size);
     }
 
     /**
@@ -198,16 +204,15 @@ class ImageTrackerGizmos implements GizmosRenderer
     /**
      * Render keypoints for testing & development purposes
      * @param ctx canvas 2D context
-     * @param keypoints keypoints to render
-     * @param screenSize AR screen size
+     * @param keypoints keypoints in Normalized Image Space (NIS)
      * @param viewportSize viewport size
      * @param color color of the rendered keypoints
      * @param size base keypoint rendering size
      */
-    private _renderKeypoints(ctx: CanvasRenderingContext2D, keypoints: SpeedyKeypoint[], screenSize: SpeedySize, viewportSize: SpeedySize, color = 'red', size = 1): void
+    private _renderKeypointsNIS(ctx: CanvasRenderingContext2D, keypoints: SpeedyKeypoint[], viewportSize: SpeedySize, color = 'red', size = 1): void
     {
-        const sx = viewportSize.width / screenSize.width;
-        const sy = viewportSize.height / screenSize.height;
+        const sx = viewportSize.width / NIS_SIZE;
+        const sy = viewportSize.height / NIS_SIZE;
 
         ctx.beginPath();
 
@@ -226,30 +231,26 @@ class ImageTrackerGizmos implements GizmosRenderer
     }
 
     /**
-     * Render polyline for testing & development purposes
+     * Render a polyline for testing & development purposes
      * @param ctx canvas 2D context
-     * @param polyline vertices
-     * @param screenSize AR screen size
+     * @param polyline vertices in NDC
      * @param viewportSize viewport size
      * @param color color of the rendered polyline
      * @param lineWidth
      */
-    private _renderPolyline(ctx: CanvasRenderingContext2D, polyline: SpeedyPoint2[], screenSize: SpeedySize, viewportSize: SpeedySize, color = '#0f0', lineWidth = 2): void
+    private _renderPolylineNDC(ctx: CanvasRenderingContext2D, polyline: SpeedyPoint2[], viewportSize: SpeedySize, color = '#0f0', lineWidth = 2): void
     {
-        if(polyline.length == 0)
-            return;
-
         const n = polyline.length;
-        const sx = viewportSize.width / screenSize.width;
-        const sy = viewportSize.height / screenSize.height;
+        const w = viewportSize.width;
+        const h = viewportSize.height;
 
-        // render polyline
-        ctx.beginPath();
+        if(n == 0)
+            return;
 
-        ctx.moveTo(polyline[n - 1].x * sx, polyline[n - 1].y * sy);
+        ctx.beginPath();
+        ctx.moveTo((polyline[n-1].x * 0.5 + 0.5) * w, (polyline[n-1].y * -0.5 + 0.5) * h);
         for(let j = 0; j < n; j++)
-            ctx.lineTo(polyline[j].x * sx, polyline[j].y * sy);
-
+            ctx.lineTo((polyline[j].x * 0.5 + 0.5) * w, (polyline[j].y * -0.5 + 0.5) * h);
         ctx.strokeStyle = color;
         ctx.lineWidth = lineWidth;
         ctx.stroke();
@@ -258,22 +259,23 @@ class ImageTrackerGizmos implements GizmosRenderer
     /**
      * Render the axes of a 3D coordinate system
      * @param ctx canvas 2D context
-     * @param cameraMatrix 3x4 camera matrix that maps normalized coordinates [-1,1]^3 to AR screen space
-     * @param screenSize AR screen size
+     * @param camera camera model
      * @param viewportSize viewport size
      * @param lineWidth
      */
-    private _renderAxes(ctx: CanvasRenderingContext2D, cameraMatrix: SpeedyMatrix, screenSize: SpeedySize, viewportSize: SpeedySize, lineWidth = 4): void
+    private _renderAxes(ctx: CanvasRenderingContext2D, camera: CameraModel, viewportSize: SpeedySize, lineWidth = 4): void
     {
         const RED = '#f00', GREEN = '#0f0', BLUE = '#00f';
         const color = [ RED, GREEN, BLUE ]; // color of each axis: (X,Y,Z)
         const length = 1; // length of each axis-corresponding line, given in normalized space units
-        const sx = viewportSize.width / screenSize.width;
-        const sy = viewportSize.height / screenSize.height;
+        const w = viewportSize.width;
+        const h = viewportSize.height;
+        const iw = 1 / (camera.imageSize.width / 2);
+        const ih = -1 / (camera.imageSize.height / 2);
 
         /*
 
-        Multiply the 3x4 camera matrix P by:
+        Multiply the 3x4 camera matrix by:
 
         [ 0  L  0  0 ]
         [ 0  0  L  0 ] , where L = length in normalized space of the lines
@@ -287,7 +289,7 @@ class ImageTrackerGizmos implements GizmosRenderer
 
         */
 
-        const p = cameraMatrix.read();
+        const p = camera.matrix.read();
         const l = length;
         const o = [ p[9], p[10], p[11] ]; // origin of the coordinate system
         const x = [ l*p[0]+p[9], l*p[1]+p[10], l*p[2]+p[11] ]; // x-axis
@@ -302,8 +304,8 @@ class ImageTrackerGizmos implements GizmosRenderer
             const x = q[0] / q[2], y = q[1] / q[2];
 
             ctx.beginPath();
-            ctx.moveTo(ox * sx, oy * sy);
-            ctx.lineTo(x * sx, y * sy);
+            ctx.moveTo((ox * iw * 0.5 + 0.5) * w, (oy * ih * 0.5 + 0.5) * h);
+            ctx.lineTo((x * iw * 0.5 + 0.5) * w, (y * ih * 0.5 + 0.5) * h);
             ctx.strokeStyle = color[i];
             ctx.lineWidth = lineWidth;
             ctx.stroke();

src/utils/errors.ts

@@ -23,7 +23,7 @@
 /**
  * Base error class
  */
-export abstract class BaseError extends Error
+export abstract class ARError extends Error
 {
     /**
      * Constructor
@@ -61,7 +61,7 @@ export abstract class BaseError extends Error
 /**
  * A method has received one or more illegal arguments
  */
-export class IllegalArgumentError extends BaseError
+export class IllegalArgumentError extends ARError
 {
     public get name(): string
     {
@@ -73,7 +73,7 @@ export class IllegalArgumentError extends BaseError
  * The method arguments are valid, but the method can't be called due to the
  * current state of the object
  */
-export class IllegalOperationError extends BaseError
+export class IllegalOperationError extends ARError
 {
     public get name(): string
     {
@@ -84,7 +84,7 @@ export class IllegalOperationError extends BaseError
 /**
  * The requested operation is not supported
  */
-export class NotSupportedError extends BaseError
+export class NotSupportedError extends ARError
 {
     public get name(): string
     {
@@ -95,7 +95,7 @@ export class NotSupportedError extends BaseError
 /**
  * Access denied
  */
-export class AccessDeniedError extends BaseError
+export class AccessDeniedError extends ARError
 {
     public get name(): string
     {
@@ -106,7 +106,7 @@ export class AccessDeniedError extends BaseError
 /**
  * Timeout
  */
-export class TimeoutError extends BaseError
+export class TimeoutError extends ARError
 {
     public get name(): string
     {
@@ -117,7 +117,7 @@ export class TimeoutError extends BaseError
 /**
  * Assertion error
  */
-export class AssertionError extends BaseError
+export class AssertionError extends ARError
 {
     public get name(): string
     {
@@ -126,9 +126,20 @@ export class AssertionError extends BaseError
 }
 
 /**
+ * Numerical error
+ */
+export class NumericalError extends ARError
+{
+    public get name(): string
+    {
+        return 'NumericalError';
+    }
+}
+
+/**
  * Tracking error
  */
-export class TrackingError extends BaseError
+export class TrackingError extends ARError
 {
     public get name(): string
     {
@@ -139,7 +150,7 @@ export class TrackingError extends BaseError
 /**
  * Detection error
  */
-export class DetectionError extends BaseError
+export class DetectionError extends ARError
 {
     public get name(): string
     {
@@ -150,7 +161,7 @@ export class DetectionError extends BaseError
 /**
  * Training error
  */
-export class TrainingError extends BaseError
+export class TrainingError extends ARError
 {
     public get name(): string
     {