Refactor the Image Tracker. Simplify and refine the calculations

src/trackers/image-tracker/image-tracker.ts

@@ -43,7 +43,8 @@ import { ImageTrackerState } from './states/state';
 import { ImageTrackerInitialState } from './states/initial';
 import { ImageTrackerTrainingState } from './states/training';
 import { ImageTrackerScanningState } from './states/scanning';
-import { ImageTrackerPreTrackingState } from './states/pre-tracking';
+import { ImageTrackerPreTrackingAState } from './states/pre-tracking-a';
+import { ImageTrackerPreTrackingBState } from './states/pre-tracking-b';
 import { ImageTrackerTrackingState } from './states/tracking';
 import { Nullable, Utils } from '../../utils/utils';
 import { AREventTarget } from '../../utils/ar-events';
@@ -53,34 +54,6 @@ import { SpeedyPoint2 } from 'speedy-vision/types/core/speedy-point';
 import { Viewer } from '../../geometry/viewer';
 import { Pose } from '../../geometry/pose';
 
-/*
-
-A few definitions:
-
-1. Viewport size:
-   size of the drawing buffer of the background canvas, which is the same as
-   the size in pixels of the input media (typically a video).
-
-2. AR screen size:
-   size in pixels used for image processing operations. It's determined by the
-   resolution of the tracker and by the aspect ratio of the input media.
-
-3. Raster space:
-   an image space whose top-left coordinate is (0,0) and whose bottom-right
-   coordinate is (w-1,h-1), where (w,h) is its size. The y-axis grows downwards.
-
-4. AR Screen Space (ASS):
-   a raster space whose size is the AR screen size.
-
-5. Normalized Image Space (NIS):
-   a raster space whose size is N x N, where N = NIS_SIZE.
-
-6. Normalized Device Coordinates (NDC):
-   the normalized 2D space [-1,1]x[-1,1]. The origin is at the center. Also,
-   the y-axis grows upwards.
-
-*/
-
 /** A trackable target */
 export interface TrackableImage extends Trackable
 {
@@ -133,7 +106,7 @@ export interface ImageTrackerOutput extends TrackerOutput
 }
 
 /** All possible states of an Image Tracker */
-export type ImageTrackerStateName = 'initial' | 'training' | 'scanning' | 'pre-tracking' | 'tracking';
+export type ImageTrackerStateName = 'initial' | 'training' | 'scanning' | 'pre-tracking-a' | 'pre-tracking-b' | 'tracking';
 
 /** A helper */
 const formatSize = (size: SpeedySize) => `${size.width}x${size.height}`;
@@ -183,7 +156,8 @@ export class ImageTracker extends AREventTarget<ImageTrackerEventType> implement
             'initial': new ImageTrackerInitialState(this),
             'training': new ImageTrackerTrainingState(this),
             'scanning': new ImageTrackerScanningState(this),
-            'pre-tracking': new ImageTrackerPreTrackingState(this),
+            'pre-tracking-a': new ImageTrackerPreTrackingAState(this),
+            'pre-tracking-b': new ImageTrackerPreTrackingBState(this),
             'tracking': new ImageTrackerTrackingState(this),
         };
 
@@ -349,21 +323,6 @@ export class ImageTracker extends AREventTarget<ImageTrackerEventType> implement
     }
 
     /**
-     * Compute the current size of the AR screen space
-     * Note that this may change over time
-     * @returns size
-     * @internal
-     */
-    _computeScreenSize(): SpeedySize
-    {
-        const media = this._source!._internalMedia;
-        const aspectRatio = media.width / media.height;
-        const screenSize = Utils.resolution(this._resolution, aspectRatio);
-
-        return screenSize;
-    }
-
-    /**
      * Get reference image
      * @param keypointIndex -1 if not found
      * @returns reference image
@@ -398,4 +357,18 @@ export class ImageTracker extends AREventTarget<ImageTrackerEventType> implement
         const training = this._state.training as ImageTrackerTrainingState;
         return training.referenceKeypoint(keypointIndex);
     }
+
+    /**
+     * Compute the current size of the AR screen space
+     * Note that this may change over time
+     * @returns size
+     */
+    private _computeScreenSize(): SpeedySize
+    {
+        const media = this._source!._internalMedia;
+        const aspectRatio = media.width / media.height;
+        const screenSize = Utils.resolution(this._resolution, aspectRatio);
+
+        return screenSize;
+    }
 }

src/trackers/image-tracker/settings.ts

@@ -32,9 +32,6 @@ export const TRAIN_IMAGE_SCALE = 0.8; // ORB is not scale invariant
 /** Width and height of the Normalized Image Space (NIS) */
 export const NIS_SIZE = 1024; // keypoint positions are stored as fixed point
 
-/** Normalized width & height of an image target, in pixels */
-export const TRAIN_TARGET_NORMALIZED_SIZE = NIS_SIZE; // keypoint positions are stored as fixed point
-
 /** Used to identify the best maches */
 export const SCAN_MATCH_RATIO = 0.7; // usually a value in [0.6, 0.8]
 
@@ -50,15 +47,12 @@ export const SCAN_PYRAMID_SCALEFACTOR = 1.19; // 2 ^ 0.25
 /** Threshold of the FAST corner detector used in the scanning/training states */
 export const SCAN_FAST_THRESHOLD = 60;
 
-/** Minimum number of accepted matches for us to move out from the scanning state */
+/** Minimum number of accepted matches for us to move out of the scanning state */
 export const SCAN_MIN_MATCHES = 20; //30;
 
 /** When in the scanning state, we require the image to be matched during a few consecutive frames before accepting it */
 export const SCAN_CONSECUTIVE_FRAMES = 30;//15;//45;
 
-/** Reprojection error, in pixels, used when estimating a motion model (scanning state) */
-//export const SCAN_RANSAC_REPROJECTIONERROR = 5;
-
 /** Reprojection error, in NIS pixels, used when estimating a motion model (scanning state) */
 export const SCAN_RANSAC_REPROJECTIONERROR_NIS = (NIS_SIZE * 0.02) | 0;
 
@@ -101,6 +95,9 @@ export const SUBPIXEL_GAUSSIAN_SIGMA = 1.0;
 /** Subpixel refinement method */
 export const SUBPIXEL_METHOD = 'bilinear-upsample'; // 'quadratic1d';
 
+/** Minimum acceptable number of matched keypoints when in a pre-tracking state */
+export const PRE_TRACK_MIN_MATCHES = 4;
+
 /** Minimum acceptable number of matched keypoints when in the tracking state */
 export const TRACK_MIN_MATCHES = 4;//10; //20;
 
@@ -122,13 +119,8 @@ export const TRACK_RECTIFIED_BORDER = 0.15; //0.20;
 /** Relative size (%) used to clip keypoints from the borders of the rectified image */
 export const TRACK_CLIPPING_BORDER = TRACK_RECTIFIED_BORDER * 1.20; //1.25; //1.15;
 
-/** Number of iterations used to refine the target image before tracking */
-export const TRACK_REFINEMENT_ITERATIONS = 3;
-
-/** Reprojection error, in pixels, used when estimating a motion model (tracking state) */
-export const TRACK_RANSAC_REPROJECTIONERROR = 3; //2.5;
-
-// FIXME
+/** Scale of the rectified image in NDC, without taking the aspect ratio into consideration */
+export const TRACK_RECTIFIED_SCALE = 1 - 2 * TRACK_RECTIFIED_BORDER;
 
 /** Reprojection error, in NIS pixels, used when estimating a motion model (tracking state) */
 export const TRACK_RANSAC_REPROJECTIONERROR_NIS = (NIS_SIZE * 0.0125) | 0;
@@ -143,4 +135,4 @@ export const TRACK_GRID_GRANULARITY = 10; //20; // the value of N
 export const TRACK_MATCH_RATIO = 0.75; // usually a value in [0.6, 0.8] - low values => strict tracking
 
 /** Number of consecutive frames in which we tolerate a  "target lost" situation */
-export const TRACK_LOST_TOLERANCE = 10;
+export const TRACK_LOST_TOLERANCE = 15;

src/trackers/image-tracker/states/initial.ts

@@ -78,6 +78,16 @@ export class ImageTrackerInitialState extends ImageTrackerState
     }
 
     /**
+     * Called when leaving the state, after update()
+     */
+    onLeaveState(): void
+    {
+        // we don't return to this state, so we can release the pipeline early
+        this._pipeline.release();
+        this._pipelineReleased = true;
+    }
+
+    /**
      * Create & setup the pipeline
      * @returns pipeline
      */

src/trackers/image-tracker/states/pre-tracking-a.ts

@@ -0,0 +1,273 @@
+/*
+ * encantar.js
+ * GPU-accelerated Augmented Reality for the web
+ * Copyright (C) 2022-2024 Alexandre Martins <alemartf(at)gmail.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * pre-tracking-a.ts
+ * Image tracker: Pre-Tracking A state
+ */
+
+import Speedy from 'speedy-vision';
+import { SpeedySize } from 'speedy-vision/types/core/speedy-size';
+import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
+import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix';
+import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
+import { SpeedyPipeline, SpeedyPipelineOutput } from 'speedy-vision/types/core/pipeline/pipeline';
+import { SpeedyPipelineNodeImageSource } from 'speedy-vision/types/core/pipeline/nodes/images/source';
+import { SpeedyPipelineNodeImageMultiplexer } from 'speedy-vision/types/core/pipeline/nodes/images/multiplexer';
+import { SpeedyPipelineNodeImagePortalSource, SpeedyPipelineNodeImagePortalSink } from 'speedy-vision/types/core/pipeline/nodes/images/portal';
+import { SpeedyPipelineNodeKeypointPortalSource, SpeedyPipelineNodeKeypointPortalSink } from 'speedy-vision/types/core/pipeline/nodes/keypoints/portal';
+import { SpeedyPipelineNodeResize } from 'speedy-vision/types/core/pipeline/nodes/transforms/resize';
+import { SpeedyPipelineNodePerspectiveWarp } from 'speedy-vision/types/core/pipeline/nodes/transforms/perspective-warp';
+import { SpeedyPipelineNodeKeypointBorderClipper } from 'speedy-vision/types/core/pipeline/nodes/keypoints/border-clipper';
+import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/transformer';
+import { SpeedyKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
+import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
+import { ImageTrackerState, ImageTrackerStateOutput } from './state';
+import { ReferenceImage } from '../reference-image';
+import { Nullable, Utils } from '../../../utils/utils';
+import { TrackingError } from '../../../utils/errors';
+import {
+    TRACK_RECTIFIED_SCALE, TRACK_CLIPPING_BORDER,
+    NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY, TRACK_WITH_NIGHTVISION,
+    ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_SIGMA,
+    TRACK_HARRIS_QUALITY, TRACK_DETECTOR_CAPACITY, TRACK_MAX_KEYPOINTS,
+    SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_SIGMA,
+    PRE_TRACK_MIN_MATCHES,
+    NIGHTVISION_QUALITY,
+    SUBPIXEL_METHOD,
+} from '../settings';
+
+
+
+/**
+ * Pre-Tracking A is a new training phase. The reference image that was found
+ * in the scanning state is transported to AR screen space, and a new training
+ * takes place there, with new keypoints and in a suitable warp.
+ */
+export class ImageTrackerPreTrackingAState extends ImageTrackerState
+{
+    /** reference image */
+    private _referenceImage: Nullable<ReferenceImage>;
+
+    /** a snapshot of the video from the scanning state and corresponding to the initial homography */
+    private _snapshot: Nullable<SpeedyPipelineNodeImagePortalSink>;
+
+    /** initial homography, from reference image to scanned image, NDC */
+    private _homography: SpeedyMatrix;
+
+
+
+    /**
+     * Constructor
+     * @param imageTracker
+     */
+    constructor(imageTracker: ImageTracker)
+    {
+        super('pre-tracking-a', imageTracker);
+
+        this._homography = Speedy.Matrix.Eye(3);
+        this._referenceImage = null;
+        this._snapshot = null;
+    }
+
+    /**
+     * Called as soon as this becomes the active state, just before update() runs for the first time
+     * @param settings
+     */
+    onEnterState(settings: Record<string,any>)
+    {
+        const homography = settings.homography as SpeedyMatrix;
+        const referenceImage = settings.referenceImage as ReferenceImage;
+        const snapshot = settings.snapshot as SpeedyPipelineNodeImagePortalSink;
+
+        // set attributes
+        this._homography = homography;
+        this._referenceImage = referenceImage;
+        this._snapshot = snapshot;
+    }
+
+    /**
+     * Called just before the GPU processing
+     * @returns promise
+     */
+    protected _beforeUpdate(): SpeedyPromise<void>
+    {
+        const screenSize = this.screenSize;
+        const source = this._pipeline.node('source') as SpeedyPipelineNodeImageSource;
+        const imageRectifier = this._pipeline.node('imageRectifier') as SpeedyPipelineNodePerspectiveWarp;
+        const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
+        const borderClipper = this._pipeline.node('borderClipper') as SpeedyPipelineNodeKeypointBorderClipper;
+
+        // set the reference image as the source image
+        source.media = this._imageTracker.database._findMedia(this._referenceImage!.name);
+
+        // clip keypoints from the borders of the target image
+        borderClipper.imageSize = screenSize;
+        borderClipper.borderSize = Speedy.Vector2(
+            screenSize.width * TRACK_CLIPPING_BORDER,
+            screenSize.height * TRACK_CLIPPING_BORDER
+        );
+
+        // convert keypoints to NIS
+        keypointScaler.transform = ImageTrackerUtils.rasterToNIS(screenSize);
+
+        // rectify the image
+        const scale = TRACK_RECTIFIED_SCALE;
+        const aspectRatio = ImageTrackerUtils.bestFitAspectRatioNDC(this._imageTracker, this._referenceImage!);
+        const shrink = ImageTrackerUtils.bestFitScaleNDC(aspectRatio, scale);
+        const toScreen = ImageTrackerUtils.NDCToRaster(screenSize);
+        const toNDC = ImageTrackerUtils.rasterToNDC(screenSize);
+
+        return imageRectifier.transform.setTo(
+            toScreen.times(shrink).times(toNDC)
+        ).then(() => void 0);
+    }
+
+    /**
+     * Post processing that takes place just after the GPU processing
+     * @param result pipeline results
+     * @returns state output
+     */
+    protected _afterUpdate(result: SpeedyPipelineOutput): SpeedyPromise<ImageTrackerStateOutput>
+    {
+        const referenceImage = this._referenceImage!;
+        const keypointPortalSink = this._pipeline.node('keypointPortalSink') as SpeedyPipelineNodeKeypointPortalSink;
+        const keypoints = result.keypoints as SpeedyKeypoint[];
+        const image = result.image as SpeedyMedia | undefined;
+
+        // tracker output
+        const trackerOutput: ImageTrackerOutput = {
+            keypointsNIS: image !== undefined ? keypoints : undefined, // debug only
+            image: image,
+        };
+
+        // not enough keypoints? something went wrong!
+        if(keypoints.length < PRE_TRACK_MIN_MATCHES) {
+            Utils.warning(`Can't pre-track "${referenceImage.name}" in ${this.name}!`);
+            return Speedy.Promise.resolve({
+                nextState: 'scanning',
+                trackerOutput: trackerOutput,
+            });
+        }
+
+        // done!
+        return Speedy.Promise.resolve({
+            nextState: 'pre-tracking-b',
+            trackerOutput: trackerOutput,
+            nextStateSettings: {
+                referenceKeypointPortalSink: keypointPortalSink,
+                referenceImage: this._referenceImage,
+                snapshot: this._snapshot,
+                homography: this._homography,
+            }
+        });
+    }
+
+    /**
+     * Create & setup the pipeline
+     * @returns pipeline
+     */
+    protected _createPipeline(): SpeedyPipeline
+    {
+        const pipeline = Speedy.Pipeline();
+
+        const source = Speedy.Image.Source('source');
+        const screen = Speedy.Transform.Resize('screen');
+        const greyscale = Speedy.Filter.Greyscale();
+        const imageRectifier = Speedy.Transform.PerspectiveWarp('imageRectifier');
+        const nightvision = Speedy.Filter.Nightvision();
+        const nightvisionMux = Speedy.Image.Multiplexer();
+        const detector = Speedy.Keypoint.Detector.Harris();
+        const descriptor = Speedy.Keypoint.Descriptor.ORB();
+        const blur = Speedy.Filter.GaussianBlur();
+        const clipper = Speedy.Keypoint.Clipper();
+        const borderClipper = Speedy.Keypoint.BorderClipper('borderClipper');
+        const denoiser = Speedy.Filter.GaussianBlur();
+        const subpixel = Speedy.Keypoint.SubpixelRefiner();
+        const keypointScaler = Speedy.Keypoint.Transformer('keypointScaler');
+        const keypointPortalSink = Speedy.Keypoint.Portal.Sink('keypointPortalSink');
+        const keypointSink = Speedy.Keypoint.Sink('keypoints');
+        //const imageSink = Speedy.Image.Sink('image');
+
+        source.media = null;
+        imageRectifier.transform = Speedy.Matrix.Eye(3);
+        screen.size = Speedy.Size(0,0);
+        nightvision.gain = NIGHTVISION_GAIN;
+        nightvision.offset = NIGHTVISION_OFFSET;
+        nightvision.decay = NIGHTVISION_DECAY;
+        nightvision.quality = NIGHTVISION_QUALITY;
+        nightvisionMux.port = TRACK_WITH_NIGHTVISION ? 1 : 0; // 1 = enable nightvision
+        blur.kernelSize = Speedy.Size(ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_KSIZE);
+        blur.sigma = Speedy.Vector2(ORB_GAUSSIAN_SIGMA, ORB_GAUSSIAN_SIGMA);
+        denoiser.kernelSize = Speedy.Size(SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_KSIZE);
+        denoiser.sigma = Speedy.Vector2(SUBPIXEL_GAUSSIAN_SIGMA, SUBPIXEL_GAUSSIAN_SIGMA);
+        detector.quality = TRACK_HARRIS_QUALITY;
+        detector.capacity = TRACK_DETECTOR_CAPACITY;
+        subpixel.method = SUBPIXEL_METHOD;
+        clipper.size = TRACK_MAX_KEYPOINTS;
+        borderClipper.imageSize = screen.size;
+        borderClipper.borderSize = Speedy.Vector2(0,0);
+        keypointScaler.transform = Speedy.Matrix.Eye(3);
+        keypointSink.turbo = false;
+
+        // prepare input
+        source.output().connectTo(screen.input());
+        screen.output().connectTo(greyscale.input());
+
+        // preprocess images
+        greyscale.output().connectTo(imageRectifier.input());
+        imageRectifier.output().connectTo(nightvisionMux.input('in0'));
+        imageRectifier.output().connectTo(nightvision.input());
+        nightvision.output().connectTo(nightvisionMux.input('in1'));
+
+        // keypoint detection & clipping
+        nightvisionMux.output().connectTo(detector.input());
+        detector.output().connectTo(borderClipper.input());
+        borderClipper.output().connectTo(clipper.input());
+
+        // keypoint refinement
+        imageRectifier.output().connectTo(denoiser.input());
+        denoiser.output().connectTo(subpixel.input('image'));
+        clipper.output().connectTo(subpixel.input('keypoints'));
+
+        // keypoint description
+        nightvisionMux.output().connectTo(blur.input());
+        blur.output().connectTo(descriptor.input('image'));
+        subpixel.output().connectTo(descriptor.input('keypoints'));
+
+        // prepare output
+        descriptor.output().connectTo(keypointScaler.input());
+        keypointScaler.output().connectTo(keypointSink.input());
+        keypointScaler.output().connectTo(keypointPortalSink.input());
+        //imageRectifier.output().connectTo(imageSink.input());
+
+        // done!
+        pipeline.init(
+            source, screen,
+            greyscale, imageRectifier,
+            nightvision, nightvisionMux,
+            detector, borderClipper, clipper,
+            denoiser, subpixel,
+            blur, descriptor,
+            keypointScaler, keypointSink, keypointPortalSink,
+            //imageSink
+        );
+
+        return pipeline;
+    }
+}

src/trackers/image-tracker/states/pre-tracking-b.ts

@@ -0,0 +1,391 @@
+/*
+ * encantar.js
+ * GPU-accelerated Augmented Reality for the web
+ * Copyright (C) 2022-2024 Alexandre Martins <alemartf(at)gmail.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ * pre-tracking-b.ts
+ * Image tracker: Pre-Tracking B state
+ */
+
+import Speedy from 'speedy-vision';
+import { SpeedySize } from 'speedy-vision/types/core/speedy-size';
+import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
+import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix';
+import { SpeedyMatrixExpr } from 'speedy-vision/types/core/speedy-matrix-expr';
+import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
+import { SpeedyPipeline, SpeedyPipelineOutput } from 'speedy-vision/types/core/pipeline/pipeline';
+import { SpeedyPipelineNodeImageSource } from 'speedy-vision/types/core/pipeline/nodes/images/source';
+import { SpeedyPipelineNodeImageMultiplexer } from 'speedy-vision/types/core/pipeline/nodes/images/multiplexer';
+import { SpeedyPipelineNodeImagePortalSource, SpeedyPipelineNodeImagePortalSink } from 'speedy-vision/types/core/pipeline/nodes/images/portal';
+import { SpeedyPipelineNodeKeypointPortalSource, SpeedyPipelineNodeKeypointPortalSink } from 'speedy-vision/types/core/pipeline/nodes/keypoints/portal';
+import { SpeedyPipelineNodeResize } from 'speedy-vision/types/core/pipeline/nodes/transforms/resize';
+import { SpeedyPipelineNodePerspectiveWarp } from 'speedy-vision/types/core/pipeline/nodes/transforms/perspective-warp';
+import { SpeedyPipelineNodeKeypointBorderClipper } from 'speedy-vision/types/core/pipeline/nodes/keypoints/border-clipper';
+import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/transformer';
+import { SpeedyKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
+import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
+import { ImageTrackerState, ImageTrackerStateOutput } from './state';
+import { ReferenceImage } from '../reference-image';
+import { Nullable, Utils } from '../../../utils/utils';
+import { TrackingError } from '../../../utils/errors';
+import {
+    TRACK_RECTIFIED_SCALE, TRACK_CLIPPING_BORDER,
+    NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY, TRACK_WITH_NIGHTVISION,
+    ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_SIGMA,
+    TRACK_HARRIS_QUALITY, TRACK_DETECTOR_CAPACITY, TRACK_MAX_KEYPOINTS,
+    SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_SIGMA,
+    PRE_TRACK_MIN_MATCHES, TRACK_MATCH_RATIO, TRACK_RANSAC_REPROJECTIONERROR_NDC,
+    NIGHTVISION_QUALITY,
+    SUBPIXEL_METHOD,
+} from '../settings';
+
+
+
+
+/**
+ * In Pre-Tracking B, we refine the homography obtained at the scanning state.
+ * We find a transformation that warps the snapshot obtained from the scanning
+ * state to an image that closely resembles the output of Pre-Tracking A.
+ */
+export class ImageTrackerPreTrackingBState extends ImageTrackerState
+{
+    /** reference image */
+    private _referenceImage: Nullable<ReferenceImage>;
+
+    /** a snapshot of the video from the scanning state and corresponding to the initial homography */
+    private _snapshot: Nullable<SpeedyPipelineNodeImagePortalSink>;
+
+    /** initial homography, from reference image to scanned image, NDC */
+    private _homography: SpeedyMatrix;
+
+    /** portal with keypoints from Pre-Tracking A */
+    private _referenceKeypointPortalSink: Nullable<SpeedyPipelineNodeKeypointPortalSink>;
+
+
+
+
+
+
+    /**
+     * Constructor
+     * @param imageTracker
+     */
+    constructor(imageTracker: ImageTracker)
+    {
+        super('pre-tracking-b', imageTracker);
+
+        this._homography = Speedy.Matrix.Eye(3);
+        this._referenceImage = null;
+        this._snapshot = null;
+        this._referenceKeypointPortalSink = null;
+    }
+
+    /**
+     * Called as soon as this becomes the active state, just before update() runs for the first time
+     * @param settings
+     */
+    onEnterState(settings: Record<string,any>)
+    {
+        const homography = settings.homography as SpeedyMatrix;
+        const referenceImage = settings.referenceImage as ReferenceImage;
+        const snapshot = settings.snapshot as SpeedyPipelineNodeImagePortalSink;
+        const referenceKeypointPortalSink = settings.referenceKeypointPortalSink as SpeedyPipelineNodeKeypointPortalSink;
+
+        // set attributes
+        this._homography = homography;
+        this._referenceImage = referenceImage;
+        this._snapshot = snapshot;
+        this._referenceKeypointPortalSink = referenceKeypointPortalSink;
+    }
+
+    /**
+     * Called just before the GPU processing
+     * @returns promise
+     */
+    protected _beforeUpdate(): SpeedyPromise<void>
+    {
+        const screenSize = this.screenSize;
+        const imageRectifier = this._pipeline.node('imageRectifier') as SpeedyPipelineNodePerspectiveWarp;
+        const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
+        const borderClipper = this._pipeline.node('borderClipper') as SpeedyPipelineNodeKeypointBorderClipper;
+        const imagePortalSource = this._pipeline.node('imagePortalSource') as SpeedyPipelineNodeImagePortalSource;
+        const referenceKeypointPortalSource = this._pipeline.node('referenceKeypointPortalSource') as SpeedyPipelineNodeKeypointPortalSource;
+
+        // get the snapshot from the scanning state
+        imagePortalSource.source = this._snapshot!;
+
+        // get the reference keypoints from Pre-Tracking A
+        referenceKeypointPortalSource.source = this._referenceKeypointPortalSink!;
+
+        // clip keypoints from the borders of the target image
+        borderClipper.imageSize = screenSize;
+        borderClipper.borderSize = Speedy.Vector2(
+            screenSize.width * TRACK_CLIPPING_BORDER,
+            screenSize.height * TRACK_CLIPPING_BORDER
+        );
+
+        // convert keypoints to NIS
+        keypointScaler.transform = ImageTrackerUtils.rasterToNIS(screenSize);
+
+        // rectify the image
+        const scale = TRACK_RECTIFIED_SCALE;
+        const aspectRatio = ImageTrackerUtils.bestFitAspectRatioNDC(this._imageTracker, this._referenceImage!);
+        const shrink = ImageTrackerUtils.bestFitScaleNDC(aspectRatio, scale);
+        const undistort = this._homography.inverse();
+        const toScreen = ImageTrackerUtils.NDCToRaster(screenSize);
+        const toNDC = ImageTrackerUtils.rasterToNDC(screenSize);
+
+        return imageRectifier.transform.setTo(
+            toScreen.times(shrink.times(undistort)).times(toNDC)
+        ).then(() => void 0);
+    }
+
+    /**
+     * Post processing that takes place just after the GPU processing
+     * @param result pipeline results
+     * @returns state output
+     */
+    protected _afterUpdate(result: SpeedyPipelineOutput): SpeedyPromise<ImageTrackerStateOutput>
+    {
+        const referenceImage = this._referenceImage!;
+        const referenceKeypoints = result.referenceKeypoints as SpeedyKeypoint[]; // from Pre-Tracking A
+        const keypoints = result.keypoints as SpeedyMatchedKeypoint[]; // from Pre-Tracking B
+        const image = result.image as SpeedyMedia | undefined;
+        const keypointPortalSink = this._pipeline.node('keypointPortalSink') as SpeedyPipelineNodeKeypointPortalSink;
+
+        // tracker output
+        const trackerOutput: ImageTrackerOutput = {
+            keypointsNIS: image !== undefined ? keypoints : undefined, // debug only
+            image: image,
+        };
+
+        return Speedy.Promise.resolve()
+        .then(() => {
+
+            // find matching pairs of keypoints
+            const pairs = this._findMatchingPairs(referenceKeypoints, keypoints);
+            //const pairs = ImageTrackerUtils.refineMatchingPairs(allPairs);
+            if(pairs.length < PRE_TRACK_MIN_MATCHES)
+                throw new TrackingError('Not enough data points');
+
+            // find a warp
+            const points = ImageTrackerUtils.compilePairsOfKeypointsNDC(pairs);
+            return this._findAffineMotionNDC(points);
+
+        })
+        .then(warp => {
+
+            // refine the homography
+            return this._homography.setTo(warp.times(this._homography));
+
+        })
+        .then(_ => ({
+            nextState: 'tracking',
+            //nextState: 'pre-tracking-b',
+            trackerOutput: trackerOutput,
+            nextStateSettings: {
+                // we export keypoints obtained in Pre-Tracking B, not in A.
+                // lighting conditions match, but what if the snapshot is too blurry?
+                templateKeypoints: keypoints,
+                templateKeypointPortalSink: keypointPortalSink,
+                referenceImage: this._referenceImage,
+                homography: this._homography,
+                initialScreenSize: this.screenSize,
+            }
+        }))
+        .catch(err => {
+            Utils.warning(`Can't pre-track "${referenceImage.name}" in ${this.name}! ${err.toString()}`);
+            return {
+                nextState: 'scanning',
+                trackerOutput: trackerOutput,
+            };
+        });
+    }
+
+    /**
+     * Find an affine motion model in NDC between pairs of keypoints in NDC
+     * given as a 2 x 2n [ src | dest ] matrix
+     * @param points compiled pairs of keypoints in NDC
+     * @returns a promise that resolves to a 3x3 warp in NDC that maps source to destination
+     */
+    private _findAffineMotionNDC(points: SpeedyMatrix): SpeedyPromise<SpeedyMatrixExpr>
+    {
+        return ImageTrackerUtils.findAffineWarpNDC(points, {
+            method: 'pransac',
+            reprojectionError: TRACK_RANSAC_REPROJECTIONERROR_NDC,
+            numberOfHypotheses: 512*4,
+            bundleSize: 128,
+            mask: undefined // score is not needed
+        }).then(([ warp, score ]) => {
+
+            const scale = TRACK_RECTIFIED_SCALE;
+            const aspectRatio = ImageTrackerUtils.bestFitAspectRatioNDC(this._imageTracker, this._referenceImage!);
+            const shrink = ImageTrackerUtils.bestFitScaleNDC(aspectRatio, scale);
+            const grow = ImageTrackerUtils.inverseBestFitScaleNDC(aspectRatio, scale);
+            const scaledWarp = grow.times(warp).times(shrink);
+
+            const distort = this._homography;
+            const undistort = distort.inverse();
+            const correctedWarp = distort.times(scaledWarp).times(undistort);
+
+            //console.log(Speedy.Matrix(warp).toString());
+            //console.log(Speedy.Matrix(scaledWarp).toString());
+            //console.log(Speedy.Matrix(correctedWarp).toString());
+
+            return correctedWarp;
+
+        });
+    }
+
+    /**
+     * Find matching pairs of two sets of keypoints matched via brute force
+     * @param srcKeypoints source (database)
+     * @param destKeypoints destination
+     * @returns an array of matching pairs [src, dest]
+     */
+    private _findMatchingPairs(srcKeypoints: SpeedyKeypoint[], destKeypoints: SpeedyMatchedKeypoint[]): ImageTrackerKeypointPair[]
+    {
+        const pairs: ImageTrackerKeypointPair[] = [];
+
+        for(let i = 0; i < destKeypoints.length; i++) {
+            const destKeypoint = destKeypoints[i];
+
+            if(destKeypoint.matches[0].index >= 0 && destKeypoint.matches[1].index >= 0) {
+                const d1 = destKeypoint.matches[0].distance;
+                const d2 = destKeypoint.matches[1].distance;
+
+                // the best match should be "much better" than the second best match,
+                // which means that they are "distinct enough"
+                if(d1 <= TRACK_MATCH_RATIO * d2) {
+                    const srcKeypoint = srcKeypoints[destKeypoint.matches[0].index];
+                    pairs.push([srcKeypoint, destKeypoint]);
+                }
+            }
+        }
+
+        return pairs;
+    }
+
+    /**
+     * Create & setup the pipeline
+     * @returns pipeline
+     */
+    protected _createPipeline(): SpeedyPipeline
+    {
+        const pipeline = Speedy.Pipeline();
+
+        const source = Speedy.Image.Source('source');
+        const imagePortalSource = Speedy.Image.Portal.Source('imagePortalSource');
+        const referenceKeypointPortalSource = Speedy.Keypoint.Portal.Source('referenceKeypointPortalSource');
+        const screen = Speedy.Transform.Resize('screen');
+        const greyscale = Speedy.Filter.Greyscale();
+        const imageRectifier = Speedy.Transform.PerspectiveWarp('imageRectifier');
+        const nightvision = Speedy.Filter.Nightvision();
+        const nightvisionMux = Speedy.Image.Multiplexer();
+        const detector = Speedy.Keypoint.Detector.Harris();
+        const descriptor = Speedy.Keypoint.Descriptor.ORB();
+        const blur = Speedy.Filter.GaussianBlur();
+        const clipper = Speedy.Keypoint.Clipper();
+        const borderClipper = Speedy.Keypoint.BorderClipper('borderClipper');
+        const denoiser = Speedy.Filter.GaussianBlur();
+        const subpixel = Speedy.Keypoint.SubpixelRefiner();
+        const matcher = Speedy.Keypoint.Matcher.BFKNN();
+        const keypointScaler = Speedy.Keypoint.Transformer('keypointScaler');
+        const keypointSink = Speedy.Keypoint.SinkOfMatchedKeypoints('keypoints');
+        const keypointPortalSink = Speedy.Keypoint.Portal.Sink('keypointPortalSink');
+        const referenceKeypointSink = Speedy.Keypoint.Sink('referenceKeypoints');
+        //const imageSink = Speedy.Image.Sink('image');
+
+        source.media = null;
+        imagePortalSource.source = null;
+        referenceKeypointPortalSource.source = null;
+        imageRectifier.transform = Speedy.Matrix.Eye(3);
+        screen.size = Speedy.Size(0,0);
+        nightvision.gain = NIGHTVISION_GAIN;
+        nightvision.offset = NIGHTVISION_OFFSET;
+        nightvision.decay = NIGHTVISION_DECAY;
+        nightvision.quality = NIGHTVISION_QUALITY;
+        nightvisionMux.port = TRACK_WITH_NIGHTVISION ? 1 : 0; // 1 = enable nightvision
+        blur.kernelSize = Speedy.Size(ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_KSIZE);
+        blur.sigma = Speedy.Vector2(ORB_GAUSSIAN_SIGMA, ORB_GAUSSIAN_SIGMA);
+        denoiser.kernelSize = Speedy.Size(SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_KSIZE);
+        denoiser.sigma = Speedy.Vector2(SUBPIXEL_GAUSSIAN_SIGMA, SUBPIXEL_GAUSSIAN_SIGMA);
+        detector.quality = TRACK_HARRIS_QUALITY;
+        detector.capacity = TRACK_DETECTOR_CAPACITY;
+        subpixel.method = SUBPIXEL_METHOD;
+        clipper.size = TRACK_MAX_KEYPOINTS;
+        borderClipper.imageSize = screen.size;
+        borderClipper.borderSize = Speedy.Vector2(0,0);
+        matcher.k = 2;
+        keypointScaler.transform = Speedy.Matrix.Eye(3);
+        keypointSink.turbo = false;
+
+        // prepare input
+        //source.output(); // ignore, but keep it in the pipeline
+        imagePortalSource.output().connectTo(screen.input());
+        screen.output().connectTo(greyscale.input());
+
+        // preprocess images
+        greyscale.output().connectTo(imageRectifier.input());
+        imageRectifier.output().connectTo(nightvisionMux.input('in0'));
+        imageRectifier.output().connectTo(nightvision.input());
+        nightvision.output().connectTo(nightvisionMux.input('in1'));
+
+        // keypoint detection & clipping
+        nightvisionMux.output().connectTo(detector.input());
+        detector.output().connectTo(borderClipper.input());
+        borderClipper.output().connectTo(clipper.input());
+
+        // keypoint refinement
+        imageRectifier.output().connectTo(denoiser.input());
+        denoiser.output().connectTo(subpixel.input('image'));
+        clipper.output().connectTo(subpixel.input('keypoints'));
+
+        // keypoint description
+        nightvisionMux.output().connectTo(blur.input());
+        blur.output().connectTo(descriptor.input('image'));
+        subpixel.output().connectTo(descriptor.input('keypoints'));
+
+        // keypoint matching
+        descriptor.output().connectTo(matcher.input('keypoints'));
+        referenceKeypointPortalSource.output().connectTo(matcher.input('database'));
+
+        // prepare output
+        descriptor.output().connectTo(keypointScaler.input());
+        keypointScaler.output().connectTo(keypointPortalSink.input());
+        keypointScaler.output().connectTo(keypointSink.input());
+        matcher.output().connectTo(keypointSink.input('matches'));
+        referenceKeypointPortalSource.output().connectTo(referenceKeypointSink.input());
+        //imageRectifier.output().connectTo(imageSink.input());
+
+        // done!
+        pipeline.init(
+            source, screen, imagePortalSource,
+            referenceKeypointPortalSource,
+            greyscale, imageRectifier,
+            nightvision, nightvisionMux,
+            detector, borderClipper, clipper,
+            denoiser, subpixel,
+            blur, descriptor,
+            matcher,
+            keypointScaler, keypointSink, keypointPortalSink, referenceKeypointSink,
+            //imageSink
+        );
+
+        return pipeline;
+    }
+}

src/trackers/image-tracker/states/scanning.ts

@@ -37,25 +37,21 @@ import { SpeedyPipelineNodeImagePortalSource, SpeedyPipelineNodeImagePortalSink
 import { SpeedyPipelineNodeStaticLSHTables } from 'speedy-vision/types/core/pipeline/nodes/keypoints/matchers/lsh-static-tables';
 import { SpeedyKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
 import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
 import { ImageTrackerState, ImageTrackerStateOutput } from './state';
-import { ImageTrackerPreTrackingState } from './pre-tracking';
 import { Nullable, Utils } from '../../../utils/utils';
-import { IllegalOperationError, IllegalArgumentError, DetectionError } from '../../../utils/errors';
+import { DetectionError } from '../../../utils/errors';
 import { 
     SCAN_MATCH_RATIO, SCAN_MIN_MATCHES, SCAN_CONSECUTIVE_FRAMES,
     ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_SIGMA,
     NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY,
     SCAN_WITH_NIGHTVISION, SCAN_PYRAMID_LEVELS, SCAN_PYRAMID_SCALEFACTOR,
     SCAN_FAST_THRESHOLD, SCAN_MAX_KEYPOINTS, SCAN_LSH_TABLES, SCAN_LSH_HASHSIZE,
-    SCAN_RANSAC_REPROJECTIONERROR,
-    TRAIN_TARGET_NORMALIZED_SIZE,
+    SCAN_RANSAC_REPROJECTIONERROR_NDC,
     NIGHTVISION_QUALITY,
 } from '../settings';
 
 
-/** Default target space size (used when training) */
-const DEFAULT_TARGET_SPACE_SIZE = Speedy.Size(TRAIN_TARGET_NORMALIZED_SIZE, TRAIN_TARGET_NORMALIZED_SIZE);
-
 /** Port of the portal multiplexer: get new data from the camera */
 const PORT_CAMERA = 0;
 
@@ -65,7 +61,7 @@ const PORT_MEMORY = 1;
 
 
 /**
- * Scanning state of the Image Tracker
+ * In the scanning state we look for a reference image in the video
  */
 export class ImageTrackerScanningState extends ImageTrackerState
 {
@@ -101,7 +97,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
     {
         const imagePortalMux = this._pipeline.node('imagePortalMux') as SpeedyPipelineNodeImageMultiplexer;
         const lshTables = this._pipeline.node('lshTables') as SpeedyPipelineNodeStaticLSHTables;
-        const keypoints = settings.keypoints as SpeedyKeypoint[] | undefined;
+        const database = settings.database as SpeedyKeypoint[] | undefined;
 
         // set attributes
         this._counter = 0;
@@ -111,8 +107,24 @@ export class ImageTrackerScanningState extends ImageTrackerState
         imagePortalMux.port = PORT_CAMERA;
 
         // prepare the keypoint matcher
-        if(keypoints !== undefined)
-            lshTables.keypoints = keypoints;
+        if(database !== undefined)
+            lshTables.keypoints = database;
+    }
+
+    /**
+     * Called just before the GPU processing
+     * @returns promise
+     */
+    protected _beforeUpdate(): SpeedyPromise<void>
+    {
+        const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
+        const screenSize = this.screenSize;
+
+        // convert keypoints to NIS
+        keypointScaler.transform = ImageTrackerUtils.rasterToNIS(screenSize);
+
+        // done!
+        return Speedy.Promise.resolve();
     }
 
     /**
@@ -124,116 +136,123 @@ export class ImageTrackerScanningState extends ImageTrackerState
     {
         const imagePortalMux = this._pipeline.node('imagePortalMux') as SpeedyPipelineNodeImageMultiplexer;
         const keypoints = result.keypoints as SpeedyMatchedKeypoint[];
-        const matchedKeypoints = this._goodMatches(keypoints);
+        const image = result.image as SpeedyMedia | undefined;
 
         // tracker output
         const trackerOutput: ImageTrackerOutput = {
-            keypoints: keypoints,
-            screenSize: this.screenSize
+            keypointsNIS: keypoints,
+            polylineNDC: [],
+            image: image,
         };
 
         // keep the last memorized image
         imagePortalMux.port = PORT_MEMORY;
 
-        // have we found enough matches...?
-        if(matchedKeypoints.length >= SCAN_MIN_MATCHES) {
-            return this._findHomography(matchedKeypoints).then(([homography, score]) => {
+        // find high quality matches
+        const matchedKeypoints = this._selectGoodMatches(keypoints);
+        if(matchedKeypoints.length < SCAN_MIN_MATCHES) {
+
+            // not enough high quality matches?
+            // we'll continue to scan the scene
+            this._counter = 0;
+            this._bestScore = 0;
 
-                // have we found the best homography so far?
-                if(score >= this._bestScore) {
-                    // store it only if we'll be running the pipeline again
-                    if(this._counter < SCAN_CONSECUTIVE_FRAMES - 1) {
-                        this._bestScore = score;
-                        this._bestHomography = homography;
+            return Speedy.Promise.resolve({
+                nextState: 'scanning',
+                trackerOutput: trackerOutput,
+            });
 
-                        // memorize the last image, corresponding to the best homography(*)
-                        imagePortalMux.port = PORT_CAMERA;
+        }
 
-                        /*
+        // we have enough high quality matches!
+        const pairs = this._findMatchingPairs(matchedKeypoints);
+        const points = ImageTrackerUtils.compilePairsOfKeypointsNDC(pairs);
 
-                        (*) technically speaking, this is not exactly the case. Since we're
-                            using turbo to download the keypoints, there's a slight difference
-                            between the data used to compute the homography and the last image.
-                            Still, assuming continuity of the video stream, this logic is
-                            good enough.
+        // find a homography
+        return this._findHomographyNDC(points).then(([homography, score]) => {
 
-                        */
-                    }
-                }
+            // have we found the best homography so far?
+            if(score >= this._bestScore) {
 
-                // find a polyline surrounding the target
-                return this._findPolyline(homography, DEFAULT_TARGET_SPACE_SIZE);
+                // store it only if we'll be running the pipeline again
+                if(this._counter < SCAN_CONSECUTIVE_FRAMES - 1) {
+                    this._bestScore = score;
+                    this._bestHomography = homography;
 
-            }).then(polyline => {
+                    // memorize the last image, corresponding to the best homography(*)
+                    imagePortalMux.port = PORT_CAMERA;
 
-                // continue a little longer in the scanning state
-                if(++this._counter < SCAN_CONSECUTIVE_FRAMES) {
-                    return {
-                        nextState: this.name,
-                        trackerOutput: {
-                            polyline: polyline,
-                            ...trackerOutput,
-                        },
-                    };
-                }
+                    /*
 
-                // this image should correspond to the best homography
-                const snapshot = this._pipeline.node('imagePortalSink') as SpeedyPipelineNodeImagePortalSink;
+                    (*) technically speaking, this is not exactly the case. Since we're
+                        using turbo to download the keypoints, there's a slight difference
+                        between the data used to compute the homography and the last image.
+                        Still, assuming continuity of the video stream, this logic is
+                        good enough.
 
-                // the reference image that we'll track
-                const referenceImage = this._imageTracker._referenceImageOfKeypoint(
-                    matchedKeypoints[0].matches[0].index
-                );
+                    */
+                }
 
-                // let's track the target!
-                return {
-                    nextState: 'pre-tracking',
-                    nextStateSettings: {
-                        homography: this._bestHomography,
-                        snapshot: snapshot,
-                        referenceImage: referenceImage,
-                    },
-                    trackerOutput: {
-                        polyline: polyline,
-                        ...trackerOutput,
-                    },
-                };
+            }
 
-            }).catch(() => {
+            // find a polyline surrounding the target
+            const polylineNDC = ImageTrackerUtils.findPolylineNDC(homography);
+            trackerOutput.polylineNDC!.push(...polylineNDC);
 
-                // continue in the scanning state
+            // continue a little longer in the scanning state
+            if(++this._counter < SCAN_CONSECUTIVE_FRAMES) {
                 return {
-                    nextState: this.name,
-                    trackerOutput: trackerOutput,
+                    nextState: 'scanning',
+                    trackerOutput: trackerOutput
                 };
+            }
 
-            });
-        }
-        else {
+            // this image should correspond to the best homography
+            const snapshot = this._pipeline.node('imagePortalSink') as SpeedyPipelineNodeImagePortalSink;
 
-            // not enough matches...!
-            this._counter = 0;
-            this._bestScore = 0;
+            // the reference image that we'll track
+            const referenceImage = this._imageTracker._referenceImageOfKeypoint(
+                matchedKeypoints[0].matches[0].index
+            );
 
-        }
+            // this shouldn't happen
+            if(!referenceImage)
+                throw new DetectionError(`Can't track an unknown reference image`);
+
+            // let's track the target!
+            return {
+                nextState: 'pre-tracking-a',
+                nextStateSettings: {
+                    homography: this._bestHomography,
+                    snapshot: snapshot,
+                    referenceImage: referenceImage,
+                },
+                trackerOutput: trackerOutput
+            };
+
+        })
+        .catch(err => {
+
+            // continue in the scanning state
+            Utils.warning(`Error when scanning: ${err.toString()}`)
+            return {
+                nextState: 'scanning',
+                trackerOutput: trackerOutput,
+            };
 
-        // we'll continue to scan the scene
-        return Speedy.Promise.resolve({
-            nextState: this.name,
-            trackerOutput: trackerOutput,
         });
     }
 
     /**
-     * Find "high quality" matches of a single reference image
-     * @param keypoints
-     * @returns high quality matches
+     * Select high quality matches of a single reference image
+     * @param keypoints matched keypoints of any quality, to any reference image
+     * @returns high quality matches of a single reference image
      */
-    private _goodMatches(keypoints: SpeedyMatchedKeypoint[]): SpeedyMatchedKeypoint[]
+    private _selectGoodMatches(keypoints: SpeedyMatchedKeypoint[]): SpeedyMatchedKeypoint[]
     {
         const matchedKeypointsPerImageIndex: Record<number,SpeedyMatchedKeypoint[]> = Object.create(null);
 
-        // filter "good matches"
+        // find high quality matches, regardless of reference image
         for(let j = keypoints.length - 1; j >= 0; j--) {
             const keypoint = keypoints[j];
             if(keypoint.matches[0].index >= 0 && keypoint.matches[1].index >= 0) {
@@ -255,7 +274,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
             }
         }
 
-        // find the image with the most matches
+        // find the reference image with the most high quality matches
         let matchedKeypoints: SpeedyMatchedKeypoint[] = [];
         for(const imageIndex in matchedKeypointsPerImageIndex) {
             if(matchedKeypointsPerImageIndex[imageIndex].length > matchedKeypoints.length)
@@ -267,71 +286,41 @@ export class ImageTrackerScanningState extends ImageTrackerState
     }
 
     /**
-     * Find a homography matrix using matched keypoints
-     * @param matchedKeypoints "good" matches only
-     * @returns homography from reference image space to AR screen space & homography "quality" score
+     * Find a homography matrix using matched keypoints in NDC
+     * @param points compiled pairs of keypoints in NDC
+     * @returns homography (from reference to matched, NDC) & "quality" score
      */
-    private _findHomography(matchedKeypoints: SpeedyMatchedKeypoint[]): SpeedyPromise<[SpeedyMatrix,number]>
+    private _findHomographyNDC(points: SpeedyMatrix): SpeedyPromise<[SpeedyMatrix,number]>
     {
-        const srcCoords: number[] = [];
-        const dstCoords: number[] = [];
-
-        // find matching coordinates of the keypoints
-        for(let i = matchedKeypoints.length - 1; i >= 0; i--) {
-            const matchedKeypoint = matchedKeypoints[i];
-            const referenceKeypoint = this._imageTracker._referenceKeypoint(matchedKeypoint.matches[0].index);
-            if(referenceKeypoint != null) {
-                srcCoords.push(referenceKeypoint.x);
-                srcCoords.push(referenceKeypoint.y);
-                dstCoords.push(matchedKeypoint.x);
-                dstCoords.push(matchedKeypoint.y);
-            }
-            else {
-                // this shouldn't happen
-                return Speedy.Promise.reject(
-                    new DetectionError(`Invalid keypoint match index: ${matchedKeypoint.matches[0].index} from ${matchedKeypoint.toString()}`)
-                );
-            }
-        }
-
-        // too few points?
-        const n = srcCoords.length / 2;
-        if(n < 4) {
-            return Speedy.Promise.reject(
-                new DetectionError(`Too few points to compute a homography`)
-            );
-        }
-
-        // compute a homography
-        const src = Speedy.Matrix(2, n, srcCoords);
-        const dst = Speedy.Matrix(2, n, dstCoords);
-        const mask = Speedy.Matrix.Zeros(1, n);
-
-        const homography = Speedy.Matrix.Zeros(3);
-        return Speedy.Matrix.findHomography(homography, src, dst, {
+        return ImageTrackerUtils.findPerspectiveWarpNDC(points, {
             method: 'pransac',
-            reprojectionError: SCAN_RANSAC_REPROJECTIONERROR,
+            reprojectionError: SCAN_RANSAC_REPROJECTIONERROR_NDC,
             numberOfHypotheses: 512,
             bundleSize: 128,
-            mask: mask,
-        }).then(homography => {
+        });
+    }
 
-            // check if this is a valid homography
-            const a00 = homography.at(0,0);
-            if(Number.isNaN(a00))
-                throw new DetectionError(`Can't compute homography`);
+    /**
+     * Find matching pairs of keypoints from reference image (src) to matched image (dest)
+     * @param matchedKeypoints
+     * @returns an array of matching pairs [src, dest]
+     */
+    private _findMatchingPairs(matchedKeypoints: SpeedyMatchedKeypoint[]): ImageTrackerKeypointPair[]
+    {
+        const pairs = new Array<ImageTrackerKeypointPair>(matchedKeypoints.length);
 
-            // count the number of inliers
-            const inliers = mask.read();
-            let inlierCount = 0;
-            for(let i = inliers.length - 1; i >= 0; i--)
-                inlierCount += inliers[i];
-            const score = inlierCount / inliers.length;
+        for(let i = matchedKeypoints.length - 1; i >= 0; i--) {
+            const matchedKeypoint = matchedKeypoints[i];
+            const referenceKeypoint = this._imageTracker._referenceKeypoint(matchedKeypoint.matches[0].index);
 
-            // done!
-            return [ homography, score ];
+            // this shouldn't happen
+            if(referenceKeypoint == null)
+                throw new DetectionError(`Invalid keypoint match index: ${matchedKeypoint.matches[0].index} from ${matchedKeypoint.toString()}`);
 
-        });
+            pairs[i] = [ referenceKeypoint, matchedKeypoint ];
+        }
+
+        return pairs;
     }
 
     /**
@@ -354,6 +343,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
         const clipper = Speedy.Keypoint.Clipper();
         const lshTables = Speedy.Keypoint.Matcher.StaticLSHTables('lshTables');
         const knn = Speedy.Keypoint.Matcher.LSHKNN();
+        const keypointScaler = Speedy.Keypoint.Transformer('keypointScaler');
         const keypointSink = Speedy.Keypoint.SinkOfMatchedKeypoints('keypoints');
         const imagePortalSink = Speedy.Image.Portal.Sink('imagePortalSink');
         const imagePortalSource = Speedy.Image.Portal.Source('imagePortalSource');
@@ -386,6 +376,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
         imagePortalMux.port = PORT_CAMERA; // 0 = camera stream; 1 = lock image
         imagePortalCopy.size = Speedy.Size(0,0);
         imagePortalCopy.scale = Speedy.Vector2(1,1);
+        keypointScaler.transform = Speedy.Matrix.Eye(3);
         keypointSink.turbo = true;
 
         // prepare input
@@ -412,7 +403,8 @@ export class ImageTrackerScanningState extends ImageTrackerState
         lshTables.output().connectTo(knn.input('lsh'));
 
         // prepare output
-        clipper.output().connectTo(keypointSink.input());
+        clipper.output().connectTo(keypointScaler.input());
+        keypointScaler.output().connectTo(keypointSink.input());
         knn.output().connectTo(keypointSink.input('matches'));
         //pyramid.output().connectTo(imageSink.input());
 
@@ -429,7 +421,7 @@ export class ImageTrackerScanningState extends ImageTrackerState
             greyscale, blur, nightvision, nightvisionMux, pyramid,
             detector, descriptor, clipper,
             lshTables, knn,
-            keypointSink,
+            keypointScaler, keypointSink,
             imagePortalSink, imagePortalSource,
             imagePortalMux, imagePortalBuffer, imagePortalCopy,
             //, imageSink

src/trackers/image-tracker/states/state.ts

@@ -33,10 +33,10 @@ import { SpeedyPipelineNodeResize } from 'speedy-vision/types/core/pipeline/node
 import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/transformer';
 import { SpeedyKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
 import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ReferenceImage } from '../reference-image';
 import { TrackerOutput } from '../../tracker';
 import { Nullable } from '../../../utils/utils';
-import { IllegalOperationError } from '../../../utils/errors';
-import { TRACK_RECTIFIED_BORDER } from '../settings';
+import { IllegalOperationError, IllegalArgumentError } from '../../../utils/errors';
 
 /** State output */
 export interface ImageTrackerStateOutput
@@ -46,7 +46,6 @@ export interface ImageTrackerStateOutput
     readonly nextStateSettings?: Record<string,any>;
 }
 
-
 /**
  * Abstract state of the Image Tracker
  */
@@ -61,6 +60,9 @@ export abstract class ImageTrackerState
     /** pipeline */
     protected _pipeline: SpeedyPipeline;
 
+    /** a flag telling whether or not the pipeline has been released */
+    protected _pipelineReleased: boolean;
+
 
     /**
      * Constructor
@@ -72,6 +74,7 @@ export abstract class ImageTrackerState
         this._name = name;
         this._imageTracker = imageTracker;
         this._pipeline = this._createPipeline();
+        this._pipelineReleased = false;
     }
 
     /**
@@ -84,6 +87,7 @@ export abstract class ImageTrackerState
 
     /**
      * AR screen size
+     * It may change over time, as when flipping a phone
      */
     get screenSize(): SpeedySize
     {
@@ -107,7 +111,12 @@ export abstract class ImageTrackerState
      */
     release(): null
     {
-        return this._pipeline.release();
+        if(!this._pipelineReleased) {
+            this._pipeline.release();
+            this._pipelineReleased = true;
+        }
+
+        return null;
     }
 
     /**
@@ -183,154 +192,4 @@ export abstract class ImageTrackerState
      * @returns pipeline
      */
     protected abstract _createPipeline(): SpeedyPipeline;
-
-
-
-    //
-    // Some utility methods common to various states
-    //
-
-    /**
-     * Find the coordinates of a polyline surrounding the target image
-     * @param homography maps the target image to the AR screen
-     * @param targetSize size of the target space
-     * @returns promise that resolves to 4 points in AR screen space
-     */
-    protected _findPolylineCoordinates(homography: SpeedyMatrix, targetSize: SpeedySize): SpeedyPromise<SpeedyMatrix>
-    {
-        const w = targetSize.width, h = targetSize.height;
-        const referenceImageCoordinates = Speedy.Matrix(2, 4, [
-            0, 0,
-            w, 0,
-            w, h,
-            0, h,
-        ]);
-
-        const polylineCoordinates = Speedy.Matrix.Zeros(2, 4);
-        return Speedy.Matrix.applyPerspectiveTransform(
-            polylineCoordinates,
-            referenceImageCoordinates,
-            homography
-        );
-    }
-
-    /**
-     * Find a polyline surrounding the target image
-     * @param homography maps the target image to the AR screen
-     * @param targetSize size of the target space
-     * @returns promise that resolves to 4 points in AR screen space
-     */
-    protected _findPolyline(homography: SpeedyMatrix, targetSize: SpeedySize): SpeedyPromise<SpeedyPoint2[]>
-    {
-        return this._findPolylineCoordinates(homography, targetSize).then(polylineCoordinates => {
-            const polydata = polylineCoordinates.read();
-            const polyline = Array.from({ length: 4 }, (_, i) => Speedy.Point2(polydata[2*i], polydata[2*i+1]));
-
-            return polyline;
-        });
-    }
-
-    /**
-     * Whether or not to rotate the warped image in order to best fit the AR screen
-     * @param media media associated with the reference image
-     * @param screenSize AR screen
-     * @returns boolean
-     */
-    protected _mustRotateWarpedImage(media: SpeedyMedia, screenSize: SpeedySize): boolean
-    {
-        const screenAspectRatio = screenSize.width / screenSize.height;
-        const mediaAspectRatio = media.width / media.height;
-        const eps = 0.1;
-
-        return (mediaAspectRatio >= 1+eps && screenAspectRatio < 1-eps) || (mediaAspectRatio < 1-eps && screenAspectRatio >= 1+eps);
-    }
-
-    /**
-     * Find a rectification matrix to be applied to an image fitting the entire AR screen
-     * @param media media associated with the reference image
-     * @param screenSize AR screen
-     * @returns promise that resolves to a rectification matrix
-     */
-    protected _findRectificationMatrixOfFullscreenImage(media: SpeedyMedia, screenSize: SpeedySize): SpeedyPromise<SpeedyMatrix>
-    {
-        const b = TRACK_RECTIFIED_BORDER;
-        const sw = screenSize.width, sh = screenSize.height;
-        const mediaAspectRatio = media.width / media.height;
-        const mustRotate = this._mustRotateWarpedImage(media, screenSize);
-
-        // compute the vertices of the target in screen space
-        // we suppose portrait or landscape mode for both screen & media
-        const c = mustRotate ? 1 / mediaAspectRatio : mediaAspectRatio;
-        const top = sw >= sh ? b * sh : (sh - sw * (1-2*b) / c) / 2;
-        const left = sw >= sh ? (sw - sh * (1-2*b) * c) / 2 : b * sw;
-        const right = sw - left;
-        const bottom = sh - top;
-
-        const targetVertices = Speedy.Matrix(2, 4, [
-            left, top,
-            right, top,
-            right, bottom,
-            left, bottom,
-        ]);
-
-        const screenVertices = Speedy.Matrix(2, 4, [
-            0, 0,
-            sw, 0,
-            sw, sh,
-            0, sh
-        ]);
-
-        const preRectificationMatrix = Speedy.Matrix.Eye(3);
-        const alignmentMatrix = Speedy.Matrix.Zeros(3);
-        const rectificationMatrix = Speedy.Matrix.Zeros(3);
-
-        return (mustRotate ? Speedy.Matrix.perspective(
-            // pre-rectifation: rotate by 90 degrees counterclockwise and scale to screenSize
-            preRectificationMatrix,
-            screenVertices,
-            Speedy.Matrix(2, 4, [ 0,sh , 0,0 , sw,0 , sw,sh ])
-        ) : Speedy.Promise.resolve(preRectificationMatrix)).then(_ =>
-            // alignment: align the target to the center of the screen
-            Speedy.Matrix.perspective(
-                alignmentMatrix,
-                screenVertices,
-                targetVertices
-            )
-        ).then(_ =>
-            // pre-rectify and then align
-            rectificationMatrix.setTo(alignmentMatrix.times(preRectificationMatrix))
-        );
-    }
-
-    /**
-     * Find a rectification matrix to be applied to the target image
-     * @param homography maps a reference image to the AR screen
-     * @param targetSize size of the target space
-     * @param media media associated with the reference image
-     * @param screenSize AR screen
-     * @returns promise that resolves to a rectification matrix
-     */
-    protected _findRectificationMatrixOfCameraImage(homography: SpeedyMatrix, targetSize: SpeedySize, media: SpeedyMedia, screenSize: SpeedySize): SpeedyPromise<SpeedyMatrix>
-    {
-        const sw = screenSize.width, sh = screenSize.height;
-        const screen = Speedy.Matrix(2, 4, [ 0, 0, sw, 0, sw, sh, 0, sh ]);
-
-        const rectificationMatrix = Speedy.Matrix.Zeros(3);
-        return this._findPolylineCoordinates(homography, targetSize).then(polyline =>
-
-            // from target space to (full)screen
-            Speedy.Matrix.perspective(rectificationMatrix, polyline, screen)
-
-        ).then(_ =>
-
-            // from (full)screen to rectified coordinates
-            this._findRectificationMatrixOfFullscreenImage(media, screenSize)
-
-        ).then(mat =>
-
-            // function composition
-            rectificationMatrix.setTo(mat.times(rectificationMatrix))
-
-        );
-    }
-}
+}

src/trackers/image-tracker/states/tracking.ts

@@ -26,6 +26,7 @@ import { SpeedyVector2 } from 'speedy-vision/types/core/speedy-vector';
 import { SpeedySize } from 'speedy-vision/types/core/speedy-size';
 import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media';
 import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix';
+import { SpeedyMatrixExpr } from 'speedy-vision/types/core/speedy-matrix-expr';
 import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise';
 import { SpeedyPipeline, SpeedyPipelineOutput } from 'speedy-vision/types/core/pipeline/pipeline';
 import { SpeedyPipelineNodeImageSource } from 'speedy-vision/types/core/pipeline/nodes/images/source';
@@ -39,40 +40,33 @@ import { SpeedyPipelineNodeKeypointBorderClipper } from 'speedy-vision/types/cor
 import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/pipeline/nodes/keypoints/transformer';
 import { SpeedyKeypoint, SpeedyTrackedKeypoint, SpeedyMatchedKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
 import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName, ImageTrackerResult, TrackableImage } from '../image-tracker';
-import { ImageTrackerState, ImageTrackerStateOutput } from './state';
+import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
 import { ImageTrackerEvent } from '../image-tracker-event';
+import { ImageTrackerState, ImageTrackerStateOutput } from './state';
 import { Nullable, Utils } from '../../../utils/utils';
 import { ReferenceImage } from '../reference-image';
 import { CameraModel } from '../../../geometry/camera-model';
 import { Viewer } from '../../../geometry/viewer';
 import { Pose } from '../../../geometry/pose';
 import { Transform } from '../../../geometry/transform';
-import { IllegalOperationError, IllegalArgumentError, TrackingError } from '../../../utils/errors';
+import { IllegalOperationError, IllegalArgumentError, TrackingError, NumericalError } from '../../../utils/errors';
 import {
-    TRACK_RECTIFIED_BORDER, TRACK_CLIPPING_BORDER, TRACK_MIN_MATCHES, TRACK_LOST_TOLERANCE,
+    TRACK_RECTIFIED_SCALE, TRACK_CLIPPING_BORDER, TRACK_MIN_MATCHES, TRACK_LOST_TOLERANCE,
     NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY, TRACK_WITH_NIGHTVISION,
     ORB_GAUSSIAN_KSIZE, ORB_GAUSSIAN_SIGMA,
     SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_SIGMA,
     TRACK_HARRIS_QUALITY, TRACK_DETECTOR_CAPACITY, TRACK_MAX_KEYPOINTS,
-    TRACK_RANSAC_REPROJECTIONERROR, TRACK_GRID_GRANULARITY, TRACK_MATCH_RATIO,
-    NIGHTVISION_QUALITY,
-    SUBPIXEL_METHOD,
+    TRACK_RANSAC_REPROJECTIONERROR_NDC, TRACK_MATCH_RATIO,
+    NIGHTVISION_QUALITY, SUBPIXEL_METHOD,
 } from '../settings';
 import { Settings } from '../../../core/settings';
 
-
 /** Whether or not we want to accelerate GPU-CPU transfers. Using turbo costs a slight delay on the tracking */
 const USE_TURBO = true;
 
 /** Number of PBOs; meaningful only when using turbo */
 const NUMBER_OF_PBOS = 2;
 
-/** Frame skipping; meaningful only when using turbo */
-const TURBO_SKIP = 2;
-
-/** A pair (a,b) of arrays of keypoints such that keypoint a[i] is a match to keypoint b[i] for all i */
-type QualityMatches = [ SpeedyMatchedKeypoint[], SpeedyKeypoint[] ];
-
 
 
 /**
@@ -91,33 +85,29 @@ export class ImageTrackerTrackingState extends ImageTrackerState
     /** current homography (for computing the pose) */
     private _poseHomography: SpeedyMatrix;
 
-    /** initial homography (i.e., the homography we found when we first started tracking) */
-    private _initialHomography: SpeedyMatrix; // from (full)screen to the actual target
-
     /** initial keypoints (i.e., the keypoints we found when we first started tracking) */
-    private _initialKeypoints: SpeedyKeypoint[];
+    private _templateKeypoints: SpeedyKeypoint[];
 
-    /** a helper */
-    private _counter: number;
-
-    /** camera model */
-    private _camera: CameraModel;
+    /** the screen size when the tracking began */
+    private _initialScreenSize: SpeedySize;
 
-    /** predicted keypoints */
-    private _predictedKeypoints: SpeedyMatchedKeypoint[];
+    /** last output of the tracker */
+    private _lastOutput: ImageTrackerOutput;
 
-    /** last pipeline output */
+    /** last output of the pipeline */
     private _lastPipelineOutput: SpeedyPipelineOutput;
 
-    /** a helper */
-    private _pipelineCounter: number;
+    /** a helper for frame skipping */
+    private _skipCounter: number;
 
-    /** last output */
-    private _lastOutput: ImageTrackerOutput;
+    /** a helper */
+    private _counter: number;
 
     /** the number of consecutive frames in which we have lost the tracking */
     private _lostCounter: number;
 
+    /** camera model */
+    private _camera: CameraModel;
 
 
 
@@ -132,18 +122,14 @@ export class ImageTrackerTrackingState extends ImageTrackerState
         this._referenceImage = null;
         this._warpHomography = Speedy.Matrix.Eye(3);
         this._poseHomography = Speedy.Matrix.Eye(3);
-        this._initialHomography = Speedy.Matrix.Eye(3);
-        this._initialKeypoints = [];
-        this._counter = 0;
-        this._camera = new CameraModel();
-        this._predictedKeypoints = [];
-        this._lastPipelineOutput = { keypoints: [] };
-        this._pipelineCounter = 0;
+        this._templateKeypoints = [];
+        this._initialScreenSize = Speedy.Size(1, 1);
         this._lastOutput = {};
+        this._lastPipelineOutput = { keypoints: [] };
+        this._skipCounter = 0;
+        this._counter = 0;
         this._lostCounter = 0;
-
-        // we need at least 4 correspondences of points to compute a homography matrix
-        Utils.assert(TRACK_MIN_MATCHES >= 4);
+        this._camera = new CameraModel();
     }
 
     /**
@@ -152,11 +138,11 @@ export class ImageTrackerTrackingState extends ImageTrackerState
      */
     onEnterState(settings: Record<string,any>)
     {
-        const homography = settings.homography as SpeedyMatrix;
+        const homography = settings.homography as SpeedyMatrix; // NDC, from reference image to video
         const referenceImage = settings.referenceImage as Nullable<ReferenceImage>;
         const templateKeypoints = settings.templateKeypoints as SpeedyKeypoint[];
-        const keypointPortalSink = settings.keypointPortalSink as SpeedyPipelineNodeKeypointPortalSink;
-        const screenSize = settings.screenSize as SpeedySize; // this.screenSize is not yet set
+        const templateKeypointPortalSink = settings.templateKeypointPortalSink as SpeedyPipelineNodeKeypointPortalSink;
+        const initialScreenSize = settings.initialScreenSize as SpeedySize; // this.screenSize is not yet set
         const keypointPortalSource = this._pipeline.node('keypointPortalSource') as SpeedyPipelineNodeKeypointPortalSource;
 
         // this shouldn't happen
@@ -167,20 +153,19 @@ export class ImageTrackerTrackingState extends ImageTrackerState
         this._referenceImage = referenceImage;
         this._warpHomography = Speedy.Matrix(homography);
         this._poseHomography = Speedy.Matrix(homography);
-        this._initialHomography = Speedy.Matrix(homography);
-        this._initialKeypoints = templateKeypoints;
-        this._counter = 0;
-        this._predictedKeypoints = [];
-        this._lastPipelineOutput = { keypoints: [] };
-        this._pipelineCounter = 0;
+        this._templateKeypoints = templateKeypoints;
+        this._initialScreenSize = Speedy.Size(initialScreenSize.width, initialScreenSize.height);
         this._lastOutput = {};
+        this._lastPipelineOutput = { keypoints: [] };
+        this._skipCounter = 0;
+        this._counter = 0;
         this._lostCounter = 0;
 
         // setup portals
-        keypointPortalSource.source = keypointPortalSink;
+        keypointPortalSource.source = templateKeypointPortalSink;
 
         // setup camera
-        this._camera.init(screenSize);
+        this._camera.init(initialScreenSize);
 
         // emit event
         const ev = new ImageTrackerEvent('targetfound', referenceImage);
@@ -197,6 +182,9 @@ export class ImageTrackerTrackingState extends ImageTrackerState
     {
         const referenceImage = this._referenceImage as ReferenceImage;
 
+        // log
+        Utils.log(`No longer tracking image "${referenceImage.name}"!`);
+
         // release the camera
         this._camera.release();
 
@@ -213,7 +201,7 @@ export class ImageTrackerTrackingState extends ImageTrackerState
     {
         const imageRectifier = this._pipeline.node('imageRectifier') as SpeedyPipelineNodePerspectiveWarp;
         const borderClipper = this._pipeline.node('borderClipper') as SpeedyPipelineNodeKeypointBorderClipper;
-        const keypointRectifier = this._pipeline.node('keypointRectifier') as SpeedyPipelineNodeKeypointTransformer;
+        const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
         const screenSize = this.screenSize;
 
         /*
@@ -230,10 +218,20 @@ export class ImageTrackerTrackingState extends ImageTrackerState
             screenSize.height * TRACK_CLIPPING_BORDER
         );
 
+        // convert keypoints to NIS
+        keypointScaler.transform = ImageTrackerUtils.rasterToNIS(screenSize);
+
         // rectify the image
-        return this._findImageWarp(this._warpHomography, screenSize).then(warp => {
-            imageRectifier.transform = warp;
-        });
+        const scale = TRACK_RECTIFIED_SCALE;
+        const aspectRatio = ImageTrackerUtils.bestFitAspectRatioNDC(this._imageTracker, this._referenceImage!);
+        const shrink = ImageTrackerUtils.bestFitScaleNDC(aspectRatio, scale);
+        const undistort = this._warpHomography.inverse();
+        const toScreen = ImageTrackerUtils.NDCToRaster(screenSize);
+        const toNDC = ImageTrackerUtils.rasterToNDC(screenSize);
+
+        return imageRectifier.transform.setTo(
+            toScreen.times(shrink.times(undistort)).times(toNDC)
+        ).then(() => void 0);
     }
 
     /**
@@ -242,35 +240,26 @@ export class ImageTrackerTrackingState extends ImageTrackerState
      */
     protected _gpuUpdate(): SpeedyPromise<SpeedyPipelineOutput>
     {
-        //return super._gpuUpdate();
-
         // No turbo?
         if(!USE_TURBO || Settings.powerPreference == 'low-power')
             return super._gpuUpdate();
 
         // When using turbo, we reduce the GPU usage by skipping every other frame
-        const counter = this._pipelineCounter;
-        this._pipelineCounter = (this._pipelineCounter + 1) % TURBO_SKIP;
-
-        // Skip frame
-        if(counter != 0) {
-            if(this._lastPipelineOutput.keypoints !== undefined) {
-                this._predictedKeypoints = this._predictKeypoints(
-                    this._lastPipelineOutput.keypoints,
-                    this._initialKeypoints
-                );
-            }
-            else
-                this._predictedKeypoints.length = 0;
+        if(0 == (this._skipCounter = 1 - this._skipCounter)) {
+            const templateKeypoints = this._templateKeypoints;
+            const previousKeypoints = this._lastPipelineOutput.keypoints as SpeedyMatchedKeypoint[];
+            //const currentKeypoints = this._predictKeypoints(previousKeypoints, templateKeypoints);
+            const currentKeypoints = previousKeypoints; // this actually works
+
+            this._lastPipelineOutput.keypoints = currentKeypoints;
 
-            this._lastPipelineOutput.keypoints = this._predictedKeypoints;
             return Speedy.Promise.resolve(this._lastPipelineOutput);
         }
 
         // Run the pipeline and store the results
-        return super._gpuUpdate().then(results => {
-            this._lastPipelineOutput = results;
-            return results;
+        return super._gpuUpdate().then(result => {
+            this._lastPipelineOutput = result;
+            return result;
         });
     }
 
@@ -281,106 +270,65 @@ export class ImageTrackerTrackingState extends ImageTrackerState
      */
     protected _afterUpdate(result: SpeedyPipelineOutput): SpeedyPromise<ImageTrackerStateOutput>
     {
-        const imageRectifier = this._pipeline.node('imageRectifier') as SpeedyPipelineNodePerspectiveWarp;
         const keypoints = result.keypoints as SpeedyMatchedKeypoint[];
         const image = result.image as SpeedyMedia | undefined;
-        const referenceImage = this._referenceImage as ReferenceImage;
+        const referenceImage = this._referenceImage!;
+        const screenSize = this.screenSize;
+
+        // track the target
+        return Speedy.Promise.resolve()
+        .then(() => {
+
+            // if a change in screen size occurs, we need to recalibrate
+            // (perform a new pre-training)
+            if(!screenSize.equals(this._initialScreenSize))
+                throw new TrackingError('Detected a change in screen size');
 
-        // find the best keypoint matches
-        return this._preprocessMatches(keypoints, this._initialKeypoints).then(matches => {
+            // find matching pairs of keypoints
+            const allPairs = this._findMatchingPairs(this._templateKeypoints, keypoints);
+            const pairs = ImageTrackerUtils.refineMatchingPairs(allPairs);
+            if(pairs.length < TRACK_MIN_MATCHES)
+                throw new TrackingError('Not enough data points to continue the tracking');
 
             // find motion models
-            return Speedy.Promise.all<SpeedyMatrix>([
-                this._findAffineMotion(matches),
-                this._findPerspectiveMotion(matches)
+            const points = ImageTrackerUtils.compilePairsOfKeypointsNDC(pairs);
+            return Speedy.Promise.all<SpeedyMatrixExpr>([
+                this._findAffineMotionNDC(points),
+                this._findPerspectiveMotionNDC(points)
             ]);
 
-        }).then(([affineMotion, perspectiveMotion]) => {
+        })
+        .then(([affineMotion, perspectiveMotion]) => {
 
             const lowPower = (Settings.powerPreference == 'low-power');
-            const frozen = !(!USE_TURBO || lowPower || this._counter % TURBO_SKIP == 0);
+            const delay = NUMBER_OF_PBOS * (!lowPower ? 2 : 1);
 
             // update warp homography
-            const delay = NUMBER_OF_PBOS * (!lowPower ? TURBO_SKIP : 1);
-            const remainder = delay >>> 1; // we want remainder > 0, so it skips the first frame
-            if(!USE_TURBO || this._counter % delay == remainder)
-                this._warpHomography.setToSync(this._warpHomography.times(affineMotion));
+            if(!USE_TURBO || this._counter % delay == 1) // skip the first frame (PBOs)
+                this._warpHomography.setToSync(affineMotion.times(this._warpHomography));
 
             // update pose homography
-            if(!frozen)
-                this._poseHomography.setToSync(this._warpHomography.times(perspectiveMotion));
+            this._poseHomography.setToSync(perspectiveMotion.times(this._warpHomography));
+            if(Number.isNaN(this._poseHomography.at(0,0)))
+                throw new NumericalError('Bad homography'); // normalize? 1 / h33
 
             // update counter
             this._counter = (this._counter + 1) % delay;
 
-            // update the camera
-            if(!frozen)
-                return this._camera.update(this._poseHomography, this.screenSize);
-            else
-                return this._camera.matrix;
+            // update camera model FIXME
+            const toNDC = ImageTrackerUtils.rasterToNDC(screenSize);
+            const toScreen = ImageTrackerUtils.NDCToRaster(screenSize);
+            const homography = Speedy.Matrix(toScreen.times(this._poseHomography).times(toNDC));
 
-        }).then(_ => {
+            //console.log("PIXL ", homography.toString());
+            //console.log("POSE ", this._poseHomography.toString());
+            //console.log("WARP ", this._warpHomography.toString());
+            //console.log("> AF ", Speedy.Matrix(affineMotion).toString());
+            //console.log("> PF ", Speedy.Matrix(perspectiveMotion).toString());
 
-            // find the inverse of the rectification matrix
-            const rectificationMatrix = imageRectifier.transform;
-            const inverseRectificationMatrix = Speedy.Matrix(rectificationMatrix.inverse());
-
-            // move keypoints from rectified space back to image space
-            const n = keypoints.length;
-            const coords: number[] = new Array(2*n);
-            for(let i = 0, j = 0; i < n; i++, j += 2) {
-                coords[j] = keypoints[i].position.x;
-                coords[j+1] = keypoints[i].position.y;
-            }
-
-            return Speedy.Matrix.applyPerspectiveTransform(
-                Speedy.Matrix.Zeros(2, n),
-                Speedy.Matrix(2, n, coords),
-                inverseRectificationMatrix
-            );
-
-            /*
-            // test image center
-            const coords2: number[] = new Array(2 * n);
-            for(let i = 0, j = 0; i < n; i++, j += 2) {
-                coords2[j] = this._imageTracker.screenSize.width / 2;
-                coords2[j+1] = this._imageTracker.screenSize.height / 2;
-                if(i % 2 == 0) {
-                    coords2[j] = this._imageTracker.screenSize.width / 4;
-                    coords2[j+1] = this._imageTracker.screenSize.height / 4;
-                }
-            }
-
-            return Speedy.Matrix.applyPerspectiveTransform(
-                Speedy.Matrix.Zeros(2, n),
-                Speedy.Matrix(2, n, coords2),
-                this._poseHomography
-                //this._warpHomography
-            );
-            */
-
-        }).then(mat => {
-
-            /*
-
-            const n = keypoints.length;
-            const coords = mat.read();
-
-            // ** this will interfere with the calculations when frame skipping is on **
-
-            // get keypoints in image space
-            for(let i = 0, j = 0; i < n; i++, j += 2) {
-                keypoints[i].position.x = coords[j];
-                keypoints[i].position.y = coords[j+1];
-            }
-
-            */
-
-            // find a polyline surrounding the target
-            return this._findPolyline(this._poseHomography, this.screenSize);
-            //return this._findPolyline(this._warpHomography, this.screenSize);
-
-        }).then(polyline => {
+            return this._camera.update(homography, screenSize);
+        })
+        .then(() => {
 
             // we let the target object be at the origin of the world space
             // (identity transform). We also perform a change of coordinates,
@@ -406,33 +354,34 @@ export class ImageTrackerTrackingState extends ImageTrackerState
                 viewer: viewer
             };
 
-            // build and save the output
-            this._lastOutput = {
+            // tracker output
+            const trackerOutput: ImageTrackerOutput = {
                 exports: result,
-                cameraMatrix: this._camera.matrix,
-                homography: this._warpHomography,
-                //keypoints: keypoints,
-                screenSize: this.screenSize,
+                //keypointsNIS: image !== undefined ? keypoints : undefined, // debug only
                 image: image,
-                polyline: polyline,
+                polylineNDC: ImageTrackerUtils.findPolylineNDC(this._poseHomography),
+                cameraMatrix: this._camera.matrix,
+                screenSize: screenSize,
             };
 
+            // save the last output
+            this._lastOutput = trackerOutput;
+
             // we have successfully tracked the target in this frame
             this._lostCounter = 0;
 
             // done!
             return {
                 nextState: 'tracking',
-                trackerOutput: this._lastOutput
+                trackerOutput: trackerOutput
             };
 
-        }).catch(err => {
+        })
+        .catch(err => {
 
             // give some tolerance to tracking errors
             if(err instanceof TrackingError) {
                 if(++this._lostCounter <= TRACK_LOST_TOLERANCE) {
-                    //console.log("ABSORB",this._lostCounter,err.toString())
-                    // absorb the error
                     return {
                         nextState: 'tracking',
                         trackerOutput: this._lastOutput
@@ -440,351 +389,144 @@ export class ImageTrackerTrackingState extends ImageTrackerState
                 }
             }
 
-            // lost tracking
+            // log
             Utils.warning(`The target has been lost! ${err.toString()}`);
-            this._camera.reset();
 
             // go back to the scanning state
             return {
                 nextState: 'scanning',
-                trackerOutput: {
-                    image: image,
-                    screenSize: this.screenSize,
-                },
+                trackerOutput: { }
             };
 
         });
     }
 
     /**
-     * Find quality matches between two sets of keypoints
-     * @param currKeypoints keypoints of the current frame
-     * @param prevKeypoints keypoints of the previous frame
-     * @returns quality matches
-     */
-    private _findQualityMatches(currKeypoints: SpeedyMatchedKeypoint[], prevKeypoints: SpeedyKeypoint[]): QualityMatches
-    {
-        const result: QualityMatches = [ [], [] ];
-        const n = currKeypoints.length;
-
-        for(let i = 0; i < n; i++) {
-            const currKeypoint = currKeypoints[i];
-
-            if(currKeypoint.matches[0].index >= 0 && currKeypoint.matches[1].index >= 0) {
-                const d1 = currKeypoint.matches[0].distance;
-                const d2 = currKeypoint.matches[1].distance;
-
-                if(d1 <= TRACK_MATCH_RATIO * d2) {
-                    const prevKeypoint = prevKeypoints[currKeypoint.matches[0].index];
-
-                    result[0].push(currKeypoint);
-                    result[1].push(prevKeypoint);
-                }
-            }
-        }
-
-        return result;
-    }
-
-    /**
-     * Find a better spatial distribution of the input matches
-     * @param matches quality matches
-     * @returns refined quality matches
-     */
-    private _refineQualityMatches(matches: QualityMatches): QualityMatches
-    {
-        const currKeypoints = matches[0];
-        const prevKeypoints = matches[1];
-
-        // find a better spatial distribution of the keypoints
-        const indices = this._distributeKeypoints(currKeypoints, TRACK_GRID_GRANULARITY);
-        const n = indices.length; // number of refined matches
-
-        // assemble output
-        const result: QualityMatches = [ new Array(n), new Array(n) ];
-        for(let i = 0; i < n; i++) {
-            result[0][i] = currKeypoints[indices[i]];
-            result[1][i] = prevKeypoints[indices[i]];
-        }
-
-        // done!
-        return result;
-    }
-
-    /**
-     * Spatially distribute keypoints over a grid
-     * @param keypoints keypoints to be distributed
-     * @param gridCells number of grid elements in each axis
-     * @returns a list of indices of keypoints[]
+     * Find an affine motion model in NDC between pairs of keypoints in NDC
+     * given as a 2 x 2n [ src | dest ] matrix
+     * @param points compiled pairs of keypoints in NDC
+     * @returns a promise that resolves to a 3x3 warp in NDC that maps source to destination
      */
-    private _distributeKeypoints(keypoints: SpeedyKeypoint[], gridCells: number): number[]
+    private _findAffineMotionNDC(points: SpeedyMatrix): SpeedyPromise<SpeedyMatrixExpr>
     {
-        // get the coordinates of the keypoints
-        const n = keypoints.length;
-        const points: number[] = new Array(2 * n);
-        for(let i = 0, j = 0; i < n; i++, j += 2) {
-            points[j] = keypoints[i].x;
-            points[j+1] = keypoints[i].y;
-        }
-
-        // normalize the coordinates to [0,1] x [0,1]
-        this._normalizePoints(points);
-
-        // distribute the keypoints over a grid
-        const numberOfCells = gridCells * gridCells;
-        const grid: number[] = (new Array(numberOfCells)).fill(-1);
-        for(let i = 0, j = 0; i < n; i++, j += 2) {
-            // find the grid location of the i-th point
-            const xg = Math.floor(points[j] * gridCells); // 0 <= xg,yg < gridCells
-            const yg = Math.floor(points[j+1] * gridCells);
-
-            // store the index of the i-th point in the grid
-            grid[yg * gridCells + xg] = i;
-        }
-
-        // retrieve points of the grid
-        const indices: number[] = [];
-        for(let g = 0; g < numberOfCells; g++) {
-            if(grid[g] >= 0) {
-                const i = grid[g];
-                indices.push(i);
-            }
-        }
+        /*
 
-        // done!
-        return indices;
-    }
+        We can probably get more accurate motion estimates if we
+        work in 3D rather than in 2D. We're currently estimating an
+        affine motion in 2D NDC space, which does not account for
+        perspective distortions. What if we projected the keypoints
+        into 3D NDC space, estimated the camera motion (rotation and
+        translation) that best describes the observed observed motion
+        of the keypoints, and then projected things back to 2D NDC
+        space? Need to figure this out; we'll get a homography matrix.
 
-    /**
-     * Normalize points to [0,1)^2
-     * @param points 2 x n matrix of points in column-major format
-     * @returns points
-     */
-    private _normalizePoints(points: number[]): number[]
-    {
-        Utils.assert(points.length % 2 == 0);
-
-        const n = points.length / 2;
-        if(n == 0)
-            return points;
-
-        let xmin = Number.POSITIVE_INFINITY, xmax = Number.NEGATIVE_INFINITY;
-        let ymin = Number.POSITIVE_INFINITY, ymax = Number.NEGATIVE_INFINITY;
-        for(let i = 0, j = 0; i < n; i++, j += 2) {
-            const x = points[j], y = points[j+1];
-            xmin = x < xmin ? x : xmin;
-            ymin = y < ymin ? y : ymin;
-            xmax = x > xmax ? x : xmax;
-            ymax = y > ymax ? y : ymax;
-        }
+        Note: work with a 6 DoF perspective transform instead of 8.
 
-        const xlen = xmax - xmin + 1; // +1 is a correction factor, so that 0 <= x,y < 1
-        const ylen = ymax - ymin + 1;
-        for(let i = 0, j = 0; i < n; i++, j += 2) {
-            points[j] = (points[j] - xmin) / xlen;
-            points[j+1] = (points[j+1] - ymin) / ylen;
-        }
+        */
 
-        return points;
-    }
+        return ImageTrackerUtils.findAffineWarpNDC(points, {
+            method: 'pransac',
+            reprojectionError: TRACK_RANSAC_REPROJECTIONERROR_NDC,
+            numberOfHypotheses: 512*4,
+            bundleSize: 128,
+            mask: undefined // score is not needed
+        }).then(([ warp, score ]) => {
 
-    /**
-     * Find a matrix with the coordinates of quality matches
-     * @param matches n quality matches
-     * @returns a 2 x 2n matrix split into two 2 x n blocks [ prevKeypoints | currKeypoints ]
-     */
-    private _findMatrixOfMatches(matches: QualityMatches): SpeedyMatrix
-    {
-        const n = matches[0].length;
-        Utils.assert(n > 0);
+            const scale = TRACK_RECTIFIED_SCALE;
+            const aspectRatio = ImageTrackerUtils.bestFitAspectRatioNDC(this._imageTracker, this._referenceImage!);
+            const shrink = ImageTrackerUtils.bestFitScaleNDC(aspectRatio, scale);
+            const grow = ImageTrackerUtils.inverseBestFitScaleNDC(aspectRatio, scale);
+            const scaledWarp = grow.times(warp).times(shrink);
 
-        // sets of keypoints
-        const currKeypoints = matches[0];
-        const prevKeypoints = matches[1];
+            const distort = this._warpHomography;
+            const undistort = distort.inverse();
+            const correctedWarp = distort.times(scaledWarp).times(undistort);
 
-        // get the coordinates of the keypoints of the set of refined matches
-        const src: number[] = new Array(2*n);
-        const dst: number[] = new Array(2*n);
+            return correctedWarp;
 
-        for(let i = 0, j = 0; i < n; i++, j += 2) {
-            src[j] = prevKeypoints[i].x;
-            src[j+1] = prevKeypoints[i].y;
+        }).catch(err => {
 
-            dst[j] = currKeypoints[i].x;
-            dst[j+1] = currKeypoints[i].y;
-        }
+            throw new TrackingError(`Can't find an affine motion`, err);
 
-        // assemble the matrix
-        return Speedy.Matrix(2, 2*n, src.concat(dst));
+        });
     }
 
     /**
-     * Preprocess keypoint matches
-     * @param currKeypoints keypoints of the current frame
-     * @param prevKeypoints keypoints of the previous frame
-     * @returns a promise that is rejected if there are not enough "good" matches, or that is resolved to a
-     *          2 x 2n matrix split into two 2 x n blocks [ source x,y coordinates | dest x,y coordinates ]
+     * Find a perspective motion model in NDC between pairs of keypoints in NDC
+     * given as a 2 x 2n [ src | dest ] matrix
+     * @param points compiled pairs of keypoints in NDC
+     * @returns a promise that resolves to a 3x3 warp in NDC that maps source to destination
      */
-    private _preprocessMatches(currKeypoints: SpeedyMatchedKeypoint[], prevKeypoints: SpeedyKeypoint[]): SpeedyPromise<SpeedyMatrix>
+    private _findPerspectiveMotionNDC(points: SpeedyMatrix): SpeedyPromise<SpeedyMatrixExpr>
     {
-        // find and refine quality matches
-        const qualityMatches = this._findQualityMatches(currKeypoints, prevKeypoints);
-        const refinedMatches = this._refineQualityMatches(qualityMatches);
-
-        // not enough matches?
-        const n = refinedMatches[0].length;
-        if(n < TRACK_MIN_MATCHES)
-            return Speedy.Promise.reject(new TrackingError('Not enough data to compute a motion model'));
-
-        // find matrix of matches
-        const matrixOfMatches = this._findMatrixOfMatches(refinedMatches);
+        return ImageTrackerUtils.findPerspectiveWarpNDC(points, {
+            method: 'pransac',
+            reprojectionError: TRACK_RANSAC_REPROJECTIONERROR_NDC,
+            numberOfHypotheses: 512*2,
+            bundleSize: 128,//128*4,
+            mask: undefined // score is not needed
+        }).then(([ warp, score ]) => {
 
-        // warp matrix of matches
-        const result = Speedy.Matrix.Zeros(2, 2*n);
-        return this._findKeypointWarp().then(transform =>
+            const scale = TRACK_RECTIFIED_SCALE;
+            const aspectRatio = ImageTrackerUtils.bestFitAspectRatioNDC(this._imageTracker, this._referenceImage!);
+            const shrink = ImageTrackerUtils.bestFitScaleNDC(aspectRatio, scale);
+            const grow = ImageTrackerUtils.inverseBestFitScaleNDC(aspectRatio, scale);
+            const scaledWarp = grow.times(warp).times(shrink);
 
-            Speedy.Matrix.applyAffineTransform(
-                result,
-                matrixOfMatches,
-                transform.block(0,1,0,2)
-            )
+            const distort = this._poseHomography;
+            const undistort = distort.inverse();
+            const correctedWarp = distort.times(scaledWarp).times(undistort);
 
-        );
-    }
+            return correctedWarp;
 
-    /**
-     * Find an affine motion model of the target image
-     * @param preprocessedMatches 2 x 2n matrix split into two 2 x n blocks [ src | dest ]
-     * @returns a promise that resolves to a 3x3 affine motion model (last row is [ 0  0  1 ])
-     */
-    private _findAffineMotion(preprocessedMatches: SpeedyMatrix): SpeedyPromise<SpeedyMatrix>
-    {
-        const model = Speedy.Matrix.Eye(3);
-        const n = preprocessedMatches.columns / 2; // number of preprocessed matches
-
-        // find motion model
-        return Speedy.Matrix.findAffineTransform(
-            model.block(0,1,0,2),
-            preprocessedMatches.block(0,1,0,n-1),
-            preprocessedMatches.block(0,1,n,2*n-1), {
-            method: 'pransac',
-            reprojectionError: TRACK_RANSAC_REPROJECTIONERROR,
-            numberOfHypotheses: 512,
-            bundleSize: 128,
-        }).then(_ => {
-
-            // validate the model
-            const a00 = model.at(0,0);
-            if(Number.isNaN(a00))
-                throw new TrackingError(`Can't compute affine motion model: bad keypoints`);
+        }).catch(err => {
 
-            // done!
-            return model;
+            throw new TrackingError(`Can't find a perspective motion`, err);
 
         });
     }
 
     /**
-     * Find a perspective motion model of the target image
-     * @param preprocessedMatches 2 x 2n matrix split into two 2 x n blocks [ src | dest ]
-     * @returns a promise that resolves to a 3x3 perspective motion model
+     * Find matching pairs of two sets of keypoints matched via brute force
+     * @param srcKeypoints source (database)
+     * @param destKeypoints destination
+     * @returns an array of matching pairs [src, dest]
      */
-    private _findPerspectiveMotion(preprocessedMatches: SpeedyMatrix): SpeedyPromise<SpeedyMatrix>
+    private _findMatchingPairs(srcKeypoints: SpeedyKeypoint[], destKeypoints: SpeedyMatchedKeypoint[]): ImageTrackerKeypointPair[]
     {
-        /*
-
-        We can probably get more accurate motion estimates if we
-        work in 3D rather than in 2D. We're currently estimating
-        an affine transform in image space. What if we projected
-        the keypoints into world space, estimated the camera motion
-        (rotation and translation) that best describes the observed
-        observed motion of the keypoints, and then projected things
-        back to image space? Need to figure this out; we'll get a
-        homography matrix.
-
-        Note: keypoints are in rectified image space.
-
-        Note: work with a 6 DoF perspective transform instead of 8.
-
-        */
-
-        const model = Speedy.Matrix.Zeros(3);
-        const n = preprocessedMatches.columns / 2; // number of preprocessed matches
-
-        // find motion model
-        return Speedy.Matrix.findHomography(
-            model,
-            preprocessedMatches.block(0,1,0,n-1),
-            preprocessedMatches.block(0,1,n,2*n-1), {
-            method: 'pransac',
-            reprojectionError: TRACK_RANSAC_REPROJECTIONERROR,
-            numberOfHypotheses: 512*2,
-            bundleSize: 128*4, //*4
-        }).then(_ => {
-
-            // validate the model
-            const a00 = model.at(0,0);
-            if(Number.isNaN(a00))
-                throw new TrackingError(`Can't compute perspective motion model: bad keypoints`);
-
-            // done!
-            return model;
+        const pairs: ImageTrackerKeypointPair[] = [];
 
-        });
-    }
+        for(let i = 0; i < destKeypoints.length; i++) {
+            const destKeypoint = destKeypoints[i];
 
-    /**
-     * Find a rectification matrix to be applied to the target image
-     * @param homography maps a reference image to the AR screen
-     * @param media target
-     * @param screenSize AR screen
-     * @returns promise that resolves to a rectification matrix
-     */
-    private _findImageWarp(homography: SpeedyMatrix, screenSize: SpeedySize): SpeedyPromise<SpeedyMatrix>
-    {
-        const referenceImage = this._referenceImage as ReferenceImage;
-        const media = this._imageTracker.database._findMedia(referenceImage.name);
-        const mat = Speedy.Matrix.Zeros(3);
+            if(destKeypoint.matches[0].index >= 0 && destKeypoint.matches[1].index >= 0) {
+                const d1 = destKeypoint.matches[0].distance;
+                const d2 = destKeypoint.matches[1].distance;
 
-        return this._findRectificationMatrixOfFullscreenImage(media, screenSize).then(warp =>
-            mat.setTo(warp.times(homography.inverse()))
-        );
-    }
+                // the best match should be "much better" than the second best match,
+                // which means that they are "distinct enough"
+                if(d1 <= TRACK_MATCH_RATIO * d2) {
+                    const srcKeypoint = srcKeypoints[destKeypoint.matches[0].index];
+                    pairs.push([srcKeypoint, destKeypoint]);
+                }
+            }
+        }
 
-    /**
-     * Find a warp to be applied to the keypoints
-     * @returns affine transform
-     */
-    private _findKeypointWarp(): SpeedyPromise<SpeedyMatrix>
-    {
-        const referenceImage = this._referenceImage as ReferenceImage;
-        const media = this._imageTracker.database._findMedia(referenceImage.name);
-        const screenSize = this.screenSize;
-        const sw = screenSize.width, sh = screenSize.height;
-        const mat = Speedy.Matrix.Eye(3, 3);
-
-        // no rotation is needed
-        if(!this._mustRotateWarpedImage(media, screenSize))
-            return Speedy.Promise.resolve(mat);
-
-        // rotate by 90 degrees clockwise and scale
-        return Speedy.Matrix.affine(
-            mat.block(0,1,0,2),
-            Speedy.Matrix(2, 3, [ 0,sh , 0,0 , sw,0  ]),
-            Speedy.Matrix(2, 3, [ 0,0 , sw,0 , sw,sh ])
-        ).then(_ => mat);
+        return pairs;
     }
 
     /**
      * Predict the keypoints without actually looking at the image
      * @param curr keypoints at time t (will modify the contents)
-     * @param initial keypoints at time t-1 (not just t = 0)
+     * @param prev keypoints at time t-1 (not just t = 0)
      * @returns keypoints at time t+1
      */
-    private _predictKeypoints(curr: SpeedyMatchedKeypoint[], initial: SpeedyKeypoint[]): SpeedyMatchedKeypoint[]
+    /*
+    private _predictKeypoints(curr: SpeedyMatchedKeypoint[], prev: SpeedyKeypoint[]): SpeedyMatchedKeypoint[]
     {
         // the target image is likely to be moving roughly in
         // the same manner as it was in the previous frame
+        const alpha = 0.8; //0.2;
         const next: SpeedyMatchedKeypoint[] = [];
         const n = curr.length;
 
@@ -793,25 +535,23 @@ export class ImageTrackerTrackingState extends ImageTrackerState
 
             if(cur.matches[0].index < 0 || cur.matches[1].index < 0)
                 continue;
-            /*
-            else if(cur.matches[0].distance > TRACK_MATCH_RATIO * cur.matches[1].distance)
-                continue;
-            */
+            //else if(cur.matches[0].distance > TRACK_MATCH_RATIO * cur.matches[1].distance)
+            //    continue;
 
-            const ini = initial[cur.matches[0].index];
-            const dx = cur.position.x - ini.position.x;
-            const dy = cur.position.y - ini.position.y;
+            const prv = prev[cur.matches[0].index];
+            const dx = cur.position.x - prv.position.x;
+            const dy = cur.position.y - prv.position.y;
 
             // a better mathematical model is needed
-            const alpha = 0.8; //0.2;
-            cur.position.x = ini.position.x + alpha * dx;
-            cur.position.y = ini.position.y + alpha * dy;
+            cur.position.x = prv.position.x + alpha * dx;
+            cur.position.y = prv.position.y + alpha * dy;
             next.push(cur);
         }
 
         // done!
         return next;
     }
+    */
 
     /**
      * Create & setup the pipeline
@@ -835,10 +575,10 @@ export class ImageTrackerTrackingState extends ImageTrackerState
         const denoiser = Speedy.Filter.GaussianBlur();
         const borderClipper = Speedy.Keypoint.BorderClipper('borderClipper');
         const clipper = Speedy.Keypoint.Clipper();
-        const keypointRectifier = Speedy.Keypoint.Transformer('keypointRectifier');
+        const keypointScaler = Speedy.Keypoint.Transformer('keypointScaler');
         const keypointPortalSource = Speedy.Keypoint.Portal.Source('keypointPortalSource');
         const keypointSink = Speedy.Keypoint.SinkOfMatchedKeypoints('keypoints');
-        const imageSink = Speedy.Image.Sink('image');
+        //const imageSink = Speedy.Image.Sink('image');
 
         source.media = null;
         screen.size = Speedy.Size(0,0);
@@ -858,7 +598,7 @@ export class ImageTrackerTrackingState extends ImageTrackerState
         clipper.size = TRACK_MAX_KEYPOINTS;
         borderClipper.imageSize = screen.size;
         borderClipper.borderSize = Speedy.Vector2(0,0);
-        keypointRectifier.transform = Speedy.Matrix.Eye(3);
+        keypointScaler.transform = Speedy.Matrix.Eye(3);
         matcher.k = 2;
         keypointPortalSource.source = null;
         keypointSink.turbo = USE_TURBO;
@@ -893,9 +633,8 @@ export class ImageTrackerTrackingState extends ImageTrackerState
         descriptor.output().connectTo(matcher.input('keypoints'));
 
         // prepare output
-        descriptor.output().connectTo(keypointRectifier.input());
-        //preMatcher.output().connectTo(keypointRectifier.input());
-        keypointRectifier.output().connectTo(keypointSink.input());
+        descriptor.output().connectTo(keypointScaler.input());
+        keypointScaler.output().connectTo(keypointSink.input());
         matcher.output().connectTo(keypointSink.input('matches'));
         //imageRectifier.output().connectTo(imageSink.input());
 
@@ -905,9 +644,10 @@ export class ImageTrackerTrackingState extends ImageTrackerState
             imageRectifier, nightvision, nightvisionMux, blur,
             detector, subpixel, borderClipper, clipper, denoiser,
             descriptor, matcher,
-            keypointPortalSource, keypointRectifier, keypointSink,
+            keypointPortalSource, keypointScaler, keypointSink,
             //imageSink
         );
+
         return pipeline;
     }
 }

src/trackers/image-tracker/states/training.ts

@@ -31,6 +31,7 @@ import { SpeedyPipelineNodeKeypointTransformer } from 'speedy-vision/types/core/
 import { SpeedyKeypoint } from 'speedy-vision/types/core/speedy-keypoint';
 import { Resolution } from '../../../utils/resolution';
 import { ImageTracker, ImageTrackerOutput, ImageTrackerStateName } from '../image-tracker';
+import { ImageTrackerUtils, ImageTrackerKeypointPair } from '../image-tracker-utils';
 import { ImageTrackerState, ImageTrackerStateOutput } from './state';
 import { ReferenceImage } from '../reference-image';
 import { ReferenceImageDatabase } from '../reference-image-database';
@@ -43,7 +44,6 @@ import {
     SCAN_WITH_NIGHTVISION, NIGHTVISION_GAIN, NIGHTVISION_OFFSET, NIGHTVISION_DECAY,
     SUBPIXEL_GAUSSIAN_KSIZE, SUBPIXEL_GAUSSIAN_SIGMA,
     TRAIN_IMAGE_SCALE,
-    TRAIN_TARGET_NORMALIZED_SIZE,
     NIGHTVISION_QUALITY,
     SUBPIXEL_METHOD,
 } from '../settings';
@@ -58,6 +58,9 @@ interface TrainingMap
 
     /** maps a keypoint index to an image index */
     readonly referenceImageIndex: number[];
+
+    /** reference images */
+    readonly referenceImages: ReferenceImage[];
 }
 
 
@@ -70,9 +73,6 @@ export class ImageTrackerTrainingState extends ImageTrackerState
     /** index of the image being used to train the tracker */
     private _currentImageIndex = 0;
 
-    /** reference images */
-    private _image: ReferenceImage[] = [];
-
     /** training map */
     private _trainingMap: TrainingMap;
 
@@ -89,7 +89,8 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         // initialize the training map
         this._trainingMap = {
             keypoints: [],
-            referenceImageIndex: []
+            referenceImageIndex: [],
+            referenceImages: [],
         };
     }
 
@@ -107,9 +108,9 @@ export class ImageTrackerTrainingState extends ImageTrackerState
 
         // prepare to train...
         this._currentImageIndex = 0;
-        this._image.length = 0;
-        this._trainingMap.referenceImageIndex.length = 0;
         this._trainingMap.keypoints.length = 0;
+        this._trainingMap.referenceImageIndex.length = 0;
+        this._trainingMap.referenceImages.length = 0;
 
         // lock the database
         Utils.log(`Image Tracker: training using ${database.count} reference image${database.count != 1 ? 's' : ''}`);
@@ -117,7 +118,17 @@ export class ImageTrackerTrainingState extends ImageTrackerState
 
         // collect all images
         for(const referenceImage of database)
-            this._image.push(referenceImage);
+            this._trainingMap.referenceImages.push(referenceImage);
+    }
+
+    /**
+     * Called when leaving the state, after update()
+     */
+    onLeaveState(): void
+    {
+        // we don't return to this state, so we can release the pipeline early
+        this._pipeline.release();
+        this._pipelineReleased = true;
     }
 
     /**
@@ -126,18 +137,13 @@ export class ImageTrackerTrainingState extends ImageTrackerState
      */
     protected _beforeUpdate(): SpeedyPromise<void>
     {
-        const arScreenSize = this.screenSize;
         const source = this._pipeline.node('source') as SpeedyPipelineNodeImageSource;
         const screen = this._pipeline.node('screen') as SpeedyPipelineNodeResize;
         const keypointScaler = this._pipeline.node('keypointScaler') as SpeedyPipelineNodeKeypointTransformer;
 
-        // this shouldn't happen
-        if(this._currentImageIndex >= this._image.length)
-            return Speedy.Promise.reject(new IllegalOperationError());
-
         // set the appropriate training media
         const database = this._imageTracker.database;
-        const referenceImage = this._image[this._currentImageIndex];
+        const referenceImage = this._trainingMap.referenceImages[this._currentImageIndex];
         const media = database._findMedia(referenceImage.name);
         source.media = media;
 
@@ -146,46 +152,12 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         const scale = TRAIN_IMAGE_SCALE; // ORB is not scale-invariant
         const aspectRatioOfTrainingImage = media.width / media.height;
 
-        /*
-        let sin = 0, cos = 1;
-
-        if((aspectRatioOfSourceVideo - 1) * (aspectRatioOfTrainingImage - 1) >= 0) {
-            // training image and source video: both in landscape mode or both in portrait mode
-            screen.size = Utils.resolution(resolution, aspectRatioOfTrainingImage);
-            screen.size.width = Math.round(screen.size.width * scale);
-            screen.size.height = Math.round(screen.size.height * scale);
-        }
-        else if(aspectRatioOfTrainingImage > aspectRatioOfSourceVideo) {
-            // training image: portrait mode; source video: landscape mode
-            screen.size = Utils.resolution(resolution, 1 / aspectRatioOfTrainingImage);
-            screen.size.width = Math.round(screen.size.width * scale);
-            screen.size.height = Math.round(screen.size.height * scale);
-            sin = 1; cos = 0; // rotate 90deg
-        }
-        else {
-            // training image: landscape mode; source video: portrait mode
-        }
-        */
         screen.size = Utils.resolution(resolution, aspectRatioOfTrainingImage);
         screen.size.width = Math.round(screen.size.width * scale);
         screen.size.height = Math.round(screen.size.height * scale);
 
-
-        // convert keypoints from the training image space to AR screen space
-        // let's pretend that trained keypoints belong to the AR screen space,
-        // regardless of the size of the target image. This will make things
-        // easier when computing the homography.
-        /*
-        const sw = arScreenSize.width / screen.size.width;
-        const sh = arScreenSize.height / screen.size.height;
-        */
-        const sw = TRAIN_TARGET_NORMALIZED_SIZE / screen.size.width;
-        const sh = TRAIN_TARGET_NORMALIZED_SIZE / screen.size.height;
-        keypointScaler.transform = Speedy.Matrix(3, 3, [
-            sw, 0,  0,
-            0,  sh, 0,
-            0,  0,  1,
-        ]);
+        // convert keypoints to NIS
+        keypointScaler.transform = ImageTrackerUtils.rasterToNIS(screen.size);
 
         // log
         Utils.log(`Image Tracker: training using reference image "${referenceImage.name}" at ${screen.size.width}x${screen.size.height}...`);
@@ -201,13 +173,19 @@ export class ImageTrackerTrainingState extends ImageTrackerState
      */
     protected _afterUpdate(result: SpeedyPipelineOutput): SpeedyPromise<ImageTrackerStateOutput>
     {
-        const referenceImage = this._image[this._currentImageIndex];
+        const referenceImage = this._trainingMap.referenceImages[this._currentImageIndex];
         const keypoints = result.keypoints as SpeedyKeypoint[];
         const image = result.image as SpeedyMedia | undefined;
 
         // log
         Utils.log(`Image Tracker: found ${keypoints.length} keypoints in reference image "${referenceImage.name}"`);
 
+        // tracker output
+        const trackerOutput: ImageTrackerOutput = {
+            keypointsNIS: image !== undefined ? keypoints : undefined, // debug only
+            image: image,
+        };
+
         // set the training map, so that we can map all keypoints of the current image to the current image
         for(let i = 0; i < keypoints.length; i++) {
             this._trainingMap.keypoints.push(keypoints[i]);
@@ -217,31 +195,22 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         // the current image has been processed!
         ++this._currentImageIndex;
 
-        // set output
-        if(this._currentImageIndex >= this._image.length) {
-
-            // finished training!
-            return Speedy.Promise.resolve({
-                //nextState: 'training',
-                nextState: 'scanning',
-                nextStateSettings: {
-                    keypoints: this._trainingMap.keypoints,
-                },
-                trackerOutput: { },
-                //trackerOutput: { image, keypoints, screenSize: this.screenSize },
-            });
-
-        }
-        else {
-
-            // we're not done yet
+        // we're not done yet
+        if(this._currentImageIndex < this._trainingMap.referenceImages.length) {
             return Speedy.Promise.resolve({
                 nextState: 'training',
-                trackerOutput: { },
-                //trackerOutput: { image, keypoints, screenSize: this.screenSize },
+                trackerOutput: trackerOutput
             });
-
         }
+
+        // finished training!
+        return Speedy.Promise.resolve({
+            nextState: 'scanning',
+            trackerOutput: trackerOutput,
+            nextStateSettings: {
+                database: this._trainingMap.keypoints,
+            }
+        });
     }
 
     /**
@@ -267,7 +236,7 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         const clipper = Speedy.Keypoint.Clipper();
         const keypointScaler = Speedy.Keypoint.Transformer('keypointScaler');
         const keypointSink = Speedy.Keypoint.Sink('keypoints');
-        const imageSink = Speedy.Image.Sink('image');
+        //const imageSink = Speedy.Image.Sink('image');
 
         source.media = null;
         screen.size = Speedy.Size(0,0);
@@ -317,7 +286,7 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         // prepare output
         descriptor.output().connectTo(keypointScaler.input());
         keypointScaler.output().connectTo(keypointSink.input());
-        nightvisionMux.output().connectTo(imageSink.input());
+        //nightvisionMux.output().connectTo(imageSink.input());
 
         // done!
         pipeline.init(
@@ -326,13 +295,13 @@ export class ImageTrackerTrainingState extends ImageTrackerState
             pyramid, detector, blur, descriptor, clipper,
             denoiser, blurredPyramid, subpixel,
             keypointScaler, keypointSink,
-            imageSink
+            //imageSink
         );
         return pipeline;
     }
 
     /**
-     * Get reference image
+     * Get the reference image associated with a keypoint index in the training map
      * @param keypointIndex -1 if not found
      * @returns reference image
      */
@@ -342,11 +311,11 @@ export class ImageTrackerTrainingState extends ImageTrackerState
         if(imageIndex < 0)
             return null;
 
-        return this._image[imageIndex];
+        return this._trainingMap.referenceImages[imageIndex];
     }
 
     /**
-     * Get reference image index
+     * Get the reference image index associated with a keypoint index in the training map
      * @param keypointIndex -1 if not found
      * @returns reference image index, or -1 if not found
      */
@@ -357,14 +326,14 @@ export class ImageTrackerTrainingState extends ImageTrackerState
             return -1;
 
         const imageIndex = this._trainingMap.referenceImageIndex[keypointIndex];
-        if(imageIndex < 0 || imageIndex >= this._image.length)
+        if(imageIndex < 0 || imageIndex >= this._trainingMap.referenceImages.length)
             return -1;
 
         return imageIndex;
     }
 
     /**
-     * Get keypoint of the trained set
+     * Get a keypoint of the trained set
      * @param keypointIndex -1 if not found
      * @returns a keypoint
      */