/* * encantar.js * GPU-accelerated Augmented Reality for the web * Copyright (C) 2022-2025 Alexandre Martins * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Lesser General Public License as published * by the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public License * along with this program. If not, see . * * image-tracker.ts * Image Tracker */ import Speedy from 'speedy-vision'; import { SpeedySize } from 'speedy-vision/types/core/speedy-size'; import { SpeedyMatrix } from 'speedy-vision/types/core/speedy-matrix'; import { SpeedyMedia } from 'speedy-vision/types/core/speedy-media'; import { SpeedyPipeline } from 'speedy-vision/types/core/pipeline/pipeline'; import { SpeedyPromise } from 'speedy-vision/types/core/speedy-promise'; import { SpeedyPipelineNode } from 'speedy-vision/types/core/pipeline/pipeline-node'; import { SpeedyPipelineNodeImageSource } from 'speedy-vision/types/core/pipeline/nodes/images/source'; import { SpeedyPipelineNodeResize } from 'speedy-vision/types/core/pipeline/nodes/transforms/resize'; import { SpeedyPipelineNodeFASTKeypointDetector } from 'speedy-vision/types/core/pipeline/nodes/keypoints/detectors/fast'; import { SpeedyKeypoint } from 'speedy-vision/types/core/speedy-keypoint'; import { VideoSource } from '../../sources/video-source'; import { CanvasSource } from '../../sources/canvas-source'; import { Tracker, TrackerOutput, TrackerResult, Trackable, TrackerType, TrackerResultType } from '../tracker'; import { Session } from '../../core/session'; import { IllegalOperationError, IllegalArgumentError } from '../../utils/errors'; import { Resolution } from '../../utils/resolution'; import { ReferenceImage, ReferenceImageWithMedia } from './reference-image'; import { ReferenceImageDatabase } from './reference-image-database'; import { ImageTrackerState } from './states/state'; import { ImageTrackerInitialState } from './states/initial'; import { ImageTrackerTrainingState } from './states/training'; import { ImageTrackerScanningState } from './states/scanning'; import { ImageTrackerPreTrackingAState } from './states/pre-tracking-a'; import { ImageTrackerPreTrackingBState } from './states/pre-tracking-b'; import { ImageTrackerTrackingState } from './states/tracking'; import { Nullable, Utils } from '../../utils/utils'; import { AREventTarget } from '../../utils/ar-events'; import { ImageTrackerEvent, ImageTrackerEventType } from './image-tracker-event'; import { SpeedyPoint2 } from 'speedy-vision/types/core/speedy-point'; import { Viewer } from '../../geometry/viewer'; import { Pose } from '../../geometry/pose'; import { CameraModel } from '../../geometry/camera-model'; /** A trackable target */ export interface TrackableImage extends Trackable { /** the pose of the target */ readonly pose: Pose; /** the reference image linked to the target */ readonly referenceImage: ReferenceImage; } /** Image Tracker result to be consumed by the user */ export class ImageTrackerResult extends TrackerResult { /** tracker */ readonly tracker: ImageTracker; /** trackable targets */ readonly trackables: TrackableImage[]; /** 3D virtual camera */ readonly viewer: Viewer; /** * Constructor * @param tracker * @param trackables * @param viewer */ constructor(tracker: ImageTracker, trackables: TrackableImage[], viewer: Viewer) { super(); this.tracker = tracker; this.trackables = trackables; this.viewer = viewer; } } /** Image Tracker output */ export interface ImageTrackerOutput extends TrackerOutput { /** tracker result to be consumed by the user */ readonly exports?: ImageTrackerResult; /** keypoints found in this framestep */ readonly keypoints?: SpeedyKeypoint[]; /** optional keypoints for visualizing & testing */ readonly keypointsNIS?: SpeedyKeypoint[]; /** optional polyline for visualizing & testing */ readonly polylineNDC?: SpeedyPoint2[]; /** optional camera model for visualizing & testing */ readonly camera?: CameraModel; } /** All possible states of an Image Tracker */ export type ImageTrackerStateName = 'initial' | 'training' | 'scanning' | 'pre-tracking-a' | 'pre-tracking-b' | 'tracking'; /** Options for instantiating an ImageTracker */ export interface ImageTrackerOptions { /** resolution of the tracker; it helps define the AR screen space */ resolution?: Resolution; } /** Default options for instantiating an ImageTracker */ const DEFAULT_OPTIONS: Readonly = { resolution: 'sm' }; /** * The ImageTracker tracks an image (one at a time) */ export class ImageTracker extends AREventTarget implements Tracker { /** session */ private _session: Nullable; /** source of data */ private _source: Nullable; /** all states */ private readonly _state: Record; /** name of the active state */ private _activeStateName: ImageTrackerStateName; /** last emitted output of the tracker */ private _lastOutput: ImageTrackerOutput; /** reference image database */ private readonly _database: ReferenceImageDatabase; /** the AR resolution size, used in GPU processing, defines the AR screen space */ private _resolution: Resolution; /** * Constructor * @param options */ constructor(options: ImageTrackerOptions) { super(); // the states this._state = { 'initial': new ImageTrackerInitialState(this), 'training': new ImageTrackerTrainingState(this), 'scanning': new ImageTrackerScanningState(this), 'pre-tracking-a': new ImageTrackerPreTrackingAState(this), 'pre-tracking-b': new ImageTrackerPreTrackingBState(this), 'tracking': new ImageTrackerTrackingState(this), }; // initial setup this._session = null; this._source = null; this._activeStateName = 'initial'; this._lastOutput = { }; this._database = new ReferenceImageDatabase(); // user settings options = Object.assign({}, DEFAULT_OPTIONS, options); this._resolution = options.resolution!; } /** * The type of the tracker * @deprecated */ get type(): keyof TrackerType { return 'image-tracker'; } /** * Check if this tracker is of a certain type */ is(type: T): this is TrackerType[T] { return type === this.type; } /** * Current state name */ get state(): ImageTrackerStateName { return this._activeStateName; } /** * Reference Image Database * Must be configured before training the tracker */ get database(): ReferenceImageDatabase { return this._database; } /** * Resolution of the tracker */ get resolution(): Resolution { return this._resolution; } /** * Resolution of the tracker * @readonly */ set resolution(resolution: Resolution) { // this property is readonly, but this setter has been kept because // it wasn't readonly in previous versions of the engine. FIXME } /** * Size of the AR screen space, in pixels * @internal */ get screenSize(): SpeedySize { return this._state[this._activeStateName].screenSize; } /** * Last emitted output * @internal */ get _output(): ImageTrackerOutput { return this._lastOutput; } /** * Stats related to this tracker * @internal */ get _stats(): string { const screenSize = this.screenSize; return `${screenSize.width}x${screenSize.height} ${this.state}`; } /** * Initialize this tracker * @param session * @returns promise that resolves after the tracker has been initialized * @internal */ _init(session: Session): SpeedyPromise { // store the session this._session = session; // find a suitable source of data // XXX also let the user specify a source manually? for(const source of session.sources) { // prefer video sources if(source._is('video')) { this._source = source; break; } else if(source._is('canvas')) this._source = source; } if(this._source === null) throw new IllegalOperationError('The image tracker requires a suitable source of data'); // initialize states for(const state of Object.values(this._state)) state.init(); // done! return Speedy.Promise.resolve(); } /** * Release this tracker * @returns promise that resolves after the tracker has been released * @internal */ _release(): SpeedyPromise { // release states for(const state of Object.values(this._state)) state.release(); // unlink session this._session = null; // done! return Speedy.Promise.resolve(); } /** * Update the tracker * @returns promise * @internal */ _update(): SpeedyPromise { // validate if(this._session == null) return Speedy.Promise.reject(new IllegalOperationError(`Uninitialized tracker`)); // compute the screen size for image processing purposes // note: this may change over time...! const media = this._source!._internalMedia; const screenSize = this._computeScreenSize(); // run the active state const activeState = this._state[this._activeStateName]; return activeState.update(media, screenSize).then(({ trackerOutput, nextState, nextStateSettings }) => { // update the output of the tracker this._lastOutput = trackerOutput; // need to change the state? if(this._activeStateName != nextState) { activeState.onLeaveState(); this._activeStateName = nextState; this._state[nextState].onEnterState(nextStateSettings || {}); } }); } /** * Get reference image * @param keypointIndex -1 if not found * @returns reference image * @internal */ _referenceImageOfKeypoint(keypointIndex: number): Nullable { const training = this._state.training as ImageTrackerTrainingState; return training.referenceImageOfKeypoint(keypointIndex); } /** * Get reference image index * @param keypointIndex -1 if not found * @returns reference image index, or -1 if not found * @internal */ _referenceImageIndexOfKeypoint(keypointIndex: number): number { const training = this._state.training as ImageTrackerTrainingState; return training.referenceImageIndexOfKeypoint(keypointIndex); } /** * Get a keypoint of the trained set * @param keypointIndex * @returns a keypoint * @internal */ _referenceKeypoint(keypointIndex: number): Nullable { const training = this._state.training as ImageTrackerTrainingState; return training.referenceKeypoint(keypointIndex); } /** * Compute the current size of the AR screen space * Note that this may change over time * @returns size */ private _computeScreenSize(): SpeedySize { const media = this._source!._internalMedia; const aspectRatio = media.width / media.height; const screenSize = Utils.resolution(this._resolution, aspectRatio); return screenSize; } }