import { findLocalPeaks } from "./evaluation.js"; import { computeLocalizationMap, createLocalizationAnalysisPlan, } from "./localization.js"; import { DEFAULT_GCC_PHAT_CROSS_SPECTRUM_BATCH_SIZE, DEFAULT_GCC_PHAT_POINT_BATCH_SIZE, } from "./gcc-phat.js"; import { isDasAlgorithm, requireLocalizationAlgorithmDefinition, } from "./localization-algorithms.js"; import { DEFAULT_MAX_COMPENSATION_DB, MAX_FREQUENCY_RESPONSE_HZ, MAX_FREQUENCY_RESPONSE_POINTS, requireMaxCompensationDb, } from "./frequency-response.js"; export const REALTIME_LOCALIZATION_PLAN_KIND = "micarrsim.realtime-localization-plan/1"; export const DEFAULT_STREAM_RESULTS_PER_CALL = 1; export const MAX_STREAM_RESULTS_PER_CALL = 32; export const DEFAULT_MAXIMUM_STREAM_INPUT_BYTES = 256 * 1024 * 1024; export const DEFAULT_REALTIME_POINT_BATCH_SIZE = 256; export const DEFAULT_REALTIME_STFT_TRANSFORM_BATCH_SIZE = 128; function requirePositiveFinite(value, label) { if (!Number.isFinite(value) || value <= 0) { throw new RangeError(`${label} must be a positive finite number.`); } return value; } function requirePositiveInteger(value, label) { if (!Number.isInteger(value) || value <= 0) { throw new RangeError(`${label} must be a positive integer.`); } return value; } function requirePowerOfTwo(value, label) { requirePositiveInteger(value, label); if (value < 2 || (value & (value - 1)) !== 0) { throw new RangeError(`${label} must be a power of two greater than one.`); } return value; } function monotonicNow() { return globalThis.performance?.now?.() ?? Date.now(); } function freezePosition(position, label) { if (!Array.isArray(position) || position.length !== 3 || !position.every(Number.isFinite)) { throw new TypeError(`${label} must be a finite [x, y, z] array.`); } return Object.freeze([...position]); } function copyFrequencyResponse(value, label) { if (value === undefined) return undefined; if ( !Array.isArray(value) || value.length < 2 || value.length > MAX_FREQUENCY_RESPONSE_POINTS ) { throw new RangeError( label + " must contain 2-" + MAX_FREQUENCY_RESPONSE_POINTS + " points.", ); } let previousFrequency = -Infinity; return Object.freeze(value.map((point, index) => { const pointLabel = label + "[" + index + "]"; if (!point || typeof point !== "object") { throw new TypeError(pointLabel + " must be an object."); } if ( !Number.isFinite(point.frequencyHz) || point.frequencyHz < 0 || point.frequencyHz > MAX_FREQUENCY_RESPONSE_HZ || point.frequencyHz <= previousFrequency ) { throw new RangeError(pointLabel + ".frequencyHz must be finite and strictly increasing."); } if (!Number.isFinite(point.gainDb) || point.gainDb < -120 || point.gainDb > 120) { throw new RangeError(pointLabel + ".gainDb must lie inside -120..120 dB."); } if ( point.phaseDeg !== undefined && (!Number.isFinite(point.phaseDeg) || point.phaseDeg < -180 || point.phaseDeg > 180) ) { throw new RangeError(pointLabel + ".phaseDeg must lie inside -180..180 degrees."); } previousFrequency = point.frequencyHz; return Object.freeze({ ...point }); })); } function copyArrayDefinition(value) { if (!value || !Array.isArray(value.microphones) || value.microphones.length === 0) { throw new TypeError("array.microphones must be a non-empty array."); } const microphoneIds = new Set(); const microphones = value.microphones.map((microphone, index) => { const id = microphone.id ?? "mic-" + index; if (microphoneIds.has(id)) throw new Error("duplicate microphone id: " + id); microphoneIds.add(id); if ( microphone.sensitivityDb !== undefined && !Number.isFinite(microphone.sensitivityDb) ) { throw new TypeError("array.microphones[" + index + "].sensitivityDb must be finite."); } if ( microphone.phaseOffsetDeg !== undefined && !Number.isFinite(microphone.phaseOffsetDeg) ) { throw new TypeError("array.microphones[" + index + "].phaseOffsetDeg must be finite."); } const frequencyResponse = copyFrequencyResponse( microphone.frequencyResponse, "array.microphones[" + index + "].frequencyResponse", ); return Object.freeze({ ...microphone, id, position: freezePosition( microphone.position, "array.microphones[" + index + "].position", ), ...(frequencyResponse ? { frequencyResponse: Object.freeze(frequencyResponse) } : {}), }); }); return Object.freeze({ ...value, microphones: Object.freeze(microphones), }); } function copyScan(scan) { if (!scan || typeof scan !== "object") throw new TypeError("scan must be an object."); const numericFields = ["xMin", "xMax", "yMin", "yMax", "z"]; for (const field of numericFields) { if (!Number.isFinite(scan[field])) throw new TypeError(`scan.${field} must be finite.`); } const columns = requirePositiveInteger(scan.columns, "scan.columns"); const rows = requirePositiveInteger(scan.rows, "scan.rows"); if (scan.xMax < scan.xMin || scan.yMax < scan.yMin) { throw new RangeError("scan bounds must satisfy min <= max."); } return Object.freeze({ ...scan, columns, rows }); } function copyBands(bands, sampleRate) { if (!Array.isArray(bands) || bands.length === 0) { throw new TypeError("bands must be a non-empty array."); } const nyquist = sampleRate / 2; return Object.freeze(bands.map((band, index) => { if ( !Number.isFinite(band?.minHz) || !Number.isFinite(band?.maxHz) || band.minHz < 0 || band.maxHz <= band.minHz || band.maxHz >= nyquist ) { throw new RangeError(`bands[${index}] must lie inside 0–${nyquist} Hz.`); } return Object.freeze({ ...band, id: band.id ?? `band-${index + 1}`, color: band.color ?? "#65f2bf", }); })); } function copyPeakDetection(value, scan) { const options = value ?? {}; if (!options || typeof options !== "object" || Array.isArray(options)) { throw new TypeError("peakDetection must be an object."); } const thresholdRatio = options.thresholdRatio ?? 0.5; if (!Number.isFinite(thresholdRatio) || thresholdRatio < 0 || thresholdRatio > 1) { throw new RangeError("peakDetection.thresholdRatio must lie inside 0..1."); } const xStep = scan.columns > 1 ? (scan.xMax - scan.xMin) / (scan.columns - 1) : 0; const yStep = scan.rows > 1 ? (scan.yMax - scan.yMin) / (scan.rows - 1) : 0; const minimumPeakDistanceM = options.minimumPeakDistanceM ?? Math.hypot(xStep, yStep); if (!Number.isFinite(minimumPeakDistanceM) || minimumPeakDistanceM < 0) { throw new RangeError("peakDetection.minimumPeakDistanceM must be a non-negative finite number."); } const maximumDetections = options.maximumDetections ?? 8; if (!Number.isInteger(maximumDetections) || maximumDetections < 1 || maximumDetections > 128) { throw new RangeError("peakDetection.maximumDetections must be an integer from 1 through 128."); } return Object.freeze({ thresholdRatio, minimumPeakDistanceM, maximumDetections }); } function validateChannels(channels, expectedChannelCount) { if (!Array.isArray(channels) || channels.length !== expectedChannelCount) { throw new Error( `channel count ${channels?.length ?? 0} does not match microphone count ${expectedChannelCount}.`, ); } if (!channels.every((channel) => channel instanceof Float32Array)) { throw new TypeError("every channel must be a Float32Array."); } const sampleCount = channels[0].length; if (channels.some((channel) => channel.length !== sampleCount)) { throw new Error("all channels must have the same sample count."); } return sampleCount; } function createAbortError(signal) { if (signal?.reason instanceof Error && signal.reason.name === "AbortError") return signal.reason; const error = new Error("localization was canceled."); error.name = "AbortError"; return error; } function throwIfAborted(signal) { if (signal?.aborted) throw createAbortError(signal); } export function detectBandPeaks(rawMap, scan, options = {}) { const rawScale = options.rawScale ?? rawMap?.scale ?? "power"; if (rawScale !== "power" && rawScale !== "score") { throw new RangeError('peak map rawScale must be "power" or "score".'); } const peakDetection = copyPeakDetection(options, scan); const localPeaks = findLocalPeaks(rawMap, scan); const threshold = (localPeaks[0]?.value ?? 0) * peakDetection.thresholdRatio; const detections = []; for (const peak of localPeaks) { if (peak.value < threshold) continue; if ( peakDetection.minimumPeakDistanceM > 0 && detections.some((selected) => Math.hypot( selected.position[0] - peak.position[0], selected.position[1] - peak.position[1], ) <= peakDetection.minimumPeakDistanceM ) ) { continue; } detections.push({ rank: detections.length + 1, index: peak.index, position: [...peak.position], rawValue: peak.value, rawScale, ...(rawScale === "power" ? { rawPower: peak.value } : { rawScore: peak.value }), }); if (detections.length === peakDetection.maximumDetections) break; } return detections; } function locationsFromBands(bands, scan, peakDetection) { return bands.flatMap((band) => detectBandPeaks(band.rawMap, scan, { ...peakDetection, rawScale: band.rawScale, }).map((peak) => ({ bandId: band.id, ...peak, })), ); } /** * Compile immutable microphone/scan/band configuration and CPU geometry caches. * * This API deliberately has no Scene or synthesis dependency: callers provide * sampled microphone channels directly to localizeMultichannel(). */ export function createLocalizationPlan(config) { if (!config || typeof config !== "object") throw new TypeError("config must be an object."); const sampleRate = requirePositiveFinite(config.sampleRate, "sampleRate"); const arrayDefinition = copyArrayDefinition(config.array ?? config.arrayDefinition); const scan = copyScan(config.scan); const peakDetection = copyPeakDetection(config.peakDetection, scan); const frequencyBands = copyBands(config.bands ?? config.frequencyBands, sampleRate); const fftSize = requirePowerOfTwo(config.stft?.fftSize ?? config.fftSize ?? 512, "fftSize"); const hopSize = requirePositiveInteger( config.stft?.hopSize ?? config.stftHopSize ?? fftSize / 2, "stftHopSize", ); if (hopSize > fftSize) throw new RangeError("stftHopSize must not exceed fftSize."); const speedOfSound = requirePositiveFinite(config.speedOfSound ?? 343, "speedOfSound"); const normalization = config.normalization ?? config.beamforming?.normalization ?? "peak"; if (normalization !== "peak" && normalization !== "none") { throw new RangeError('normalization must be "peak" or "none".'); } const compensateFrequencyResponse = config.compensateFrequencyResponse ?? config.beamforming?.compensateFrequencyResponse ?? true; if (typeof compensateFrequencyResponse !== "boolean") { throw new TypeError("compensateFrequencyResponse must be boolean."); } const maxCompensationDb = requireMaxCompensationDb( config.maxCompensationDb ?? config.beamforming?.maxCompensationDb ?? DEFAULT_MAX_COMPENSATION_DB, ); const algorithm = requireLocalizationAlgorithmDefinition( config.algorithm ?? config.beamforming?.algorithm, ); const phatRelativeFloor = config.phatRelativeFloor ?? config.beamforming?.phatRelativeFloor; const phatFloorRatio = config.phatFloorRatio ?? config.beamforming?.phatFloorRatio; const scene = Object.freeze({ speedOfSound }); const experiment = Object.freeze({ stft: Object.freeze({ fftSize, hopSize, window: "hann" }), scan, frequencyBands, beamforming: Object.freeze({ algorithm: algorithm.id, backend: "cpu", normalization, compensateFrequencyResponse, maxCompensationDb, }), }); const analysisPlan = createLocalizationAnalysisPlan({ sampleRate, arrayDefinition, scene, experiment, maximumSteeringCacheBytes: config.maximumSteeringCacheBytes, maximumPlanBytes: config.maximumPlanBytes, }); return Object.freeze({ kind: REALTIME_LOCALIZATION_PLAN_KIND, algorithm: algorithm.id, sampleRate, channelCount: arrayDefinition.microphones.length, arrayDefinition, scan, bands: frequencyBands, peakDetection, stft: experiment.stft, speedOfSound, normalization, maxCompensationDb, scene, experiment, analysisPlan, ...(isDasAlgorithm(algorithm.id) ? { dasPlan: analysisPlan } : { gccPhatPlan: analysisPlan }), computeOptions: Object.freeze({ pointBatchSize: isDasAlgorithm(algorithm.id) ? DEFAULT_REALTIME_POINT_BATCH_SIZE : DEFAULT_GCC_PHAT_POINT_BATCH_SIZE, stftTransformBatchSize: DEFAULT_REALTIME_STFT_TRANSFORM_BATCH_SIZE, ...(!isDasAlgorithm(algorithm.id) ? { crossSpectrumBatchSize: DEFAULT_GCC_PHAT_CROSS_SPECTRUM_BATCH_SIZE } : {}), ...(phatRelativeFloor !== undefined ? { phatRelativeFloor } : {}), ...(phatFloorRatio !== undefined ? { phatFloorRatio } : {}), ...(config.computeOptions ?? {}), }), }); } function requireLocalizationPlan(plan) { if (plan?.kind !== REALTIME_LOCALIZATION_PLAN_KIND) { throw new TypeError("plan must be created by createLocalizationPlan()."); } return plan; } /** * Localize one already-sampled multichannel block. */ export async function localizeMultichannel(options) { if (!options || typeof options !== "object") throw new TypeError("options must be an object."); const plan = options.plan ? requireLocalizationPlan(options.plan) : createLocalizationPlan(options); const sampleCount = validateChannels(options.channels, plan.channelCount); if (sampleCount === 0) throw new RangeError("channels must contain at least one sample."); throwIfAborted(options.signal); const processingStartedAt = monotonicNow(); const localization = await computeLocalizationMap({ ...plan.computeOptions, ...(options.computeOptions ?? {}), channels: options.channels, sampleRate: plan.sampleRate, arrayDefinition: plan.arrayDefinition, scene: plan.scene, experiment: plan.experiment, analysisPlan: plan.analysisPlan, signal: options.signal, onProgress: options.onProgress, }); const locations = locationsFromBands( localization.bands, plan.scan, plan.peakDetection, ); const processingTimeMs = Math.max(0, monotonicNow() - processingStartedAt); return { ...localization, sampleRate: plan.sampleRate, sampleCount, duration: sampleCount / plan.sampleRate, processingTimeMs, locations, }; } /** * A fixed-capacity chunk buffer for real microphone adapters. * * append() is synchronous and remains safe while an analysis is awaiting CPU * batches. analyzeAvailable() copies one window before yielding, commits its * hop only after successful localization, and returns no more than * maximumResultsPerCall maps. */ export class StreamingDasLocalizer { constructor(config) { this.plan = createLocalizationPlan(config); this.analysisWindowSamples = requirePositiveInteger( config.analysisWindowSamples ?? Math.max(this.plan.stft.fftSize, Math.ceil(this.plan.sampleRate / 10)), "analysisWindowSamples", ); if (this.analysisWindowSamples < this.plan.stft.fftSize) { throw new RangeError("analysisWindowSamples must be at least fftSize."); } this.analysisHopSamples = requirePositiveInteger( config.analysisHopSamples ?? this.analysisWindowSamples, "analysisHopSamples", ); if (this.analysisHopSamples > this.analysisWindowSamples) { throw new RangeError("analysisHopSamples must not exceed analysisWindowSamples."); } this.analysisDeadlineMs = this.analysisHopSamples / this.plan.sampleRate * 1000; this.maximumBufferedSamples = requirePositiveInteger( config.maximumBufferedSamples ?? this.analysisWindowSamples * 4, "maximumBufferedSamples", ); if (this.maximumBufferedSamples < this.analysisWindowSamples) { throw new RangeError("maximumBufferedSamples must be at least analysisWindowSamples."); } this.maximumResultsPerCall = requirePositiveInteger( config.maximumResultsPerCall ?? DEFAULT_STREAM_RESULTS_PER_CALL, "maximumResultsPerCall", ); if (this.maximumResultsPerCall > MAX_STREAM_RESULTS_PER_CALL) { throw new RangeError( `maximumResultsPerCall must not exceed ${MAX_STREAM_RESULTS_PER_CALL}.`, ); } this.maximumInputBufferBytes = requirePositiveInteger( config.maximumInputBufferBytes ?? DEFAULT_MAXIMUM_STREAM_INPUT_BYTES, "maximumInputBufferBytes", ); this.retainedInputBytes = this.plan.channelCount * (this.maximumBufferedSamples + this.analysisWindowSamples) * Float32Array.BYTES_PER_ELEMENT; if ( !Number.isSafeInteger(this.retainedInputBytes) || this.retainedInputBytes > this.maximumInputBufferBytes ) { throw new RangeError( "stream input buffers require " + this.retainedInputBytes.toLocaleString() + " bytes, above maximumInputBufferBytes " + this.maximumInputBufferBytes.toLocaleString() + ".", ); } this.buffers = Array.from( { length: this.plan.channelCount }, () => new Float32Array(this.maximumBufferedSamples), ); this.analysisChannels = Array.from( { length: this.plan.channelCount }, () => new Float32Array(this.analysisWindowSamples), ); this.startIndex = 0; this.bufferedSamples = 0; this.bufferStartSample = 0; this.totalReceivedSamples = 0; this.droppedSamples = 0; this.analysisAttempts = 0; this.completedAnalyses = 0; this.failedAnalyses = 0; this.lastProcessingTimeMs = null; this.maximumProcessingTimeMs = 0; this.deadlineMisses = 0; this.nextSequence = 0; this.pendingAnalysis = null; this.busy = false; } availableAnalysisCount() { const bufferEndSample = this.bufferStartSample + this.bufferedSamples; if (!this.pendingAnalysis) { if (this.bufferedSamples < this.analysisWindowSamples) return 0; return 1 + Math.floor( (this.bufferedSamples - this.analysisWindowSamples) / this.analysisHopSamples, ); } // The failed window remains in analysisChannels even if capture overflow // advances the ring while localization is running. Count it once, then // count windows that will remain after its successful hop commit. const nextRingStart = Math.max( this.bufferStartSample, this.pendingAnalysis.startSample + this.analysisHopSamples, ); const futureSamples = bufferEndSample - nextRingStart; const futureCount = futureSamples < this.analysisWindowSamples ? 0 : 1 + Math.floor( (futureSamples - this.analysisWindowSamples) / this.analysisHopSamples, ); return 1 + futureCount; } getState() { return { sampleRate: this.plan.sampleRate, algorithm: this.plan.algorithm, channelCount: this.plan.channelCount, analysisWindowSamples: this.analysisWindowSamples, analysisHopSamples: this.analysisHopSamples, analysisDeadlineMs: this.analysisDeadlineMs, maximumBufferedSamples: this.maximumBufferedSamples, maximumInputBufferBytes: this.maximumInputBufferBytes, maximumResultsPerCall: this.maximumResultsPerCall, bufferedSamples: this.bufferedSamples, availableAnalysisCount: this.availableAnalysisCount(), bufferStartSample: this.bufferStartSample, totalReceivedSamples: this.totalReceivedSamples, droppedSamples: this.droppedSamples, analysisAttempts: this.analysisAttempts, completedAnalyses: this.completedAnalyses, failedAnalyses: this.failedAnalyses, lastProcessingTimeMs: this.lastProcessingTimeMs, maximumProcessingTimeMs: this.maximumProcessingTimeMs, deadlineMisses: this.deadlineMisses, nextSequence: this.nextSequence, busy: this.busy, retainedInputBytes: this.retainedInputBytes, }; } consume(sampleCount, countAsDropped) { if (sampleCount <= 0) return; this.startIndex = (this.startIndex + sampleCount) % this.maximumBufferedSamples; this.bufferedSamples -= sampleCount; this.bufferStartSample += sampleCount; if (countAsDropped) this.droppedSamples += sampleCount; } append(channels) { const chunkSamples = validateChannels(channels, this.plan.channelCount); if (chunkSamples === 0) return this.getState(); const capacity = this.maximumBufferedSamples; const receivedBefore = this.totalReceivedSamples; if (chunkSamples >= capacity) { const retainedStart = chunkSamples - capacity; for (let channelIndex = 0; channelIndex < this.plan.channelCount; channelIndex += 1) { this.buffers[channelIndex].set(channels[channelIndex].subarray(retainedStart)); } this.droppedSamples += this.bufferedSamples + retainedStart; this.startIndex = 0; this.bufferedSamples = capacity; this.bufferStartSample = receivedBefore + retainedStart; this.totalReceivedSamples += chunkSamples; return this.getState(); } const overflow = Math.max(0, this.bufferedSamples + chunkSamples - capacity); this.consume(overflow, true); const writeIndex = (this.startIndex + this.bufferedSamples) % capacity; const firstLength = Math.min(chunkSamples, capacity - writeIndex); for (let channelIndex = 0; channelIndex < this.plan.channelCount; channelIndex += 1) { const source = channels[channelIndex]; const target = this.buffers[channelIndex]; target.set(source.subarray(0, firstLength), writeIndex); if (firstLength < chunkSamples) { target.set(source.subarray(firstLength), 0); } } this.bufferedSamples += chunkSamples; this.totalReceivedSamples += chunkSamples; return this.getState(); } copyAnalysisWindow() { const firstLength = Math.min( this.analysisWindowSamples, this.maximumBufferedSamples - this.startIndex, ); for (let channelIndex = 0; channelIndex < this.plan.channelCount; channelIndex += 1) { const source = this.buffers[channelIndex]; const target = this.analysisChannels[channelIndex]; target.set(source.subarray(this.startIndex, this.startIndex + firstLength), 0); if (firstLength < this.analysisWindowSamples) { target.set( source.subarray(0, this.analysisWindowSamples - firstLength), firstLength, ); } } } async analyzeAvailable(options = {}) { if (this.busy) throw new Error("analyzeAvailable() is already running."); const maximumResults = options.maximumResults ?? this.maximumResultsPerCall; requirePositiveInteger(maximumResults, "maximumResults"); if (maximumResults > this.maximumResultsPerCall) { throw new RangeError( `maximumResults cannot exceed configured bound ${this.maximumResultsPerCall}.`, ); } this.busy = true; const results = []; try { while ( results.length < maximumResults && (this.pendingAnalysis || this.bufferedSamples >= this.analysisWindowSamples) ) { throwIfAborted(options.signal); const retry = this.pendingAnalysis; const startSample = retry?.startSample ?? this.bufferStartSample; const endSample = startSample + this.analysisWindowSamples; if (!retry) this.copyAnalysisWindow(); this.analysisAttempts += 1; let localized; try { localized = await localizeMultichannel({ plan: this.plan, channels: this.analysisChannels, signal: options.signal, onProgress: options.onProgress, computeOptions: options.computeOptions, }); } catch (error) { this.pendingAnalysis = { startSample, endSample }; this.failedAnalyses += 1; throw error; } const result = { sequence: this.nextSequence, startSample, endSample, startTime: startSample / this.plan.sampleRate, endTime: endSample / this.plan.sampleRate, centerTime: (startSample + endSample) / (2 * this.plan.sampleRate), ...localized, deadlineMs: this.analysisDeadlineMs, deadlineMet: localized.processingTimeMs <= this.analysisDeadlineMs, }; this.pendingAnalysis = null; const commitThroughSample = startSample + this.analysisHopSamples; this.consume( Math.min( this.bufferedSamples, Math.max(0, commitThroughSample - this.bufferStartSample), ), false, ); this.nextSequence += 1; this.completedAnalyses += 1; this.lastProcessingTimeMs = localized.processingTimeMs; this.maximumProcessingTimeMs = Math.max( this.maximumProcessingTimeMs, localized.processingTimeMs, ); if (!result.deadlineMet) this.deadlineMisses += 1; results.push(result); // Consumer failures must not turn a successfully committed analysis // into a localization failure or make its window eligible for retry. options.onResult?.(result); } return results; } finally { this.busy = false; } } async push(channels, options = {}) { this.append(channels); return this.analyzeAvailable(options); } reset() { if (this.busy) throw new Error("cannot reset while analysis is running."); this.startIndex = 0; this.bufferedSamples = 0; this.bufferStartSample = 0; this.totalReceivedSamples = 0; this.droppedSamples = 0; this.analysisAttempts = 0; this.completedAnalyses = 0; this.failedAnalyses = 0; this.lastProcessingTimeMs = null; this.maximumProcessingTimeMs = 0; this.deadlineMisses = 0; this.nextSequence = 0; this.pendingAnalysis = null; } } /** Algorithm-neutral alias; the older class name remains for compatibility. */ export { StreamingDasLocalizer as StreamingLocalizer }; export function createStreamingLocalizer(config) { return new StreamingDasLocalizer(config); }