544 lines
18 KiB
JavaScript
544 lines
18 KiB
JavaScript
import { synthesizeMicrophoneSignals } from "/packages/acoustics-core/propagation.js";
|
|
import { sampleQuasiStaticWindow } from "/packages/acoustics-core/trajectory.js";
|
|
import { createLocalizationAnalysisPlan } from "/packages/acoustics-core/localization.js";
|
|
import { localizationAlgorithmSupportsBackend } from "/packages/acoustics-core/localization-algorithms.js";
|
|
import { computeLocalizationWithBackend } from "/packages/webgpu-kernels/compute.js";
|
|
import { isMusicUnavailableError } from "/packages/acoustics-core/music.js";
|
|
import { getCompleteFrameCount } from "/packages/acoustics-core/stft.js";
|
|
import { analyzeCfarActivity } from "/packages/acoustics-core/activity-monitor.js";
|
|
import {
|
|
computeDixonSpectralFlux,
|
|
pickCausalSpectralFluxPeaks,
|
|
} from "/packages/acoustics-core/spectral-flux.js";
|
|
|
|
function post(type, jobId, payload = {}, transfer = []) {
|
|
self.postMessage({ type, jobId, ...payload }, transfer);
|
|
}
|
|
|
|
function serializeError(error) {
|
|
return {
|
|
name: error?.name ?? "Error",
|
|
code: error?.code ?? "",
|
|
message: error instanceof Error ? error.message : String(error),
|
|
stack: error?.stack ?? "",
|
|
causes: error instanceof AggregateError
|
|
? error.errors.map((cause) => ({ name: cause?.name, code: cause?.code, message: cause?.message }))
|
|
: [],
|
|
};
|
|
}
|
|
|
|
function signalStats(synthesis) {
|
|
return {
|
|
preNormalizationPeak: synthesis.preNormalizationPeak,
|
|
appliedScale: synthesis.appliedScale,
|
|
clippedSampleCount: synthesis.clippedSampleCount,
|
|
peak: synthesis.peak,
|
|
acousticModel: synthesis.acousticModel,
|
|
acousticPathCount: synthesis.acousticPathCount,
|
|
acousticPathCounts: synthesis.acousticPathCounts,
|
|
workingSet: synthesis.workingSet,
|
|
windowing: synthesis.windowing,
|
|
sampling: synthesis.sampling,
|
|
};
|
|
}
|
|
|
|
function collectBandTransfers(run, transfer) {
|
|
const knownBuffers = new Set(transfer);
|
|
for (const band of run.bands ?? []) {
|
|
for (const map of [band.rawMap, band.displayMap]) {
|
|
const buffer = map?.buffer;
|
|
if (buffer instanceof ArrayBuffer && !knownBuffers.has(buffer)) {
|
|
knownBuffers.add(buffer);
|
|
transfer.push(buffer);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
function collectTypedArrayTransfers(value, transfer, seen = new Set(), buffers = new Set()) {
|
|
if (value === null || typeof value !== "object" || seen.has(value)) return;
|
|
seen.add(value);
|
|
if (ArrayBuffer.isView(value) && !(value instanceof DataView)) {
|
|
if (!buffers.has(value.buffer)) {
|
|
buffers.add(value.buffer);
|
|
transfer.push(value.buffer);
|
|
}
|
|
return;
|
|
}
|
|
if (Array.isArray(value)) {
|
|
for (const item of value) collectTypedArrayTransfers(item, transfer, seen, buffers);
|
|
return;
|
|
}
|
|
for (const item of Object.values(value)) {
|
|
collectTypedArrayTransfers(item, transfer, seen, buffers);
|
|
}
|
|
}
|
|
|
|
function activitySignalStats(synthesis, scene) {
|
|
const channel = synthesis.channels[0];
|
|
let selectedChannelPeak = 0;
|
|
for (const sample of channel) {
|
|
selectedChannelPeak = Math.max(selectedChannelPeak, Math.abs(sample));
|
|
}
|
|
const workingSetWithoutSourceIdentity = {
|
|
model: synthesis.workingSet.model,
|
|
channelBytes: synthesis.workingSet.channelBytes,
|
|
maximumTransientBytes: synthesis.workingSet.maximumTransientBytes,
|
|
maximumTransientScope: synthesis.workingSet.maximumTransientScope,
|
|
pathSummary: synthesis.workingSet.pathSummary,
|
|
summaryPathPairEvaluations:
|
|
synthesis.workingSet.summaryPathPairEvaluations,
|
|
synthesisPathBuildPairEvaluations:
|
|
synthesis.workingSet.synthesisPathBuildPairEvaluations,
|
|
retainedPathObjects: synthesis.workingSet.retainedPathObjects,
|
|
remainingNonToneTransformModel:
|
|
synthesis.workingSet.remainingNonToneTransformModel,
|
|
};
|
|
const windowingWithoutSourceTruth = {
|
|
model: synthesis.windowing.model,
|
|
renderStartSample: synthesis.windowing.renderStartSample,
|
|
outputStartSample: synthesis.windowing.outputStartSample,
|
|
outputSampleCount: synthesis.windowing.outputSampleCount,
|
|
preRollSamples: synthesis.windowing.preRollSamples,
|
|
renderMaximumPropagationDelaySeconds:
|
|
synthesis.windowing.renderMaximumPropagationDelaySeconds,
|
|
activePathMaximumDelaySeconds:
|
|
synthesis.windowing.activePathMaximumDelaySeconds,
|
|
requestedMaximumPropagationDelaySeconds:
|
|
synthesis.windowing.requestedMaximumPropagationDelaySeconds,
|
|
geometryModel: synthesis.windowing.geometryModel,
|
|
sourcePositionSampleTime: synthesis.windowing.sourcePositionSampleTime,
|
|
dopplerMode: synthesis.windowing.dopplerMode,
|
|
delayedTailPolicy: synthesis.windowing.delayedTailPolicy,
|
|
arrivalOverlapModel: synthesis.windowing.arrivalOverlapModel,
|
|
};
|
|
const peakNormalization = scene.mix?.normalization === "peak";
|
|
return {
|
|
preNormalizationPeak: synthesis.preNormalizationPeak,
|
|
appliedScale: synthesis.appliedScale,
|
|
clippedSampleCount: synthesis.clippedSampleCount,
|
|
selectedChannelPeak,
|
|
acousticModel: synthesis.acousticModel,
|
|
acousticPathCount: synthesis.acousticPathCount,
|
|
acousticPathCounts: synthesis.acousticPathCounts,
|
|
workingSet: workingSetWithoutSourceIdentity,
|
|
windowing: windowingWithoutSourceTruth,
|
|
preprocessing: {
|
|
configuredNormalization: scene.mix?.normalization ?? "none",
|
|
normalizationScope: peakNormalization
|
|
? "selected-physical-microphone-whole-render"
|
|
: "none",
|
|
futureDependentNormalization: peakNormalization,
|
|
detectorCausalAfterPreprocessing: true,
|
|
endToEndCausal: !peakNormalization,
|
|
},
|
|
samplingModel: "ideal-real-sampler-no-antialias-filter",
|
|
sampleRate: synthesis.sampleRate,
|
|
nyquistHz: synthesis.sampleRate / 2,
|
|
explicitSceneSourceIdentitiesAndArrivalsIncluded: false,
|
|
sourceDerivedAggregatePhysicalAuditIncluded: true,
|
|
};
|
|
}
|
|
|
|
async function runActivity({
|
|
jobId,
|
|
scene,
|
|
arrayDefinition,
|
|
experiment,
|
|
assetBuffers,
|
|
activityConfig,
|
|
}) {
|
|
if (!activityConfig || typeof activityConfig !== "object") {
|
|
throw new TypeError("activityConfig must be an object.");
|
|
}
|
|
const channelIndex = activityConfig.channelIndex;
|
|
if (
|
|
!Number.isInteger(channelIndex) ||
|
|
channelIndex < 0 ||
|
|
channelIndex >= arrayDefinition.microphones.length
|
|
) {
|
|
throw new RangeError("activityConfig.channelIndex is outside the microphone array.");
|
|
}
|
|
const microphone = arrayDefinition.microphones[channelIndex];
|
|
const totalStartedAt = performance.now();
|
|
post("progress", jobId, {
|
|
stage: "activity-signal",
|
|
message: "전 대역 감시용 마이크 수신 신호를 합성하고 있습니다…",
|
|
});
|
|
const signalStartedAt = performance.now();
|
|
const selectedMicrophoneArray = {
|
|
...arrayDefinition,
|
|
microphones: [microphone],
|
|
};
|
|
const synthesis = synthesizeMicrophoneSignals({
|
|
scene,
|
|
arrayDefinition: selectedMicrophoneArray,
|
|
assetBuffers,
|
|
minimumDistance: experiment.beamforming.minimumDistance,
|
|
});
|
|
const signalGeneration = performance.now() - signalStartedAt;
|
|
const samples = synthesis.channels[0];
|
|
const detectorStartedAt = performance.now();
|
|
let analysis;
|
|
let peakPicking = null;
|
|
|
|
if (
|
|
activityConfig.algorithm === "ca-cfar" ||
|
|
activityConfig.algorithm === "os-cfar"
|
|
) {
|
|
analysis = await analyzeCfarActivity({
|
|
samples,
|
|
sampleRate: synthesis.sampleRate,
|
|
fftSize: experiment.stft.fftSize,
|
|
hopSize: experiment.stft.hopSize,
|
|
window: experiment.stft.window,
|
|
channelId: microphone.id,
|
|
detector: activityConfig.algorithm,
|
|
trainingCellsPerSide: activityConfig.trainingCellsPerSide,
|
|
guardCellsPerSide: activityConfig.guardCellsPerSide,
|
|
nominalFalseAlarmProbability:
|
|
activityConfig.nominalFalseAlarmProbability,
|
|
...(activityConfig.algorithm === "os-cfar"
|
|
? { rank: activityConfig.rank }
|
|
: {}),
|
|
onProgress: (progress) => post("progress", jobId, {
|
|
stage: progress.stage,
|
|
progress,
|
|
}),
|
|
});
|
|
} else if (activityConfig.algorithm === "dixon-spectral-flux") {
|
|
post("progress", jobId, {
|
|
stage: "activity-spectral-flux",
|
|
message: "Dixon spectral flux 원시 onset novelty를 계산하고 있습니다…",
|
|
});
|
|
analysis = computeDixonSpectralFlux({
|
|
samples,
|
|
sampleRate: synthesis.sampleRate,
|
|
fftSize: experiment.stft.fftSize,
|
|
hopSize: experiment.stft.hopSize,
|
|
channelId: microphone.id,
|
|
maximumOutputBytes: activityConfig.maximumOutputBytes,
|
|
});
|
|
if (activityConfig.peakPicker) {
|
|
peakPicking = pickCausalSpectralFluxPeaks(
|
|
analysis.rawOdf,
|
|
activityConfig.peakPicker,
|
|
);
|
|
}
|
|
} else {
|
|
throw new RangeError(
|
|
'activityConfig.algorithm must be "ca-cfar", "os-cfar", or "dixon-spectral-flux".',
|
|
);
|
|
}
|
|
|
|
const detectorTime = performance.now() - detectorStartedAt;
|
|
const report = {
|
|
kind: "micarrsim.activity-worker-report/1",
|
|
algorithm: activityConfig.algorithm,
|
|
channel: {
|
|
index: channelIndex,
|
|
id: microphone.id,
|
|
position: [...microphone.position],
|
|
selection: "user-selected-single-physical-microphone",
|
|
microphoneFusion: "none",
|
|
},
|
|
input: {
|
|
sampleRate: synthesis.sampleRate,
|
|
sampleCount: samples.length,
|
|
fftSize: experiment.stft.fftSize,
|
|
hopSize: experiment.stft.hopSize,
|
|
experimentWindow: experiment.stft.window,
|
|
adcRangeHz: [0, synthesis.sampleRate / 2],
|
|
analogAliasOrderResolved: false,
|
|
sceneSourceTruthUsedByPhysicalSimulator: true,
|
|
sceneSourceTruthPassedToDetector: false,
|
|
synthesizedMicrophoneCount: 1,
|
|
},
|
|
signalStats: activitySignalStats(synthesis, scene),
|
|
analysis,
|
|
...(peakPicking ? { peakPicking } : {}),
|
|
timingMs: {
|
|
total: performance.now() - totalStartedAt,
|
|
signalGeneration,
|
|
detector: detectorTime,
|
|
},
|
|
};
|
|
const transfer = [];
|
|
collectTypedArrayTransfers(report, transfer);
|
|
post("activity-result", jobId, { activityReport: report }, transfer);
|
|
}
|
|
|
|
async function computeLocalizationOrUnavailable(options) {
|
|
try {
|
|
return await computeLocalizationWithBackend(options);
|
|
} catch (error) {
|
|
if (!isMusicUnavailableError(error)) throw error;
|
|
return {
|
|
algorithm: "music",
|
|
backend: "cpu",
|
|
status: "unavailable",
|
|
unavailable: {
|
|
code: error.code,
|
|
reason: error.reason,
|
|
message: error.message,
|
|
details: { ...error.details },
|
|
},
|
|
frameCount: getCompleteFrameCount(
|
|
options.channels[0]?.length ?? 0,
|
|
options.experiment.stft.fftSize,
|
|
options.experiment.stft.hopSize,
|
|
),
|
|
microphoneCount:
|
|
options.arrayDefinition.microphones.length,
|
|
pointCount:
|
|
options.experiment.scan.rows *
|
|
options.experiment.scan.columns,
|
|
objectiveDirection: "minimum",
|
|
rawScale: "noise-projection",
|
|
bands: [],
|
|
};
|
|
}
|
|
}
|
|
|
|
async function runTimeline({ jobId, scene, arrayDefinition, experiment, assetBuffers, framePlan }) {
|
|
if (!Array.isArray(framePlan?.frames) || framePlan.frames.length === 0) {
|
|
throw new TypeError("framePlan.frames must be a non-empty array.");
|
|
}
|
|
|
|
const totalStartedAt = performance.now();
|
|
const totalFrames = framePlan.frames.length;
|
|
const frames = [];
|
|
const transfer = [];
|
|
let signalGeneration = 0;
|
|
let localizationTime = 0;
|
|
const localizationScene = { speedOfSound: scene.speedOfSound };
|
|
const algorithm = experiment.beamforming.algorithm ?? "das";
|
|
const requestedBackend = experiment.beamforming.backend ?? "auto";
|
|
const reuseCpuPlan =
|
|
!localizationAlgorithmSupportsBackend(algorithm, "webgpu") ||
|
|
requestedBackend === "cpu";
|
|
let localizationExperiment = experiment;
|
|
let analysisPlan = reuseCpuPlan
|
|
? createLocalizationAnalysisPlan({
|
|
sampleRate: scene.sampleRate,
|
|
arrayDefinition,
|
|
scene: localizationScene,
|
|
experiment: localizationExperiment,
|
|
})
|
|
: undefined;
|
|
|
|
post("progress", jobId, {
|
|
stage: "timeline-start",
|
|
message: `라이브 맵 ${totalFrames}개를 준비하고 있습니다…`,
|
|
completedFrames: 0,
|
|
totalFrames,
|
|
});
|
|
|
|
for (let frameIndex = 0; frameIndex < totalFrames; frameIndex += 1) {
|
|
const plannedFrame = framePlan.frames[frameIndex];
|
|
const synthesisWindow = sampleQuasiStaticWindow(
|
|
scene,
|
|
plannedFrame.startTime,
|
|
plannedFrame.endTime,
|
|
{
|
|
arrayDefinition,
|
|
assetBuffers,
|
|
minimumDistance: experiment.beamforming.minimumDistance,
|
|
},
|
|
);
|
|
post("progress", jobId, {
|
|
stage: "timeline-signal",
|
|
message: `라이브 신호 합성 중 · ${frameIndex + 1}/${totalFrames}`,
|
|
frameIndex,
|
|
completedFrames: frameIndex,
|
|
totalFrames,
|
|
});
|
|
|
|
const frameStartedAt = performance.now();
|
|
const signalStartedAt = performance.now();
|
|
const synthesis = synthesizeMicrophoneSignals({
|
|
scene: synthesisWindow.scene,
|
|
arrayDefinition,
|
|
assetBuffers,
|
|
minimumDistance: experiment.beamforming.minimumDistance,
|
|
signalOffsets: synthesisWindow.signalOffsets,
|
|
});
|
|
const frameSignalGeneration = performance.now() - signalStartedAt;
|
|
signalGeneration += frameSignalGeneration;
|
|
|
|
const localizationStartedAt = performance.now();
|
|
const localization = await computeLocalizationOrUnavailable({
|
|
channels: synthesis.channels,
|
|
sampleRate: synthesis.sampleRate,
|
|
arrayDefinition,
|
|
scene: localizationScene,
|
|
experiment: localizationExperiment,
|
|
...(analysisPlan ? { analysisPlan } : {}),
|
|
onFallback: ({ error }) =>
|
|
post("fallback", jobId, {
|
|
code: error.code,
|
|
message: error.message,
|
|
frameIndex,
|
|
totalFrames,
|
|
}),
|
|
onProgress: (progress) =>
|
|
post("progress", jobId, {
|
|
stage: `timeline-${progress.stage}`,
|
|
frameIndex,
|
|
completedFrames: frameIndex,
|
|
totalFrames,
|
|
progress,
|
|
}),
|
|
});
|
|
const frameLocalization = performance.now() - localizationStartedAt;
|
|
localizationTime += frameLocalization;
|
|
collectBandTransfers(localization, transfer);
|
|
|
|
if (
|
|
!analysisPlan &&
|
|
localization.backend === "cpu" &&
|
|
localization.fallback?.from === "webgpu"
|
|
) {
|
|
// Auto has established that WebGPU is unavailable for this worker. Keep
|
|
// the first frame's honest fallback metadata, then avoid retrying the GPU
|
|
// and rebuilding CPU geometry/workspaces on every remaining live frame.
|
|
localizationExperiment = {
|
|
...experiment,
|
|
beamforming: {
|
|
...experiment.beamforming,
|
|
backend: "cpu",
|
|
},
|
|
};
|
|
analysisPlan = createLocalizationAnalysisPlan({
|
|
sampleRate: scene.sampleRate,
|
|
arrayDefinition,
|
|
scene: localizationScene,
|
|
experiment: localizationExperiment,
|
|
});
|
|
}
|
|
|
|
frames.push({
|
|
index: plannedFrame.index ?? frameIndex,
|
|
startTime: synthesisWindow.startTime,
|
|
endTime: synthesisWindow.endTime,
|
|
sampleTime: synthesisWindow.sampleTime,
|
|
duration: synthesisWindow.duration,
|
|
run: {
|
|
...localization,
|
|
signalStats: signalStats(synthesis),
|
|
timingMs: {
|
|
total: performance.now() - frameStartedAt,
|
|
signalGeneration: frameSignalGeneration,
|
|
localization: frameLocalization,
|
|
beamforming: frameLocalization,
|
|
},
|
|
},
|
|
});
|
|
|
|
post("progress", jobId, {
|
|
stage: "timeline-frame-complete",
|
|
message: `라이브 맵 준비 · ${frameIndex + 1}/${totalFrames}`,
|
|
frameIndex,
|
|
completedFrames: frameIndex + 1,
|
|
totalFrames,
|
|
});
|
|
}
|
|
|
|
const backends = new Set(frames.map((frame) => frame.run.backend));
|
|
const fallbacks = frames
|
|
.filter((frame) => frame.run.fallback)
|
|
.map((frame) => ({ frameIndex: frame.index, ...frame.run.fallback }));
|
|
post(
|
|
"timeline-result",
|
|
jobId,
|
|
{
|
|
timeline: {
|
|
backend: backends.size === 1 ? frames[0].run.backend : "mixed",
|
|
algorithm: experiment.beamforming.algorithm ?? "das",
|
|
requestedBackend: experiment.beamforming.backend ?? "auto",
|
|
duration: scene.duration,
|
|
frameCount: frames.length,
|
|
frames,
|
|
...(fallbacks.length > 0 ? { fallbacks } : {}),
|
|
timingMs: {
|
|
total: performance.now() - totalStartedAt,
|
|
signalGeneration,
|
|
localization: localizationTime,
|
|
beamforming: localizationTime,
|
|
},
|
|
},
|
|
},
|
|
transfer,
|
|
);
|
|
}
|
|
|
|
self.addEventListener("message", async (event) => {
|
|
if (
|
|
event.data?.type !== "run" &&
|
|
event.data?.type !== "run-timeline" &&
|
|
event.data?.type !== "detect-activity"
|
|
) return;
|
|
const { jobId, scene, arrayDefinition, experiment, assetBuffers } = event.data;
|
|
const totalStartedAt = performance.now();
|
|
try {
|
|
if (event.data.type === "detect-activity") {
|
|
await runActivity(event.data);
|
|
return;
|
|
}
|
|
if (event.data.type === "run-timeline") {
|
|
await runTimeline(event.data);
|
|
return;
|
|
}
|
|
post("progress", jobId, { stage: "signal", message: "마이크 수신 신호를 합성하고 있습니다…" });
|
|
const signalStartedAt = performance.now();
|
|
const synthesis = synthesizeMicrophoneSignals({
|
|
scene,
|
|
arrayDefinition,
|
|
assetBuffers,
|
|
minimumDistance: experiment.beamforming.minimumDistance,
|
|
});
|
|
const signalGeneration = performance.now() - signalStartedAt;
|
|
post("progress", jobId, {
|
|
stage: "signal-complete",
|
|
message: `마이크 신호 합성 완료 · ${signalGeneration.toFixed(0)} ms`,
|
|
});
|
|
|
|
const localizationStartedAt = performance.now();
|
|
const localization = await computeLocalizationOrUnavailable({
|
|
channels: synthesis.channels,
|
|
sampleRate: synthesis.sampleRate,
|
|
arrayDefinition,
|
|
scene,
|
|
experiment,
|
|
onFallback: ({ error }) =>
|
|
post("fallback", jobId, {
|
|
code: error.code,
|
|
message: error.message,
|
|
}),
|
|
onProgress: (progress) => post("progress", jobId, { stage: progress.stage, progress }),
|
|
});
|
|
const localizationTime = performance.now() - localizationStartedAt;
|
|
const transfer = [];
|
|
collectBandTransfers(localization, transfer);
|
|
post(
|
|
"result",
|
|
jobId,
|
|
{
|
|
run: {
|
|
...localization,
|
|
signalStats: signalStats(synthesis),
|
|
timingMs: {
|
|
total: performance.now() - totalStartedAt,
|
|
signalGeneration,
|
|
localization: localizationTime,
|
|
beamforming: localizationTime,
|
|
},
|
|
},
|
|
},
|
|
transfer,
|
|
);
|
|
} catch (error) {
|
|
post("error", jobId, { error: serializeError(error) });
|
|
}
|
|
});
|