This document discusses processing audio data at a low level in JavaScript. It provides examples of reading audio data from the browser using the Mozilla Audio Data API, writing audio data, and processing it concurrently using Web Workers to avoid blocking the main thread. Code samples are given for setting up audio streams, reading and writing audio data, and analyzing samples in real-time.

1. JS: Audio Data Processing
Ingvar Stepanyan
Web Developer, MSP
@RReverser (http://rreverser.com/)

2. Native implementations:
[Mozilla] Audio Data API (low-level processing)
[WebKit] Web Audio API (+ some high-level sugar)
[Flash] Sound API (low-level processing)
Unifying wrappers:
dynamicaudio.js (Audio Data API + Flash)
XAudioJS (Audio Data API + Web Audio API + Flash)
sink.js & co. (Audio Data API + Web Audio API)
Current status
Low
Level

3. <audio id=“audio” ... />
....................................
var audio = document.getElementById(‘audio’),
channels, rate, bufferLength;
audio.addEventListener('loadedmetadata', function () {
channels = audio.mozChannels;
rate = audio.mozSampleRate;
bufferLength = audio.mozFrameBufferLength;
// ...
});
Reading audio data

4. audio.addEventListener('MozAudioAvailable', function () {
var buffer = event.frameBuffer,
time = event.time;
for (var i = 0; i < bufferLength / channels; i++ ) {
// use buffer[i * channels + channelNumber]
}
});
Reading audio data

5. 0.409862111297023, 0.259322117331362, -0.008777887762307, -0.984780922682458, -0.739880137997811, -0.393993360534229, -
0.698508825498069, -0.393512581774658, 0.073465971474621, -0.774029495474732, -0.068926253092157, 0.698052197232118,
0.450745893413682, 0.731625352383412, -0.633931102362129, 0.882738533608791, -0.493848309462582, -0.668645515465273,
0.271953939580002, 0.246314079650827, -0.645711462859856, 0.736913698005765, 0.872459503967142, -0.482284551106221,
0.661431378511110, -0.291951234239424, 0.669216884142705, -0.341110225783700, -0.165754738005617, -0.986556649848787,
0.803597140689702, -0.813939097068497, 0.297290844923532, -0.434373616658307, -0.820145776933725, -0.351005880456336,
0.380713105392354, -0.721515454501945, 0.066484189654290, 0.642101001764223, -0.597205735590506, -0.200460217995576,
0.491272422617811, 0.480988687186248, 0.227128301563507, 0.685168802023231, 0.075880282193273, -0.749673188605156,
0.702223692201174, -0.184544172051190, -0.506130812375945, -0.686331843733017, -0.791173541823365, 0.886386883125611,
0.042702665517532, 0.573126318821646, 0.576012637739739, -0.329392317046949, -0.560594547513062, -0.223420931279321,
0.652487040645743, 0.403985003945232, 0.881201502984618, 0.992330609078359, 0.157417714187382, 0.133725348839930,
0.343408728622162, 0.005933505310489, -0.914475827505964, 0.154820823526593, 0.148971641643792, 0.841135222645066,
0.095968894389784, -0.234891692775066, -0.055960466199662, -0.133320485223315, -0.945514113185999, 0.485242740877991,
0.933864235656065, 0.022960704882891, -0.139667183659487, 0.618401839172406, -0.195349340573667, 0.587291626042564,
0.158475194222853, 0.565554351499445, 0.101859803696333, 0.593813833398595, -0.349654793639986, 0.665640345371317,
0.814509967148215, 0.533843529155366, -0.631997214437279, -0.503973088121093, -0.180033876163095, -0.715287970025233,
0.739453218066262, -0.452880703839526, -0.112778265491663, -0.650074534874509, -0.925708286102557, 0.976021772837429,
0.768202886746198, -0.559265634778471, 0.004451442257993, 0.747659266125223, -0.692840193163558, -0.811031246931423,
0.064833012476722, -0.112213849236344, -0.045264398268911, 0.353260515196007, 0.150881652218966, -0.199176332305722,
0.680000507648084, 0.265540158726514, -0.692481536406930, -0.353516025899132, 0.548221585427517, 0.461960884430420,
0.983277137292909, 0.622653444743655, 0.480297639150934, 0.452168428463840, -0.823544922845037, -0.331975377216007,
0.358569546837979, -0.596899627797886, 0.301496127117938, 0.576480504549117, 0.150743674405126, -0.851734411492563,
0.127175797857987, 0.806251938739738, 0.360201361643523, 0.380717789492329, 0.759847187611911, -0.629616985016364,
0.179681096205048, 0.147705467636446, 0.793359439243774, -0.888943630149254, -0.768200353474536, 0.837243925007694,
0.601330942636268, -0.611155192762695, 0.176371159373866, 0.731621908979866, 0.294121659309740, 0.758972133758613,
0.226735505828943, -0.099879270352797, 0.716827038775917, -0.439818898014638, -0.845310989782625, 0.712989203634548,
0.040939664050535, -0.492177661587544, -0.611892969959734, -0.419016380589113, 0.526549046576405, -0.638706291747430,
0.611630316089939, -0.340452683930807, 0.833273571181310, 0.526974765301767, 0.462530036158461, -0.796029844105523,
0.604456871305559, 0.061317570172945, 0.642369077263937, -0.978365729885714, -0.468053997163355, 0.954769747262937,
0.446717706860744, -0.960283148953476, -0.751094692845392, 0.805721468166348, -0.290676951057867, 0.391488310207055,
0.099343420640492, 0.034684411420110, 0.463771493683684, -0.378392748975848, -0.830120263808777, -0.357574609519144,
0.755175701837563, -0.379417718802803, 0.295501263679325, 0.181793810178941, -0.116752805641118, -0.279166790263324,
0.938534135910232, 0.091496919686446, -0.107904426032210, -0.978447352332630, -0.180865072441111, 0.831093166130408,
0.099057141660298, 0.456832343000489, -0.652587456595693, 0.359989313981682, -0.951405226971930, 0.381285879196237,
0.182679971280062, -0.580757635269123, 0.879272667221741, -0.480336023774189, 0.833228235340136, .......................................
Low-level Data Format
WTF???

6. Samples ( not “examples” :) )
𝑆(𝑡) =
𝐴(𝑡)
𝐴 𝑚𝑎𝑥
𝑇𝑠𝑎𝑚𝑝𝑙𝑒 =
1
𝑓𝑠𝑎𝑚𝑝𝑙𝑒
𝑡 =
𝑁𝑠𝑎𝑚𝑝𝑙𝑒𝑠
𝑓𝑠𝑎𝑚𝑝𝑙𝑒

7. Sample analysis
https://github.com/corbanbrook/dsp.js/

8. Nyquist–Shannon sampling theorem:
𝑓𝑠𝑎𝑚𝑝𝑙𝑒 ≥ 2𝑓𝑚𝑎𝑥
Sample rate
http://bartus.org/akustyk/adc.html

9. audio.mozSetup(channels, sampleRate);
// …
audio.mozWriteAudio(samples1);
// …
audio.mozWriteAudio(samples2);
Writing audio

10. function AudioStream(readFn, bufSize) {
var audio = new Audio();
audio.mozSetup(1, sampleRate);
readFn = readFn.bind(audio, bufSize);
audio.writeAudio = function(soundData, callback) {
var written = this.mozWriteAudio(soundData),
length = soundData.length;
written < length
? setTimeout(this.writeAudio.bind(this, soundData.subarray(written), callback),
written ? 1000 * written / sampleRate : 0)
: callback(readFn());
}
readFn();
return audio;
}
Buffered writing

11. Main
Thread
Write
Audio
setTimeout
Read
Function
Calculation
Worker
Concurrency

12. Demonstration
http://rreverser.com/dev/piano/