Nowadays we have native options to deal with media signals on javascript! How can we use the available APIs to process it? This talk is about amazing uses to Web Audio API and the Video tag like sampling, quantization, filtering, etc. Can we synthesize guitar effects?

1. Digital Signal
Processing w/
Javascript
@rafael_sps

2. Who?
rafael
specht da silva
Telecommunication
Technologist
Javascript enthusiast

5. “... mathematical
manipulation of an
information signal to
modify or improve it”

7. Sampling

11. Why
this is
important?

12. var playBuffer = function (buffer) {
source = context.createBufferSource();
source.buffer = buffer;
var processor = context.createScriptProcessor(512, 2, 2);
processor.onaudioprocess = onAudioProcess;
source.connect(processor);
processor.connect(context.destination);
source.start(0, startOffset % buffer.duration);
}

13. var onAudioProcess = function (ev) {
inputBuffer = ev.inputBuffer;
outputBuffer = ev.outputBuffer;
inputData = inputBuffer.getChannelData(0);
outputData0 = outputBuffer.getChannelData(0);
outputData1 = outputBuffer.getChannelData(1);
for (var sample = 0, len = inputData.length; sample < len; sample++) {
outputData0[sample] = (1 - panelX) * inputData[sample];
outputData1[sample] = panelX * inputData[sample];
}
}

14. DEMO

17. Quantization

19. COMPRESS

20. // Original (37)
[1.1, 1.8, 2.3, 2.4, 1.9, 2.7, 3.2, 3.4, 3.1]
// Quantized (19)
[1, 2, 2, 2, 2, 3, 3, 3, 3]
// Compressed (15)
[1, ‘2 4’, ‘3 4’]

21. Images

22. var canvas = document.getElementById('canvas');
var ctx = canvas.getContext('2d');
function make_base(src) {
base_image = new Image();
base_image.src = src;
base_image.onload = function(){
ctx.drawImage(base_image, 0, 0);
imageLoaded(base_image);
}
}

23. var img = ctx.getImageData(
0,
0,
canvas.width,
canvas.height
);

24. img.height // number
img.width // number
img.data // Uint8ClampedArray
r g b a r g b a
[65, 113, 130, 255, 65, 113, 130, 255, ...

26. Compressing

27. var red = [];
var green = [];
var blue = [];
var alpha = [];
for (var i = 4; i < imgData.data.length; i+=4) {
red.push(imgData.data[i])
green.push(imgData.data[i+1])
blue.push(imgData.data[i+2])
alpha.push(imgData.data[i+3])
}

28. Image Diff

29. Video

30. Video Diff

31. “Always
bet on JS”
Brendan Eich

33. Thank you
@rafael_sps
github.com/rssilva

34. References
Images:
https://farm7.staticflickr.com/6075/6068692939_3fa9b50838_b.jpg
http://www.curtopoa.com.br/images/albuns/fotos-aereas-porto-alegre/cais_porto_alegre.jpg
http://wvwri.org/wp-content/uploads/2014/01/1391018991.jpg
https://www.flickr.com/photos/murilocardoso/6020194265
https://www.flickr.com/photos/josephkennelty/9554300993
https://www.flickr.com/photos/takeabreakwithme/3234099003
https://www.flickr.com/photos/busiguy6/457381735/in/photolist-nQthL-oN8PmF-f29NoU-dCtU8y-pvt5S-9KoqQh-23cgG-
35D6wU-ccbYUC-GqcDZ-azBSCm-p5sZXB-6ib9Wb-56Srho-hH5RD-fc6RYp-gfcqpQ-vmaEe-56NhmD-azUZWy-
5Y95gU-bUu2s-eDhQLn-6YW9JX-8YQVX1-8M8pUQ-bCnrz-8AsQiz-imtb9d-7N9J1y-ogg8U-5NtoYg-4cELXj-bUoJRy-
vmaEc-wGktP-fpNZBa-bGQycT-bGQaTk-5KSTpm-dqtWeP-ruZnP4-9cM7SK-qrm1Lr-7rDrFc-7QRNWn-9KEpL9-
oVwQbZ-9PxYEY-pDkEs3
https://www.flickr.com/photos/wdig/7176747886
https://www.flickr.com/photos/poulomee/16140153959
https://www.flickr.com/photos/voulez-vous/12076930664

35. References
Projects:
http://trackingjs.com/
https://github.com/felipenmoura/js-chroma-key
http://fellipe.com/demos/lena-js/
http://dashersw.github.io/pedalboard.js/