Using Node.js to               Build Great           Streaming Services@sh1mmer
Tom Hughes-Croucher       @sh1mmer
@sh1mmer
@sh1mmer
Scalable Server-Side Code with JavaScript                  Node                        Up and Running                     ...
A story@sh1mmer
factory@sh1mmer
@sh1mmer
@sh1mmer
var http = require(http);   var server = http.createServer();   server.listen(8000);   server.on(request, function(req,res...
Making Doodads@sh1mmer
Knobs      Screens   Antennas                           & buttons@sh1mmer
@sh1mmer
@sh1mmer
@sh1mmer
@sh1mmer
@sh1mmer
var http = require(http), fs = require(fs);   var server = http.createServer();   server.listen(8000);   server.on(request...
Improving the factory         with streaming@sh1mmer
@sh1mmer
@sh1mmer
@sh1mmer
var http = require(http);   var server = http.createServer();   server.listen(8000);   server.on(request, function(req,res...
Streams API@sh1mmer
Streams     •   Readable              •   Writable         •   data event          •   write()         •   end event      ...
var stream = require(stream);   var s = new stream.Stream(); //I am an EventEmitter   s.readable = true; //now I implement...
var stream = require(stream);   var s = new stream.Stream(); //I am an EventEmitter   s.writable = true; //now I implement...
The too many parts       problem and why     phone calls dont work@sh1mmer
@sh1mmer
nextTick                      Event Loop FailnextTick                     Run stuff                     TCP               ...
stream.pause()
nextTick                      Event Loop FailnextTick                      stream.pause()                     TCP         ...
Managing Backpressure     • Manage Buffer sizes     • Use pause() / resume() to control back       pressure     • Assume t...
//manage Nodes write Buffers   if(socket.bufferSize > maxSafe) {     writeStream.pause();   }   //Manage RS read stream Bu...
New API@sh1mmer
New Streams TBD•   Writable streams    •   Same•   Readable (https://github.com/isaacs/readable-stream)    •   configurable...
@sh1mmer
Types of Streaming@sh1mmer
Simplex@sh1mmer
var fs = require(fs);   var rs = fs.createReadStream(/my/file/path);   rs.on(data, function(d) {     console.log(d);   });...
Throughput@sh1mmer
var fs = require(fs);   var rs = fs.createReadStream(/my/file/path);   rs.pipe(process.stdout);@sh1mmer
Duplex@sh1mmer
var bot1 = new chatterBot();   var bot2 = new chatterBot();   bot1.pipe(bot2).pipe(bot1);@sh1mmer
Libraries@sh1mmer
JSONStream@sh1mmer
{"total_rows":129,"offset":0,"rows":[      { "id":"change1_0.6995461115147918"      , "key":"change1_0.6995461115147918"  ...
var stream = JSONStream.parse([rows, true, doc]) //rows,   ANYTHING, doc   stream.on(data, function(data) {     console.lo...
node-libexpat@sh1mmer
Application@sh1mmer
Questions?@sh1mmer
Upcoming SlideShare
Loading in …5
×

Using Node.js to Build Great Streaming Services - HTML5 Dev Conf

24,771 views
24,884 views

Published on

Published in: Technology
0 Comments
15 Likes
Statistics
Notes
  • Be the first to comment

No Downloads
Views
Total views
24,771
On SlideShare
0
From Embeds
0
Number of Embeds
15
Actions
Shares
0
Downloads
158
Comments
0
Likes
15
Embeds 0
No embeds

No notes for slide

Using Node.js to Build Great Streaming Services - HTML5 Dev Conf

  1. 1. Using Node.js to Build Great Streaming Services@sh1mmer
  2. 2. Tom Hughes-Croucher @sh1mmer
  3. 3. @sh1mmer
  4. 4. @sh1mmer
  5. 5. Scalable Server-Side Code with JavaScript Node Up and Running Tom Hughes-Croucher http://ofps.oreilly.com/titles/9781449398583/ http://shop.oreilly.com/product/0636920015956.do@sh1mmer
  6. 6. A story@sh1mmer
  7. 7. factory@sh1mmer
  8. 8. @sh1mmer
  9. 9. @sh1mmer
  10. 10. var http = require(http); var server = http.createServer(); server.listen(8000); server.on(request, function(req,res) { var doodad = new Doodad(); res.writeHead(200,{Content-Type:text/doodad}); res.end(doodad); });@sh1mmer
  11. 11. Making Doodads@sh1mmer
  12. 12. Knobs Screens Antennas & buttons@sh1mmer
  13. 13. @sh1mmer
  14. 14. @sh1mmer
  15. 15. @sh1mmer
  16. 16. @sh1mmer
  17. 17. @sh1mmer
  18. 18. var http = require(http), fs = require(fs); var server = http.createServer(); server.listen(8000); server.on(request, function(req,res) { var screen = fs.readFileSync(/factory1/screen); var dial = fs.readFileSync(/factory2/dial); var doodad = new Doodad(screen, dial); res.writeHead(200,{Content-Type:text/doodad}); res.end(doodad); });@sh1mmer
  19. 19. Improving the factory with streaming@sh1mmer
  20. 20. @sh1mmer
  21. 21. @sh1mmer
  22. 22. @sh1mmer
  23. 23. var http = require(http); var server = http.createServer(); server.listen(8000); server.on(request, function(req,res) { var parts = 0; http.get(factory1/screen/, function(res) { var screen = ""; res.on(data, function(d) { screen += d }); res.on(end, finish); parts++; }; http.get(factory2/dial, function(res) { var dial = "" res.on(data, function(d) { dial += d }); res.on(end, finish); parts++; } function finish() { if(parts == 2) { var doodad = new Doodad(screen, dial); res.writeHead(200,{Content-Type:text/doodad}); res.end(doodad); } } });@sh1mmer
  24. 24. Streams API@sh1mmer
  25. 25. Streams • Readable • Writable • data event • write() • end event • end() • pause() • resume() • drain event • destroy() • destroy() • pipe() • destroySoon()@sh1mmer
  26. 26. var stream = require(stream); var s = new stream.Stream(); //I am an EventEmitter s.readable = true; //now I implement read API s.emit(data, "my data"); s.emit(end); //Im done sending data@sh1mmer
  27. 27. var stream = require(stream); var s = new stream.Stream(); //I am an EventEmitter s.writable = true; //now I implement write API s.data = ""; s.write = function(d) { s.data += d; }; s.end = function() { if(s.data) { console.log(s.data) } }; s.destroy = function() { s.writable = false; }@sh1mmer
  28. 28. The too many parts problem and why phone calls dont work@sh1mmer
  29. 29. @sh1mmer
  30. 30. nextTick Event Loop FailnextTick Run stuff TCP Conn node.cc FS Read add-ons v8 TCP Conn Timeout libuv TCPInterval Conn libev iocp FS Timeout Read FS Read
  31. 31. stream.pause()
  32. 32. nextTick Event Loop FailnextTick stream.pause() TCP Conn node.cc FS Read add-ons v8 TCP Conn Timeout libuv TCPInterval Conn libev iocp FS Timeout Read FS Read
  33. 33. Managing Backpressure • Manage Buffer sizes • Use pause() / resume() to control back pressure • Assume that data event may happen after pause • Remember node buffer -> kernel buffer@sh1mmer
  34. 34. //manage Nodes write Buffers if(socket.bufferSize > maxSafe) { writeStream.pause(); } //Manage RS read stream Buffers fs.createReadStream(./path/to/file, { flags: r, encoding: null, fd: null, mode: 0666, bufferSize: 64 * 1024 });@sh1mmer
  35. 35. New API@sh1mmer
  36. 36. New Streams TBD• Writable streams • Same• Readable (https://github.com/isaacs/readable-stream) • configurable stream buffer waterline • readable event • read(len) -> returns data• Pipe • Subscribers must implement readable stream • No more direct access with Pipe
  37. 37. @sh1mmer
  38. 38. Types of Streaming@sh1mmer
  39. 39. Simplex@sh1mmer
  40. 40. var fs = require(fs); var rs = fs.createReadStream(/my/file/path); rs.on(data, function(d) { console.log(d); });@sh1mmer
  41. 41. Throughput@sh1mmer
  42. 42. var fs = require(fs); var rs = fs.createReadStream(/my/file/path); rs.pipe(process.stdout);@sh1mmer
  43. 43. Duplex@sh1mmer
  44. 44. var bot1 = new chatterBot(); var bot2 = new chatterBot(); bot1.pipe(bot2).pipe(bot1);@sh1mmer
  45. 45. Libraries@sh1mmer
  46. 46. JSONStream@sh1mmer
  47. 47. {"total_rows":129,"offset":0,"rows":[ { "id":"change1_0.6995461115147918" , "key":"change1_0.6995461115147918" , "value":{"rev":"1-e240bae28c7bb3667f02760f6398d508"} , "doc":{ "_id": "change1_0.6995461115147918" , "_rev": "1-e240bae28c7bb3667f02760f6398d508","hello":1} }, { "id":"change2_0.6995461115147918" , "key":"change2_0.6995461115147918" , "value":{"rev":"1-13677d36b98c0c075145bb8975105153"} , "doc":{ "_id":"change2_0.6995461115147918" , "_rev":"1-13677d36b98c0c075145bb8975105153" , "hello":2 } }, ]}@sh1mmer
  48. 48. var stream = JSONStream.parse([rows, true, doc]) //rows, ANYTHING, doc stream.on(data, function(data) { console.log(received:, data); }); stream.on(root, function(root, count) { if (!count) { console.log(no matches found:, root); } });@sh1mmer
  49. 49. node-libexpat@sh1mmer
  50. 50. Application@sh1mmer
  51. 51. Questions?@sh1mmer

×