Real Time Web with Node

REAL TIME WEB WITH NODE By Tim Caswell Saturday, June 5, 2010

Live Interaction The web is about doing things, not just tourism anymore. Saturday, June 5, 2010

Uses of Live Interaction Chat Widget Twitter Feed Stock Ticker Real-Time Game Collaborative Documents TXJS Demos Saturday, June 5, 2010

Why we need non-blocking Polling is too slow and ineﬃcient. Live interaction requires the server to push data. In order to push data, the connections need to be persistent. The server needs to handle thousands of persistent connections Threads aren’t going to scale. Saturday, June 5, 2010

This is your server. (with blocking I/O) Saturday, June 5, 2010

This is your server with low concurrency. Saturday, June 5, 2010

This is your server with high concurrency. Saturday, June 5, 2010

Connect We’ll use a new node framework that “connects” the web users to each other. Saturday, June 5, 2010

It’s like Japanese Legos Saturday, June 5, 2010

Pre-Built Blocks Connect.createServer([ {filter: "log"}, {filter: "body-decoder", route: "/stream"}, {provider: "pubsub", route: "/stream"}, {filter: "conditional-get"}, {filter: "cache"}, {filter: "gzip"}, {provider: "cache-manifest", root: root}, {provider: "static", root: root} ]); Saturday, June 5, 2010

And easy too! Saturday, June 5, 2010

method-override.js var key; // Initialize any state (on server startup) exports.setup = function (env) { key = this.key || "_method"; }; // Modify the request stream (on request) exports.handle = function(err, req, res, next){ if (key in req.body) { req.method = req.body[key].toUpperCase(); } next(); }; Saturday, June 5, 2010

response-time.js exports.handle = function(err, req, res, next){ var start = new Date, writeHead = res.writeHead; res.writeHead = function(code, headers){ res.writeHead = writeHead; headers['X-Response-Time'] = (new Date - start) + "ms"; res.writeHead(code, headers); }; next(); }; Saturday, June 5, 2010

Well, actually, it’s not always easy. Saturday, June 5, 2010

static.js Saturday, June 5, 2010

static.js var fs = require('fs'), Url = require('url'), Path = require('path'); var lifetime = 1000 * 60 * 60; // 1 hour browser cache lifetime var DEFAULT_MIME = 'application/octet-stream'; module.exports = { setup: function (env) { this.root = this.root || process.cwd(); }, handle: function (err, req, res, next) { // Skip on error if (err) { next(); return; } var url = Url.parse(req.url); var pathname = url.pathname.replace(/..+/g, '.'), filename = Path.join(this.root, pathname); if (filename[filename.length - 1] === "/") { filename += "index.html"; } Saturday, June 5, 2010

static.js // Buffer any events that fire while waiting on the stat. var fs = require('fs'), var events = []; Url = require('url'), function onData() { Path = require('path'); events.push(["data"].concat(Array.prototype.slice.call(arguments))); } var lifetime = 1000 * 60 * 60; // 1 hourfunction onEnd() { browser cache lifetime events.push(["end"].concat(Array.prototype.slice.call(arguments))); } var DEFAULT_MIME = 'application/octet-stream'; req.addListener("data", onData); module.exports = { req.addListener("end", onEnd); setup: function (env) { fs.stat(filename, function (err, stat) { this.root = this.root || process.cwd(); }, // Stop buffering events req.removeListener("data", onData); handle: function (err, req, res, next) {req.removeListener("end", onEnd); // Skip on error if (err) { // Fall through for missing files, thow error for other problems next(); if (err) { return; if (err.errno === process.ENOENT) { } next(); var url = Url.parse(req.url); // Refire the buffered events events.forEach(function (args) { req.emit.apply(req, args); var pathname = url.pathname.replace(/..+/g, '.'), }); filename = Path.join(this.root, pathname); return; if (filename[filename.length - 1] === "/") { filename += "index.html"; } Saturday, June 5, 2010

static.js // Buffer any events that fire while waiting on the stat. var fs = require('fs'), var events = []; Url = require('url'), function onData() { Path = require('path'); events.push(["data"].concat(Array.prototype.slice.call(arguments))); } var lifetime = 1000 * 60 * 60; // 1 hourfunction onEnd() { browser cache lifetime events.push(["end"].concat(Array.prototype.slice.call(arguments))); } var DEFAULT_MIME = 'application/octet-stream'; req.addListener("data", onData); module.exports = { req.addListener("end", onEnd); (err); setup: function (env) { fs.stat(filename, function (err, stat) { rn; this.root = this.root || process.cwd(); }, // Stop buffering events req.removeListener("data", onData); req.removeListener("end", onEnd); the file directly using (err, req, res, next) { handle: function buffers ile(filename, function error data) { // Skip on (err, if (err) { // Fall through for missing files, thow error for other problems err) { next(); if (err) { next(err); return; if (err.errno === process.ENOENT) { return; } next(); var url = Url.parse(req.url); // Refire the buffered events writeHead(200, { events.forEach(function (args) { ontent-Type": Mime.type(filename), req.emit.apply(req, args); ontent-Length": pathname = url.pathname.replace(/..+/g, '.'), var data.length, }); filename = Path.join(this.root, pathname); ast-Modified": stat.mtime.toUTCString(), return; / Cache in browser for 1 year ache-Control": (filename[filename.length - 1] === "/") { if "public max-age=" + 31536000 filename += "index.html"; end(data); } Saturday, June 5, 2010

static.js // Buffer any events that fire while waiting on the stat. var fs = require('fs'), var events = []; Url = require('url'), function onData() { Path = require('path'); events.push(["data"].concat(Array.prototype.slice.call(arguments))); } var lifetime = 1000 * 60 * 60; // 1 hourfunction onEnd() { browser cache lifetime }; events.push(["end"].concat(Array.prototype.slice.call(arguments))); } var DEFAULT_MIME = 'application/octet-stream'; // Mini mime module for static file serving req.addListener("data", onData); module.exports = { var req.addListener("end", onEnd); Mime = { (err); type: function getMime(path) (err, stat) { setup: function (env) { fs.stat(filename, function { rn; var index = path.lastIndexOf("."); this.root = this.root || process.cwd(); }, if (index < buffering events // Stop 0) { return DEFAULT_MIME; req.removeListener("data", onData); } req.removeListener("end", onEnd); the file directly using (err, req, res, next) { type = Mime.TYPES[path.substring(index).toLowerCase()] || DEFAULT_MIME; handle: function buffers var ile(filename, // function error data) { Skip on (err, if (err) { return Fall through for missing files, thow error "; charset=utf-8" : type; // (/(text|javascript)/).test(type) ? type + for other problems err) { }, next(); if (err) { next(err); return; if (err.errno === process.ENOENT) { return; TYPES : { ".3gp" } next(); "video/3gpp", : var url = Url.parse(req.url); ".a" Refire the buffered events // : "application/octet-stream", writeHead(200, { ".ai" : "application/postscript", events.forEach(function (args) { ontent-Type": Mime.type(filename), ".aif" req.emit.apply(req, args); : "audio/x-aiff", ontent-Length": pathname = url.pathname.replace(/..+/g, '.'), "audio/x-aiff", var data.length, ".aiff" : }); filename = Path.join(this.root, pathname); ast-Modified": stat.mtime.toUTCString(), ".asc" return; "application/pgp-signature", : / Cache in browser for 1 year ".asf" : "video/x-ms-asf", ache-Control": (filename[filename.length - 1] === "/") { if "public max-age=" + 31536000 ".asm" : "text/x-asm", filename += "index.html"; ".asx" : "video/x-ms-asf", end(data); } ".atom" : "application/atom+xml", ".au" : "audio/basic", Saturday, June 5, 2010 ".avi" : "video/x-msvideo",

static.js // Buffer any events that fire while waiting on the stat. var fs = require('fs'), : "video/x-flv",var events = []; ".flv" ".for" : function onData() { Url = require('url'), "text/x-fortran", Path = require('path'); ".gem" events.push(["data"].concat(Array.prototype.slice.call(arguments))); : "application/octet-stream", } ".gemspec" : "text/x-script.ruby", var lifetime = ".gif" 60: *"image/gif", function onEnd() { 1000 * 60; // 1 hour browser cache lifetime }; events.push(["end"].concat(Array.prototype.slice.call(arguments))); ".gz" : "application/x-gzip", ".h" : "text/x-c", } var DEFAULT_MIME = 'application/octet-stream'; // Mini mime module for static file serving req.addListener("data", onData); ".hh" : "text/x-c", module.exports ".htm" = { var Mime = { : "text/html", req.addListener("end", onEnd); (err); ".html" : "text/html", rn; setup: function (env) "image/vnd.microsoft.icon", getMime(path) (err, stat) { ".ico" : { type: function fs.stat(filename, function { var index = path.lastIndexOf("."); this.root = this.root || process.cwd(); ".ics" : "text/calendar", }, ".ifb" : "text/calendar", (index < buffering events if // Stop 0) { return DEFAULT_MIME; req.removeListener("data", onData); ".iso" : "application/octet-stream", } : req, res, next) {req.removeListener("end", onEnd); the file directly ".jar" (err,"application/java-archive", handle: function buffers using var type = Mime.TYPES[path.substring(index).toLowerCase()] || DEFAULT_MIME; ile(filename, // function error: data) { Skip".java" on (err, "text/x-java-source", if (err) { return Fall through for missing files, thow error "; charset=utf-8" : type; // (/(text|javascript)/).test(type) ? type + for other problems ".jnlp" : "application/x-java-jnlp-file", err) { }, next(); ".jpeg" : "image/jpeg", if (err) { next(err); return; ".jpg" : "image/jpeg", if (err.errno === process.ENOENT) { return; TYPES : { ".3gp" : "video/3gpp", } ".js" : "application/javascript", next(); var url = Url.parse(req.url); ".a" Refire the buffered events // : "application/octet-stream", writeHead(200, { ".json" : "application/json", ".ai" : "application/postscript", ".log" : "text/plain", events.forEach(function (args) { ontent-Type": Mime.type(filename), ".aif" req.emit.apply(req, args); : "audio/x-aiff", ontent-Length": pathname = url.pathname.replace(/..+/g, '.'), "audio/x-aiff", var data.length, "audio/x-mpegurl", ".m3u" : ".aiff" : filename = Path.join(this.root, pathname); ".m4v" : "video/mp4", }); ast-Modified": stat.mtime.toUTCString(), ".asc" return; "application/pgp-signature", : / Cache in browser ".man" year "text/troff", for 1 : ".asf" : "video/x-ms-asf", ache-Control": (filename[filename.length - 1] === "/") { if "public max-age=" + 31536000 ".mathml" : "application/mathml+xml", ".asm" : "text/x-asm", filename += : "application/mbox", ".mbox" "index.html"; ".asx" : "video/x-ms-asf", end(data); } ".mdoc" : "text/troff", ".me" : "text/troff", ".atom" : "application/atom+xml", ".au" : "audio/basic", ".mid" : "audio/midi", Saturday, June 5, 2010 ".avi" : "video/x-msvideo",

static.js // Buffer any events that fire while waiting "text/x-c", ".cc" : on the stat. var fs = require('fs'), : "video/x-flv",var events = []; ".flv" ".chm" : "application/vnd.ms-htmlhelp", Url = require('url'), "text/x-fortran", ".for" : function onData() { ".class" : "application/octet-stream", Path = require('path'); ".gem" events.push(["data"].concat(Array.prototype.slice.call(arguments))); : "application/octet-stream", ".com" : "application/x-msdownload", } ".gemspec" : "text/x-script.ruby", ".conf" : "text/plain", var lifetime = ".gif" 60: *"image/gif", function onEnd() { 1000 * 60; // 1 hour browser cache lifetime ".cpp" : "text/x-c", }; events.push(["end"].concat(Array.prototype.slice.call(arguments))); ".crt" : "application/x-x509-ca-cert", ".gz" : "application/x-gzip", var DEFAULT_MIME = 'application/octet-stream'; ".h" : "text/x-c", } ".css" : "text/css", // Mini mime module for static file serving req.addListener("data", onData); ".csv" : "text/csv", ".hh" : "text/x-c", module.exports ".htm"= { var Mime = { : "text/html", req.addListener("end", onEnd); ".cxx" : "text/x-c", ".html" : "text/html", ".deb" : "application/x-debian-package", (err); rn; setup: function (env) "image/vnd.microsoft.icon", getMime(path) (err, stat) { ".ico" : { type: function fs.stat(filename, function { ".der" : "application/x-x509-ca-cert", var index = path.lastIndexOf(".");".diff" : "text/x-diff", this.root = this.root || process.cwd(); ".ics" : "text/calendar", }, ".ifb" : "text/calendar", (index < buffering events if // Stop 0) { ".djv" : "image/vnd.djvu", return DEFAULT_MIME; req.removeListener("data", onData); ".djvu" : "image/vnd.djvu", ".iso" : "application/octet-stream", } : req, res, next) {req.removeListener("end", onEnd); ".dll" : "application/x-msdownload", the file directly ".jar" (err,"application/java-archive", handle: function buffers using var type = Mime.TYPES[path.substring(index).toLowerCase()] || DEFAULT_MIME; // function error: data) { Skip".java" on (err, "text/x-java-source", ".dmg" : "application/octet-stream", ile(filename, return Fall through for missing files, thow error "; charset=utf-8" : type; if (err) { // (/(text|javascript)/).test(type) ? : "application/msword", ".jnlp" : "application/x-java-jnlp-file", ".doc" type + for other problems err) { }, next(); ".jpeg" : "image/jpeg", if (err) { ".dot" : "application/msword", next(err); return; ".jpg" : "image/jpeg", if (err.errno === process.ENOENT) { "application/xml-dtd", ".dtd" : return; TYPES : { ".3gp" : "video/3gpp", ".dvi" } ".js" : "application/javascript", next(); : "application/x-dvi", var url = Url.parse(req.url); ".a" Refire the buffered events : "application/java-archive", // : "application/octet-stream", ".ear" writeHead(200, { ".json" : "application/json", ".ai" : "application/postscript", ".log" : "text/plain", events.forEach(function (args) : "message/rfc822", ".eml" { ontent-Type": Mime.type(filename), ".aif" req.emit.apply(req, args); : "application/postscript", : "audio/x-aiff", ".eps" ontent-Length": pathname = url.pathname.replace(/..+/g, '.'), "audio/x-aiff", var data.length, "audio/x-mpegurl", ".m3u" : ".aiff" : filename = Path.join(this.root, pathname); ".m4v" : "video/mp4", }); ".exe" : "application/x-msdownload", ast-Modified": stat.mtime.toUTCString(), ".asc" return; "application/pgp-signature", : ".f" : "text/x-fortran", / Cache in browser ".man" year "text/troff", for 1 : ".asf" : "video/x-ms-asf", ".f77" : "text/x-fortran", ache-Control": (filename[filename.length - 1] === "/") { if "public max-age=" + 31536000 ".mathml" : "application/mathml+xml", ".asm" : "text/x-asm", ".f90" filename += : "application/mbox", ".mbox" "index.html"; : "text/x-fortran", ".asx" : "video/x-ms-asf", end(data); } ".mdoc" : "text/troff", ".me" : "text/troff", ".atom" : "application/atom+xml", ".au" : "audio/basic", ".mid" : "audio/midi", Saturday, June 5, 2010 ".avi" : "video/x-msvideo",

static.js // Buffer any events that fire while waiting "text/x-c", ".cc" : on the stat. var fs = require('fs'), : "video/x-flv",var events = []; ".flv" ".chm" : "application/vnd.ms-htmlhelp", Url = require('url'), "text/x-fortran", ".for" : function onData() { ".class" : "application/octet-stream", Path = require('path'); ".gem" events.push(["data"].concat(Array.prototype.slice.call(arguments))); : "application/octet-stream", ".com" : "application/x-msdownload", } ".gemspec" : "text/x-script.ruby", ".conf" : "text/plain", var lifetime = ".gif" 60: *"image/gif", function onEnd() { 1000 * 60; // 1 hour browser cache lifetime ".cpp" : "text/x-c", }; events.push(["end"].concat(Array.prototype.slice.call(arguments))); ".crt" : "application/x-x509-ca-cert", ".gz" : "application/x-gzip", var DEFAULT_MIME = 'application/octet-stream'; ".h" : "text/x-c", } ".css" : "text/css", // Mini mime module for static file serving req.addListener("data", onData); ".csv" : "text/csv", ".hh" : "text/x-c", module.exports ".htm"= { var Mime = { : "text/html", req.addListener("end", onEnd); ".cxx" : "text/x-c", ".html" : "text/html", ".deb" : "application/x-debian-package", (err); rn; setup: function (env) "image/vnd.microsoft.icon", getMime(path) (err, stat) { ".ico" : { type: function fs.stat(filename, function { ".der" : "application/x-x509-ca-cert", var index = path.lastIndexOf(".");".diff" : "text/x-diff", this.root = this.root || process.cwd(); ".ics" : "text/calendar", }, ".ifb" : "text/calendar", (index < buffering events if // Stop 0) { ".djv" : "image/vnd.djvu", return DEFAULT_MIME; req.removeListener("data", onData); ".djvu" : "image/vnd.djvu", ".iso" : "application/octet-stream", } req.removeListener("end", onEnd); ".dll" : "application/x-msdownload", the file directly ".jar"".bmp" req, "image/bmp", type = Mime.TYPES[path.substring(index).toLowerCase()] || DEFAULT_MIME; handle: function (err,"application/java-archive", using buffers : res, next) { : var // function error: data) { Skip".java" on (err, "text/x-java-source", ".bz2" : "application/x-bzip2", ".dmg" : "application/octet-stream", ile(filename, return (/(text|javascript)/).test(type) ? : "application/msword", : type; type + "; charset=utf-8" err) { if (err) { : "text/x-c", // Fall through for missing files, thow error for other problems ".jnlp" : "application/x-java-jnlp-file", ".c" ".doc" }, if (err) { ".dot" : "application/msword", next(err); next(); ".cab" ".jpeg" : "image/jpeg", : "application/vnd.ms-cab-compressed", return; ".jpg" : "image/jpeg", if (err.errno === process.ENOENT) { "application/xml-dtd", ".dtd" : return; TYPES : { ".3gp" : "video/3gpp", ".dvi" } ".js" : "application/javascript", next(); : "application/x-dvi", var url = Url.parse(req.url); ".a" Refire the buffered events : "application/java-archive", // : "application/octet-stream", ".ear" writeHead(200, { ".json" : "application/json", ".ai" : "application/postscript", ".log" : "text/plain", events.forEach(function (args) : "message/rfc822", ".eml" { ontent-Type": Mime.type(filename), ".aif" req.emit.apply(req, args); : "application/postscript", : "audio/x-aiff", ".eps" ontent-Length": pathname = url.pathname.replace(/..+/g, '.'), "audio/x-aiff", var data.length, "audio/x-mpegurl", ".m3u" : ".aiff" : filename = Path.join(this.root, pathname); ".m4v" : "video/mp4", }); ".exe" : "application/x-msdownload", ast-Modified": stat.mtime.toUTCString(), ".asc" return; "application/pgp-signature", : ".f" : "text/x-fortran", / Cache in browser ".man" year "text/troff", for 1 : ".asf" : "video/x-ms-asf", ".f77" : "text/x-fortran", ache-Control": (filename[filename.length - 1] === "/") { if "public max-age=" + 31536000 ".mathml" : "application/mathml+xml", ".asm" : "text/x-asm", ".f90" filename += : "application/mbox", ".mbox" "index.html"; : "text/x-fortran", ".asx" : "video/x-ms-asf", end(data); } ".mdoc" : "text/troff", ".me" : "text/troff", ".atom" : "application/atom+xml", ".au" : "audio/basic", ".mid" : "audio/midi", Saturday, June 5, 2010 ".avi" : "video/x-msvideo",

Built-in Filter Modules Authentication Error Handler Authorization Gzip Body Decoder Log Cache Method Override Conditional Get Response Time Debug Session Saturday, June 5, 2010

Built-in Data Providers Static Cache Manifest Rest Direct Router JSON-RPC PubSub More... Saturday, June 5, 2010

Demo Time! Saturday, June 5, 2010

app.js (stack) require.paths.unshift("./lib"); var Connect = require('connect'); var root = __dirname + "/public"; module.exports = Connect.createServer([ {filter: "log"}, {filter: "body-decoder"}, {provider: "pubsub", route: "/stream", logic: Backend}, {filter: "conditional-get"}, {filter: "cache"}, {filter: "gzip"}, {provider: "cache-manifest", root: root}, {provider: "static", root: root} ]); Saturday, June 5, 2010

app.js (Backend) var Backend = { subscribe: function (subscriber) { if (subscribers.indexOf(subscriber) < 0) { subscribers.push(subscriber); } }, unsubscribe: function (subscriber) { var pos = subscribers.indexOf(subscriber); if (pos >= 0) { subscribers.slice(pos); } }, publish: function (message, callback) { subscribers.forEach(function (subscriber) { subscriber.send(message); }); callback(); } }; Saturday, June 5, 2010

http://github.com/extjs/connect http://twitter.com/creationix http://raphaeljs.com http://nodejs.org Saturday, June 5, 2010

Any Questions ? Saturday, June 5, 2010