Async data pipelines for client-side JavaScript

966 views

Published on

Patterns like data pipelines, queuing and multiplexing are familiar to backend developers working on distributed and high-traffic systems. Projects such as Node.js or ZeroMQ make the concepts of streams, queues and pipelines first-order primitives that allow you to compose software in an organic and declarative way.

In this talk, I will attempt to bring together these patterns and principles and explore how they can be applied to everyday client-side JavaScript programming.

Published in: Software, Business, Technology
0 Comments
0 Likes
Statistics
Notes
  • Be the first to comment

  • Be the first to like this

No Downloads
Views
Total views
966
On SlideShare
0
From Embeds
0
Number of Embeds
11
Actions
Shares
0
Downloads
5
Comments
0
Likes
0
Embeds 0
No embeds

No notes for slide

Async data pipelines for client-side JavaScript

  1. 1. ASYNC PIPELINES in client-side JavaScript (an experiment) ismael celis . new-bamboo.co.uk . github.com/ismasan
  2. 2. V/Capp.AppView = Backbone.View.extend({ initialize: function () { ... }, render: function () { ... }, ! addOne: function (todo) { ... }, addAll: function () { ... }, filterOne: function (todo) { todo.trigger('visible'); }, filterAll: function () { ... } ! } Mapp.Todo = Backbone.Model.extend({ ! defaults: { title: '', completed: false }, toggle: function () { this.save({ completed: !this.get('completed') }); }, processAndSave: function (data, callback) { this.set(data) var self = this this.save(function () { callback() self.trigger('processed') Todo.trigger('processed', self) }) } }); T <h1>{{ count }} productos encontrados</h1> ! <ul> {{#products}} <li><a href="{{ href }}">{{name}}</a></li> {{/products}} </ul>
  3. 3. + verbs - nouns
  4. 4. How my JS apps should be built
  5. 5. How my JS apps end up being built
  6. 6. ...
  7. 7. BASIC OBJECT var Person = BasicObject.extend({
 initialize: function (name) {
 this.name = name
 },
 talk: function () {
 alert("What's up!")
 }
 })
 
 var Programmer = Person.extend({
 talk: function () {
 alert("Go away, I'm busy writing code!")
 }
 })
  8. 8. STRUCT var mario = new Struct({name: 'Mario'}) ! mario.set('name', 'Luigi') // triggers `change:name` event ! mario.get('name') // “Luigi" ! mario.uid() // "uid13983601141350.9170819637365639"
  9. 9. var p1 = new Pipe() var p2 = new Pipe() var p3 = new Pipe() ! p1.pipe(p2).pipe(p3) ! p3.on('add', function (struct) { console.log('p3 received data!', struct) }) ! p1.add(mario) // forwards mario to other pipes downstream, // including p2 and then p3 PIPE
  10. 10. PIPE pipe(anotherPipe) ! add(struct) ! remove(struct)
  11. 11. PIPE ! add(struct) ! filter(struct, filterPromise) ! _add(struct, addPromise) ! _formwardAdd(struct)
  12. 12. PIPE + FILTER var MarioFilter = Pipe.extend({ ! filter: function (struct, filterPromise) { if(struct.get('name') == 'Mario') filterPromise.resolve(struct) else filterPromise.reject(struct) } ! })
  13. 13. PIPE + FILTER var filter = new MarioFilter() ! filter.pipe(p2).pipe(p3) ! filter.add(new Struct({name: 'Mario'})) // p2 and p3 get struct added ! filter.add(new Struct({name: 'Luigi'})) // filter is rejected. p2 and p3 DO NOT get struct added
  14. 14. PIPE + FILTER + AJAX var MarioFilter = Pipe.extend({ filter: function (struct, filterPromise) { $.get( 'http://some.api.com/valid_mario', {name: struct.get('name')} ).then( function () { filterPromise.resolve(struct) }, // success function () { filterPromise.reject(struct) } // error ) } })
  15. 15. PIPE api.jquery.com/jQuery.when/ $.when( pipe1.add(struct), pipe2.add(struct), pipe3.add(struct) ).then(...)
  16. 16. INDEX var Index = Pipe.extend({
 
 initialize: function () {
 this._index = {} this._list = []
 }, ! …
 })
  17. 17. INDEX var Index = Pipe.extend({
 
 …
 
 _add: function (struct, addPromise) {
 
 if(! this._index[struct.uid()]) {
 this._index[struct.uid()] = struct this._list.push(struct)
 addPromise.resolve(struct)
 }
 
 }
 })
  18. 18. INDEX var index = new Index() index.add(mario) index.add(luigi) ! index.pipe(another_pipe) // ‘another_pipe’ will get mario and luigi added to it now ! index.add(princess) // ‘another_pipe’ gets princess added to it.
  19. 19. INDEX index.add(mario) // forwards to other pipes ! mario.set(age: 30) ! index.add(mario) // does not forward.
  20. 20. CAPPED INDEX var CappedIndex = Index.extend({ ! limit: 10, ! _add: function (struct, promise) { // remove first if limit reached if(this._list.length > this.limit - 1) this.remove(this._list[0]) // add next return Index.prototype._add.call(this, struct, promise) } ! })
  21. 21. DEVICES
  22. 22. CHOKE POINT var p1 = new Pipe() var p2 = new SomeAjaxPipe() var results = new Pipe() var choke = new ChokePoint(p1, p2) ! choke.pipe(results) ! choke.add(struct)
  23. 23. PIPELINE var p1 = new Pipe() var p2 = new SomeAjaxPipe() var results = new Pipe() var pipeline = new LeakyPipeline(p1, p2) ! pipeline.add(struct) // will forward struct to p1, p2 and finally pipe on to results pipe
  24. 24. FAN IN var in1 = new Pipe() var in2 = new Pipe() var results = new Pipe() var fanIn = new Ventilator([in1, in2]) ! fanIn.pipe(results) ! in1.add(struct) // forwards struct to results
  25. 25. FAN OUT var out1 = new Pipe() var out2 = new Pipe() ! var fanOut = new Ventilator(null, [out1, out2]) ! fanOut.add(struct) // forwards struct to out1 and out2
  26. 26. MANY-TO-MANY var ventilator = new Ventilator( [in1, in2], [out1, out2] ) ! in1.add(struct) // forwards struct to out1 and out2 in2.add(struct) // forwards struct to out1 and out2
  27. 27. ROUTER var p1 = new Pipe() var p2 = new Pipe() var default = new Pipe() ! var router = new Router() ! router .route(p1, function (struct, promise) { if(struct.get('name') == 'Joe') promise.resolve() else promise.reject() }) .route(p2, function (struct, promise) { if(struct.get('name') == 'Jane') promise.resolve() else promise.reject() }) .default(default)
  28. 28. ROUTER router.add(new Struct({name: 'Joe'})) // forwards struct to p1 ! router.add(new Struct({name: 'Jane'})) // forwards struct to p2 ! router.add(new Struct({name: 'Paul'})) // forwards struct to default ! router.pipe(other) // pipes all data to `other`.
  29. 29. CUSTOM DEVICE var MyCustomDevice = Pipe.extend({ ! filter: function (struct, filterPromise) {...}, ! _add: function (struct, addPromise) {...}, ! _remove: function (struct, removePromise) {...}, ! _forwardAdd: function (struct) {...}, ! _forwardRemove: function (struct) {...} })
  30. 30. CUSTOM DEVICE Throttle, Buffer,Aggregator, Counter, Identity Map, State Machine
  31. 31. REPOSITORY // users.pipe(results).get('http://some.api.com/users') var Users = Pipe.extend({ ! // Get from remote API get: function (url) { var self = this ! $.getJSON(url).then(function (data) { data.forEach(function (user) { self.add(user) }) }) return this } })
  32. 32. REPOSITORY var Users = Pipe.extend({ ! // A repository can have its own index initialize: function () { this.__index = new Index(UserStruct) // pipe index to itself so repository can pipe to other pipes transparently this.__index.pipe(this) }, ! // Get from remote API get: function (url) { var self = this ! $.getJSON(url).then(function (data) { data.forEach(function (user) { self.__index.add(user) }) }) return this } })
  33. 33. REPOSITORY var Users = Pipe.extend({ ! _add: function (struct, addPromise) { // send data to server $.ajax(this.url, { type: 'post', dataType: 'json', data: struct.attributes }).then(function (data) { struct.set(data) // update struct with data coming from server addPromise.resolve(struct) }) } ! })
  34. 34. FRAMEWORK ?
  35. 35. VIEW var View = Pipe.extend({ initialize: function ($e) { ... this._children = {} }, _add: function (item, promise) { // create and index child view this._children[item.uid()] = new ChildElement(this.$itemTemplate) promise.resolve(item) }, _remove: function (item, promise) { // unbind, destroy and de-index child-view var child = this._children[item.uid()].destroy() delete this._children[item.uid()] promise.resolve(item) } })
  36. 36. VIEW rivetsjs.com <div id="items"> <ul data-item> <li data-text="item.name"></li> </ul> </div>
  37. 37. VIEW var view = new View($('#items')) ! var mario = new Struct({name: 'Mario'}) ! view.add(mario)
  38. 38. CAPPED VIEW function cappedView($e, limit) { var index = new CappedIndex(limit) var view = new View($e) index.pipe(view) return index }
  39. 39. APP var timeline = cappedView($('#timeline'), 10) ! var socket = new SocketRepository('ws://some.server.com') ! socket.pipe(timeline) gist.github.com/ismasan/5848735
  40. 40. github.com/ismasan/plumber.js reactive-extensions.github.io/RxJS/ ismael celis . new-bamboo.co.uk . github.com/ismasan

×