0
Ismael Celis
Networked pub/sub with Go, Ruby and ZMQ
bootic.net - Hosted e-commerce in South America
Store frontsTheme editor
Checkout
Dashboard
Image resizer API
… and more in the pipeline.
• analytics
• automated backups
• activity dashboard
• audit trail
Ancillary services
Also: Play with cool toys
Publisher 1
Publisher 2
Publisher 3
Events hub Events bus
pub/sub
Publisher 1
Publisher 2
Publisher 3
Events hub Events bus
pub/sub
JSON
UDP
msgpack ZMQ
Events hub Events bus
pub/sub
JSON
UDP
msgpack ZMQ
Stats
Backups
Logs
?
Websocket
pub / sub
Publisher 1
pub/sub
require 'socket'



socket = UDPSocket.new



message = {

time: Time.now.to_s,

type: 'pageview',

ap...
Events hub
pub/sub
github.com/bootic/bootic_data_collector
func (daemon *Daemon) ReceiveDatagrams() {



for {

buffer := ...
Events hub
pub/sub
github.com/bootic/bootic_data_collector
// Start up UDP daemon
daemon, err := udp.NewDaemon(udpHost)


...
Events hub
pub/sub
github.com/bootic/bootic_data_collector
daemon, err := udp.NewDaemon(udpHost)
Events hub
pub/sub
github.com/bootic/bootic_data_collector
type Daemon struct {

Conn *net.UDPConn

observers map[string][...
Events hub
pub/sub
github.com/bootic/bootic_data_collector
go daemon.ReceiveDatagrams()
Events hub
pub/sub
github.com/bootic/bootic_data_collector


daemon, err := udp.NewDaemon(udpHost)



// Start up PUB ZMQ ...
Events hub
pub/sub
github.com/bootic/bootic_data_collector
daemon.Subscribe(zmqObserver.Notifier)
Events hub
pub/sub
github.com/bootic/bootic_data_collector
// Start up UDP daemon

daemon, err := udp.NewDaemon(udpHost)

...
pub / sub
pageviews
tracker
Events hub
1px tracking .gif JSON / UDP
github.com/bootic/bootic_pageviews
Events hub
Redis
pub sub
aggregator HTTP API
internets
“pageview”
Stats
aggregates
pub/sub
// Setup ZMQ subscriber

daemon, _ := booticzmq.NewZMQSubscriber(zmqAddress)



// Setup Redis tr...
Stats
aggregates
pub/sub
github.com/bootic/bootic_stats_aggregates
year := now.Year()

month := now.Month()

day := now.Da...
Stats
aggregates
pub/sub
github.com/bootic/bootic_stats_aggregates
GET /api/stats/track/acme/pageview/2013/12/16
{

"accou...
Stats
aggregates
pub/sub
github.com/bootic/bootic_stats_aggregates
Git backups
pub/sub
github.com/bootic/bootic_themes_backup
Events hub
pub sub
Git backups
Themes API
“theme”
Git
Git
git clone tufte:/home/git/git_themes/acme
Git backups
pub/sub
doneChan := make(chan string)

bufferChan := make(chan int, 20)



stores := make(map[string]*ThemeSto...
Git backups
pub/sub
github.com/bootic/bootic_themes_backup
go store.DelayedWrite(bufferChan, doneChan)
doneChan := make(ch...
Git backups
pub/sub
github.com/bootic/bootic_themes_backup
func (store *ThemeStore) DelayedWrite(bufferChan chan int, done...
The future
The future
• Searchable events history
• Per-account secure websocket
• More stats!
• Webhooks
Ismael Celis @ismasan
bit.ly/1fYmUff
Blog post
bit.ly/1fYmUff
Ismael Celis @ismasan
bootic/bootic_data_collector
bootic/bootic_pageviews
bootic/bootic_stats_aggregates
b...
Upcoming SlideShare
Loading in...5
×

Networked pub/sub with UDP, Go, Ruby and ZMQ

1,760

Published on

Building a network of publishers and subscribers with Golang, Ruby, ZMQ (with some Git and Redis in the mix). Presented at London Ruby User Group in March 2014.

Blog post: http://new-bamboo.co.uk/blog/2013/09/17/micro-network-daemons-in-go

Published in: Technology
0 Comments
2 Likes
Statistics
Notes
  • Be the first to comment

No Downloads
Views
Total Views
1,760
On Slideshare
0
From Embeds
0
Number of Embeds
8
Actions
Shares
0
Downloads
19
Comments
0
Likes
2
Embeds 0
No embeds

No notes for slide

Transcript of "Networked pub/sub with UDP, Go, Ruby and ZMQ"

  1. 1. Ismael Celis Networked pub/sub with Go, Ruby and ZMQ
  2. 2. bootic.net - Hosted e-commerce in South America
  3. 3. Store frontsTheme editor Checkout Dashboard Image resizer API … and more in the pipeline.
  4. 4. • analytics • automated backups • activity dashboard • audit trail Ancillary services
  5. 5. Also: Play with cool toys
  6. 6. Publisher 1 Publisher 2 Publisher 3 Events hub Events bus pub/sub
  7. 7. Publisher 1 Publisher 2 Publisher 3 Events hub Events bus pub/sub JSON UDP msgpack ZMQ
  8. 8. Events hub Events bus pub/sub JSON UDP msgpack ZMQ Stats Backups Logs ? Websocket
  9. 9. pub / sub
  10. 10. Publisher 1 pub/sub require 'socket'
 
 socket = UDPSocket.new
 
 message = {
 time: Time.now.to_s,
 type: 'pageview',
 app: 'store_fronts',
 data: {
 account: 'acme',
 user: 'Joe Bloggs',
 domain: 'www.acme.com',
 path: '/about/us',
 ua: 'Mozilla/5.0 (Windows NT 6.2; Win64; x64)...'
 }
 }
 
 json = ActiveSupport::JSON.encode(message)
 
 socket.send(json, 0, 'events_hub_host', 5555)
  11. 11. Events hub pub/sub github.com/bootic/bootic_data_collector func (daemon *Daemon) ReceiveDatagrams() {
 
 for {
 buffer := make([]byte, 1024)
 
 daemon.Conn.ReadFromUDP(buffer)
 
 event, err := data.DecodeJSON(buffer[:c])
 daemon.Dispatch(event)
 
 }
 
 panic("should never have got myself into this.")
 }
  12. 12. Events hub pub/sub github.com/bootic/bootic_data_collector // Start up UDP daemon daemon, err := udp.NewDaemon(udpHost)
 
 // Start up PUB ZMQ client
 zmqObserver := fanout.NewZmq(zmqAddress)
 
 // Push incoming UDP events down ZMQ pub/sub socket
 daemon.Subscribe(zmqObserver.Notifier)
  13. 13. Events hub pub/sub github.com/bootic/bootic_data_collector daemon, err := udp.NewDaemon(udpHost)
  14. 14. Events hub pub/sub github.com/bootic/bootic_data_collector type Daemon struct {
 Conn *net.UDPConn
 observers map[string][]data.EventsChannel
 }
 
 func NewDaemon(udpHostAndPort string) (daemon *Daemon, err error) {
 conn, err := createUDPListener(udpHostAndPort)
 
 if err != nil {
 return
 }
 
 daemon = &Daemon{
 Conn: conn,
 observers: make(map[string][]data.EventsChannel),
 }
 
 go daemon.ReceiveDatagrams()
 
 return
 }
  15. 15. Events hub pub/sub github.com/bootic/bootic_data_collector go daemon.ReceiveDatagrams()
  16. 16. Events hub pub/sub github.com/bootic/bootic_data_collector 
 daemon, err := udp.NewDaemon(udpHost)
 
 // Start up PUB ZMQ client
 zmqObserver := fanout.NewZmq(zmqAddress)
 
 // Push incoming UDP events down ZMQ pub/sub socket
 daemon.Subscribe(zmqObserver.Notifier)
  17. 17. Events hub pub/sub github.com/bootic/bootic_data_collector daemon.Subscribe(zmqObserver.Notifier)
  18. 18. Events hub pub/sub github.com/bootic/bootic_data_collector // Start up UDP daemon
 daemon, err := udp.NewDaemon(udpHost)
 
 
 // Setup Websockets server
 wshub := ws.HandleWebsocketsHub("/ws")
 
 // Push incoming UDP messages to multiple listeners
 daemon.Subscribe(wshub.Notifier)
 
 // Start up PUB ZMQ client
 zmqObserver := fanout.NewZmq(zmqAddress)
 
 // Push incoming UDP events down ZMQ pub/sub socket
 daemon.Subscribe(zmqObserver.Notifier)
 
 log.Fatal("HTTP server error: ", http.ListenAndServe(wsHost, nil))
  19. 19. pub / sub
  20. 20. pageviews tracker Events hub 1px tracking .gif JSON / UDP github.com/bootic/bootic_pageviews
  21. 21. Events hub Redis pub sub aggregator HTTP API internets “pageview”
  22. 22. Stats aggregates pub/sub // Setup ZMQ subscriber
 daemon, _ := booticzmq.NewZMQSubscriber(zmqAddress)
 
 // Setup Redis tracker
 tracker, err := redis_stats.NewTracker(redisAddress)
 
 
 // Redis subscribes to these events
 daemon.SubscribeToType(tracker.Notifier, "pageview") github.com/bootic/bootic_stats_aggregates
  23. 23. Stats aggregates pub/sub github.com/bootic/bootic_stats_aggregates year := now.Year()
 month := now.Month()
 day := now.Day()
 hour := now.Hour()
 
 go func () {
 
 // increment current month in year: “track:acme/pageview/2013”
 yearKey := fmt.Sprintf("track:%s/%s/%s", account, evtType, year)
 self.Conn.HIncrBy(yearKey, month, 1)
 
 // increment current day in month: “track:acme/pageview/2013/12”
 monthKey := fmt.Sprintf("track:%s/%s/%s/%s", key, evtType, year, month)
 self.Conn.HIncrBy(monthKey, day, 1)
 
 // increment current hour in day: “track:acme/pageview/2013/12/16”
 dayKey := fmt.Sprintf("track:%s/%s/%s/%s/%s", key, evtType, year, month, day)
 self.Conn.HIncrBy(dayKey, hour, 1)
 
 }()
  24. 24. Stats aggregates pub/sub github.com/bootic/bootic_stats_aggregates GET /api/stats/track/acme/pageview/2013/12/16 {
 "account": "acme",
 "event": "pageview",
 "year": "2013",
 "month": "12",
 "day": "16",
 "data": {
 "0": 2693,
 "1": 1215,
 "2": 341,
 "3": 176,
 "4": 80,
 "5": 89,
 "6": 333,
 "7": 779,
 "8": 1506,
 "9": 2553,
 "10": 3734
 }
 }

  25. 25. Stats aggregates pub/sub github.com/bootic/bootic_stats_aggregates
  26. 26. Git backups pub/sub github.com/bootic/bootic_themes_backup
  27. 27. Events hub pub sub Git backups Themes API “theme” Git
  28. 28. Git git clone tufte:/home/git/git_themes/acme
  29. 29. Git backups pub/sub doneChan := make(chan string)
 bufferChan := make(chan int, 20)
 
 stores := make(map[string]*ThemeStore)
 
 for {
 select {
 case event := <-writer.Notifier:
 account := event.Get("account")
 
 store := stores[account]
 // Register store and start delayed writing 
 // if not already registered
 if store == nil {
 store = NewThemeStore(account)
 stores[account] = store
 go store.DelayedWrite(bufferChan, doneChan)
 }
 
 case account := <-doneChan:
 // A store is done writing. 
 // Un-register it so it can be registered again.
 delete(stores, account)
 }
 } github.com/bootic/bootic_themes_backup
  30. 30. Git backups pub/sub github.com/bootic/bootic_themes_backup go store.DelayedWrite(bufferChan, doneChan) doneChan := make(chan string)
 bufferChan := make(chan int, 20)
 
 …
  31. 31. Git backups pub/sub github.com/bootic/bootic_themes_backup func (store *ThemeStore) DelayedWrite(bufferChan chan int, doneChan chan string) {
 
 time.Sleep(10)
 
 // Start work. This will block if buffer is full.
 bufferChan <- 1
 
 store.Backup()
 
 // Done. Free space in the buffer
 <-bufferChan
 
 doneChan <- store.Account
 }()
 } doneChan := make(chan string)
 bufferChan := make(chan int, 20)
 
 …
  32. 32. The future
  33. 33. The future • Searchable events history • Per-account secure websocket • More stats! • Webhooks
  34. 34. Ismael Celis @ismasan bit.ly/1fYmUff Blog post
  35. 35. bit.ly/1fYmUff Ismael Celis @ismasan bootic/bootic_data_collector bootic/bootic_pageviews bootic/bootic_stats_aggregates bootic/bootic_themes_backup Githubs bootic/bootic_stathat Blog post Ismael Celis @ismasan
  1. A particular slide catching your eye?

    Clipping is a handy way to collect important slides you want to go back to later.

×