Networked pub/sub with UDP, Go, Ruby and ZMQ10. Publisher 1
pub/sub
require 'socket'
socket = UDPSocket.new
message = {
time: Time.now.to_s,
type: 'pageview',
app: 'store_fronts',
data: {
account: 'acme',
user: 'Joe Bloggs',
domain: 'www.acme.com',
path: '/about/us',
ua: 'Mozilla/5.0 (Windows NT 6.2; Win64; x64)...'
}
}
json = ActiveSupport::JSON.encode(message)
socket.send(json, 0, 'events_hub_host', 5555)
14. Events hub
pub/sub
github.com/bootic/bootic_data_collector
type Daemon struct {
Conn *net.UDPConn
observers map[string][]data.EventsChannel
}
func NewDaemon(udpHostAndPort string) (daemon *Daemon, err error) {
conn, err := createUDPListener(udpHostAndPort)
if err != nil {
return
}
daemon = &Daemon{
Conn: conn,
observers: make(map[string][]data.EventsChannel),
}
go daemon.ReceiveDatagrams()
return
}
18. Events hub
pub/sub
github.com/bootic/bootic_data_collector
// Start up UDP daemon
daemon, err := udp.NewDaemon(udpHost)
// Setup Websockets server
wshub := ws.HandleWebsocketsHub("/ws")
// Push incoming UDP messages to multiple listeners
daemon.Subscribe(wshub.Notifier)
// Start up PUB ZMQ client
zmqObserver := fanout.NewZmq(zmqAddress)
// Push incoming UDP events down ZMQ pub/sub socket
daemon.Subscribe(zmqObserver.Notifier)
log.Fatal("HTTP server error: ", http.ListenAndServe(wsHost, nil))
22. Stats
aggregates
pub/sub
// Setup ZMQ subscriber
daemon, _ := booticzmq.NewZMQSubscriber(zmqAddress)
// Setup Redis tracker
tracker, err := redis_stats.NewTracker(redisAddress)
// Redis subscribes to these events
daemon.SubscribeToType(tracker.Notifier, "pageview")
github.com/bootic/bootic_stats_aggregates
23. Stats
aggregates
pub/sub
github.com/bootic/bootic_stats_aggregates
year := now.Year()
month := now.Month()
day := now.Day()
hour := now.Hour()
go func () {
// increment current month in year: “track:acme/pageview/2013”
yearKey := fmt.Sprintf("track:%s/%s/%s", account, evtType, year)
self.Conn.HIncrBy(yearKey, month, 1)
// increment current day in month: “track:acme/pageview/2013/12”
monthKey := fmt.Sprintf("track:%s/%s/%s/%s", key, evtType, year, month)
self.Conn.HIncrBy(monthKey, day, 1)
// increment current hour in day: “track:acme/pageview/2013/12/16”
dayKey := fmt.Sprintf("track:%s/%s/%s/%s/%s", key, evtType, year, month, day)
self.Conn.HIncrBy(dayKey, hour, 1)
}()
24. Stats
aggregates
pub/sub
github.com/bootic/bootic_stats_aggregates
GET /api/stats/track/acme/pageview/2013/12/16
{
"account": "acme",
"event": "pageview",
"year": "2013",
"month": "12",
"day": "16",
"data": {
"0": 2693,
"1": 1215,
"2": 341,
"3": 176,
"4": 80,
"5": 89,
"6": 333,
"7": 779,
"8": 1506,
"9": 2553,
"10": 3734
}
}
29. Git backups
pub/sub
doneChan := make(chan string)
bufferChan := make(chan int, 20)
stores := make(map[string]*ThemeStore)
for {
select {
case event := <-writer.Notifier:
account := event.Get("account")
store := stores[account]
// Register store and start delayed writing
// if not already registered
if store == nil {
store = NewThemeStore(account)
stores[account] = store
go store.DelayedWrite(bufferChan, doneChan)
}
case account := <-doneChan:
// A store is done writing.
// Un-register it so it can be registered again.
delete(stores, account)
}
}
github.com/bootic/bootic_themes_backup
31. Git backups
pub/sub
github.com/bootic/bootic_themes_backup
func (store *ThemeStore) DelayedWrite(bufferChan chan int, doneChan chan string) {
time.Sleep(10)
// Start work. This will block if buffer is full.
bufferChan <- 1
store.Backup()
// Done. Free space in the buffer
<-bufferChan
doneChan <- store.Account
}()
}
doneChan := make(chan string)
bufferChan := make(chan int, 20)
…