Successfully reported this slideshow.
We use your LinkedIn profile and activity data to personalize ads and to show you more relevant ads. You can change your ad preferences anytime.

MicroCPH: Observability and Product Release

174 views

Published on

Using the three pillars of observability to build, test, and release new products.

Published in: Software
  • Be the first to comment

MicroCPH: Observability and Product Release

  1. 1. ✔
  2. 2.
  3. 3.
  4. 4. package prometheus // Collector is the interface implemented by anything that can be used by // Prometheus to collect metrics. A Collector has to be registered for // collection. See Registerer.Register. type Collector interface { // Describe sends the super-set of all possible descriptors of metrics // collected by this Collector to the provided channel and returns once // the last descriptor has been sent. Describe(chan<- *Desc) // Collect is called by the Prometheus registry when collecting // metrics. The implementation sends each collected metric via the // provided Collect(chan<- Metric) }
  5. 5.
  6. 6. Service A Service B Service C Queue Datastore Service B Service B Service C Service C
  7. 7. dorpc.SetLogger(loglvl.Level( log.KV("component", "grpc"), loglvl.Error, ))
  8. 8. mm, metricsHandler := metrics.Prometheus(metrics.PrometheusNamespace(app)) mm = mm.Labels(metrics.L{ "app": app, "env": env.Env, }) metrics.SetRoot(mm)
  9. 9. func (s *server) initalizeMetrics() { s.metrics = metricsConfig{ attemptedConvergeChassis: s.metricsNode.Gauge("attempted_converge_chassis", "number of chassis converger attempting to converge"), failedConvergeChassis: s.metricsNode.Gauge("failed_converge_chassis", "number of chassis that failed to converge"), } }
  10. 10. dbDriverName, traceOpt := tracemysql.MySQL() // note: default buffer size is 64 * 1 << 10 tracerOpt := tracer.WithBufferSize(64 * 1 << 14) bufferOpt := trace.UseTracerOptions(tracerOpt) tr := trace.InitTracer(traceOpt, bufferOpt) defer tr.Close()
  11. 11. Service A Service B Service C QueueDatastore Service B Service B Service C Service C
  12. 12. Service B DHCP server Virtual Switch Service A RPC calls Iface Y DHCPv4 gRPC DHCPv6 Iface X dropletX DHCP traffic RPC calls Hypervisor
  13. 13. package dhcp4conn var _ prometheus.Collector = &collector{} type collector struct { ReadBytesTotal *prometheus.Desc ReadPacketsTotal *prometheus.Desc WriteBytesTotal *prometheus.Desc WritePacketsTotal *prometheus.Desc }
  14. 14. workC := make(chan request, Workers) for i := 0; i < Workers; i++ { go func() { defer workWG.Done() for r := range workC { s.serve(r.buf, r.from) } }() }
  15. 15. type RateMap struct { mu sync.Mutex ... rateMap map[string]*rate } type RateMapCollector struct { RequestRate *prometheus.Desc rm *RateMap buckets []float64 } func (r *RateMapCollector) Collect(ch chan<- prometheus.Metric) { ... ch <- prometheus.MustNewConstHistogram( r.RequestRate, count, sum, rateCount) }
  16. 16. Rate Limiter Centralized LoggingCentralized LoggingCentralized LoggingCentralized Logging Elastalert emits log line
  17. 17. ● ● ●

×