This document discusses implementing a job queue in Golang. It begins by explaining buffered and unbuffered channels, and shows examples of using channels to coordinate goroutines. It then demonstrates how to build a job queue that uses a channel to enqueue jobs and have worker goroutines process jobs from the channel concurrently. It also discusses ways to gracefully shutdown workers using contexts and wait groups. Finally, it covers topics like auto-scaling agents, communicating between servers and agents, and handling job cancellation.
2. About me
• Software Engineer in Mediatek
• Member of Drone CI/CD Platform
• Member of Gitea Platform
• Member of Gin Golang Framework
• Maintain Some GitHub Actions Plugins.
• Teacher of Udemy Platform: Golang + Drone
21. func worker(jobChan <-chan Job) {
for job := range jobChan {
process(job)
}
}
// make a channel with a capacity of 1024.
jobChan := make(chan Job, 1024)
// start the worker
go worker(jobChan)
// enqueue a job
jobChan <- job
22. func worker(jobChan <-chan Job) {
for job := range jobChan {
process(job)
}
}
// make a channel with a capacity of 1024.
jobChan := make(chan Job, 1024)
// start the worker
go worker(jobChan)
// enqueue a job
jobChan <- job
23. Block if there already are 1024 jobs
jobChan := make(chan Job, 1024)
46. cancelChan := make(chan struct{})
go worker(jobChan, cancelChan)
func worker(jobChan <-chan Job, cancelChan <-chan struct{}) {
for {
select {
case <-cancelChan:
return
case job := <-jobChan:
process(job)
}
}
}
// to cancel the worker, close the cancel channel
close(cancelChan)
Create a cancel channel
47. cancelChan := make(chan struct{})
go worker(jobChan, cancelChan)
func worker(jobChan <-chan Job, cancelChan <-chan struct{}) {
for {
select {
case <-cancelChan:
return
case job := <-jobChan:
process(job)
}
}
}
// to cancel the worker, close the cancel channel
close(cancelChan)
Create a cancel channel
close(cancelChan)
73. Watch the Cancel event
go func() {
done, _ := r.Manager.Watch(ctx, id)
if done {
cancel()
}
}()
74. Handle cancel event on Server
subscribers: make(map[chan struct{}]int64),
cancelled: make(map[int64]time.Time),
75. User cancel running job
c.Lock()
c.cancelled[id] = time.Now().Add(time.Minute * 5)
for subscriber, build := range c.subscribers {
if id == build {
close(subscriber)
}
}
c.Unlock()
76. Agent subscribe the cancel event
for {
select {
case <-ctx.Done():
return false, ctx.Err()
case <-time.After(time.Minute):
c.Lock()
_, ok := c.cancelled[id]
c.Unlock()
if ok {
return true, nil
}
case <-subscriber:
return true, nil
}
}