Using Goroutine and channel to realize Master-worker mode

This is a creation in Article, where the information may have evolved or changed.

What is Master-worker mode?

If you have done Java server, then you must be not unfamiliar with Master-worker mode, it is a parallel mode.

The primary function of the Master-worker mode is to schedule the master task to be computed on multiple workers, and then return the results to master for consolidation. The goal is to reduce the stress of a process on task handling in master.

Master-worker Mode Flowchart:

The graph originates from the network

Master-worker Detailed Flowchart:

The graph originates from the network

Golang Implementing Master-worker Mode

Worker

Package worker

Import (

"FMT"

)

Encapsulates the data structures that need to be processed

Type Job struct {

Num INT

}

Func newjob (num int) Job {

Return Job{num:num}

}

Type Worker struct {

ID int//workerid

Workerpool Chan Chan Job//worker Pool

Jobchannel Chan job//worker get job from Jobchannel for processing

Result Map[interface{}]int//worker The processing result into reuslt

Quit Chan bool//STOP worker signal

}

Func Newworker (Workerpool Chan Chan Job, result map[interface{}]int, id int) Worker {

Return worker{

Id:id,

Workerpool:workerpool,

Jobchannel:make (Chan Job),

Result:result,

Quit:make (chan bool),

}

}

Func (w Worker) Start () {

Go func () {

for {

Put the worker's Jobchannel in the master's Workerpool

W.workerpool <-W.jobchannel

Select {

Get the job from Jobchannel, Jobchannel is a synchronous channel that blocks this

Case Job: = <-w.jobchannel:

Handle this Job

and deposit the resulting results into the result set in master

x: = Job.num * Job.num

Fmt. Println (W.id, ":", X)

W.RESULT[X] = w.id

Stop signal

Case <-w.quit:

Return

}

}

}()

}

Func (w Worker) Stop () {

Go func () {

W.quit <-True

}()

}

Master

Package Master

Import (

"Masterworkerpattern/worker"

)

Type Master struct {

Workerpool Chan Chan worker. Job//worker Pool

Result Map[interface{}]int//store worker processed results set

Jobqueue Chan worker. Job//Pending Tasks Chan

Workerlist []worker. Worker//Store worker list to stop worker

}

var maxworker int

Maxworkers: Number of open threads

Result: Results set

Func newmaster (maxworkers int, result map[interface{}]int) *master {

Pool: = Make (Chan chan worker. Job, Maxworkers)

Maxworker = Maxworkers

Return &master{workerpool:pool, Result:result, Jobqueue:make (Chan worker. Job, 2*maxworkers)}

}

Func (M *master) Run () {

Start all the worker

For I: = 0; i < Maxworker; i++ {

Work: = worker. Newworker (M.workerpool, M.result, i)

M.workerlist = Append (m.workerlist, work)

Work. Start ()

}

Go M.dispatch ()

}

Func (M *master) dispatch () {

for {

Select {

Case Job: = <-m.jobqueue:

Go func (Job worker. JOB) {

Remove a worker's jobchannel from a Workerpool

Jobchannel: = <-m.workerpool

The receive pairing operation in sending Job,worker to this jobchannel will be awakened

Jobchannel <-Job

} (Job)

}

}

}

Add task to Task channel

Func (M *master) addjob (num int) {

Job: = worker. Newjob (num)

Send a task to a task channel

M.jobqueue <-Job

}

Stop All Tasks

Func (M *master) Stop () {

For _, V: = Range M.workerlist {

V.stop ()

}

}

Test

Masterworkerpattern Project Main.go

Package Main

Import (

"Masterworkerpattern/master"

"FMT"

"Time"

)

Func Main () {

Result: = map[interface{}]int{}

Mas: = Master. Newmaster (4, result)

Mas. Run ()

For I: = 0; I < 10; i++ {

Mas. AddJob (i)

}

Time. Sleep (Time.millisecond)

Mas. Stop ()

Fmt. Println ("result=", result)

}

Run results

0:81

3:0

0:4

0:36

0:25

0:49

0:64

1:1

3:9

2:16

result= map[81:0 36:0 64:0 1:1 0:3 4:0 25:0 49:0 9:3 16:2]

The result of this operation is indeterminate, with a line of code in WORKER.GO:

X:=job.num*job.num

FMT. Println (W.id, ":", X), which prints the square of Num in the Workerid and job, can be seen from the final test results that the worker of Id=0 calculates a number of jobs, while the others are very few, and this has no effect, and the execution of multiple times can be entirely different results. Also, how many workers can I open to work for master? If you are 2 cores, suggest 2-4, if you are 4 cores, suggest 4-8, i.e. I-2*i, try to squeeze the CPU.