Twitter's ID generator's snowflake algorithm's self-made Golang version

The snowflake algorithm is particularly efficient when generating IDs, for reference to: 1190000011282426

The result of the snowflake algorithm generation ID is an integer of 64bit size, which is structured as:

• 1位No. The highest 1 of the binary is negative, but the IDs we generate are usually integers, so this is 0.

• 41位, which is used to record the timestamp (in milliseconds).

  • 41-bit can represent < Span id= "mathjax-span-4" class= "mn" >2 41 −1 number,
  • If it is used only to represent positive integers (a positive number in the computer contains 0), the range of values that can be represented is: 0 to 2-1, minus 1 because the range of values that can be represented is calculated from 0, Rather than 1.
  • This means that 41 bits can represent2–1 The number of milliseconds to convert into a unit year is(21)/(∗∗∗∗365)= theYears

• 10位To record the work machine ID.

  • Can be deployed at 2 x=1024x768 nodes, including 5位datacenterId and5位workerId
  • 5-bit (bit) The maximum positive integer that can be represented is 25−1=31 , which means you can use 0, 1, 2, 3 、.... 31 these 32 numbers to represent a different datecenterid or Workerid

• 12位, serial number, used to record the different IDs produced in the same millisecond.

  • 12位（bit）The maximum positive integer that can be represented is 2−1=4096, which can be 0, 1, 2, 3 、.... 4095 of these 4,096 numbers to represent the 4,096 ID numbers generated in the same machine in the same time intercept (milliseconds)

Snowflake can guarantee:

• All generated IDs are incremented by time trend
• Duplicate IDs are not generated in the entire distributed system (because there are datacenterid and Workerid to differentiate)

But in the shadow of a datacenterid, workerid do not need to occupy so many bits, or the machine is not so much. I wrote a bit of the individual fields can be customized settings.

Https://github.com/liuyongshuai/tofuutils/blob/master/snowflake.go

/** * @author Liu yongshuai<[email protected]> * @package tofuutils * @date 2018-01-25 19:19 * * Package Tofuutilsimport ("Sync" "FMT" "Time")/** See Test cases: Structure type of Go Test-test.run testnewidgenerator*///snowflake snowflakeidgenerator struct {workerid Int64//Current workeridworkeridaftershift Int64//Shift Workerid, can be directly followed by timestamp, ordinal take or operation Lastmstimestamp Int64//Last timestamp cursequence Int64//Current ordinal timebitsize uint8//timestamp accounted for the number of bits, default is 41 bits, maximum not more than 60 bits Workerid Bitsize uint8//workerid occupies the number of digits, default 10, maximum no more than 60 bits sequencebitsize uint8//serial number occupies the number of digits, default 12, maximum not more than 60 bit lock *sync. Mutex//Synchronous ishaveinit BOOL//whether the maximum value of Maxworkerid Int64//workerid has been initialized, calculated maxsequence Int64//at initialization After the maximum value of the serial number, the initialization of the calculated Workeridleftshift Uint8//generated Workerid only take the lowest few, here to move left, give the serial number, the initialization of the calculated Timestampleftshift Uint8// The resulting timestamp is shifted left several, workid, the sequence number, the initialization of the computed}//instantiation of an ID generator func newidgenerator () *snowflakeidgenerator {return &     Snowflakeidgenerator{workerid:0,lastmstimestamp:0,cursequence:   0,TIMEBITSIZE:41,//default timestamp occupies the number of bits workeridbitsize:10,//default Workerid occupies the number of bits sequencebitsize:12,//default number of digits accounted for Maxwo rkerid:0,//MAX Workerid, computed at initialization maxsequence:0,//maximum ordinal value, initialized when calculated workeridleftshift:0,//worker ID left shift Number of digits timestampleftshift:0,lock:new (sync. Mutex), ishaveinit:false,}}//set worker Idfunc (SFG *snowflakeidgenerator) Setworkerid (w Int64) *snowflakeidgenerato R {sfg.lock.Lock () defer sfg.lock.Unlock () Sfg.ishaveinit = Falsesfg.workerid = Wreturn sfg}//Set the number of bits for the timestamp func (SFG * Snowflakeidgenerator) settimebitsize (n uint8) *snowflakeidgenerator {sfg.lock.Lock () defer sfg.lock.Unlock () Sfg.ishaveinit = Falsesfg.timebitsize = Nreturn sfg}//Set the number of digits of the worker ID func (SFG *snowflakeidgenerator) Setworkeridbitsize (n uint8) *snowflakeidgenerator {sfg.lock.Lock () defer sfg.lock.Unlock () Sfg.ishaveinit = Falsesfg.workeridbitsize = Nreturn sfg}//Set number of digits in the number of bits func (SFG *snowflakeidgenerator) setsequencebitsize (n uint8) * Snowflakeidgenerator {Sfg.lock.Lock () deferSfg.lock.Unlock () Sfg.ishaveinit = Falsesfg.sequencebitsize = Nreturn sfg}//initialization operation func (SFG *snowflakeidgenerator) Init ( ) (*snowflakeidgenerator, error) {Sfg.lock.Lock () defer sfg.lock.Unlock ()//if the sfg.ishaveinit {return SFG, nil} is already initialized If sfg.sequencebitsize < 1 | | Sfg.sequencebitsize > {return nil, fmt. Errorf ("Init failed:\tinvalid sequence bit size, should (1,60)")}if sfg.timebitsize < 1 | | Sfg.timebitsize > {return nil, fmt. Errorf ("Init failed:\tinvalid time bit size, should (1,60)")}if sfg.workeridbitsize < 1 | | Sfg.workeridbitsize > {return nil, fmt. Errorf ("Init failed:\tinvalid worker ID bit size, should (1,60)")}if sfg.workeridbitsize+sfg.sequencebitsize+ Sfg.timebitsize! = + + return nil, FMT. Errorf ("Init failed:\tinvalid sum of all bit size, should EQ 63")}//determines the number of shifts sfg.workeridleftshift = sfg.sequencebitsizesfg.t Imestampleftshift = sfg.sequencebitsize + sfg.workeridbitsize//determine serial number and workerid maximum Sfg.maxworkerid =-1 ^ ( -1 << Sfg.workeridbitsize) Sfg.maxsEquence = 1 ^ ( -1 << sfg.sequencebitsize)//post-shift Workerid, return results can be directly with the timestamp, sequence number or operation can be sfg.workeridaftershift = Sfg.workerid << sfg.workeridleftshift//Determine if the current Workerid is legal if Sfg.workerid > Sfg.maxworkerid {return nil, fmt. Errorf ("Init failed:\tinvalid worker ID, should not greater than%d", Sfg.maxworkerid)}//initialization complete sfg.ishaveinit = truesfg.la Stmstimestamp = 0sfg.cursequence = 0return SFG, nil}//generate a timestamp, according to the bit size set up several//that is, the resulting timestamp first shifted right several, and then left a few, the highest specified number of bits is retained func (SFG *snowflakeidgenerator) Gents () Int64 {rawts: = time. Now (). Unixnano () diff: = 64-sfg.timebitsizeret: = (rawts >> diff) << Diffreturn ret}//generates the next timestamp, if the number of bits in the timestamp is small and the sequence number is exhausted here The wait time will be longer func (SFG *snowflakeidgenerator) Gennextts (last Int64) int64 {for {cur: = sfg.gents () If cur > last {return cur} }}//generates the next Idfunc (SFG *snowflakeidgenerator) NextID () (Int64, error) {Sfg.lock.Lock () defer sfg.lock.Unlock ()// If you have not initialized the IF!sfg.ishaveinit {return 0, FMT. Errorf ("Gen nextid failed:\tplease Execute Init () First")}//determine the current timestamp, if it is smaller than the previous one, indicatingSomething went wrong curts: = Sfg.gents () if Curts < Sfg.lastmstimestamp {return 0, FMT. Errorf ("Gen nextid failed:\tunknown error, the system clock occur some wrong")}//if the timestamp is the same as the last time stamp, increase the ordinal if curts = = Sfg.lastmsti Mestamp {sfg.cursequence = (sfg.cursequence + 1) & sfg.maxsequence//serial number 0 is run out, regenerate timestamp if sfg.cursequence = = 0 {curts = Sfg.gennextts (Sfg.lastmstimestamp)}} else {//If two timestamps are different, then 0 sequence number sfg.cursequence = 0}sfg.lastmstimestamp = curts// Assemble the processed bits into a int64 type Curts = Curts | Sfg.workeridaftershift | Sfg.cursequencereturn Curts, nil}//parse generated idfunc (SFG *snowflakeidgenerator) parse (ID Int64) (Int64, Int64, int64, error) { If not yet initialized if!sfg.ishaveinit {return 0, 0, 0, FMT. Errorf ("Parse failed:\tplease execute Init () First")}//extracts the timestamp section shift: = Sfg.sequencebitsize + Sfg.sequencebitsizetimestamp: = (ID & ( -1 << shift)) >> shift//extract Workerid section shift = Sfg.sequencebitsizeworkerid: = (ID & (Sfg.maxworkerid << shift)) >> shift//ordinal part sequence: = ID & sfg.ma xsequence//Parse Error if workeriD! = Sfg.workerid | | Workerid > Sfg.maxworkerid {fmt. Printf ("workerbitsize=%d\tmaxworkerid=%d\n", Sfg.workeridbitsize, Sfg.maxworkerid) return 0, 0, 0, FMT. Errorf ("Parse failed:invalid ID, originworkerid=%d\tparseworkerid=%d\n", Sfg.workerid, Workerid)}if sequence < 0 | | Sequence > Sfg.maxsequence {fmt. Printf ("sequesncebitsize=%d\tmaxsequence=%d\n", Sfg.sequencebitsize, Sfg.maxsequence) return 0, 0, 0, FMT. Errorf ("Parse failed:invalid ID, parsesequence=%d\n", sequence)}return timestamp, workerid, sequence, nil}

Test code: Https://github.com/liuyongshuai/tofuutils/blob/master/snowflake_test.go

About a continuous generation of the 1亿三千多万个 ID is written to the file, and no duplicates are found for the time being.

Package Tofuutilsimport ("Testing" "FMT" "Time" "OS") func Testnewidgenerator (t *testing. T) {b: = "\t\t\t" B2: = "\t\t\t\t\t" D: = "====================================="//First generator Gentor1, err: = Newidgenerator (). Setworkerid (100). Init () if err! = Nil {fmt. PRINTLN (Err) t.error (ERR)}//Second generator Gentor2, err: = Newidgenerator (). Settimebitsize (48). Setsequencebitsize (10). Setworkeridbitsize (5). Setworkerid (30). Init () if err! = Nil {fmt. PRINTLN (Err) t.error (err)}fmt. Printf ("%s%s%s\n", D, B, D) fmt. Printf ("workerid=%d lasttimestamp=%d%s workerid=%d lasttimestamp=%d\n", Gentor1.workerid, Gentor1.lastmstimestamp, b , Gentor2.workerid, Gentor2.lastmstimestamp) fmt. Printf ("sequencebitsize=%d timebitsize=%d%s sequencebitsize=%d timebitsize=%d\n", Gentor1.sequencebitsize, Gentor1.timebitsize, B,gentor2.sequencebitsize, gentor2.timebitsize) fmt. Printf ("workerbitsize=%d sequencebitsize=%d%s workerbitsize=%d sequencebitsize=%d\n", Gentor1.workeridbitsize, Gentor1.sequencebitsize, B,gentor2.workeridbitsize, GENTOR2.SEQUENCEBItsize) fmt. Printf ("%s%s%s\n", D, B, D) var IDs []int64for I: = 0; I < 100; i++ {id1, err: = Gentor1. NextID () if err! = Nil {fmt. PRINTLN (Err) Return}id2, err: = Gentor2. NextID () if err! = Nil {fmt. PRINTLN (err) return}ids = append (IDs, Id2) fmt. Printf ("%d%s%d\n", Id1, B2, Id2)}//Parse idfor _, id: = range ids {ts, workerid, seq, err: = Gentor2. Parse (ID) fmt. Printf ("id=%d\ttimestamp=%d\tworkerid=%d\tsequence=%d\terr=%v\n", ID, ts, workerid, seq, err)}}//multithreaded test func Testsnowflakeidgenerator_multithread (t *testing. T) {f: = "./snowflake.txt"//ready to write the file fp, err: = OS. OpenFile (f, OS. O_wronly|os. O_append|os. O_create, 0755) if err! = Nil {fmt. PRINTLN (Err) t.error (ERR)}//initializes the ID generator with default parameters Gentor, err: = Newidgenerator (). Setworkerid (100). Init () if err! = Nil {fmt. PRINTLN (Err) t.error (err)}//start 10 threads, the error is reported for I: = 0; I < 10; i++ {go func () {for {gid, err: = Gentor. NextID () if err! = Nil {panic (err)}n, err: = FP. WriteString (FMT. Sprintf ("%d\n", GID)) if err! = Nil | | N <= 0 {panic (err)}}} ()}time. Sleep (Ten * time. Second)//time. Sleep (* time. Second)}

Twitter's ID generator's snowflake algorithm's self-made Golang version