In a word, compared with the ordinary hashing algorithm, the consistent hashing algorithm has some fault tolerance and expansibility for the dynamic deletion of nodes.
/**
* MurMurHash algorithm is a non-encrypted HASH algorithm with high performance,
* Compared with the traditional CRC32, MD5, SHA-1 (both algorithms are encrypted HASH algorithm, the complexity itself is very high, the performance damage caused is also inevitable)
* Wait for HASH algorithm to be much faster, and it is said that the collision rate of this algorithm is very low
* http://murmurhash.googlepages.com/
*/
private Long hash(String key) {
ByteBuffer buf = ByteBuffer.wrap(key.getBytes());
int seed = 0x1234ABCD;
ByteOrder byteOrder = buf.order();
buf.order(ByteOrder.LITTLE_ENDIAN);
long m = 0xc6a4a7935bd1e995L;
int r = 47;
long h = seed ^ (buf.remaining() * m);
long k;
while (buf.remaining() >= 8) {
k = buf.getLong();
k *= m;
k ^= k >>> r;
k *= m;
h ^= k;
h *= m;
}
if (buf.remaining()> 0) {
ByteBuffer finish = ByteBuffer.allocate(8).order(
ByteOrder.LITTLE_ENDIAN);
// for big-endian version, do this first:
// finish.position(8-buf.remaining());
finish.put(buf).rewind();
h ^= finish.getLong();
h *= m;
}
h ^= h >>> r;
h *= m;
h ^= h >>> r;
buf.order(byteOrder);
return h;
}
package redis.cn;
import java.nio.charset.Charset;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
import com.google.common.hash.HashFunction;
import com.google.common.hash.Hashing;
public class ConsistentHash {
// ------------------ java implementation of consistent hashing algorithm ------------------
private SortedMap<Long,String> ketamaNodes = new TreeMap<Long,String>();
private int numberOfReplicas = 1024;
// Google's jar package is used here - guava-18.0.jar
private HashFunction hashFunction = Hashing.md5();
private List<String> nodes;
private volatile boolean init = false; //Whether the flag is initialized
// parameterized constructor
public ConsistentHash(int numberOfReplicas,List<String> nodes){
this.numberOfReplicas = numberOfReplicas;
this.nodes = nodes;
init();
}
// According to the hash value of the key, find the nearest node (server)
public String getNodeByKey(String key){
if(!init)
throw new RuntimeException("init uncomplete...");
// Note that here is the NIO package java.nio.charset.Charset
byte[] digest = hashFunction.hashString(key, Charset.forName("UTF-8")).asBytes();
long hash = hash(digest,0);
//If this node is found, take the node directly and return
if(!ketamaNodes.containsKey(hash)){
//Get the sub-Map larger than the current key, and then take the first key from it, which is the key larger than and closest to it
SortedMap<Long,String> tailMap = ketamaNodes.tailMap(hash);
if(tailMap.isEmpty()){
hash = ketamaNodes.firstKey();
}else{
hash = tailMap.firstKey();
}
}
return ketamaNodes.get(hash);
}
// Add node
public synchronized void addNode(String node){
init = false;
nodes.add(node);
init();
}
private void init(){
//For all nodes, generate numberOfReplicas virtual nodes
for(String node:nodes){
//Every four virtual nodes are 1 group
for(int i=0;i<numberOfReplicas/4;i++){
//Get a unique name for this group of virtual nodes
byte[] digest = hashFunction.hashString(node+i, Charset.forName("UTF-8")).asBytes();
//Md5 is a 16-byte array, and every 16 bytes of the 16-byte array corresponds to a virtual node, which is why the four virtual nodes are divided into a group
for(int h=0;h<4;h++){
Long k = hash(digest,h);
ketamaNodes.put(k,node);
}
}
}
init = true;
}
public void printNodes(){
for(Long key:ketamaNodes.keySet()){
System.out.println(ketamaNodes.get(key));
}
}
// Hash algorithm
public static long hash(byte[] digest, int nTime)
{
long rv = ((long)(digest[3 + nTime * 4] & 0xFF) << 24)
| ((long)(digest[2 + nTime * 4] & 0xFF) << 16)
| ((long)(digest[1 + nTime * 4] & 0xFF) << 8)
| ((long)digest[0 + nTime * 4] & 0xFF);
return rv;
}
}
//Small amount of optimized performance
public ThreadLocal<MessageDigest> md5Holder = new ThreadLocal<MessageDigest>();
public static final Hashing MD5 = new Hashing() {
public long hash(String key) {
return hash(SafeEncoder.encode(key));
}
// The hash algorithm used by sharding is MD5
public long hash(byte[] key) {
try {
if (md5Holder.get() == null) {
md5Holder.set(MessageDigest.getInstance("MD5"));
}
}
catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("++++ no md5 algorythm found");
}
MessageDigest md5 = md5Holder.get();
md5.reset();
md5.update(key);
//Get MD5 byte sequence
byte[] bKey = md5.digest();
//The first four bytes are used as calculation parameters, and finally a 32-bit int value is obtained.
//This calculation method can ensure that the hash value of the key is more "random"/"discrete"
//If the hash value is too dense, it is not conducive to the realization of consistent hashing (especially when there is a "virtual node" design)
long res = ((long) (bKey[3] & 0xFF) << 24)
| ((long) (bKey[2] & 0xFF) << 16)
| ((long) (bKey[1] & 0xFF) << 8)
| (long) (bKey[0] & 0xFF);
return res;
}
};
//The shards list provides all redis-server configuration information for the client, including: ip, port, weight, name
//Where weight is the weight, it will directly determine the "proportion" (density) of the "virtual node", the higher the weight, the higher the probability of being hit by the hash in storage
//--- the more data stored on it.
//Where name is "node name", jedis uses name as a calculation parameter of "node hash value".
//---
// Consistent hash algorithm requires that each "virtual node" must have a "hash value", and each actual server can have multiple "virtual nodes" (API level)
//The number of virtual nodes = "logical interval length" * weight, the "virtual nodes" of each server will be distributed in the global area in a "hash" way
//The total length of the global zone is 2^32. Each "virtual node" is mapped in the global zone by means of a hash value.
// Ring: 0-->vnode1(:1230)-->vnode2(:2800)-->vnode3(400000)---2^32-->0
//All "virtual nodes" will be arranged in the order of their "node hash" (both positive and reverse order), so there must be a difference in hash value between two adjacent "virtual nodes",
//Then this difference is the data hash value range loaded by the previous (or the latter, depending on the implementation) "virtual node".
//For example, data whose hash value is "2000" will be accepted by vnode1.
private void initialize(List<S> shards){
//Virtual node, adopt TreeMap storage: sort, binary tree
nodes = new TreeMap<Long, S>();
for (int i = 0; i != shards.size(); ++i) {
final S shardInfo = shards.get(i);
if (shardInfo.getName() == null)
//When "name" is not set, use "SHARD-NODE" as the parameter of the "virtual node" hash value calculation
//"Logical interval step" is 160, why?
//Finally, the "virtual nodes" of multiple servers will be staggered, not necessarily very uniform.
for (int n = 0; n <160 * shardInfo.getWeight(); n++) {
nodes.put(this.algo.hash("SHARD-" + i + "-NODE-" + n), shardInfo);
}
else
for (int n = 0; n <160 * shardInfo.getWeight(); n++) {
nodes.put(this.algo.hash(shardInfo.getName() + "*" + shardInfo.getWeight() + n), shardInfo);
}
resources.put(shardInfo, shardInfo.createResource());
}
}
Public R Getshard (String key) {
return Resources.get (Getshardinfo (key));
}
Public S Getshardinfo (byte[] key) {
//Get the list of "virtual nodes" for >=key
sortedmap<long, s> tail = Nodes.tailmap ( Algo.hash (key));
If the virtual node does not exist, the first node is returned.
if (tail.size () = = 0) {
return Nodes.get (Nodes.firstkey ());
}
If present, returns the first node of the "virtual node" that meets the (>=key) condition
return Tail.get (Tail.firstkey ());
}
package redis.cn;
import java.util.ArrayList;
import java.util.List;
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
import redis.clients.jedis.JedisShardInfo;
import redis.clients.jedis.ShardedJedis;
import redis.clients.jedis.ShardedJedisPool;
/**
* @author yangcq
* @category jedis is also an implementation of the consistent hash algorithm. To build a redis distributed cluster, you can use jedis.
*/
public class ShardedRedis {
// In addition to the toolkit that comes with jdk, you need to import the following 2 jar packages
// commons-pool2-2.0.jar
// jedis-2.4.2.jar
The
public static void main(String[] args){
// jedis configuration parameters
GenericObjectPoolConfig genericObjectPoolConfig = new GenericObjectPoolConfig();
genericObjectPoolConfig.setMaxTotal(1000);
genericObjectPoolConfig.setMaxIdle(500);
List<JedisShardInfo> jedisShardInfoList = new ArrayList<JedisShardInfo>();
JedisShardInfo jedisShardInfo1 = new JedisShardInfo("127.0.0.1",1234);
JedisShardInfo jedisShardInfo2 = new JedisShardInfo("127.0.0.1",1235);
JedisShardInfo jedisShardInfo3 = new JedisShardInfo("127.0.0.1",1236);
jedisShardInfoList.add(jedisShardInfo1);
jedisShardInfoList.add(jedisShardInfo2);
jedisShardInfoList.add(jedisShardInfo3);
ShardedJedisPool shardedJedisPool = new ShardedJedisPool(genericObjectPoolConfig,jedisShardInfoList);
set("key1","value1",shardedJedisPool);
set("key2","value2",shardedJedisPool);
set("key3","value3",shardedJedisPool);
set("key4","value4",shardedJedisPool);
set("key5","value5",shardedJedisPool);
// jedis hides the details of implementing a consistent hash algorithm, but only provides us with a simple interface call to implement the construction of a redis distributed cluster
// So how does jedis implement a consistent hash algorithm?
}
public static void set(String key, String value, ShardedJedisPool pool){
// Get the redis instance from the shared resource pool
ShardedJedis shardedJedis = pool.getResource();
// Assignment
shardedJedis.set(key,value);
pool.returnResource(shardedJedis);
}
}
public Sharded(List<S> shards, Hashing algo, Pattern tagPattern) {
this.algo = algo;
this.tagPattern = tagPattern;
initialize(shards);
}
//Initialize the hash ring
private void initialize(List<S> shards) {
nodes = new TreeMap<Long, S>();
for (int i = 0; i != shards.size(); ++i) {
final S shardInfo = shards.get(i);
if (shardInfo.getName() == null)
for (int n = 0; n <160 * shardInfo.getWeight(); n++) {
nodes.put(this.algo.hash("SHARD-" + i + "-NODE-" + n),
shardInfo);
}
else
for (int n = 0; n <160 * shardInfo.getWeight(); n++) {
nodes.put(
this.algo.hash(shardInfo.getName() + "*"
+ shardInfo.getWeight() + n), shardInfo);
}
resources.put(shardInfo, shardInfo.createResource());
}
}
//Store key and value to the corresponding shard
public String set(String key, String value) {
Jedis j = getShard(key);
return j.set(key, value);
}
public R getShard(String key) {
return resources.get(getShardInfo(key));
}
//Acquire shard according to key
public S getShardInfo(byte[] key) {
SortedMap<Long, S> tail = nodes.tailMap(algo.hash(key));
if (tail.isEmpty()) {
return nodes.get(nodes.firstKey());
}
return tail.get(tail.firstKey());
}
Start building with 50+ products and up to 12 months usage for Elastic Compute Service
1 on 1 presale consultation
24/7 Technical Support 6 Free Tickets per Quarter Faster Response
Alibaba Cloud offers highly flexible support services tailored to meet your exact needs.
A comprehensive suite of global cloud computing services to power your business
Payment Methods We Support
