Key-value Cache There are many, with more memcache, Redis, they are operating in the form of independent services, sometimes in the work of the need to embed a local Key-value cache, of course, there are leveldb, but the feeling is still too heavyweight.
This article implements a super lightweight cache,
1, the implementation of the code requires only 400 lines;
2, high performance, value length at 1K test speed at about 2 million per second
3, the cache is mapped to the file, so there is no malloc, free overhead, as well as the memory leak, memory fragments, etc.
4, if the service is dead, the cache content continues to exist after the reboot;
5, if the cache map to disk files even if the machine hangs, the contents of the cache will still exist, of course, there may be data corruption;
6, to a certain extent, the implementation of the LRU elimination algorithm, the realization of the LRU is not the global only a chain, so can only say that in a certain procedure to achieve;
7, stable, has been used in a number of projects, online deployment of the machine has dozens of units, running a large six months has not been a problem;
8, the general cache key, value is the form of strings, this cache key, value can be class, struct object structure more convenient;
The old rules go directly to code:
Copy Code code as follows:
Template<typename K, TypeName v>
Class HashTable
{
Public
HashTable (const char *tablename, uint32_t Tablelen, uint32_t nodetotal);
Virtual ~hashtable ();
BOOL Add (K &key, V &value)
{
Autolock Autolock (M_mutexlock);
Check is exist
uint32_t nodeId = Getidbykey (key);
if (nodeId!= m_invalidid) return false;
NodeId = Getfreenode ();
if (nodeId = = M_invalidid) return false;
uint32_t hashcode = key. Hashcode ();
Entry *tmpnode = m_entryaddr + nodeId;
Tmpnode->m_key = Key;
Tmpnode->m_code = hashcode;
Tmpnode->m_value = Value;
uint32_t index = hashcode% m_headaddr->m_tablelen;
Addnodetohead (index, NODEID);
return true;
}
BOOL Del (K &key)
{
Autolock Autolock (M_mutexlock);
uint32_t nodeId = Getidbykey (key);
if (nodeId = = M_invalidid) return false;
uint32_t index = key. Hashcode ()% m_headaddr->m_tablelen;
Return Recyclenode (index, NODEID);
}
BOOL Set (K &key, V &value)
{
Autolock Autolock (M_mutexlock);
uint32_t nodeId = Getidbykey (key);
if (nodeId = = M_invalidid) return false;
(M_entryaddr + nodeId)->m_value = Value;
return true;
}
BOOL Get (K &key, V &value)
{
Autolock Autolock (M_mutexlock);
uint32_t nodeId = Getidbykey (key);
if (nodeId = = M_invalidid) return false;
Value = (m_entryaddr + nodeId)->m_value;
return true;
}
BOOL Exist (K &key)
{
Autolock Autolock (M_mutexlock);
uint32_t nodeId = Getidbykey (key);
if (nodeId = = M_invalidid) return false;
return true;
}
uint32_t Count ()
{
Autolock Autolock (M_mutexlock);
Return m_headaddr->m_usedcount;
}
If exist set else add
BOOL Replace (K &key, V &value)
{
Autolock Autolock (M_mutexlock);
if (Exist (key)) return Set (key, value);
else return ADD (key, value);
}
/***********************************************
Lru:when visit a node, move it to head * * *
************************************************/
If no empty place,recycle tail
BOOL Lruadd (k &key, v &value, K &recykey, v &recyvalue, BOOL &recycled)
{
Autolock Autolock (M_mutexlock);
if (Exist (key)) return false;
if (ADD (key, value)) return true;
uint32_t index = key. Hashcode ()% m_headaddr->m_tablelen;
uint32_t tailid = Gettailnodeid (index);
if (Tailid = = M_invalidid) return false;
Entry *tmpnode = m_entryaddr + tailid;
Recykey = tmpnode->m_key;
Recyvalue = tmpnode->m_value;
Recycled = true;
Recyclenode (index, tailid);
Return Add (key, value);
}
BOOL Lruset (K &key, V &value)
{
Autolock Autolock (M_mutexlock);
if (Set (key, value)) return Movetohead (key);
else return false;
}
BOOL Lruget (K &key, V &value)
{
Autolock Autolock (M_mutexlock);
if (key, value) return Movetohead (key);
else return false;
}
If exist set else add; If add failed recycle tail than add
BOOL Lrureplace (k &key, v &value, K &recykey, v &recyvalue, BOOL &recycled)
{
Autolock Autolock (M_mutexlock);
Recycled = false;
if (Exist (key)) return Lruset (key, value);
else return Lruadd (key, value, Recykey, recyvalue, recycled);
}
void Clear ()
{
Autolock Autolock (M_mutexlock);
m_headaddr->m_freebase = 0;
M_headaddr->m_recyclehead = 0;
M_headaddr->m_usedcount = 0;
for (uint32_t i = 0; i < m_headaddr->m_tablelen; ++i)
{
(m_arrayaddr+i)->m_head = M_invalidid;
(m_arrayaddr+i)->m_tail = M_invalidid;
}
}
int Getrowkeys (vector<k> &keys, uint32_t index)
{
Autolock Autolock (M_mutexlock);
if (index >= m_headaddr->m_tablelen) return-1;
Keys.clear ();
Keys.reserve (16);
int count = 0;
Array *tmparray = m_arrayaddr + index;
uint32_t nodeId = tmparray->m_head;
while (NodeId!= m_invalidid)
{
Entry *tmpnode = m_entryaddr + nodeId;
Keys.push_back (Tmpnode->m_key);
NodeId = tmpnode->m_next;
++count;
}
return count;
}
void *padding (uint32_t size)
{
Autolock Autolock (M_mutexlock);
if (Size > m_headsize-sizeof (tablehead)) return NULL;
else return m_headaddr->m_padding;
}
Private
static const uint32_t M_invalidid = 0xFFFFFFFF;
static const uint32_t m_headsize = 1024;
struct Tablehead
{
uint32_t M_tablelen;
uint32_t m_nodetotal;
uint32_t m_freebase;
uint32_t M_recyclehead;
uint32_t M_usedcount;
Char m_tablename[256];
uint32_t M_padding[0];
};
struct Array
{
uint32_t M_head;
uint32_t M_tail;
};
struct Entry
{
V m_value;
K M_key;
uint32_t M_code;
uint32_t M_next;
uint32_t M_prev;
};
size_t M_memsize;
uint8_t *m_memaddr;
Tablehead *m_headaddr;
Array *m_arrayaddr;
Entry *m_entryaddr;
Threadmutex M_mutexlock;
BOOL Movetohead (K &key);
uint32_t Getidbykey (K &key);
void Addnodetohead (uint32_t index, uint32_t nodeId);
BOOL Movenodetohead (uint32_t index, uint32_t nodeId);
BOOL Recyclenode (uint32_t index, uint32_t nodeId);
uint32_t Gettailnodeid (uint32_t index);
uint32_t Getfreenode ();
Disable_copy_and_assign (HashTable);
};
Template<typename K, TypeName v>
Hashtable<k, v>::hashtable (const char *tablename, uint32_t Tablelen, uint32_t nodetotal)
{
Abortassert (tablename!= NULL);
M_memsize = m_headsize + tablelen*sizeof (Array) + nodetotal*sizeof (Entry);
M_memaddr = (uint8_t*) memfile::realloc (tablename, m_memsize);
Abortassert (m_memaddr!= NULL);
M_headaddr = (tablehead*) (M_MEMADDR);
M_arrayaddr = (array*) (m_memaddr + m_headsize);
M_entryaddr = (entry*) (m_memaddr + m_headsize + tablelen*sizeof (Array));
M_headaddr->m_tablelen = Tablelen;
M_headaddr->m_nodetotal = Nodetotal;
strncpy (M_headaddr->m_tablename, tablename, sizeof (m_headaddr->m_tablename));
if (M_headaddr->m_usedcount = = 0)//if the ' I ' init array to invalid ID
{
for (uint32_t i = 0; i < Tablelen; ++i)
{
(m_arrayaddr+i)->m_head = M_invalidid;
(m_arrayaddr+i)->m_tail = M_invalidid;
}
m_headaddr->m_freebase = 0;
M_headaddr->m_recyclehead = 0;
}
}
Template<typename K, TypeName v>
Hashtable<k, V>::~hashtable ()
{
Memfile::release (M_MEMADDR, m_memsize);
}
Template<typename K, TypeName v>
BOOL Hashtable<k, V>::movetohead (K &key)
{
uint32_t nodeId = Getidbykey (key);
uint32_t index = key. Hashcode ()% m_headaddr->m_tablelen;
Return Movenodetohead (index, NODEID);
}
Template<typename K, TypeName v>
uint32_t hashtable<k, V>::getidbykey (K &key)
{
uint32_t hashcode = key. Hashcode ();
uint32_t index = hashcode% m_headaddr->m_tablelen;
Array *tmparray = m_arrayaddr + index;
uint32_t nodeId = tmparray->m_head;
while (NodeId!= m_invalidid)
{
Entry *tmpnode = m_entryaddr + nodeId;
if (Tmpnode->m_code = = Hashcode && key. Equals (Tmpnode->m_key)) break;
NodeId = tmpnode->m_next;
}
return nodeId;
}
Template<typename K, TypeName v>
void Hashtable<k, V>::addnodetohead (uint32_t index, uint32_t nodeId)
{
if (Index >= M_headaddr->m_tablelen | | | nodeId >= m_headaddr->m_nodetotal) return;
Array *tmparray = m_arrayaddr + index;
Entry *tmpnode = m_entryaddr + nodeId;
if (M_invalidid = = Tmparray->m_head)
{
Tmparray->m_head = nodeId;
Tmparray->m_tail = nodeId;
}
Else
{
Tmpnode->m_next = tmparray->m_head;
(M_entryaddr + tmparray->m_head)->m_prev = nodeId;
Tmparray->m_head = nodeId;
}
}
Template<typename K, TypeName v>
BOOL Hashtable<k, V>::movenodetohead (uint32_t index, uint32_t nodeId)
{
if (Index >= M_headaddr->m_tablelen | | | | nodeId >= m_headaddr->m_nodetotal) return false;
Array *tmparray = m_arrayaddr + index;
Entry *tmpnode = m_entryaddr + nodeId;
Already head
if (Tmparray->m_head = = nodeId)
{
return true;
}
uint32_t Nodeprev = tmpnode->m_prev;
uint32_t Nodenext = tmpnode->m_next;
(m_entryaddr+nodeprev)->m_next = Nodenext;
if (Nodenext!= m_invalidid)
{
(m_entryaddr+nodenext)->m_prev = Nodeprev;
}
Else
{
Tmparray->m_tail = Nodeprev;
}
Tmpnode->m_prev = M_invalidid;
Tmpnode->m_next = tmparray->m_head;
(M_entryaddr + tmparray->m_head)->m_prev = nodeId;
Tmparray->m_head = nodeId;
return true;
}
Template<typename K, TypeName v>
BOOL Hashtable<k, V>::recyclenode (uint32_t index, uint32_t nodeId)
{
if (Index >= M_headaddr->m_tablelen | | | | nodeId >= m_headaddr->m_nodetotal) return false;
Array *tmparray = m_arrayaddr + index;
Entry *tmpnode = m_entryaddr + nodeId;
uint32_t Nodeprev = tmpnode->m_prev;
uint32_t Nodenext = tmpnode->m_next;
if (Nodeprev!= m_invalidid)
{
(M_entryaddr + nodeprev)->m_next = Nodenext;
}
Else
{
Tmparray->m_head = Nodenext;
}
if (Nodenext!= m_invalidid)
{
(M_entryaddr + nodenext)->m_prev = Nodeprev;
}
Else
{
Tmparray->m_tail = Nodeprev;
}
(m_entryaddr+nodeid)->m_next = m_headaddr->m_recyclehead;
M_headaddr->m_recyclehead = nodeId;
--(M_headaddr->m_usedcount);
return true;
}
Template<typename K, TypeName v>
uint32_t hashtable<k, V>::gettailnodeid (uint32_t index)
{
if (index >= m_headaddr->m_tablelen) return m_invalidid;
Array *tmparray = m_arrayaddr + index;
Return tmparray->m_tail;
}
Template<typename K, TypeName v>
uint32_t hashtable<k, V>::getfreenode ()
{
uint32_t nodeId = M_invalidid;
if (M_headaddr->m_usedcount < m_headaddr->m_freebase)//get from recycle list
{
NodeId = m_headaddr->m_recyclehead;
M_headaddr->m_recyclehead = (M_entryaddr+nodeid)->m_next;
+ + (M_headaddr->m_usedcount);
}
else if (M_headaddr->m_usedcount < m_headaddr->m_nodetotal)//get from free mem
{
NodeId = m_headaddr->m_freebase;
+ + (m_headaddr->m_freebase);
+ + (M_headaddr->m_usedcount);
}
Else
{
NodeId = M_invalidid;
}
Init node
if (NodeId < m_headaddr->m_nodetotal)
{
Entry *tmpnode = m_entryaddr + nodeId;
memset (tmpnode, 0, sizeof (Entry));
Tmpnode->m_next = M_invalidid;
Tmpnode->m_prev = M_invalidid;
}
return nodeId;
}
