Mysql implementation of local keyvalue database cache example _ MySQL

Mysql implements the local keyvalue database cache example bitsCN.com

There are many Key-Value caches, most of which are memcache and redis. they all run in the form of independent services. at work, sometimes a local key-value cache needs to be embedded, of course there are already LevelDb and so on, but it still feels too heavyweight.

This article implements a super lightweight cache,

1. only 400 lines of code are required;

2. high performance. the testing speed is about 2 million per second when the value length is 1 K.

3. the cache is mapped to files, so there is no malloc or free overhead, as well as memory leakage and memory fragmentation;

4. if the service fails, the cached content continues to exist after restart;

5. if the cache is mapped to a disk file, even if the machine crashes, the cached content will still exist. of course, data corruption may occur;

6. to a certain extent, the LRU elimination algorithm is implemented. the LRU is implemented on a chain instead of a global one, so it can only be said to be implemented in a certain program;

7. it is stable and has been used in multiple projects. There are dozens of machines deployed online, and no problems have occurred after nearly half a year of operation;

8. common cache keys and values are strings. the cached keys and values can be class and struct object structures for ease of use;

The old rule goes directly to the code:

Template
Class HashTable
{
Public:
HashTable (const char * tablename, uint32_t tableLen, uint32_t nodeTotal );
Virtual ~ HashTable ();

Bool Add (K & key, V & value)
{
AutoLock autoLock (m_MutexLock );

// Check is exist
Uint32_t nodeId = GetIdByKey (key );
If (nodeId! = M_InvalidId) return false;

NodeId = GetFreeNode ();
If (nodeId = m_InvalidId) return false;

Uint32_t hashCode = key. HashCode ();
Entry * tmpNode = m_EntryAddr + nodeId;
TmpNode-> m_Key = key;
TmpNode-> m_Code = hashCode;
TmpNode-> m_Value = value;

Uint32_t index = hashCode % m_HeadAddr-> m_TableLen;
AddNodeToHead (index, nodeId );

Return true;
}

Bool Del (K & key)
{
AutoLock autoLock (m_MutexLock );

Uint32_t nodeId = GetIdByKey (key );
If (nodeId = m_InvalidId) return false;

Uint32_t index = key. HashCode () % m_HeadAddr-> m_TableLen;

Return RecycleNode (index, nodeId );
}

Bool Set (K & key, V & value)
{
AutoLock autoLock (m_MutexLock );

Uint32_t nodeId = GetIdByKey (key );
If (nodeId = m_InvalidId) return false;

(M_EntryAddr + nodeId)-> m_Value = value;

Return true;
}

Bool Get (K & key, V & value)
{
AutoLock autoLock (m_MutexLock );

Uint32_t nodeId = GetIdByKey (key );
If (nodeId = m_InvalidId) return false;

Value = (m_EntryAddr + nodeId)-> m_Value;

Return true;
}

Bool Exist (K & key)
{
AutoLock autoLock (m_MutexLock );

Uint32_t nodeId = GetIdByKey (key );
If (nodeId = m_InvalidId) return false;

Return true;
}

Uint32_t Count ()
{
AutoLock autoLock (m_MutexLock );
Return m_HeadAddr-> m_UsedCount;
}

// If exist set else add
Bool Replace (K & key, V & value)
{
AutoLock autoLock (m_MutexLock );

If (Exist (key) return Set (key, value );
Else return Add (key, value );
}

/*************************************** ********
* *** LRU: when visit a node, move it to head ****
**************************************** ********/
// If no empty place, recycle tail
Bool LruAdd (K & key, V & value, K & recyKey, V & recyValue, bool & recycled)
{
AutoLock autoLock (m_MutexLock );

If (Exist (key) return false;

If (Add (key, value) return true;

Uint32_t index = key. HashCode () % m_HeadAddr-> m_TableLen;
Uint32_t tailId = GetTailNodeId (index );

If (tailId = m_InvalidId) return false;

Entry * tmpNode = m_EntryAddr + tailId;
RecyKey = tmpNode-> m_Key;
RecyValue = tmpNode-> m_Value;
Recycled = true;

RecycleNode (index, tailId );

Return Add (key, value );
}

Bool LruSet (K & key, V & value)
{
AutoLock autoLock (m_MutexLock );

If (Set (key, value) return MoveToHead (key );
Else return false;
}

Bool LruGet (K & key, V & value)
{
AutoLock autoLock (m_MutexLock );

If (Get (key, value) return MoveToHead (key );
Else return false;
}

// If exist set else add; if add failed recycle tail than add
Bool LruReplace (K & key, V & value, K & recyKey, V & recyValue, bool & recycled)
{
AutoLock autoLock (m_MutexLock );

Recycled = false;

If (Exist (key) return LruSet (key, value );
Else return LruAdd (key, value, recyKey, recyValue, recycled );
}

Void Clear ()
{
AutoLock autoLock (m_MutexLock );

M_HeadAddr-> m_FreeBase = 0;
M_HeadAddr-> m_RecycleHead = 0;
M_HeadAddr-> m_UsedCount = 0;
For (uint32_t I = 0; I <m_HeadAddr-> m_TableLen; ++ I)
{
(M_ArrayAddr + I)-> m_Head = m_InvalidId;
(M_ArrayAddr + I)-> m_Tail = m_InvalidId;
}
}

Int GetRowKeys (vector & Keys, uint32_t index)
{
AutoLock autoLock (m_MutexLock );

If (index> = m_HeadAddr-> m_TableLen) return-1;

Keys. clear ();
Keys. reserve (16 );

Int count = 0;
Array * tmpArray = m_ArrayAddr + index;
Uint32_t nodeId = tmpArray-> m_Head;
While (nodeId! = M_InvalidId)
{
Entry * tmpNode = m_EntryAddr + nodeId;
Keys. push_back (tmpNode-> m_Key );
NodeId = tmpNode-> m_Next;
++ Count;
}

Return count;
}

Void * Padding (uint32_t size)
{
AutoLock autoLock (m_MutexLock );

If (size> m_HeadSize-sizeof (TableHead) return NULL;
Else return m_HeadAddr-> m_Padding;
}

Private:
Static const uint32_t m_InvalidId = 0 xffffffff;
Static const uint32_t m_HeadSize = 1024;
Struct TableHead
{
Uint32_t m_TableLen;
Uint32_t m_NodeTotal;
Uint32_t m_FreeBase;
Uint32_t m_RecycleHead;
Uint32_t m_UsedCount;
Char m_TableName [256];
Uint32_t m_Padding [0];
};

Struct Array
{
Uint32_t m_Head;
Uint32_t m_Tail;
};

Struct Entry
{
V m_Value;
K m_Key;
Uint32_t m_Code;
Uint32_t m_Next;
Uint32_t m_Prev;
};

Size_t m_MemSize;
Uint8_t * m_MemAddr;
TableHead * m_HeadAddr;
Array * m_ArrayAddr;
Entry * m_EntryAddr;

ThreadMutex m_MutexLock;

Bool MoveToHead (K & key );
Uint32_t GetIdByKey (K & key );
Void AddNodeToHead (uint32_t index, uint32_t nodeId );
Bool MoveNodeToHead (uint32_t index, uint32_t nodeId );
Bool RecycleNode (uint32_t index, uint32_t nodeId );
Uint32_t GetTailNodeId (uint32_t index );
Uint32_t GetFreeNode ();

DISABLE_COPY_AND_ASSIGN (HashTable );
};

Template
HashTable : HashTable (const char * tablename, uint32_t tableLen, uint32_t nodeTotal)
{
AbortAssert (tablename! = NULL );

M_MemSize = m_HeadSize + tableLen * sizeof (Array) + nodeTotal * sizeof (Entry );
M_MemAddr = (uint8_t *) MemFile: Realloc (tablename, m_MemSize );
AbortAssert (m_MemAddr! = NULL );

M_HeadAddr = (TableHead *) (m_MemAddr );
M_ArrayAddr = (Array *) (m_MemAddr + m_HeadSize );
M_EntryAddr = (Entry *) (m_MemAddr + m_HeadSize + tableLen * sizeof (Array ));

M_HeadAddr-> m_TableLen = tableLen;
M_HeadAddr-> m_NodeTotal = nodeTotal;
Strncpy (m_HeadAddr-> m_TableName, tablename, sizeof (m_HeadAddr-> m_TableName ));

If (m_HeadAddr-> m_UsedCount = 0) // if first use init array to invalid
{
For (uint32_t I = 0; I <tableLen; ++ I)
{
(M_ArrayAddr + I)-> m_Head = m_InvalidId;
(M_ArrayAddr + I)-> m_Tail = m_InvalidId;
}

M_HeadAddr-> m_FreeBase = 0;
M_HeadAddr-> m_RecycleHead = 0;
}
}

Template
HashTable ::~ HashTable ()
{
MemFile: Release (m_MemAddr, m_MemSize );
}

Template
Bool HashTable : MoveToHead (K & key)
{
Uint32_t nodeId = GetIdByKey (key );
Uint32_t index = key. HashCode () % m_HeadAddr-> m_TableLen;

Return MoveNodeToHead (index, nodeId );
}

Template
Uint32_t HashTable : GetIdByKey (K & key)
{
Uint32_t hashCode = key. HashCode ();
Uint32_t index = hashCode % m_HeadAddr-> m_TableLen;
Array * tmpArray = m_ArrayAddr + index;

Uint32_t nodeId = tmpArray-> m_Head;
While (nodeId! = M_InvalidId)
{
Entry * tmpNode = m_EntryAddr + nodeId;
If (tmpNode-> m_Code = hashCode & key. Equals (tmpNode-> m_Key) break;

NodeId = tmpNode-> m_Next;
}

Return nodeId;
}

Template
Void HashTable : AddNodeToHead (uint32_t index, uint32_t nodeId)
{
If (index> = m_HeadAddr-> m_TableLen | nodeId> = m_HeadAddr-> m_NodeTotal) return;

Array * tmpArray = m_ArrayAddr + index;
Entry * tmpNode = m_EntryAddr + nodeId;
If (m_InvalidId = tmpArray-> m_Head)
{
TmpArray-> m_Head = nodeId;
TmpArray-> m_Tail = nodeId;
}
Else
{
TmpNode-> m_Next = tmpArray-> m_Head;
(M_EntryAddr + tmpArray-> m_Head)-> m_Prev = nodeId;
TmpArray-> m_Head = nodeId;
}
}

Template
Bool HashTable : MoveNodeToHead (uint32_t index, uint32_t nodeId)
{
If (index> = m_HeadAddr-> m_TableLen | nodeId> = m_HeadAddr-> m_NodeTotal) return false;

Array * tmpArray = m_ArrayAddr + index;
Entry * tmpNode = m_EntryAddr + nodeId;

// Already head
If (tmpArray-> m_Head = nodeId)
{
Return true;
}

Uint32_t nodePrev = tmpNode-> m_Prev;
Uint32_t nodeNext = tmpNode-> m_Next;
(M_EntryAddr + nodePrev)-> m_Next = nodeNext;
If (nodeNext! = M_InvalidId)
{
(M_EntryAddr + nodeNext)-> m_Prev = nodePrev;
}
Else
{
TmpArray-> m_Tail = nodePrev;
}

TmpNode-> m_Prev = m_InvalidId;
TmpNode-> m_Next = tmpArray-> m_Head;
(M_EntryAddr + tmpArray-> m_Head)-> m_Prev = nodeId;
TmpArray-> m_Head = nodeId;

Return true;
}

Template
Bool HashTable : RecycleNode (uint32_t index, uint32_t nodeId)
{
If (index> = m_HeadAddr-> m_TableLen | nodeId> = m_HeadAddr-> m_NodeTotal) return false;

Array * tmpArray = m_ArrayAddr + index;
Entry * tmpNode = m_EntryAddr + nodeId;

Uint32_t nodePrev = tmpNode-> m_Prev;
Uint32_t nodeNext = tmpNode-> m_Next;

If (nodePrev! = M_InvalidId)
{
(M_EntryAddr + nodePrev)-> m_Next = nodeNext;
}
Else
{
TmpArray-> m_Head = nodeNext;
}

If (nodeNext! = M_InvalidId)
{
(M_EntryAddr + nodeNext)-> m_Prev = nodePrev;
}
Else
{
TmpArray-> m_Tail = nodePrev;
}

(M_EntryAddr + nodeId)-> m_Next = m_HeadAddr-> m_RecycleHead;
M_HeadAddr-> m_RecycleHead = nodeId;
-- (M_HeadAddr-> m_UsedCount );

Return true;
}

Template
Uint32_t HashTable : GetTailNodeId (uint32_t index)
{
If (index> = m_HeadAddr-> m_TableLen) return m_InvalidId;

Array * tmpArray = m_ArrayAddr + index;

Return tmpArray-> m_Tail;
}

Template
Uint32_t HashTable : GetFreeNode ()
{
Uint32_t nodeId = m_InvalidId;
If (m_HeadAddr-> m_UsedCount <m_HeadAddr-> m_FreeBase) // get from recycle list
{
NodeId = m_HeadAddr-> m_RecycleHead;
M_HeadAddr-> m_RecycleHead = (m_EntryAddr + nodeId)-> m_Next;
++ (M_HeadAddr-> m_UsedCount );
}
Else if (m_HeadAddr-> m_UsedCount <m_HeadAddr-> m_NodeTotal) // get from free mem
{
NodeId = m_HeadAddr-> m_FreeBase;
++ (M_HeadAddr-> m_FreeBase );
++ (M_HeadAddr-> m_UsedCount );
}
Else
{
NodeId = m_InvalidId;
}

// Init node
If (nodeId <m_HeadAddr-> m_NodeTotal)
{
Entry * tmpNode = m_EntryAddr + nodeId;
Memset (tmpNode, 0, sizeof (Entry ));

TmpNode-> m_Next = m_InvalidId;
TmpNode-> m_Prev = m_InvalidId;
}

Return nodeId;
}

BitsCN.com