The vision of Redis memory management code

Zmalloc.h

/* Zmalloc-total amount of allocated memory aware version of malloc () * * Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail Dot com> * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, is permitted provided that the FOL Lowing conditions is met: * * * redistributions of source code must retain the above copyright notice, * This list of conditions and the following disclaimer. * * redistributions in binary form must reproduce the above copyright * Notice, this list of conditions and the foll Owing disclaimer in the * documentation and/or other materials provided with the distribution.  * * Neither the name of the Redis nor the names of its contributors is used * to endorse or promote products derived From this software without * specific prior written permission. * * This software are provided by the COPYRIGHT holders and CONTRIBUTORS "as are" * and any EXPRESS OR implied warranties, INcluding, LIMITED to, the * implied warranties of merchantability and FITNESS for A particular PURPOSE * is DISCL Aimed. In NO EVENT shall the COPYRIGHT OWNER OR CONTRIBUTORS is * liable for any DIRECT, INDIRECT, incidental, special, exemplary , or * consequential damages (including, but not LIMITED to, procurement of * substitute GOODS OR SERVICES; LOSS of Use, DATA, OR profits; or business * interruption) however caused and on any theory of liability, WHETHER in * contract, STRICT liability, OR TOR T (including negligence OR OTHERWISE) * arising in any-of-the-----out of the-of-this software, even IF advised of the * Possi Bility of SUCH DAMAGE. */#ifndef __zmalloc_h#define __zmalloc_h/* Double expansion needed for stringification of macro values. */#define __XSTR (s) __str (s) #define __STR (s) #s//tcmalloc memory Allocator # if defined (use_tcmalloc) #define ZMALLOC_LIB (" tcmalloc-"__xstr (tc_version_major)". "__xstr (Tc_version_minor)) #include <google/tcmalloc.h> #if (tc_version_ MAJOR = = 1 && tc_version_minor >= 6) | | (Tc_version_major > 1)//version number required # define Have_malloc_size 1//tcmalloc built-in get the function of allocating memory Space # # Zmalloc_size (P) tc_malloc _size (P) #else//Compile error handling #error "Newer version of Tcmalloc required" #endif #elif defined (use_jemalloc) #define Zmalloc_lib ("jemalloc-" __xstr (jemalloc_version_major) "." __xstr (Jemalloc_version_minor) "." __xstr (Jemalloc_version_bugfix)) #include <jemalloc/jemalloc.h> #if (jemalloc_version_major = = 2 && jemalloc_version_minor >= 1) | | (Jemalloc_version_major > 2) #define have_malloc_size 1#define zmalloc_size (P) je_malloc_usable_size (p) #else #error "Newer version of Jemalloc required" #endif #elif defined (__apple__) #include <malloc/malloc.h> #define Have_malloc _size 1#define zmalloc_size (P) malloc_size (P) #endif//using the default libc allocator #ifndef zmalloc_lib#define zmalloc_lib "libc" # Endifvoid *zmalloc (size_t size), void *zcalloc (size_t size), void *zrealloc (void *ptr, size_t size), void zfree (void *ptr); Char *zstrdup (const char *s); size_t zmalloc_used_memory (void), void zmalloc_enable_thread_safeness (void), void Zmalloc_set_oom_handler (void (*oom_ Handler) (size_t)); float Zmalloc_get_fragmentation_ratio (size_t RSS); size_t zmalloc_get_rss (void); size_t zmalloc_ Get_private_dirty (void); size_t Zmalloc_get_smap_bytes_by_field (char *field); void Zlibc_free (void *ptr); #ifndef has _malloc_sizesize_t zmalloc_size (void *ptr), #endif #endif/* __zmalloc_h */

Zmalloc.c

/* Zmalloc-total amount of allocated memory aware version of malloc () * * Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail Dot com> * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, is permitted provided that the FOL Lowing conditions is met: * * * redistributions of source code must retain the above copyright notice, * This list of conditions and the following disclaimer. * * redistributions in binary form must reproduce the above copyright * Notice, this list of conditions and the foll Owing disclaimer in the * documentation and/or other materials provided with the distribution.  * * Neither the name of the Redis nor the names of its contributors is used * to endorse or promote products derived From this software without * specific prior written permission. * * This software are provided by the COPYRIGHT holders and CONTRIBUTORS "as are" * and any EXPRESS OR implied warranties, INcluding, LIMITED to, the * implied warranties of merchantability and FITNESS for A particular PURPOSE * is DISCL Aimed. In NO EVENT shall the COPYRIGHT OWNER OR CONTRIBUTORS is * liable for any DIRECT, INDIRECT, incidental, special, exemplary , or * consequential damages (including, but not LIMITED to, procurement of * substitute GOODS OR SERVICES; LOSS of Use, DATA, OR profits; or business * interruption) however caused and on any theory of liability, WHETHER in * contract, STRICT liability, OR TOR T (including negligence OR OTHERWISE) * arising in any-of-the-----out of the-of-this software, even IF advised of the * Possi Bility of SUCH DAMAGE. */#include <stdio.h> #include <stdlib.h>/* This function provide us access to the original libc free (). This was useful * for instance to free results obtained by backtrace_symbols (). We need * to define this function before including zmalloc.h so may shadow the * free implementation if we use Jemalloc or another non standard allOcator. *//The free () interface for LIBC is retained. Used to release backtrace_symbols () obtained result of void zlibc_free (void *ptr) {free (PTR);} #include <string.h> #include <pthread.h> #include "config.h" #include "zmalloc.h" #ifdef have_malloc_size// Tc_malloc,je_malloc,mac Platform # define PREFIX_SIZE (0) #else/* Other Platform memory allocation * * Memory model: data_size (prefix_size) +data** front prefix_ The size byte stores the request space. Back space for the requested memory */#if defined (__sun) | | Defined (__SPARC) | | Defined (__sparc__) #define PREFIX_SIZE (sizeof (long Long)) #else//linux#define prefix_size (sizeof (size_t )) #endif #endif/* explicitly override Malloc/free etc when using Tcmalloc. */#if defined (USE_TCMALLOC) #define malloc (size) tc_malloc (size) #define CALLOC (count,size) Tc_calloc (count,size) # Define REALLOC (Ptr,size) tc_realloc (ptr,size) #define FREE (PTR) tc_free (PTR) #elif defined (use_jemalloc) #define Malloc (size) je_malloc (size) #define CALLOC (count,size) je_calloc (count,size) #define REALLOC (ptr,size) Je_realloc (ptr,size ) #define FREE (PTR) je_free (PTR) #endif//Atomic manipulation changes total use of memory # if defined (__atomic_relaxed) #define UPDATE_ZMALLOC_STAT_ADD (__n) __atomic_add_fetch (&used_memory, (__n), __ATOMIC_RELAXED) # Define UPDATE_ZMALLOC_STAT_SUB (__n) __atomic_sub_fetch (&used_memory, (__n), __atomic_relaxed) #elif defined (with _atomic) #define UPDATE_ZMALLOC_STAT_ADD (__n) __sync_add_and_fetch (&used_memory, (__n)) #define UPDATE_ZMALLOC_ Stat_sub (__n) __sync_sub_and_fetch (&used_memory, (__n)) #else # define UPDATE_ZMALLOC_STAT_ADD (__n) do {Pthread_     Mutex_lock (&used_memory_mutex);     Used_memory + = (__n); Pthread_mutex_unlock (&used_memory_mutex);     } while (0) #define UPDATE_ZMALLOC_STAT_SUB (__n) do {pthread_mutex_lock (&used_memory_mutex);     Used_memory-= (__n); Pthread_mutex_unlock (&used_memory_mutex); } while (0) #endif//__n is the allocated memory byte.     Update the value of used_memory, note long alignment # define UPDATE_ZMALLOC_STAT_ALLOC (__n) do {size_t _n = (__n); if (_n& (sizeof (long)-1)) _n + = sizeof (long)-(_n& (sizeof (long)-1)); \//Allocate bytes less than integer multiples of long to be added to integer multiples if (zmalloc_thread_saFe) {\//thread safe Mode Update_zmalloc_stat_add (_n);     } else {\//non-thread safe mode used_memory + = _n;     }} while (0)//__n is the freed memory byte, update the value of used_memory, note long alignment # define UPDATE_ZMALLOC_STAT_FREE (__n) do {size_t _n = (__n);     if (_n& (sizeof (long)-1)) _n + = sizeof (long)-(_n& (sizeof (long)-1));     if (Zmalloc_thread_safe) {update_zmalloc_stat_sub (_n);     } else {used_memory-= _n;          }} while (0) static size_t used_memory = 0;     Initially, the total memory space used is 0static int zmalloc_thread_safe = 0;  The default thread-Safe mode is not started pthread_mutex_t Used_memory_mutex = Pthread_mutex_initializer; Defines and initializes a mutually exclusive//out-of-memory default handler for static void Zmalloc_default_oom (size_t size) {fprintf (stderr, "Zmalloc:out for Memory tryin    G to allocate%zu bytes\n ", size);    Fflush (stderr); Abort ();} Insufficient memory to handle function pointers. Initial default function static void (*zmalloc_oom_handler) (size_t) = Zmalloc_default_oom;//malloc Implement memory request function void *zmalloc (size_t size) {V    OID *ptr = malloc (size+prefix_size); if (!ptR) Zmalloc_oom_handler (size); The request failed to invoke the out of memory handler function, terminating the program #ifdef have_malloc_size//tc_malloc,je_malloc,mac Platform Update_zmalloc_stat_alloc (    Zmalloc_size (PTR));    return ptr; #else//Other Platform * ((size_t*) ptr) = size;    Update_zmalloc_stat_alloc (size+prefix_size); Return (char*) ptr+prefix_size; #endif}//calloc implement memory deallocation function void *zcalloc (size_t SIZE) {void *ptr = Calloc (1, Size+prefix_siz    E);    if (!ptr) zmalloc_oom_handler (size), #ifdef have_malloc_size update_zmalloc_stat_alloc (Zmalloc_size (PTR));    return ptr; #else * ((size_t*) ptr) = size;    Update_zmalloc_stat_alloc (size+prefix_size); Return (char*) ptr+prefix_size; #endif}void *zrealloc (void *ptr, size_t SIZE) {#ifndef have_malloc_size void *realptr; #en    DIF size_t oldsize;    void *newptr;    if (ptr = = NULL) return zmalloc (size), #ifdef have_malloc_size oldsize = Zmalloc_size (PTR);    Newptr = ReAlloc (ptr,size);    if (!newptr) zmalloc_oom_handler (size); Update_zmaLloc_stat_free (oldsize);    Update_zmalloc_stat_alloc (Zmalloc_size (newptr));    return newptr; #else realptr = (char*) ptr-prefix_size;    oldsize = * ((size_t*) realptr);    Newptr = ReAlloc (realptr,size+prefix_size);    if (!newptr) zmalloc_oom_handler (size);    * ((size_t*) newptr) = size;    Update_zmalloc_stat_free (oldsize);    Update_zmalloc_stat_alloc (size); Return (char*) newptr+prefix_size; #endif}/* provide zmalloc_size () for systems where this function was not provided by * MAL LOC itself, given that in, the We store a header with this * information as the first bytes of every allocation.    */#ifndef have_malloc_sizesize_t zmalloc_size (void *ptr) {void *realptr = (char*) ptr-prefix_size;    size_t size = * ((size_t*) realptr); /* Assume at least-all the allocations is padded at sizeof (long) by * the underlying allocator.    *///If all allocators are allocated with long-aligned if (size& (sizeof (long)-1)) size + sizeof (long)-(size& (sizeof (long)-1)); return size+prefix_size;} #endif//releases the memory space that PTR points to void zfree (void *ptr) {#ifndef have_malloc_size void *realptr;    size_t oldsize; #endif if (ptr = = NULL) return; #ifdef have_malloc_size update_zmalloc_stat_free (Zmalloc_size (PTR));    Free (PTR); #else realptr = (char*) ptr-prefix_size;    oldsize = * ((size_t*) realptr);    Update_zmalloc_stat_free (oldsize+prefix_size);    Free (realptr); #endif}char *zstrdup (const char *s) {size_t L = strlen (s) +1;    Char *p = Zmalloc (l);    memcpy (p,s,l); return p;}    Returns the total use of memory size_t zmalloc_used_memory (void) {size_t um; if (zmalloc_thread_safe) {#if defined (__atomic_relaxed) | | defined (have_atomic) um = update_zmalloc_stat_add (0); #el        Se pthread_mutex_lock (&used_memory_mutex);        Um = used_memory;    Pthread_mutex_unlock (&used_memory_mutex); #endif} else {um = used_memory; } return um; Set thread mode void zmalloc_enable_thread_safeness (void) {zmalloc_thread_safe = 1;} Set the out of memory processing function void Zmalloc_set_oom_handler (void (*ooM_handler) (size_t)) {Zmalloc_oom_handler = Oom_handler;} /* Get The RSS information in an os-specific. * * Warning:the function Zmalloc_get_rss () is not designed to being fast * and may not being called in the busy loops where Red is tries to release * memory expiring or swapping out objects. * For this kind of "fast RSS reporting" usages use instead the * function Redisestimaterss () so is a much faster (and Less precise) * version of the function. */#if defined (have_proc_stat) #include <unistd.h> #include <sys/types.h> #include <sys/stat.h>#   Include <fcntl.h>//gets the resident set size of the current process size_t zmalloc_get_rss (void) {int page = sysconf (_sc_pagesize);    Get page size size_t RSS;    Char buf[4096];    Char filename[256];    int FD, Count;    Char *p, *x;    snprintf (filename,256, "/proc/%d/stat", Getpid ());    if (fd = open (filename,o_rdonly)) = =-1) return 0;        if (read (fd,buf,4096) <= 0) {close (FD);    return 0;    } close (FD);    p = buf; Count = 23; /* RSS is the 24th field In/proc/<pid>/stat */while (P && count--) {p = STRCHR (P, ');    if (p) p++;    } if (!p) return 0;    x = STRCHR (P, ');    if (!x) return 0;    *x = ' + ';    rss = Strtoll (p,null,10);    RSS *= page; return RSS;} #elif defined (have_taskinfo) #include <unistd.h> #include <stdio.h> #include <stdlib.h> #include < sys/types.h> #include <sys/sysctl.h> #include <mach/task.h> #include <mach/mach_init.h>size_t    Zmalloc_get_rss (void) {task_t task = Mach_port_null;    struct Task_basic_info t_info;    mach_msg_type_number_t t_info_count = Task_basic_info_count;    if (Task_for_pid (Current_task (), Getpid (), &task)! = kern_success) return 0;    Task_info (Task, Task_basic_info, (task_info_t) &t_info, &t_info_count); return t_info.resident_size;} #elsesize_t Zmalloc_get_rss (void) {/* If we can ' t get the RSS in an os-specific-in-the-system just * return The Memory USAGE We estimated in Zmalloc (). * * Fragmentation'll appear to being always 1 (no fragmentation) * Of course ... */return zmalloc_used_memory (); } #endif/* fragmentation = rss/allocated-bytes *///compute fragmentation rate float zmalloc_get_fragmentation_ratio (size_t RSS) {return (fl OAT) rss/zmalloc_used_memory ();} /* Get The sum of the specified field (converted form KB to bytes) in */proc/self/smaps. The field must is specified with trailing ":" as it * Apperas in the smaps output. * * Example:zmalloc_get_smap_bytes_by_field ("RSS:");    */#if defined (have_proc_smaps) size_t Zmalloc_get_smap_bytes_by_field (char *field) {char line[1024];    size_t bytes = 0;    FILE *FP = fopen ("/proc/self/smaps", "R");    int flen = strlen (field);    if (!FP) return 0; while (Fgets (line,sizeof (line), fp)! = NULL) {if (strncmp (line,field,flen) = = 0) {Char *p = STRCHR (line,            ' K ');                if (p) {*p = ' + ';         Bytes + = strtol (line+flen,null,10) * 1024;   }}} fclose (FP); return bytes;}    #elsesize_t Zmalloc_get_smap_bytes_by_field (char *field) {(void) field); return 0;} #endif//Get private_dirty size, Rss=shared_clean+shared_dirty+private_clean+private_dirty//shared_clean: Reference greater than 1, unaltered// Shared_dirty: Reference greater than 1, modified//private_clean: Reference equals 1, not modified//private_dirty: Reference equals 1, changed size_t zmalloc_get_private_dirty (void) {return Zmalloc_get_smap_bytes_by_field ("Private_dirty:");}

Copyright notice: This article Bo Master original articles, blogs, without consent may not be reproduced.

The vision of Redis memory management code