Redis memory management code comment, redis memory management comment

Redis memory management code comment, redis memory management comment

Zmalloc. h

/* Zmalloc-total amount of allocated memory aware version of malloc () ** Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com> * All rights reserved. ** Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * ** Redistributions of source code must retain the above copyright notice, * this list of conditions and the following disclawing. ** Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclawing in the * documentation and/or other materials provided with the distribution. ** Neither the name of Redis nor the names of its contributors may be used * to endorse or promote products derived from this software without * specific prior written permission. ** this software is provided by the copyright holders and contributors "as is" * and any express or implied warranties, INCLUDING, but not limited, THE * implied warranties of merchantability and fitness for a special PURPOSE * are disclaimed. in no event shall the copyright owner or contributors be * liable for any direct, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * consequential damages (INCLUDING, but not limited, procurement of * substitute goods or services; loss of use, DATA, or profits; or business * INTERRUPTION) however caused and on any theory of liability, whether in * CONTRACT, strict liability, or tort (including negligence or otherwise) * arising in any way out of the use of this software, even if advised of the * possibility of such damage. */# ifndef _ ZMALLOC_H # define _ ZMALLOC_H/* Double expansion needed for stringification of macro values. */# define _ xstr (s) _ str (s) # define _ str (s) # s // TCMALLOC memory distributor # if defined (USE_TCMALLOC) # define ZMALLOC_LIB ("tcmalloc-" _ xstr (TC_VERSION_MAJOR )". "_ xstr (TC_VERSION_MINOR) # include <google/tcmalloc. h> # if (TC_VERSION_MAJOR = 1 & TC_VERSION_MINOR> = 6) | (TC_VERSION_MAJOR> 1) // version requirements # define HAVE_MALLOC_SIZE 1 // TCMALLOC built-in function for obtaining the size of allocated memory # define zmalloc_size (p) tc_malloc_size (p) # else // compile error handling # error "Newer version of tcmalloc required" # endif # elif defined (USE_JEMALLOC) # define ZMALLOC_LIB ("jemalloc-" _ xstr (callback )". "_ xstr (JEMALLOC_VERSION_MINOR )". "_ xstr (JEMALLOC_VERSION_BUGFIX) # include <jemalloc/jemalloc. h ># if (Bytes = 2 & JEMALLOC_VERSION_MINOR> = 1) | (Bytes> 2) # define HAVE_MALLOC_SIZE 1 # define zmalloc_size (p) je_malloc_usable_size (p) # else # error "Newer version of jemalloc required" # endif # elif defined (_ APPLE _) # include <malloc/malloc. h> # define HAVE_MALLOC_SIZE 1 # define zmalloc_size (p) malloc_size (p) # endif // use the default libc distributor # ifndef ZMALLOC_LIB # define ZMALLOC_LIB "libc" # endifvoid * zmalloc (size_t size); void * zcalloc (size_t size ); void * zrealloc (void * ptr, size_t size); void zfree (void * ptr); char * zstrdup (const char * s); size_t zmalloc_used_memory (void ); void Merge (void); void Merge (void (* oom_handler) (size_t); float aggregate (size_t rss); size_t zmalloc_get_rss (void); size_t Merge (void ); size_t convert (char * field); void zlibc_free (void * ptr); # ifndef HAVE_MALLOC_SIZEsize_t zmalloc_size (void * ptr); # endif/* _ ZMALLOC_H */

Zmalloc. c

/* Zmalloc-total amount of allocated memory aware version of malloc () ** Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com> * All rights reserved. ** Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * ** Redistributions of source code must retain the above copyright notice, * t His list of conditions and the following disclawing. ** Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclawing in the * documentation and/or other materials provided with the distribution. ** Neither the name of Redis nor the names of its contributors may be used * to endorse or promote products derived from this software Without * specific prior written permission. ** this software is provided by the copyright holders and contributors "as is" * and any express or implied warranties, INCLUDING, but not limited, THE * implied warranties of merchantability and fitness for a special PURPOSE * are disclaimed. in no event shall the copyright owner or contributors be * liable for any direct, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * consequential damages (INCLUDING, but not limited to, procurement of * substitute goods or services; loss of use, DATA, or profits; or business * INTERRUPTION) however caused and on any theory of liability, whether in * CONTRACT, strict liability, or tort (including negligence or otherwise) * arising in any way out of the use of this software, even if advised of the * POSS Ibility of such damage. */# include <stdio. h> # include <stdlib. h>/* This function provide us access to the original libc free (). this is useful * for instance to free results obtained by backtrace_symbols (). we need * to define this function before including zmalloc. h that may shadow the * free implementation if we use jemalloc or another non standard allocator. * /// retain the libc free () interface, used to release backtrace_s Void zlibc_free (void * ptr) {free (ptr) ;}# include <string. h> # include <pthread. h> # include "config. h "# include" zmalloc. h "# ifdef HAVE_MALLOC_SIZE // tc_malloc, je_malloc, Mac platform # define PREFIX_SIZE (0) # else/* Memory Allocation on other platforms ** Memory Model: data_size (PREFIX_SIZE) + data ** PREFIX_SIZE: the size of the applied byte storage space, followed by the Applied memory */# if defined (_ sun) | defined (_ iSCSI) | defined (_ iSCSI _) # define PREFIX_SIZE (sizeof (long) # else // l Inux # define PREFIX_SIZE (sizeof (size_t) # endif/* Explicitly override malloc/free etc when using tcmalloc. */# if defined (USE_TCMALLOC) # define malloc (size) tc_malloc (size) # define calloc (count, size) tc_calloc (count, size) # define realloc (ptr, size) tc_realloc (ptr, size) # define free (ptr) tc_free (ptr) # elif defined (USE_JEMALLOC) # define malloc (size) je_malloc (size) # define calloc (count, size) je_calloc (c Ount, size) # define realloc (ptr, size) je_realloc (ptr, size) # define free (ptr) je_free (ptr) # endif // modify the total memory used by atomic operations # if defined (_ ATOMIC_RELAXED) # define update_zmalloc_stat_add (_ n) _ atomic_add_fetch (& used_memory, (_ n ), _ ATOMIC_RELAXED) # define update_zmalloc_stat_sub (_ n) _ atomic_sub_fetch (& used_memory, (_ n), _ ATOMIC_RELAXED) # elif defined (HAVE_ATOMIC) # define update_zmalloc_stat_add (_ n) _ sync_add_and _ Fetch (& used_memory, (_ n) # define update_zmalloc_stat_sub (_ n) _ sync_sub_and_fetch (& used_memory, (_ n )) # else # define aggregate (_ n) do {\ pthread_mutex_lock (& used_memory_mutex); \ used_memory + = (_ n); \ pthread_mutex_unlock (& used_memory_mutex ); \} while (0) # define update_zmalloc_stat_sub (_ n) do {\ pthread_mutex_lock (& used_memory_mutex); \ used_memory-= (_ n); \ pthread_mutex_unl Ock (& used_memory_mutex) ;\} while (0) # endif // _ n is the allocated memory byte, which updates the value of used_memory, note long-type alignment # define update_zmalloc_stat_alloc (_ n) do {\ size_t _ n = (_ n); \ if (_ n & (sizeof (long)-1 )) _ n + = sizeof (long)-(_ n & (sizeof (long)-1 )); \\// the allocated bytes do not multiply the Integers of the long type by an integer multiple if (zmalloc_thread_safe) {\// thread security mode update_zmalloc_stat_add (_ n ); \} else {\// non-thread security mode used_memory + = _ n ;\}\} while (0) // _ n is the memory byte released, update the value of used_memory. Pay attention to lon. G-type alignment # define update_zmalloc_stat_free (_ n) do {\ size_t _ n = (_ n); \ if (_ n & (sizeof (long)-1 )) _ n + = sizeof (long)-(_ n & (sizeof (long)-1); \ if (zmalloc_thread_safe) {\ update_zmalloc_stat_sub (_ n ); \} else {\ used_memory-= _ n; \}\} while (0) static size_t used_memory = 0; // The total memory space used at the initial stage is 0 static int zmalloc_thread_safe = 0; // The thread security mode pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER is disabled by default.; // Define and initialize a mutex // default handler for memory insufficiency static void zmalloc_default_oom (size_t size) {fprintf (stderr, "zmalloc: out of memory trying to allocate % zu bytes \ n ", size); fflush (stderr); abort () ;}// handler pointer for memory insufficiency, initially, the default function is static void (* zmalloc_oom_handler) (size_t) = zmalloc_default_oom; // The Memory application function void * zmalloc (size_t size) {void * ptr = malloc (size + PREFIX_SIZE); if (! Ptr) zmalloc_oom_handler (size); // application failed call the memory insufficiency handler and terminate the Program # ifdef HAVE_MALLOC_SIZE // tc_malloc, je_malloc, Mac platform update_zmalloc_stat_alloc (zmalloc_size (ptr )); return ptr; # else // other platforms * (size_t *) ptr) = size; update_zmalloc_stat_alloc (size + PREFIX_SIZE); return (char *) ptr + PREFIX_SIZE; # endif} // The memory release function void * zcalloc (size_t size) {void * ptr = calloc (1, size + PREFIX_SIZE); if (! Ptr) zmalloc_oom_handler (size); # ifdef HAVE_MALLOC_SIZE update_zmalloc_stat_alloc (zmalloc_size (ptr); return ptr; # else * (size_t *) ptr) = size; update_zmalloc_stat_alloc (size + PREFIX_SIZE); return (char *) ptr + PREFIX_SIZE; # endif} void * zrealloc (void * ptr, size_t size) {# ifndef HAVE_MALLOC_SIZE void * realptr; # endif size_t oldsize; void * newptr; if (ptr = NULL) return zmalloc (size); # ifdef HAVE_MALLOC_SIZE Oldsize = zmalloc_size (ptr); newptr = realloc (ptr, size); if (! Newptr) zmalloc_oom_handler (size); update_zmalloc_stat_free (oldsize); update_zmalloc_stat_alloc (zmalloc_size (newptr); return newptr; # else realptr = (char *) ptr-PREFIX_SIZE; oldsize = * (size_t *) realptr); newptr = realloc (realptr, size + PREFIX_SIZE); if (! Newptr) zmalloc_oom_handler (size); * (size_t *) newptr) = size; update_zmalloc_stat_free (oldsize); handler (size); return (char *) newptr + fixpre_size; # endif}/* Provide zmalloc_size () for systems where this function is not provided by * malloc itself, given that in that case we store a header with this * information as the first bytes of every allocation. */# ifndef HAVE_MALLOC_SIZEsize _ T zmalloc_size (void * ptr) {void * realptr = (char *) ptr-PREFIX_SIZE; size_t size = * (size_t *) realptr ); /* Assume at least that all the allocations are padded at sizeof (long) by * the underlying allocator. * /// assume that the long alignment if (size & (sizeof (long)-1) size + = sizeof (long) is used for all alignment) -(size & (sizeof (long)-1); return size + PREFIX_SIZE;} # endif // release the void zfree (void * ptr) memory space pointed to by ptr) {# ifndef HAVE_MALLOC_SIZE voi D * realptr; size_t oldsize; # endif if (ptr = NULL) return; # ifdef HAVE_MALLOC_SIZE update_zmalloc_stat_free (zmalloc_size (ptr); free (ptr ); # else realptr = (char *) ptr-PREFIX_SIZE; oldsize = * (size_t *) realptr); update_zmalloc_stat_free (oldsize + PREFIX_SIZE); free (realptr ); # endif} char * zstrdup (const char * s) {size_t l = strlen (s) + 1; char * p = zmalloc (l); memcpy (p, s, l); return p;} // returns the total used memory size_t zmal Loc_used_memory (void) {size_t um; if (zmalloc_thread_safe) {# if defined (_ ATOMIC_RELAXED) | defined (HAVE_ATOMIC) um = update_zmalloc_stat_add (0 ); # else pthread_mutex_lock (& used_memory_mutex); um = used_memory; pthread_mutex_unlock (& used_memory_mutex); # endif} else {um = used_memory;} return um ;} // sets the safe thread mode void zmalloc_enable_thread_safeness (void) {zmalloc_thread_safe = 1;} // sets the void zmall function for Processing Memory insufficiency. Oc_set_oom_handler (void (* oom_handler) (size_t) {zmalloc_oom_handler = oom_handler;}/* Get the RSS information in an OS-specific way. ** WARNING: the function zmalloc_get_rss () is not designed to be fast * and may not be called in the busy loops where Redis tries to release * memory expiring or swapping out objects. ** For this kind of "fast RSS reporting" usages use instead the * function Redi SEstimateRSS () that is a much faster (and less precise) * version of the function. */# if defined (HAVE_PROC_STAT) # include <unistd. h> # include <sys/types. h> # include <sys/stat. h> # include <fcntl. h> // obtain the size_t zmalloc_get_rss (void) {int page = sysconf (_ SC _PAGESIZE); // obtain the page size size_t rss; char buf [4096]; char filename [256]; int fd, count; char * p, * x; snprintf (filename, 256, "/proc/% d/stat", getp Id (); if (fd = open (filename, O_RDONLY) =-1) return 0; if (read (fd, buf, 4096) <= 0) {close (fd); return 0;} close (fd); p = buf; count = 23; /* RSS is the 24th field in/proc/<pid>/stat */while (p & count --) {p = strchr (p ,''); if (p) p ++;} if (! P) return 0; x = strchr (p, ''); if (! X) return 0; * x = '\ 0'; rss = strtoll (p, NULL, 10); rss * = page; return rss ;}# elif defined (HAVE_TASKINFO) # include <unistd. h> # include <stdio. h> # include <stdlib. h> # include <sys/types. h> # include <sys/sysctl. h> # include <mach/task. h ># include <mach/mach_init.h> size_t zmalloc_get_rss (void) {task_t task = MACH_PORT_NULL; struct task_basic_info t_info; effect_info_count = TASK_BASIC_INFO_CO UNT; if (task_for_pid (current_task (), getpid (), & task )! = KERN_SUCCESS) return 0; task_info (task, TASK_BASIC_INFO, (task_info_t) & t_info, & t_info_count); return t_info.resident_size ;#elsesize_t zmalloc_get_rss (void) {/* If we can't get the RSS in an OS-specific way for this system just * return the memory usage we estimated in zmalloc ().. ** Fragmentation will appear to be always 1 (no fragmentation) * of course... */return zmalloc_used_memory ();} # en Dif/* Fragmentation = RSS/allocated-bytes * // calculate the fragment rate float zmalloc_get_fragmentation_ratio (size_t rss) {return (float) rss/zmalloc_used_memory ();} /* Get the sum of the specified field (converted form kb to bytes) in */proc/self/smaps. the field must be specified with trailing ":" as it * apperas in the smaps output. ** Example: zmalloc_get_smap_bytes_by_field ("Rss:"); */# if defined (HAVE_PROC_SMA PS) size_t zmalloc_get_smap_bytes_by_field (char * field) {char line [1024]; size_t bytes = 0; FILE * fp = fopen ("/proc/self/smaps ", "r"); int flen = strlen (field); if (! Fp) return 0; while (fgets (line, sizeof (line), fp )! = NULL) {if (strncmp (line, field, flen) = 0) {char * p = strchr (line, 'k'); if (p) {* p = '\ 0'; bytes + = strtol (line + flen, NULL, 10) * 1024 ;}} fclose (fp); return bytes ;} # elsesize_t zmalloc_get_smap_bytes_by_field (char * field) {(void) field); return 0 ;}# endif // get Private_Dirty size, RSS = Shared_Clean + Shared_Dirty + Private_Clean + Private_Dirty // Shared_Clean: Reference greater than 1, not modified // Shared_Dirty: Reference greater than 1, modified // Private_Clean: Reference equal to 1, not modified // Private_Dirty: Reference is equal to 1, size_t zmalloc_get_private_dirty (void) {return zmalloc_get_smap_bytes_by_field ("Private_Dirty :");}