/* zmalloc - total amount of allocated memory aware version of malloc()
*
* Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef __ZMALLOC_H
#define __ZMALLOC_H
/* Double expansion needed for stringification of macro values. */
//宏的雙重定義，實作宏的字元串化
//參考http://my.oschina.net/moooofly/blog/126187
//此處使用#将宏(這裡是#緊跟的那個s)“字元串化”；使用##能将左右兩邊的标簽組合在一起，而且兩個标簽之間不存在空格
#define __xstr(s) __str(s)
#define __str(s) #s
/*
#define __xstr(s) __str(s)
#define __str(s) #s
#define str "hello" __xstr(raiet)
int main(){
printf("str : %s\n",str);
exit(0);
}
output helloraiet
*/
#if defined(USE_TCMALLOC)
#define ZMALLOC_LIB ("tcmalloc-" __xstr(TC_VERSION_MAJOR) "." __xstr(TC_VERSION_MINOR))
#include <google/tcmalloc.h>
//版本高于1.6
#if (TC_VERSION_MAJOR == 1 && TC_VERSION_MINOR >= 6) || (TC_VERSION_MAJOR > 1)
#define HAVE_MALLOC_SIZE 1
//分别定義出記憶體占有量的計算方法
#define zmalloc_size(p) tc_malloc_size(p)
#else
#error "Newer version of tcmalloc required"
#endif
#elif defined(USE_JEMALLOC)
#define ZMALLOC_LIB ("jemalloc-" __xstr(JEMALLOC_VERSION_MAJOR) "." __xstr(JEMALLOC_VERSION_MINOR) "." __xstr(JEMALLOC_VERSION_BUGFIX))
#include <jemalloc/jemalloc.h>
//版本高于2.1
#if (JEMALLOC_VERSION_MAJOR == 2 && JEMALLOC_VERSION_MINOR >= 1) || (JEMALLOC_VERSION_MAJOR > 2)
#define HAVE_MALLOC_SIZE 1
//分别定義出記憶體占有量的計算方法
#define zmalloc_size(p) je_malloc_usable_size(p)
#else
#error "Newer version of jemalloc required"
#endif
#elif defined(__APPLE__)
#include <malloc/malloc.h>
#define HAVE_MALLOC_SIZE 1
//分别定義出記憶體占有量的計算方法
#define zmalloc_size(p) malloc_size(p)
#endif
//如果沒有使用tcmalloc或是jemalloc庫，則使用glibc庫
#ifndef ZMALLOC_LIB
#define ZMALLOC_LIB "libc"
#endif
//申明一些函數原型
void *zmalloc(size_t size);
void *zcalloc(size_t size);
void *zrealloc(void *ptr, size_t size);
void zfree(void *ptr);
char *zstrdup(const char *s);
size_t zmalloc_used_memory(void);
void zmalloc_enable_thread_safeness(void);
void zmalloc_set_oom_handler(void (*oom_handler)(size_t));
float zmalloc_get_fragmentation_ratio(void);
size_t zmalloc_get_rss(void);
size_t zmalloc_get_private_dirty(void);
void zlibc_free(void *ptr);
#ifndef HAVE_MALLOC_SIZE
size_t zmalloc_size(void *ptr);
#endif
#endif /* __ZMALLOC_H */
/* zmalloc - total amount of allocated memory aware version of malloc()
*
* Copyright (c) 2009-2010, Salvatore Sanfilippo <antirez at gmail dot com>
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* * Redistributions of source code must retain the above copyright notice,
* this list of conditions and the following disclaimer.
* * Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* * Neither the name of Redis nor the names of its contributors may be used
* to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include <stdio.h>
#include <stdlib.h>
/* This function provide us access to the original libc free(). This is useful
* for instance to free results obtained by backtrace_symbols(). We need
* to define this function before including zmalloc.h that may shadow the
* free implementation if we use jemalloc or another non standard allocator. */
//将這個函數定義在"zmalloc.h"之前就是為了避免調用jemallor或者是其他庫的标準操作器，而是調用原始C語言函數庫"libc"中的free()函數
void zlibc_free(void *ptr) {
free(ptr);
}
#include <string.h>
#include <pthread.h>
#include "config.h"
#include "zmalloc.h"
//PREFIX_SIZE用來記錄malloc配置設定到的記憶體大小
//tc_malloc/je_malloc/Mac平台分别采用tc_malloc_size/malloc_size/je_malloc_usable_size（不需要單獨開空間計算得到的記憶體大小，PREFIX_SIZE值置為0）
//linux和sun平台分别采用sizeof(size_t)位元組和sizeof(long long)定長字段記錄，是以要記錄配置設定空間的大小
#ifdef HAVE_MALLOC_SIZE
#define PREFIX_SIZE (0)
#else
#if defined(__sun) || defined(__sparc) || defined(__sparc__)
#define PREFIX_SIZE (sizeof(long long))
#else
#define PREFIX_SIZE (sizeof(size_t))
#endif
#endif
/* Explicitly override malloc/free etc when using tcmalloc. */
#if defined(USE_TCMALLOC)
#define malloc(size) tc_malloc(size)
#define calloc(count,size) tc_calloc(count,size)
#define realloc(ptr,size) tc_realloc(ptr,size)
#define free(ptr) tc_free(ptr)
#elif defined(USE_JEMALLOC)
#define malloc(size) je_malloc(size)
#define calloc(count,size) je_calloc(count,size)
#define realloc(ptr,size) je_realloc(ptr,size)
#define free(ptr) je_free(ptr)
#endif
//使用原子操作更新記憶體使用量(如果定義了原子操作，則使用原子操作，否則使用互斥量保護)
#ifdef HAVE_ATOMIC
#define update_zmalloc_stat_add(__n) __sync_add_and_fetch(&used_memory, (__n))
#define update_zmalloc_stat_sub(__n) __sync_sub_and_fetch(&used_memory, (__n))
#else
#define update_zmalloc_stat_add(__n) do { \
pthread_mutex_lock(&used_memory_mutex); \
used_memory += (__n); \
pthread_mutex_unlock(&used_memory_mutex); \
} while(0)
#define update_zmalloc_stat_sub(__n) do { \
pthread_mutex_lock(&used_memory_mutex); \
used_memory -= (__n); \
pthread_mutex_unlock(&used_memory_mutex); \
} while(0)
#endif
//定義在配置設定和釋放記憶體的時候更新used_memory的值（用到上面記憶體配置設定的原子操作add和sub函數）
//if結構判斷語句中傳回的是對_n末兩位進行截斷的結果：sizeof(long)在32位系統下的二進制表現形式為：100
//if結構成立證明目前配置設定的記憶體大小有碎片（即_n值不是記憶體配置設定單元的整數倍），是以後面有個+=sizeof(long)-(_n&(sizeof(long)-1)的過程就是為了補齊
//這個函數是最上面幾個函數的封(在配置設定記憶體時使用)
#define update_zmalloc_stat_alloc(__n) do { \
size_t _n = (__n); \
if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
if (zmalloc_thread_safe) { \
update_zmalloc_stat_add(_n); \
} else { \
used_memory += _n; \
} \
} while(0)
//同上面(在釋放記憶體時使用)
#define update_zmalloc_stat_free(__n) do { \
size_t _n = (__n); \
if (_n&(sizeof(long)-1)) _n += sizeof(long)-(_n&(sizeof(long)-1)); \
if (zmalloc_thread_safe) { \
update_zmalloc_stat_sub(_n); \
} else { \
used_memory -= _n; \
} \
} while(0)
static size_t used_memory = 0;
static int zmalloc_thread_safe = 0;
pthread_mutex_t used_memory_mutex = PTHREAD_MUTEX_INITIALIZER;
//異常處理(Out of memory)的預設操作，直接退出
static void zmalloc_default_oom(size_t size) {
fprintf(stderr, "zmalloc: Out of memory trying to allocate %zu bytes\n",
size);
fflush(stderr);
abort();
}
static void (*zmalloc_oom_handler)(size_t) = zmalloc_default_oom;
void *zmalloc(size_t size) {
//PREFIX_SIZE代表是否需要存儲額外的變量prefix所占的記憶體位元組數
void *ptr = malloc(size+PREFIX_SIZE);
if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
//如果有size函數可以直接獲得記憶體使用量，則直接調用zmalloc_size函數獲得記憶體使用量
update_zmalloc_stat_alloc(zmalloc_size(ptr));
return ptr;
#else
//否則一般是使用了glibc，需要自己封裝記憶體使用量
*((size_t*)ptr) = size;
update_zmalloc_stat_alloc(size+PREFIX_SIZE);
//指針向後移動，傳回真正可用的位址
return (char*)ptr+PREFIX_SIZE;
#endif
}
//同上
void *zcalloc(size_t size) {
//配置設定1個長度為size+PREFIX_SIZE大小的連續空間
void *ptr = calloc(1, size+PREFIX_SIZE);
if (!ptr) zmalloc_oom_handler(size);
#ifdef HAVE_MALLOC_SIZE
update_zmalloc_stat_alloc(zmalloc_size(ptr));
return ptr;
#else
*((size_t*)ptr) = size;
update_zmalloc_stat_alloc(size+PREFIX_SIZE);
return (char*)ptr+PREFIX_SIZE;
#endif
}
void *zrealloc(void *ptr, size_t size) {
#ifndef HAVE_MALLOC_SIZE
void *realptr;
#endif
size_t oldsize;
void *newptr;
if (ptr == NULL) return zmalloc(size);
#ifdef HAVE_MALLOC_SIZE
//先取得之前記憶體的大小，用于統計記憶體使用量(需要先減去這個大小)
oldsize = zmalloc_size(ptr);
//重新配置設定
newptr = realloc(ptr,size);
if (!newptr) zmalloc_oom_handler(size);
//更新記憶體使用量
update_zmalloc_stat_free(oldsize);
update_zmalloc_stat_alloc(zmalloc_size(newptr));
return newptr;
#else
//指針先前移動找到真正的位址起始位址，這樣才能正确釋放
realptr = (char*)ptr-PREFIX_SIZE;
oldsize = *((size_t*)realptr);
//再次配置設定size+PREFIX_SIZE大小的記憶體
newptr = realloc(realptr,size+PREFIX_SIZE);
if (!newptr) zmalloc_oom_handler(size);
*((size_t*)newptr) = size;
update_zmalloc_stat_free(oldsize);
//這裡應該要加上PREFIX_SIZE，原本的代碼沒有，應該是疏忽了
//update_zmalloc_stat_alloc(size + PREFIX_SIZE);
update_zmalloc_stat_alloc(size);
return (char*)newptr+PREFIX_SIZE;
#endif
}
/* Provide zmalloc_size() for systems where this function is not provided by
* malloc itself, given that in that case we store an header with this
* information as the first bytes of every allocation. */
//如果使用的庫(glibc)沒有定義取得記憶體的size方法，則需要自己定義，這樣就可以對外提供統一的調用接口了
#ifndef HAVE_MALLOC_SIZE
size_t zmalloc_size(void *ptr) {
void *realptr = (char*)ptr-PREFIX_SIZE;
size_t size = *((size_t*)realptr);
/* Assume at least that all the allocations are padded at sizeof(long) by
* the underlying allocator. */
if (size&(sizeof(long)-1)) size += sizeof(long)-(size&(sizeof(long)-1));
return size+PREFIX_SIZE;
}
#endif
void zfree(void *ptr) {
#ifndef HAVE_MALLOC_SIZE
void *realptr;
size_t oldsize;
#endif
if (ptr == NULL) return;
#ifdef HAVE_MALLOC_SIZE
update_zmalloc_stat_free(zmalloc_size(ptr));
free(ptr);
#else
//指針偏移
realptr = (char*)ptr-PREFIX_SIZE;
oldsize = *((size_t*)realptr);
update_zmalloc_stat_free(oldsize+PREFIX_SIZE);
free(realptr);
#endif
}
//字元串複制
char *zstrdup(const char *s) {
size_t l = strlen(s)+1;
char *p = zmalloc(l);
memcpy(p,s,l);
return p;
}
//傳回目前已經使用的記憶體大小
size_t zmalloc_used_memory(void) {
size_t um;
//區分線程安全與否
if (zmalloc_thread_safe) {
#ifdef HAVE_ATOMIC
um = __sync_add_and_fetch(&used_memory, 0);
#else
pthread_mutex_lock(&used_memory_mutex);
um = used_memory;
pthread_mutex_unlock(&used_memory_mutex);
#endif
}
else {
um = used_memory;
}
return um;
}
//設定線程安全
void zmalloc_enable_thread_safeness(void) {
zmalloc_thread_safe = 1;
}
//設定自定義的異常處理函數
void zmalloc_set_oom_handler(void (*oom_handler)(size_t)) {
zmalloc_oom_handler = oom_handler;
}
/* Get the RSS information in an OS-specific way.
*
* WARNING: the function zmalloc_get_rss() is not designed to be fast
* and may not be called in the busy loops where Redis tries to release
* memory expiring or swapping out objects.
*
* For this kind of "fast RSS reporting" usages use instead the
* function RedisEstimateRSS() that is a much faster (and less precise)
* version of the funciton. */
//以下的函數都是和系統相關的
#if defined(HAVE_PROC_STAT)
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
/*
RES：resident memory usage 常駐記憶體
1、程序目前使用的記憶體大小，但不包括swap out
2、包含其他程序的共享
3、如果申請100m的記憶體，實際使用10m，它隻增長10m，與VIRT相反
4、關于庫占用記憶體的情況，它隻統計加載的庫檔案所占記憶體大小
res的值是 /proc/<pid>/stat檔案的第24個字段
*/
size_t zmalloc_get_rss(void) {
int page = sysconf(_SC_PAGESIZE);
size_t rss;
char buf[4096];
char filename[256];
int fd, count;
char *p, *x;
snprintf(filename,256,"/proc/%d/stat",getpid());
if ((fd = open(filename,O_RDONLY)) == -1) return 0;
if (read(fd,buf,4096) <= 0) {
close(fd);
return 0;
}
close(fd);
p = buf;
count = 23; /* RSS is the 24th field in /proc/<pid>/stat */
while(p && count--) {
p = strchr(p,' ');
if (p) p++;
}
if (!p) return 0;
x = strchr(p,' ');
if (!x) return 0;
*x = '\0';
rss = strtoll(p,NULL,10);
rss *= page;
return rss;
}
#elif defined(HAVE_TASKINFO)
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/sysctl.h>
#include <mach/task.h>
#include <mach/mach_init.h>
size_t zmalloc_get_rss(void) {
task_t task = MACH_PORT_NULL;
struct task_basic_info t_info;
mach_msg_type_number_t t_info_count = TASK_BASIC_INFO_COUNT;
if (task_for_pid(current_task(), getpid(), &task) != KERN_SUCCESS)
return 0;
task_info(task, TASK_BASIC_INFO, (task_info_t)&t_info, &t_info_count);
return t_info.resident_size;
}
#else
//否則直接傳回實際的記憶體使用量
size_t zmalloc_get_rss(void) {
/* If we can't get the RSS in an OS-specific way for this system just
* return the memory usage we estimated in zmalloc()..
*
* Fragmentation will appear to be always 1 (no fragmentation)
* of course... */
return zmalloc_used_memory();
}
#endif
//計算碎片率
/* Fragmentation = RSS / allocated-bytes */
float zmalloc_get_fragmentation_ratio(void) {
return (float)zmalloc_get_rss()/zmalloc_used_memory();
}
#if defined(HAVE_PROC_SMAPS)
size_t zmalloc_get_private_dirty(void) {
char line[1024];
size_t pd = 0;
FILE *fp = fopen("/proc/self/smaps","r");
if (!fp) return 0;
while(fgets(line,sizeof(line),fp) != NULL) {
if (strncmp(line,"Private_Dirty:",14) == 0) {
char *p = strchr(line,'k');
if (p) {
*p = '\0';
pd += strtol(line+14,NULL,10) * 1024;
}
}
}
fclose(fp);
return pd;
}
#else
size_t zmalloc_get_private_dirty(void) {
return 0;
}
#endif
總結：
redis的記憶體配置設定實際上使用了facebook的jemmalloc記憶體配置設定庫，隻不過為了提高系統的擴張性，将整個記憶體配置設定進行了一次封裝，對外提供統一的接口，而内部使用了不同的實作。上面的代碼也可以看出，他主要封裝了現在流行的兩個個記憶體配置設定庫：google的tcmalloc和facebook的jemalloc庫。
這一套對記憶體的封裝的代碼，很值得在以後的系統中使用，一來對記憶體管理會有更高的可控性，比如，可以利用内部的實作監控記憶體，檢測記憶體洩露等等，二來利用更加優秀的記憶體配置設定庫也可以在一定程度上提高整個系統的性能。
![CentOs7 安裝redis4.0 遇到的坑一、CentOs7連接配接網絡二、make檔案的時候出錯三、安裝ruby的redis插件時候報錯最後[圖]](data:image/gif;base64,R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=)