高效能伺服器c開發庫-minac工作筆記

背景
Gleasy部分高效能中介軟體（比如分布式nosql叢集cloudRedis，使用JAVA)，經過線上一段時間的驗證，發現在CPU佔用方面始終偏高，而且nosql叢集本身並無複雜的商務邏輯，多數情況下都是直接PROXY到後端的redis。出於對效能的的嚴苛要求，決定使用C對cloudRedis進行重構。在重構的過程中，由於cloudRedis的java版基於mina2，特別是codec環節大量依賴於IoBuffer和IoSession這兩個類，確實十分好用。因此衍生出來使用c語言實現完整的mina2介面的想法，借鑒MINA2本身的優秀設計，避免了很多彎路，最終實現了一套完整的開發庫，為了對mina2致敬，取名為minac。

minac介紹
minac基於epoll,使用event-loop-per-thread結構，IO thread個數預設為Cpu的核心數。
熟悉mina2的同學基本上拿過來就可以直接使用，沒有任何學習成本。
以下是提供的一些基本類庫
1. iobuffer
完整實現了java的IoBuffer類,所有方法跟java版相容（所有寫入buf的基本類型會被被轉換為big endian的，讀出時會轉換回系統支援的endian模式）提供了以下方法:

extern iobuffer_t *iobuffer_create(int32_t);
extern void iobuffer_destroy(iobuffer_t *);
extern void iobuffer_set_auto_expend(iobuffer_t *,int flag);
extern void iobuffer_set_auto_shrink(iobuffer_t *,int flag);
extern void iobuffer_set_increment(iobuffer_t *,int32_t);

extern void iobuffer_flip(iobuffer_t *);
extern void iobuffer_compact(iobuffer_t *);
extern void iobuffer_clear(iobuffer_t *);
extern int32_t iobuffer_set_position(iobuffer_t *,int32_t);
extern int32_t iobuffer_get_position(iobuffer_t *);
extern int32_t iobuffer_set_limit(iobuffer_t *,int32_t);
extern int32_t iobuffer_get_limit(iobuffer_t *);
extern void iobuffer_skip(iobuffer_t *,int32_t);

extern int32_t iobuffer_remaining(iobuffer_t *);
extern int32_t iobuffer_put_int(iobuffer_t *,int32_t);
extern int32_t iobuffer_get_int(iobuffer_t *,int32_t *);
extern int32_t iobuffer_put_double(iobuffer_t *,double);
extern int32_t iobuffer_get_double(iobuffer_t *,double *);
extern int32_t iobuffer_put_long(iobuffer_t *,int64_t);
extern int32_t iobuffer_get_long(iobuffer_t *,int64_t *);
extern int32_t iobuffer_put(iobuffer_t *,char *,int32_t);
extern int32_t iobuffer_get(iobuffer_t *,char *,int32_t);
extern int32_t iobuffer_put_by_read(iobuffer_t *,int fd,int32_t);
extern int32_t iobuffer_get_and_write(iobuffer_t *,int fd,int32_t);
/**
 * 從src中擷取內容寫入另一個www.111cn.net dest
 * 注意：此函數不支援src和dest為同一個iobuffer的情況
 */
extern int32_t iobuffer_put_iobuffer(iobuffer_t *dest,iobuffer_t *src);

2. iosession
實現了mina2的IoSession準系統，主要介面如下：

extern iosession_t *iosession_create(int connfd, mina_session_config_t *sessionConfig);
extern void iosession_destroy(iosession_t *session);
extern int iosession_write(iosession_t *session, iobuffer_t *buf);
extern int iosession_write_wait(iosession_t *session, iobuffer_t *buf,int seconds);
extern int iosession_close(iosession_t *session);
extern int iosession_flush(iosession_t *session);

3. socket_acceptor
實現了mina2的IoSocketAcceptor類，主要介面如下

extern mina_socket_acceptor_t* mina_socket_acceptor_create();
extern void mina_socket_acceptor_destroy(mina_socket_acceptor_t *acceptor);
extern int mina_socket_acceptor_bind(mina_socket_acceptor_t *acceptor,struct sockaddr_in *addr);
extern void mina_socket_acceptor_unbind(mina_socket_acceptor_t *acceptor);

4. socket_connector
實現了mina2的IoSockectConnector類，主要介面如下：

extern mina_socket_connector_t* mina_socket_connector_create();
extern void mina_socket_connector_destroy(mina_socket_connector_t *acceptor);
extern iosession_t *mina_socket_connector_connect(mina_socket_connector_t *acceptor,struct sockaddr *addr);
extern void mina_socket_connector_stop(mina_socket_connector_t *acceptor);

5. socket_handler
實現了mina2的IoSocketHandlerAdaptor，定義如下：

typedef void (*_messageReceived)(iosession_t *,iobuffer_t *);
typedef void (*_messageSent)(iosession_t *,iobuffer_t *);
typedef void (*_sessionCreated)(iosession_t *);
typedef void (*_sessionOpened)(iosession_t *);
typedef void (*_sessionClosed)(iosession_t *);
typedef void (*_exceptionCaught)(iosession_t *,mina_exception_t *);
typedef void (*_sessionIdle)(iosession_t *,mina_iosession_idle_t );

typedef struct mina_socket_handler_t{
 _messageReceived messageReceived;
 _messageSent messageSent;
 _sessionCreated sessionCreated;
 _sessionOpened sessionOpened;
 _sessionClosed sessionClosed;
 _exceptionCaught exceptionCaught;
 _sessionIdle sessionIdle;
}mina_socket_handler_t;

6. session_config
實現了mina2的SessionConfig，主要介面如下：

typedef struct mina_session_config_t{
 int readBufferSize;
 int minReadBufferSize;
 int maxReadBufferSize;
 int writeBufferSize;
 int minWriteBufferSize;
 int maxWriteBufferSize;

 int tcpNoDelay;
 int sendBufSize;
 int revBufSize;

 int readerIdleTime;//time in sec
 int writerIdleTime;//time in sec
}mina_session_config_t;

extern mina_session_config_t *mina_session_config_create();
extern void mina_session_config_destroy(mina_session_config_t *);

7. 使用例子：
用幾句代碼就輕鬆實現伺服器程式：

#include "minac.h"
#include "common.h"
#include <zlog.h>

static void messageReceived(iosession_t *session,iobuffer_t *buf){
 //dzlog_info("接收到一行,size:%d",iobuffer_remaining(buf));
 int32_t remaining = iobuffer_remaining(buf);
 if(remaining>0){
  iobuffer_t *rbuf = iobuffer_create(remaining);
  iobuffer_compact(rbuf);
  iobuffer_put_iobuffer(rbuf,buf);
  iobuffer_flip(rbuf);
  iosession_write(session,rbuf);
  iobuffer_destroy(rbuf);
 }
}

static void messageSent(iosession_t *session,iobuffer_t *buf){
 //dzlog_info("寫了www.111Cn.net資料,size:%d",iobuffer_remaining(buf));
}
static void sessionClosed(iosession_t *session){
 //dzlog_info("session is closed.%d",session->connfd);
}
static void sessionOpened(iosession_t *session){
 //dzlog_info("session is opened.%d",session->connfd);
}
static void exceptionCaught(iosession_t *session,mina_exception_t *exception){
 dzlog_info("exception met, code:%d, messag:%s",exception->errorCode,exception->errorMessage);
 iosession_close(session);
}
static void sessionIdle(iosession_t *session,mina_iosession_idle_t idleType){
 //dzlog_info("session idle, fd:%d, idleType:%d",session->connfd,idleType);
}
static void exit_handler(void *arg){
 mina_socket_acceptor_t *acceptor = (mina_socket_acceptor_t *)arg;
 mina_socket_acceptor_unbind(acceptor);
}
extern void test_acceptor(){
 mina_socket_acceptor_t *acceptor = mina_socket_acceptor_create();
 acceptor->sessionConfig->tcpNoDelay = 1;
 acceptor->sessionConfig->readerIdleTime = 2;
 acceptor->sessionConfig->writerIdleTime = 2;

 acceptor->socketHandler->messageReceived = messageReceived;
 acceptor->socketHandler->messageSent = messageSent;
 acceptor->socketHandler->sessionOpened = sessionOpened;
 acceptor->socketHandler->sessionClosed = sessionClosed;
 acceptor->socketHandler->exceptionCaught = exceptionCaught;
 acceptor->socketHandler->sessionIdle = sessionIdle;

 register_exit_handler(exit_handler,(void *)acceptor);

 struct sockaddr_in servaddr;
 servaddr.sin_family = AF_INET;
 servaddr.sin_port = htons(6666);
 servaddr.sin_addr.s_addr = htonl(INADDR_ANY);

 mina_socket_acceptor_bind(acceptor,&servaddr);
}

熟悉mina2的人，基本上對上面的代碼極為熟悉了。

效能如何
功能上面是好用了，效能如何呢？以echo server為例，效能測試結果如下：

熟悉mina2的人，基本上對上面的代碼極為熟悉了。

效能如何
功能上面是好用了，效能如何呢？以echo server為例，效能測試結果如下：
1. 實驗1(400用戶端)

400並發用戶端，TPS最大41萬，平均18萬，CPU佔用164%

2. 實驗2(1000用戶端)

1000並發用戶端，TPS最大55萬，平均30萬，CPU佔用169%

原文來自:Gleasy團隊部落格