網路資料同步開發庫libsync

 

"資料同步演算法研究"一文提出了一種改進的資料同步演算法，我在實現的原型系統基礎上，將檔案切分、差異編碼、檔案同步等關鍵演算法抽取出來封裝成動態開發庫libsync，方便自己的開發應用。在本人開發的deduputil，WSIO, wsync等軟體中，均使用了libsync動態函數庫，現已將libsync發布至google code。libsync函數庫由提供三個API，原型描述如下：

1、int file_chunk(char *src_filename, char *chunk_filename, int chunk_algo)
功能：對件進行切分產生分塊描述檔案。
參數：src_filename為切分檔案，chunk_filename為產生的塊資訊描述檔案，chunk_algo為檔案切分演算法，目前支援定長FSP切分(Ffixed-size partition)、CDC切分(content-defined chunking)和滑動塊(sliding block)切分三種。

2、int file_delta(char *src_filename, char *chunk_filename, char *delta_filename, int chunk_algo)
功能：使用產生的塊描述資訊對檔案進行差異編碼。
參數：src_filename為待編碼檔案，chunk_filename為通過函數file_chunk產生的塊描述檔案，chunk_algo為檔案切分演算法。

3、int file_sync(char *src_filename, char *delta_filename)
功能：使用差異編碼檔案將源檔案同步至目標檔案。
參數：src_filename為基本檔案，delta_filename為通過函數file_delta產生的差異編碼檔案。

/*<br /> Copyright (C) 2010, Aigui Liu</p><p> Filename: sync.h<br /> Author: Aigui.Liu@gmail.com<br /> */<br />#ifndef _SYNC_H<br />#define _SYNC_H<br />#include <stdint.h><br />#define BLOCK_SZ 2048<br />#define BLOCK_MIN_SZ 1024<br />#define BLOCK_MAX_SZ 4096<br />#define BLOCK_WIN_SZ 48<br />#define NAME_MAX_SZ256<br />#define BUF_MAX_SZ32768<br />#define HASHTABLE_BUCKET_SZ1024<br />#define CHUNK_CDC_D BLOCK_SZ<br />#define CHUNK_CDC_R 13<br />enum chunk_algo {<br />CHUNK_FSP = 0, /* fixed-size partition */<br />CHUNK_CDC, /* content-define chunking */<br />CHUNK_SBC /* slide block chunking */<br />};<br />/* define chunk file header and block entry */<br />typedef struct _chunk_file_header {<br />uint32_t block_sz;<br />uint32_t block_nr;<br />} chunk_file_header;<br />#define CHUNK_FILE_HEADER_SZ(sizeof(chunk_file_header))<br />typedef struct _chunk_block_entry {<br />uint64_t offset;<br />uint32_t len;<br />uint8_t md5[16 + 1];<br />uint8_t csum[10 + 1];<br />} chunk_block_entry;<br />#define CHUNK_BLOCK_ENTRY_SZ(sizeof(chunk_block_entry))<br />/* define delta file header and block entry */<br />typedef struct _delta_file_header {<br />uint32_t block_nr;<br />uint32_t last_block_sz;<br />uint64_t last_block_offset; /* offset in delta file */<br />} delta_file_header;<br />#define DELTA_FILE_HEADER_SZ(sizeof(delta_file_header))<br />typedef struct _delta_block_entry {<br />uint64_t offset;<br />uint32_t len;<br />uint8_t embeded; /* 1, block in delta file; 0, block in source file. */<br />} delta_block_entry;<br />#define DELTA_BLOCK_ENTRY_SZ(sizeof(delta_block_entry))<br />int file_chunk(char *src_filename, char *chunk_filename, int chunk_algo);<br />int file_delta(char *src_filename, char *chunk_filename, char *delta_filename, int chunk_algo);<br />int file_sync(char *src_filename, char *delta_filename);<br />#endif<br />

資料同步有PULL和PUSH兩種應用模式，PULL是將遠端資料同步到本地，而PUSH是將本機資料同步到遠程。對應到同步演算法，主要區別在於資料分塊和差異編碼位置不同。PULL和PUSH同步模式步驟分別如下所述。

PULL同步模式流程：
1、本地對檔案A進行資料切分，產生資料區塊描述檔案chunk；
2、上傳chunk檔案至遠程伺服器；
3、遠程伺服器對檔案B進行差異編碼，產生差異編碼檔案delta；
4、下載delta檔案至本地；
5、本地同步檔案A至檔案B，相當於下載檔案B到本地檔案A。

例子程式：
/*<br /> Copyright (c) 2010, Aigui Liu<br /> All rights reserved.</p><p> Filename: wspull.c<br /> Author: Aigui.Liu@gmail.com<br />*/<br />#include <stdio.h><br />#include <stdlib.h><br />#include <string.h><br />#include <fcntl.h><br />#include <unistd.h><br />#include "wsioclient.h"<br />#include "wsio_api.h"<br />#include "sync.h"<br />/* main function */<br />int main(int argc, char **argv)<br />{<br />char fap[32] = {0};<br />char server[256] = {0};<br />char *local_filename = NULL;<br />char remote_filename[256] = {0};<br />char local_chunkname[256] = {0};<br />char remote_chunkname[256] = {0};<br />char local_deltaname[256] = {0};<br />char remote_deltaname[256] = {0};<br />int ret = 0;<br />if (argc < 3) {<br />fprintf(stderr, "Usage: wspull remote_filename local_filename/n");<br />exit(0);<br />}</p><p>if(argv[1]) {<br />if(parseURL(argv[1], fap, server, remote_filename) == -1) {<br />fprintf(stderr, "wsio address is illegal/n");<br />exit(1);<br />}<br />}<br />wsio_init(fap, server);<br />local_filename = argv[2];<br />if (0 == access(local_filename, F_OK)) {<br />sprintf(local_chunkname, "%s_%d", tmpnam(NULL), getpid());<br />sprintf(remote_chunkname, "%s_%d", tmpnam(NULL), getpid());<br />sprintf(local_deltaname, "%s_%d", tmpnam(NULL), getpid());<br />sprintf(remote_deltaname, "%s_%d", tmpnam(NULL), getpid());<br />ret = file_chunk(local_filename, local_chunkname, CHUNK_CDC);<br />if (ret == -1) {<br />fprintf(stderr, "local file chunking failed/n");<br />return -1;<br />}<br />ret = wsio_putfile(local_chunkname, remote_chunkname);<br />if (ret == -1) {<br />fprintf(stderr, "upload chunking file failed/n");<br />return -1;<br />}<br />ret = wsio_deltafile(remote_filename, remote_chunkname, remote_deltaname, CHUNK_CDC);<br />if (ret == -1) {<br />fprintf(stderr, "remote file delta coding failed/n");<br />return -1;<br />}<br />ret = wsio_getfile(remote_deltaname, local_deltaname);<br />if (ret == -1) {<br />fprintf(stderr, "download delta file failed/n");<br />}<br />ret = file_sync(local_filename, local_deltaname);<br />if (ret == -1) {<br />fprintf(stderr, "local file sync failed/n");<br />return -1;<br />}<br />} else {<br />ret = wsio_getfile(remote_filename, local_filename);<br />if (ret == -1) {<br />fprintf(stderr, "download file failed/n");<br />return -1;<br />}<br />}<br />wsio_unlink(remote_chunkname);<br />wsio_unlink(remote_deltaname);<br />unlink(local_chunkname);<br />unlink(local_deltaname);<br />wsio_destroy();<br />fprintf(stderr, "pull %s to %s %s/n", argv[1], local_filename, (ret == 0)? "sucessfully" : "failed");<br />return 0;<br />}<br />  

PUSH同步模式流程：
1、遠程伺服器對檔案B進行資料切分，產生資料區塊描述檔案chunk；
2、下載chunk檔案至本地；
3、本地對檔案A進行差異編碼，產生差異編碼檔案delta；
4、上傳delta檔案至遠程伺服器；
5、遠程同步檔案B到A，相當於上傳檔案A到遠程檔案B。

例子程式：
/*<br /> Copyright (c) 2010, Aigui Liu<br /> All rights reserved.</p><p> Filename: wspush.c<br /> Author: Aigui.Liu@gmail.com<br />*/<br />#include <stdio.h><br />#include <stdlib.h><br />#include <string.h><br />#include <fcntl.h><br />#include <unistd.h><br />#include "wsioclient.h"<br />#include "wsio_api.h"<br />#include "sync.h"<br />/* main function */<br />int main(int argc, char **argv)<br />{<br />char fap[32] = {0};<br />char server[256] = {0};<br />char remote_filename[256] = {0};<br />char *local_filename = NULL;<br />char local_chunkname[256] = {0};<br />char remote_chunkname[256] = {0};<br />char local_deltaname[256] = {0};<br />char remote_deltaname[256] = {0};<br />int ret = 0;<br />if (argc < 3) {<br />fprintf(stderr, "Usage: wspush local_filename remote_filename/n");<br />exit(0);<br />}</p><p>if(argv[2]) {<br />if(parseURL(argv[2], fap, server, remote_filename) == -1) {<br />fprintf(stderr, "wsio address is illegal/n");<br />exit(1);<br />}<br />}<br />wsio_init(fap, server);<br />local_filename = argv[1];<br />if (0 == wsio_access(remote_filename, F_OK)) {<br />sprintf(local_chunkname, "%s_%d", tmpnam(NULL), getpid());<br />sprintf(remote_chunkname, "%s_%d", tmpnam(NULL), getpid());<br />sprintf(local_deltaname, "%s_%d", tmpnam(NULL), getpid());<br />sprintf(remote_deltaname, "%s_%d", tmpnam(NULL), getpid());<br />ret = wsio_chunkfile(remote_filename, remote_chunkname, CHUNK_CDC);<br />if (ret == -1) {<br />fprintf(stderr, "remote file chunking failed/n");<br />return -1;<br />}<br />ret = wsio_getfile(remote_chunkname, local_chunkname);<br />if (ret == -1) {<br />fprintf(stderr, "download chunking file failed/n");<br />return -1;<br />}<br />ret = file_delta(local_filename, local_chunkname, local_deltaname, CHUNK_CDC);<br />if (ret == -1) {<br />fprintf(stderr, "local file delta coding failed/n");<br />return -1;<br />}<br />ret = wsio_putfile(local_deltaname, remote_deltaname);<br />if (ret == -1) {<br />fprintf(stderr, "upload delta file failed/n");<br />}<br />ret = wsio_syncfile(remote_filename, remote_deltaname);<br />if (ret == -1) {<br />fprintf(stderr, "remote file sync failed/n");<br />return -1;<br />}<br />} else {<br />ret = wsio_putfile(local_filename, remote_filename);<br />if (ret == -1) {<br />fprintf(stderr, "upload file failed/n");<br />return -1;<br />}<br />}<br />wsio_unlink(remote_chunkname);<br />wsio_unlink(remote_deltaname);<br />unlink(local_chunkname);<br />unlink(local_deltaname);<br />wsio_destroy();<br />fprintf(stderr, "push %s to %s %s/n", local_filename, remote_filename, (ret == 0)? "sucessfully" : "failed");<br />return 0;<br />}<br />