Centos 安裝 Moosefs檔案系統

標籤：

一.環境介紹
Moosefs master:192.168.55.148
Moosefs Metalogger:192.168.55.149
Moosefs Chunk-01:192.168.55.150
Moosefs Chunk-02:192.168.55.151

Moosefs Client:192.168.55.152

 

 二.安裝

#########################################################
1) master安裝

$ yum install -y wget gcc zlib zlib-devel$ cd /usr/local/src$ groupadd mfs$ useradd -g mfs mfs -s /sbin/nologin -M$ wget http://www.moosefs.org/tl_files/mfscode/mfs-1.6.27-1.tar.gz$ tar zxvf mfs-1.6.27-1.tar.gz$ cd cd mfs-1.6.27$ ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfschunkserver --disable-mfsmount$ make$ make install

　　

 

$ cp rpm/rh/mfsmaster.init /etc/init.d/mfsmaster$ cp rpm/rh/mfscgiserv.init /etc/init.d/mfscgiserv$ chmod 755 /etc/init.d/mfsmaster$ chmod 755 /etc/init.d/mfscgiserv$ ln -s /usr/local/mfs/sbin/mfsmaster /usr/sbin/$ ln -s /usr/local/mfs/sbin/mfscgiserv /usr/sbin/$ ln -s /usr/local/mfs/etc/mfs /etc/mfs$ cd /usr/local/mfs/etc/mfs$ cp mfsmaster.cfg.dist mfsmaster.cfg$ cp mfsmetalogger.cfg.dist mfsmetalogger.cfg$ cp mfsexports.cfg.dist mfsexports.cfg$ cp mfstopology.cfg.dist mfstopology.cfg

　　

[[email protected] mfs]# more mfsexports.cfg | grep -v "^#"*/rw,alldirs,maproot=0*.rw

　　

 

 

$ cd /usr/local/mfs/var/mfs/$ cp metadata.mfs.empty metadata.mfs$ vi /etc/hosts1921.68.55.148 mfsmaster

　　

 

 

$ vi /etc/init.d/mfsmaster #!/bin/bash## Init file for the MooseFS master service## chkconfig: - 92 84## description: MooseFS master## processname: mfsmaster# config: @[email protected]/mfsmaster.cfg# Source function library.. /etc/init.d/functions# Source networking configuration.. /etc/sysconfig/network# Check that networking is up.[ "${NETWORKING}" == "no" ] && exit 0[ -x "/usr/sbin/mfsmaster" ] || exit 1[ -r "/etc/mfs/mfsexports.cfg" -o -r "/etc/mfs/mfsexports.cfg" ] || exit 1RETVAL=0prog="mfsmaster"start () {echo -n $"Starting $prog: "daemon $prog $OPTIONSRETVAL=$?echo[ $RETVAL -eq 0 ] && touch /var/lock/subsys/$progreturn $RETVAL}stop () {echo -n $"Stopping $prog: "$prog stop $OPTIONS || killproc $prog >/dev/null 2>&1RETVAL=$?echo[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$progreturn $RETVAL}restart () {stopstart}case "$1" instart)start;;stop)stop;;restart|reload)restart;;condrestart)[ -e /var/lock/subsys/$prog ] && restartRETVAL=$?;;status)status $progRETVAL=$?;;*)echo $"Usage: $0 {start|stop|restart|condrestart|status}"RETVAL=1esacexit $RETVAL

　　

 

$ touch /usr/local/mfs/var/mfs/.mfscgiserv.lock$ chmod 777 /usr/local/mfs/var/mfs/.mfscgiserv.lock

　　

$ vi /etc/init.d/mfscgiserv #!/bin/bash## Init file for the MooseFS CGI monitor service## chkconfig: - 93 83## description: MooseFS CGI server## processname: mfscgiserv# Source function library.. /etc/init.d/functions# Source networking configuration.. /etc/sysconfig/networkMFSCGISERV_USER=nobody# Source initialization configuration.# Check that networking is up.[ "${NETWORKING}" == "no" ] && exit 0[ -x "/usr/sbin/mfscgiserv" ] || exit 1RETVAL=0prog="mfscgiserv"start () {echo -n $"Starting $prog: "daemon --user $MFSCGISERV_USER $prog $OPTIONSRETVAL=$?echo[ $RETVAL -eq 0 ] && touch /var/lock/subsys/$progreturn $RETVAL}stop () {echo -n $"Stopping $prog: "$prog stop $OPTIONS || killproc $prog >/dev/null 2>&1RETVAL=$?echo[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$progreturn $RETVAL}restart () {stopstart}case "$1" instart)start;;stop)stop;;restart|reload)restart;;condrestart)[ -e /var/lock/subsys/$prog ] && restartRETVAL=$?;;status)status $progRETVAL=$?;;*)echo $"Usage: $0 {start|stop|restart|condrestart|status}"RETVAL=1esacexit $RETVAL

　　

 

$ /etc/init.d/mfsmaster$ /etc/init.d/mfscgiserv start$ chkconfig --add mfsmaster$ chkconfig -add mfscgiserv$ chkconfig mfsmaster on$ chkconfig mfscgiserv on

　　

 

 

#################################################################
2)Metalogger Server install

$ yum install -y wget gcc zlib zlib-devel$ cd /usr/local/src/$ groupadd mfs$ useradd -g mfs mfs -s /sbin/nologin -M$ tar zxvf mfs-1.6.27-1.tar.gz$ cd mfs-1.6.27$ ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfschunkserver --disable-mfsmount$ make$ make install

　　

$ ln -s /usr/local/mfs/sbin/mfsmetalogger /usr/sbin/$ ln -s /usr/local/mfs/etc/mfs /etc/mfs$ cp rpm/rh/mfsmetalogger.init /etc/init.d/mfsmetalogger$ chmod 755 /etc/init.d/mfsmetalogger$ cd /usr/local/mfs/etc/mfs/$ cp mfsmetalogger.cfg.dist mfsmetalogger.cfg$ vi /etc/hosts$ 192.168.55.149 mfsmaster

　　

 

$ vi /etc/init.d/mfsmetalogger #!/bin/bash## Init file for the MooseFS metalogger service## chkconfig: - 92 84## description: MooseFS metalogger## processname: mfsmetalogger# config: @[email protected]/mfsmetalogger.cfg# Source function library.. /etc/init.d/functions# Source networking configuration.. /etc/sysconfig/network# Source initialization configuration.#[ -r "/usr/sbin/mfsmetalogger" ] && . /usr/sbin/mfsmetalogger# Check that networking is up.[ "${NETWORKING}" == "no" ] && exit 0[ -x "/usr/sbin/mfsmetalogger" ] || exit 1RETVAL=0prog="mfsmetalogger"start () {echo -n $"Starting $prog: "daemon $prog $OPTIONSRETVAL=$?echo[ $RETVAL -eq 0 ] && touch /var/lock/subsys/$progreturn $RETVAL}stop () {echo -n $"Stopping $prog: "$prog stop $OPTIONS || killproc $prog >/dev/null 2>&1RETVAL=$?echo[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$progreturn $RETVAL}restart () {stopstart}case "$1" instart)start;;stop)stop;;restart|reload)restart;;condrestart)[ -e /var/lock/subsys/$prog ] && restartRETVAL=$?;;status)status $progRETVAL=$?;;*)echo $"Usage: $0 {start|stop|restart|condrestart|status}"RETVAL=1esacexit $RETVAL

　　

 

$ chkconfig --add mfsmetalogger$ chkconfig mfsmetalogger on

　　

 

############################################################
3) Chunk Server install

$ yum install -y wget gcc zlib zlib-devel$ cd /usr/local/src/$ groupadd mfs$ useradd -g mfs mfs -s /sbin/nologin -M$ tar zxvf mfs-1.6.27-1.tar.gz$ cd mfs-1.6.27$ ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --disable-mfsmaster$ make$ make install

　　

 

$ ln -s /usr/local/mfs/etc/mfs /etc/mfs$ ln -s /usr/local/mfs/sbin/mfschunkserver /usr/sbin/$ cp rpm/rh/mfschunkserver.init /etc/init.d/mfschunkserver$ chmod 755 /etc/init.d/mfschunkserver

　　

$ cd /usr/local/mfs/etc/mfs$ cp mfshdd.cfg.dist mfshdd.cfg$ cp mfschunkserver.cfg.dist mfschunkserver.cfg

　　

 

#建立2個掛載磁碟$ mkdir -p /storage/mfschunks$ dd if=/dev/zero of=/storage/mfschunks/mfschunks1 bs=1024 count=1 seek=$((2*1024*1024-1))$ mkfs -t ext4 /storage/mfschunks/mfschunks1$ mkdir -p /mnt/mfschunks1$ mount -t ext4 -o loop /storage/mfschunks/mfschunks1 /mnt/mfschunks1$ dd if=/dev/zero of=/storage/mfschunks/mfschunks2 bs=1024 count=1 seek=$((2*1024*1024-1))$ mkfs -t ext4 /storage/mfschunks/mfschunks2$ mkdir -p /mnt/mfschunks2$ mount -t ext4 -o loop /storage/mfschunks/mfschunks2 /mnt/mfschunks2$ chown -R mfs:mfs /mnt/mfschunks1 /mnt/mfschunks2

　　

#編輯添加掛載分區$ vi /etc/mfs/mfshdd.cfg# mount points of HDD drives##/mnt/hd1#/mnt/hd2#etc./mnt/mfschunks1/mnt/mfschunks2

　　

#寫入hosts$ vi /etc/hosts192.168.55.148 mfsmaster

　　

 

 

$ vi /etc/init.d/mfschunkserver #!/bin/bash## Init file for the MooseFS chunkserver service## chkconfig: - 93 83## description: MooseFS chunkserver## processname: mfschunkserver# config: @[email protected]/mfschunkserver.cfg# Source function library.. /etc/init.d/functions# Source networking configuration.. /etc/sysconfig/network# Source initialization configuration.#[ -r "/usr/sbin/mfschunkserver" ] && . /usr/sbin/mfschunkserver# Check that networking is up.[ "${NETWORKING}" == "no" ] && exit 0[ -x "/usr/sbin/mfschunkserver" ] || exit 1[ -r "/etc/mfs/mfshdd.cfg" -o -r "/etc/mfs/mfshdd.cfg" ] || exit 1RETVAL=0prog="mfschunkserver"start () {echo -n $"Starting $prog: "daemon $prog $OPTIONSRETVAL=$?echo[ $RETVAL -eq 0 ] && touch /var/lock/subsys/$progreturn $RETVAL}stop () {echo -n $"Stopping $prog: "$prog stop $OPTIONS || killproc $prog >/dev/null 2>&1RETVAL=$?echo[ $RETVAL -eq 0 ] && rm -f /var/lock/subsys/$progreturn $RETVAL}restart () {stopstart}case "$1" instart)start;;stop)stop;;restart|reload)restart;;condrestart)[ -e /var/lock/subsys/$prog ] && restartRETVAL=$?;;status)status $progRETVAL=$?;;*)echo $"Usage: $0 {start|stop|restart|condrestart|status}"RETVAL=1esacexit $RETVAL

　　

$ chkconfig --add mfschunkserver$ chkconfig mfschunkserver on

###############################################
4) Client install

$ yum install -y wget gcc zlib zlib-devel$ cd /usr/local/src/$ groupadd mfs$ useradd -g mfs mfs -s /sbin/nologin -M$ wget http://nchc.dl.sourceforge.net/project/buluoos/0.2/src/fuse-2.8.5.tar.gz$ tar zxvf fuse-2.8.5.tar.gz$ cd fuse-2.8.5$ ./configure$ make && make install$ echo ‘export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$PKG_CONFIG_PATH‘ >>/etc/profile$ source /etc/profile$ lsmod |grep fusefuse 73530 0

　　

#################################
#lsmod |grep fuse #檢查fuse是否載入到核心，若沒有，執行下面命令
#modprobe fuse&&lsmod |grep fuse
###############################

$ cd /usr/local/src/$ tar zxvf mfs-1.6.27-1.tar.gz$ cd mfs-1.6.27$ ./configure --prefix=/usr/local/mfs --with-default-user=mfs --with-default-group=mfs --$ disable-mfsmaster --disable-mfschunkserver$ make$ make install$ ln -s /usr/local/mfs/bin/* /usr/sbin/

　　

$ vi /etc/hosts192.168.1.1 mfsmaster$ mkdir -p /mnt/mfs$ /usr/local/mfs/bin/mfsmount /mnt/mfs -H mfsmaster$ df -hFilesystem Size Used Avail Use% Mounted on/dev/mapper/VolGroup-lv_root 11G 948M 8.8G 10% /tmpfs 116M 0 116M 0% /dev/shm/dev/sda1 485M 32M 428M 7% /bootmfsmaster:9421 6.3G 0 6.3G 0% /mnt/mfs

　　

###TEST###
write:
time dd if=/dev/zero of=/mnt/mfs/test500m bs=1024k count=500
500+0 records in
500+0 records out
524288000 bytes (524 MB) copied, 6.14798 s, 85.3 MB/s

real0m6.173s
user0m0.001s
sys0m0.973s

 

read:
time dd if=/mnt/mfs/test500m of=/dev/null

 

 

[[email protected] mfs]# mkdir 10 #建立目錄

　　

[[email protected] mfs]# mfssetgoal 2 /mnt/mfs/10 #設定副本份數/mnt/mfs/10: 2

　　

[[email protected] mfs]# mfsgetgoal -r /mnt/mfs/10 #查看副本數/mnt/mfs/10:directories with goal 2 : 1

　　

[[email protected] mfs]# mfssettrashtime -r 0 /mnt/mfs/10 #設定延遲刪除時間/mnt/mfs/10:inodes with trashtime changed: 1inodes with trashtime not changed: 0inodes with permission denied: 0

　　

[[email protected] mfs]# mfsgettrashtime -r /mnt/mfs/10 #查看延遲刪除時間/mnt/mfs/10:directories with trashtime 0 : 1

　　

[[email protected] mfs]# mfscheckfile /mnt/mfs/10/d.jpg #查看檔案副本數/mnt/mfs/10/d.jpg:chunks with 2 copies: 1

　　

[[email protected] mfs]# mfsfileinfo /mnt/mfs/10/d.jpg #查看檔案儲存詳細資料/mnt/mfs/10/d.jpg:chunk 0: 00000000000000C5_00000001 / (id:197 ver:1)copy 1: 192.168.55.150:9422copy 2: 192.168.55.151:9422

　　

[[email protected] mfs]# mfsdirinfo /mnt/mfs/10/mnt/mfs/10/:inodes: 2directories: 1files: 1chunks: 1length: 186050size: 201728realsize: 403456

　　

#查看日誌，預設列印在/var/log/messages下面
tail -f /var/log/messages

 

 

#故障切換(來自網路中總結)
(1)metalogger伺服器中需要備份master的兩個設定檔，由於設定檔不是經常變化，因此通過定時指令碼進行檔案同步即可。

(2)當metalogger沒有下載metadata之前，不能使用期接管master。此時metarestore程式會運行失敗。

(3)metarestore程式是根據metalogger中定期下載的metadata和changelog來恢複master掛掉時刻master所記錄的整個mfs的資訊。

metalogger的注意事項

1、部署metalogger的伺服器的open files數目必須大於或者等於5000，

2、metalogger在啟動時不會下載metadata，而是等到第一個下載周期的下載時間點時再去下載，metalogger下載metadata的時間點是每小時的10分30秒，時間間隔是1小時的整數倍。

3、metalogger不是在啟動的時候就取下載metadata，而是必須等到下載時間點(2中所述)才會去下載。也就是說，metalogger為確保正確性，要在啟動後最少一個小時以內master和metalogger都要保持良好的狀態。

Centos 安裝 Moosefs檔案系統