Thinking and implementation of disk script in large Data environment

Demand:

Consider a scenario, such as the disk required for storage nodes (Datanode) in HDFs, when the cluster size becomes larger, assuming that each node has 20 disks, the cluster has 200 nodes, then the actual physical disk is 4000; Assuming the disk year failure rate is 1%, Can we assume that 1 disk failures are required to be replaced or repaired every day?

Now, in order to liberate productivity, the specific requirement is that when a disk failure is discovered, only physical disks need to be replaced, and system-level operations are fully automated.

Some questions to think about:

1): Delete Its information to write Fstab failure disk information

2): The new disk is not required to partition the direct format of

3): Failure disk if repaired, and put back, if the disk data is also retained, it must not be formatted

4): Cannot operate on a disk with partitions

5): Can not be mounted on the disk operation (consider the failure of the disk hot plug)

6): Assuming that the total number of disks is indeterminate, the number of mount points should correspond to the number of disks

7): Can not be used for data can be formatted operation

8): The prerequisite for disk operation must be that the disk is not partitioned, there is no data

9): Mount operation on disks that are formatted but not mounted and compliant

More Wonderful content: http://www.bianceng.cnhttp://www.bianceng.cn/database/storage/

10): In the case of data security, please add ...

System environment: CentOS 6.3 x86_64

The shell code is as follows:

#!/bin/bash
# Script Name:check_disk.sh
# Date & time:2013-07-08/21:06:08
# Description:
# Author:mos
Mlog=${mlog:-/var/log/check_disk.log}
Date= "/bin/date +%k:%m:%s/%y-%m-%d"
Nul=/dev/null
# The parent directory of the Mount point, which is automatically created
Sdir=/mpoint
# UMT function: Uninstall Invalid disk
UMT () {
ut= (' grep ' $SDIR '/etc/fstab|awk ' {print $} ')
ud= (' grep ' $SDIR "/etc/fstab|awk-f\" ' {print $} ')
For d in ${ut[@]};d o
If [-D $d];then
#Dk = ' df $d |awk ' {print $} ' |/usr/bin/tail-1 '
dk= ' mount|grep ' $d \> ' |awk ' {print '} '
Ud= '/sbin/blkid $Dk |awk-f\ ' ' {print $} '
Td= ' grep $d/etc/fstab|awk-f\ ' ' {print $} '
[-B "$Dk"] && uft= '/sbin/parted $Dk Print|/usr/bin/tail-2|/usr/bin/head-1|awk ' {print $} '
tft= ' grep $d/etc/fstab|awk ' {print $} '
if [[$Ud!= $Td | | $Uft!= $Tft]];then
/bin/umount $d &> $NUL
/bin/sed-i "/$d/d"/etc/fstab &> $NUL
Fi
Fi
Done
For d in ${ud[@]};d o
Unull= '/sbin/blkid|grep $d '
if [[Z $Unull]];then
mpit= ' grep $d/etc/fstab|awk ' {print $} '
/bin/umount $Mpit &> $NUL
/bin/sed-i "/$d/d"/etc/fstab &> $NUL
Fi
Done
}
# MMU Function: mount a valid disk
Mmu () {
Declare-i P=1
Tb=/etc/fstab
FTB () {
sft= '/sbin/blkid $Fph |awk-f\ ' ' {print $} '
ft= ' grep ' \<disk$p\> ' $Tb '
Gud= ' grep ' \< $Sft \> ' $Tb '
[[n $Ft]] &&/bin/sed-i "/\<disk$p\>/" D $Tb
[[n $Gud]] &&/bin/sed-i "/\< $Sft \>/" D $Tb
}
Add () {
ef= '/sbin/blkid $Fph '
[[Z $Ef]] &&/bin/echo y|/sbin/mkfs.ext4 $Fph &> $NUL
Uuid= '/sbin/blkid-s UUID $Fph |awk ' {print $} '
Fs= '/sbin/blkid-s TYPE $Fph |awk-f\ ' ' {print $} '
Fr= "Defaults,noatime"
Echo-en "$Uuid \ T" $SDIR "/disk$p\t$fs\t$fr\t0 0\n" >> $Tb
/bin/mount-a &> $NUL
Break
}
Fe () {
mtp= ' Mount|grep ', ' $SDIR '/disk$p\> '
if [[-N ' $Mtp ']];then
echo "' $Date ' $Mp is used." >> $Mlog
Else
Ftb
Add
Fi
}
While:;d o
if [[d ' $SDIR/disk$p]];then
Fe
Else
/bin/mkdir-p "$SDIR"/disk$p && FTB
Add
Fi
                                                                                                    
P=p+1
Done
}
# Chech_disk function: Distinguish the validity of the disk and call the corresponding function processing
Check_disk () {
[-x/sbin/partprobe] &&/sbin/partprobe &> $NUL
Umt
/bin/mount-a &> $NUL
For b in ${exist[@]};d o
Fph= "$Dph $b"
[[!-B "$Fph"]] && echo "' $Date ' $Fph is isn't block device!!!" >> $Mlog && continue
cte= ' ls ' $Fph ' *|/usr/bin/wc-l '
[["$CTE"-ne 1]] && echo "' $Date ' $Fph mismatches." >> $Mlog && continue
Dfm= ' Mount|awk ' {print $} ' |grep ' ^ $Fph $ ' 2> $NUL '
[[-N ' $DFM]] && echo "' $Date ' $Fph is used." >> $Mlog && continue
Ugd= '/sbin/parted $Fph print|/usr/bin/wc-l '
[["$UGD"-ge 8 &&-n "$DFM"]] && echo "' $Date ' parted $Fph >= 8." >> $Mlog && continue
Lvm= '/sbin/blkid $Fph |grep-e ' Lvm2_member|swap '
[[-N ' $LVM]] && echo ' $Date ' $Fph is LVM or swap. >> $Mlog && Continue
Fgd= '/sbin/fdisk-l $Fph |/usr/bin/wc-l '
[["$FGD"-ge]] && echo "' $Date ' fdisk-l $Fph >=10." >> $Mlog && continue
Mmu
Done
}
if [["$" = "s" &&-Z "$"]];then
if [[-b/dev/cciss/c0d0]];then
Ah=hp
dph=/dev/cciss/
exist= (' Ls/dev/cciss/|grep "^\<c0d[0-9][0-9]\{0,1\}\>")
Check_disk
Else
exist= (' Ls/dev/|grep-e "^\<sd[a-z]{0,1}\>|^\dph=/dev/
Check_disk
Fi
Fi

Use remember to add-S, Format:./check_disk.sh-s, the script can be directly attached to the download.

It's not easy to write.

This article from the "Self-Improvement" blog, please be sure to retain this source http://mos1989.blog.51cto.com/4226977/1243650