抓取
#!/bin/bash. push.sh# 错误处理:尝试查找备份文件function onError(){ local errFile="err" local serverInfo=($1 $2 $3) # ip username password local logInfo=($4 $5) # srcPath destPath local srcPath=${logInfo[0]} logInfo[0]="disk1/"${srcPath##*/}".gz" #备份文件路径 logInfo[1]=${logInfo[1]}".gz" push ${serverInfo[*]} ${logInfo[*]} $errFile if [ -s $errFile ]; then echo "the file ${logInfo[0]} was not found" fi}# 抓取指定日期指定时间的日志文件function catchLogByHour(){ local errFile="err" local serverInfo=($1 $2 $3) # serverInfo=(ip username password) local logInfo=($4 $5) # logInfo=(srcPath destPath) local endHour=$6 local i=0 while [ $i -le $endHour ]; do hour=$i if [ $hour -lt 10 ]; then hour="0"$hour fi local hourLogInfo hourLogInfo[0]=${logInfo[0]}"."$hour hourLogInfo[1]=${logInfo[1]}"."$hour push ${serverInfo[*]} ${hourLogInfo[*]} $errFile if [ -s $errFile ]; then onError ${serverInfo[*]} ${hourLogInfo[*]} fi i=`expr $[$i+1]` done}# for test# logDate="20131229"# serverInfo=([1]="172.18.11.200" [2]="log" [3]="wappplog")# logInfo=([1]="logs/syslog/project_log."$logDate [2]="./data/project_log."$logDate [3]="2")# catchLogByHour ${serverInfo[*]} ${logInfo[*]}
推送
#!/bin/bashMAX_RE_CONNECT=3SLEEP_TIME=2# 实际推送动作function pushAction(){ local ip=$1 local user=$2 local password=$3 local path=$4 local localPath=$5 local error=$6 :>$error # 清空文件相当于rm,touch命令 export RSYNC_PASSWORD="$password" rsync -Cavz $user@$ip::$path $localPath > $error 2>&1 # 判断是否发送错误 grep "rsync error" $error > $error"_copy" cp $error"_copy" $error rm -f $error"_copy"}# 失败重新尝试，一旦超过最大尝试次数认为无法完成function push(){ local serverInfo=($1 $2 $3) # ip username password local logInfo=($4 $5) # srcPath destPath local error=$6 local i=0 while [ $i -lt $MAX_RE_CONNECT ] do pushAction ${serverInfo[*]} ${logInfo[*]} $error # no error if [ ! -s $error ]; then break fi echo "try again..." sleep $SLEEP_TIME i=`expr $[$i+1]` done}# for test# serverInfo=([1]="172.18.11.201" [2]="log" [3]="wappplog")# fileInfo=([1]="logs/syslog/project_log.20131229.16" [2]="./lishujun.log" [3]="aaa")# push ${serverInfo[*]} ${fileInfo[*]}
main.sh
#! /bin/shcd `dirname $0`. catch.shif [ $# -eq 0 ]; then # 取昨天数据 logDate=`date -d"1 day ago" +'%Y%m%d'` hour=23else # 取指定日期时间 today=`date +'%Y%m%d'` logDate=$1 if [ "$1" == "$today" ]; then # 取当天数据已产生的数据 hour=`date -d "1 hour ago" +"%H"` else # 取历史数据 hour=23 fifimkdir -p "./logs/project/$logDate"serverInfo=([1]="172.18.10.104" [2]="stat" [3]="3gppstat")logInfo=([1]="logs/project/project_log."$logDate [2]="./logs/project/$logDate/project_log."$logDate [3]=$hour)catchLogByHour ${serverInfo[*]} ${logInfo[*]}