巧用shell+rsync实现日志自动过滤处理压缩上传分类(在斯凯的最终版本下周离职)

  1. 云栖社区>
  2. 博客>
  3. 正文

巧用shell+rsync实现日志自动过滤处理压缩上传分类(在斯凯的最终版本下周离职)

天飞.h 2016-04-13 11:04:46 浏览281
展开阅读全文
上次的版本http://blog.chinaunix.net/uid-21142030-id-5613029.html


点击(此处)折叠或打开

  1. [10:58 t ~]$ cat rsync.t
  2. #!/usr/bin/env bash
  3. #set -x
  4. #2015-08-18:    修复直接压缩201目录的bug
  5. #2015-09-28:    新加项目分类目录,新加多重判断
  6. #2015-10-19:    修复与glusterfs冲突问题.
  7. #2016-04-08:    处理磁盘空紧张时无法处理日志压缩上传导致空间满问题。
  8. #2016-04-11:    解决目录非单独挂载时识别为/无限循环问题。
  9. #2016-04-12:    优化搜索逻辑,添加输出日志。

  10. LOGPATH=$1
  11. PROJECT=$2

  12. SLEEPTIME=1s
  13. DATETIME=$(date +%F_%T)
  14. DATE=$(date +%F)
  15. IP=$(LC_ALL=C /sbin/ifconfig|grep "inet addr:"|grep -v "127.0.0.1"|cut -d: -f2|awk '{print $1}'|sed ':a;$!N;s/\n/_/g;ta;P;D')
  16. LINE=`seq -s= 80|sed 's/[0-9]*//g'`

  17. USER=admin
  18. export RSYNC_PASSWORD=skymobi
  19. RSYNCIP=192.168.1.2
  20. RSYNCMODULE=chss-t
  21. RSYNCCONFMODULE=chss-tc

  22. RSYNCDIR=${PROJECT}/"$IP"
  23. FINDLOGFILE=/tmp/find_"$IP".log.$DATE
  24. RSYNCLOGFILE=/tmp/rsync_"$IP".log.$DATE
  25. RSYNCEXELOGFILE=/tmp/rsync.exe.log.$DATE

  26. REGEX="-regex .*logs/\(.*log\)?.*201.*\(.*log\)?\(.gz\)?"
  27. REGEX2="-regex .*tar.gz"
  28. TIME="-mtime +${DAYS:=1}"
  29. DEL="-exec rm -rfv {} ;"
  30. COMPRESSION="-exec tar -zcvf `basename {}`_${RSYNCMODULE}.tar.gz {} --remove-files --exclude=*tar.gz ;"

  31. #创建目录
  32. echo -e "$LINE\n$DATETIME : Create directory for upload."
  33. mkdir -p /tmp/$RSYNCDIR
  34. echo -e "$DATETIME : Upload create directory.\n$LINE"
  35. for rspath in $RSYNCMODULE $RSYNCCONFMODULE
  36. do
  37.     rsync -atv --log-file=${RSYNCLOGFILE} /tmp/${PROJECT} ${USER}@"$RSYNCIP"::${rspath}
  38. done

  39. #扫描日志
  40. for Path in $LOGPATH
  41. do
  42.   echo -e "$LINE \n$DATETIME : Operation on the [ $Path ] directory "
  43.   #1.是否存在目录
  44.   if [ -d /$Path ];then
  45. #########################################
  46.     RSYNC(){
  47.       if test $# -ne 0;then
  48.         rsync -aERtvz --no-p --chmod=Do+rX,Fo-X --log-file=${RSYNCLOGFILE} --remove-source-files --skip-compress=*.gz "$@" ${USER}@"$RSYNCIP"::${ttype:=$RSYNCMODULE}/${RSYNCDIR}
  49.       else
  50.         echo "$DATETIME : no $RSYNCMODULE.tar.gz file found."
  51.         return 1
  52.       fi
  53.     }
  54.     FINDER(){
  55.       find -L /${Path} $@
  56.     }
  57.     SPACECOUNT(){
  58.     df -P|grep /${Path} |tail -n 1 |awk '{print $5}'|awk -F% '{print $1}'
  59.     }
  60. ##########################################


  61.     #2.是否空间紧张
  62.     echo "$DATETIME : Start while."
  63.     starttime=`date`
  64.     SPACE=`SPACECOUNT`
  65.     Count=0
  66.     while test -n "$SPACE" -a "${SPACE:=0}" -gt 20 -a $Count -lt 2
  67.     do
  68.         
  69.         
  70.       if [ $SPACE -gt 80 ];then
  71.     echo "$DATETIME : Start del."
  72.         TIME="-mmin +720"
  73.         FINDER "$REGEX" "-a" "$TIME" "$DEL" >> $FINDLOGFILE
  74.         SPACE=`SPACECOUNT`
  75.         continue
  76.       elif [ $SPACE -gt 60 ];then
  77.     echo "$DATETIME : Start upload file direct."
  78.         echo '60%
  79.         TIME="-mmin +60"
  80.         RSYNC `FINDER "$REGEX" "-a" "$TIME"`
  81.         SPACE=`SPACECOUNT`
  82.         ((Count+=1))
  83.         continue
  84.       #2.
  85.       else
  86.     echo "$DATETIME : Start compression."
  87.         echo 'space
  88.         TIME="-mmin +0"
  89.         FINDER "$REGEX" "-a ! $REGEX2" "-a" "$TIME" "$COMPRESSION" >> $FINDLOGFILE
  90.         SPACE=`SPACECOUNT`
  91.     echo "new space is $SPACE"
  92.         break
  93.       fi
  94.     done

  95.     endtime=`date`
  96.     haldertime=$((`date -d "$endtime" +%s`-`date -d "$starttime" +%s`))
  97.     echo "haldertime:$haldertime"
  98.     #3.执行上传操作
  99.     echo "$DATETIME : Start upload tar.gz file."
  100.     starttime=`date`
  101. # for ttype in ${RSYNCMODULE} #${RSYNCCONFMODULE}
  102. # do
  103.           RSYNC $(find /$Path -type f -name "*_${RSYNCMODULE}.tar.gz" -a ! -regex '.*/brick/.*')
  104.           #RSYNC $(find /$Path -type f -name "*_$ttype.tar.gz" -a ! -regex '.*/brick/.*')
  105. # done
  106.     endtime=`date`
  107.     haldertime=$((`date -d "$endtime" +%s`-`date -d "$starttime" +%s`))
  108.     echo "upload_tar_gz_haldertime:$haldertime"
  109.   #1.
  110.   else
  111.     echo /${Path} does not exist...
  112.   fi
  113.   echo -e "$DATETIME : [ $Path ] To the end of the directory operation. \n$LINE"
  114.   #1.是否存在目录
  115. done >>$RSYNCEXELOGFILE


点击(此处)折叠或打开

  1. [10:58 t ~]$ cat rsync.t
  2. #!/usr/bin/env bash
  3. #set -x
  4. #2015-08-18:    修复直接压缩201目录的bug
  5. #2015-09-28:    新加项目分类目录,新加多重判断
  6. #2015-10-19:    修复与glusterfs冲突问题.
  7. #2016-04-08:    处理磁盘空紧张时无法处理日志压缩上传导致空间满问题。
  8. #2016-04-11:    解决目录非单独挂载时识别为/无限循环问题。
  9. #2016-04-12:    优化搜索逻辑,添加输出日志。

  10. LOGPATH=$1
  11. PROJECT=$2

  12. SLEEPTIME=1s
  13. DATETIME=$(date +%F_%T)
  14. DATE=$(date +%F)
  15. IP=$(LC_ALL=C /sbin/ifconfig|grep "inet addr:"|grep -v "127.0.0.1"|cut -d: -f2|awk '{print $1}'|sed ':a;$!N;s/\n/_/g;ta;P;D')
  16. LINE=`seq -s= 80|sed 's/[0-9]*//g'`

  17. USER=admin
  18. export RSYNC_PASSWORD=skymobi
  19. RSYNCIP=192.168.165.88
  20. RSYNCMODULE=chss-t
  21. RSYNCCONFMODULE=chss-tc

  22. RSYNCDIR=${PROJECT}/"$IP"
  23. FINDLOGFILE=/tmp/find_"$IP".log.$DATE
  24. RSYNCLOGFILE=/tmp/rsync_"$IP".log.$DATE
  25. RSYNCEXELOGFILE=/tmp/rsync.exe.log.$DATE

  26. REGEX="-regex .*logs/\(.*log\)?.*201.*\(.*log\)?\(.gz\)?"
  27. REGEX2="-regex .*tar.gz"
  28. TIME="-mtime +${DAYS:=1}"
  29. DEL="-exec rm -rfv {} ;"
  30. COMPRESSION="-exec tar -zcvf `basename {}`_${RSYNCMODULE}.tar.gz {} --remove-files --exclude=*tar.gz ;"

  31. #创建目录
  32. echo -e "$LINE\n$DATETIME : Create directory for upload."
  33. mkdir -p /tmp/$RSYNCDIR
  34. echo -e "$DATETIME : Upload create directory.\n$LINE"
  35. for rspath in $RSYNCMODULE $RSYNCCONFMODULE
  36. do
  37.     rsync -atv --log-file=${RSYNCLOGFILE} /tmp/${PROJECT} ${USER}@"$RSYNCIP"::${rspath}
  38. done

  39. #扫描日志
  40. for Path in $LOGPATH
  41. do
  42.   echo -e "$LINE \n$DATETIME : Operation on the [ $Path ] directory "
  43.   #1.是否存在目录
  44.   if [ -d /$Path ];then
  45. #########################################
  46.     RSYNC(){
  47.       if test $# -ne 0;then
  48.         rsync -aERtvz --no-p --chmod=Do+rX,Fo-X --log-file=${RSYNCLOGFILE} --remove-source-files --skip-compress=*.gz "$@" ${USER}@"$RSYNCIP"::${ttype:=$RSYNCMODULE}/${RSYNCDIR}
  49.       else
  50.         echo "$DATETIME : no $RSYNCMODULE.tar.gz file found."
  51.         return 1
  52.       fi
  53.     }
  54.     FINDER(){
  55.       find -L /${Path} $@
  56.     }
  57.     SPACECOUNT(){
  58.     df -P|grep /${Path} |tail -n 1 |awk '{print $5}'|awk -F% '{print $1}'
  59.     }
  60. ##########################################


  61.     #2.是否空间紧张
  62.     echo "$DATETIME : Start while."
  63.     starttime=`date`
  64.     SPACE=`SPACECOUNT`
  65.     Count=0
  66.     while test -n "$SPACE" -a "${SPACE:=0}" -gt 20 -a $Count -lt 2
  67.     do
  68.         
  69.         
  70.       if [ $SPACE -gt 80 ];then
  71.     echo "$DATETIME : Start del."
  72.         TIME="-mmin +720"
  73.         FINDER "$REGEX" "-a" "$TIME" "$DEL" >> $FINDLOGFILE
  74.         SPACE=`SPACECOUNT`
  75.         continue
  76.       elif [ $SPACE -gt 60 ];then
  77.     echo "$DATETIME : Start upload file direct."
  78.         echo '60%
  79.         TIME="-mmin +60"
  80.         RSYNC `FINDER "$REGEX" "-a" "$TIME"`
  81.         SPACE=`SPACECOUNT`
  82.         ((Count+=1))
  83.         continue
  84.       #2.
  85.       else
  86.     echo "$DATETIME : Start compression."
  87.         echo 'space
  88.         TIME="-mmin +0"
  89.         FINDER "$REGEX" "-a ! $REGEX2" "-a" "$TIME" "$COMPRESSION" >> $FINDLOGFILE
  90.         SPACE=`SPACECOUNT`
  91.     echo "new space is $SPACE"
  92.         break
  93.       fi
  94.     done

  95.     endtime=`date`
  96.     haldertime=$((`date -d "$endtime" +%s`-`date -d "$starttime" +%s`))
  97.     echo "haldertime:$haldertime"
  98.     #3.执行上传操作
  99.     echo "$DATETIME : Start upload tar.gz file."
  100.     starttime=`date`
  101. # for ttype in ${RSYNCMODULE} #${RSYNCCONFMODULE}
  102. # do
  103.           RSYNC $(find /$Path -type f -name "*_${RSYNCMODULE}.tar.gz" -a ! -regex '.*/brick/.*')
  104.           #RSYNC $(find /$Path -type f -name "*_$ttype.tar.gz" -a ! -regex '.*/brick/.*')
  105. # done
  106.     endtime=`date`
  107.     haldertime=$((`date -d "$endtime" +%s`-`date -d "$starttime" +%s`))
  108.     echo "upload_tar_gz_haldertime:$haldertime"
  109.   #1.
  110.   else
  111.     echo /${Path} does not exist...
  112.   fi
  113.   echo -e "$DATETIME : [ $Path ] To the end of the directory operation. \n$LINE"
  114.   #1.是否存在目录
  115. done >>$RSYNCEXELOGFILE

点击(此处)折叠或打开

  1. [11:00 t ~]$ cat while.t
  2. #!/bin/bash
  3. set -x
  4. LOGPATH='opt app data app1'
  5. PROJECT=talentest
  6. sleeptime=0.5h

  7. while :
  8. do
  9. time /bin/bash /bin/rsync.t "$LOGPATH" $PROJECT
  10. sleep $sleeptime
  11. done


网友评论

登录后评论
0/500
评论
天飞.h
+ 关注