查找错误日志脚本

需求：

需要定期查找各个微服务Warnlog日志的ERROR，给到开发分析，已知的错误可以忽略，只找新出现的，如果是bug的就让开发排期解决，。

 

思路：

1、sync同步各个微服务上Warnlog日志到一台服务器上

2、将同一服务的不同节点日志，合并成一个日志文件

3、grep -v 排除已知可以忽略的错误信息

 

合并同步过来的日志

merge_warnlog.sh

[appdeploy@DeployServer sync-log]$ cat merge_warnlog.sh 
#!/bin/bash
 
read -p "Please input the time interval from now [default:720 min]:" t
 
logfiles=`find /app/sync-log/WarnLog/ -type f -mmin -${t}|sort`
for n in $logfiles
do
   echo $n
done
 
 
echo "###############过滤掉没有ERROR的文件###########################################"
 
>file2
 
for n2 in $logfiles
do
     res=`grep -w ERROR $n2|grep -v msg|wc -l`
     if [ $res != 0 ];then
         echo $n2|tee -a file2
     fi
done
 
 
echo "###############合并同一个服务的文件###########################################"
 
x=`awk -F "/" '{gsub(/[1-9]+/,"",$5);print $5}' file2|uniq`
for i in $x
do
  result=`echo $i | grep "_"|wc -l`
  ser_log=/app/sync-log/log/$i
  if [[ $result -eq 0 ]];then
        echo "$i服务"
        h=`grep $i file2|grep -v _`
  else
        echo "$i服务"
        h=`grep $i file2`
  fi
 
  for m in $h
   do
     echo "  " >>$ser_log
     echo "$i  " >>$ser_log
     echo "$m"|tee -a $ser_log
     echo "  " >>$ser_log
     cat -n $m >>$ser_log
   done
 
done
 
mv log/ana-data log/data

　　

然后，排除已知和可以忽略的错误，减小排除范围。

#!/bin/bash
 
 
activity='"wwwww"'
data='"DuplicateKeyException\|dispatcherServlet"'
groupapi='"wwwww"'
warehouse='"wwwwww"'
groupsub='"qwer\|福利派发抛出异常"'
base='"默认私钥解密\|获取网络图片资源异常\|用AES256解密"'
groupadminapi='"msg\|导入收货人快递信息\|查询产品失败\|/supplyProduct/modIntroduce\|/mallRefund/agreeRefund\|/mall/product/bind"'
group_fill_order='"创建补货需求出错\|该商品条码对应多种商品"'
member='"messageShelfReminderDTO\|JSONException\|getWxUserInfo\|密码解密失败\|超时\|补交款订单已付款\|积分兑换扣除积分出错\|AsyncMessageTaskService\|添加积分返回值"'
openapi='"请求超时\|/execute\|/msg/customer/reply\|FeignException: status 500\|/wxNotify/notifyContractPay\|/payBack/notify/WeiXinPayJSAPI/3\|/error"'
order='"创建补货需求出错\|统一下单出错\|该商品条码对应多种商品\|申请电子发票用户信息异常\|推送订单失败\|获取招行公钥异常\|支付回调失败"'
product='"ProductFillOrderItemExtendPO"'
shelf='"未找到对应货道信息"'
group_pay='"wwwwwww"'
group_product='"qwer"'
shopapi='"/product/getProduct"'
webadmin='"BaseController.java:188\|用户手机号已经存在\|银行业务不支持\|查询商品库\|查询仓库信息失败\|用户名称已经存在\|QiniuUpload\|ExceptionHandlerExceptionResolver.java:324\|ShelfInspectionTaskController.java:559\|数据量超过\|银行卡号有误\|BaseController.java:187\|此用户丰声CODE已经存在\|查询盘点记录明细出错"'
wxadminapi='"msg\|/shelf/logisticsTaskExecutor/uploadBackStockPic\|/myShelf/orderSign\|/user/wxUserinfo\|/myShelf/submitCheckProduct\|/prewarehouse/goodsTransfer/getPrewarehouseDetailIntersection\|/prewarehouse/goodsTransfer/listPrewarehouseGoodsTransferFillOrder\|/user/getBankCardInfo\|/user/getIdentityCardInfo\|/shelfManager/fillOrder/signShelfFillOrderFromPreWarehouse\|/shelfSaleStatistics/daySaleStatistics\|/myShelf/activate\|/myShelf/queryMyShelf\|/prewarehouse/shelfManagerApplyOrder/getShelfPrewarehouseProductSummary\|/shelf/goodsTransferShelf/signGoodsTransferOrder"'
 
wxappapi='"msg\|/shelf/scanCode\|errMsg\|/foodCard/getBalance\|/payBack/notify/WeiXinPayJSAPI/goodsOrder\|/user/getUserBeanByThree\|优惠券\|根据货架随机码\|货架编码无效\|coupon/getUserCouponMsgNew\|qn-cdn-pic.feng2.com\|wxUserInfo\|/user/checkIn\|assembleShelfData\|ouponRecordDTO转gourmetFestivalVO失败\|/coupon/getCouponCentreCoupons\|/wx/getWxJsapiSignature\|/payBack/notify/cmbPay/goodsOrder\|/shelf/apply\|/copywriting/getCopywriting\|/orderPayment/cancel"'
 
 
 
 
 
 
 
filerlog()
{
for i in `seq 1 20`
do
    echo -e "\e[40;35m 请输入想要过滤的值,或者按q退出 \e[40;37m"
    read -p  "$i:Please input filter value :" k
    if [ -z "${k}" ] ; then
        continue
    elif [ ${k} = "q" ] || [ ${k} = "Q" ] ; then
        break
    elif [ ${k} = "a" ] || [ ${k} = "A" ] ; then
        echo -e "\e[40;35m 请输入错误之后的多少行 \e[40;37m"
        read -p  "Please input after the few lines:" time line
        grep -w $time -A ${line:=7} log/$logfile
    else
        gre_e="$gre_e $gre_v $k"
        echo -e "\033[34m $gre_e  \033[0m"
        eval $gre_e
    fi
done
}
 
 
 
 
exit_log()
{
        read -n1 -p "press Key \"q\" to exit or any other Key to continue:" k
        echo ""
        if [ -z "${k}" ] ; then
            continue
        elif [ ${k} = "q" ] || [ ${k} = "Q" ] ; then
            break
        else
            continue
        fi
}
 
 
 
while :
do
    gre_e="grep -w ERROR"
    gre_v="|grep -v"
    select n in `ls log/`
      do
          logfile=$n
          eval pai=$(echo \$$logfile)
          gre_e="$gre_e log/$logfile $gre_v $pai"
          eval $gre_e
 
          filerlog
 
       break
    done
 
    exit_log
done

　

 

 

　

 收集分析访问日志，插入mysql脚本

[appdeploy@DeployServer scripts]$ cat fetch_api_data.sh
#!/bin/bash


lists=`grep api /app/deploy/scripts/distribute.cnf|grep -v -E "#|gray"|cut -d \| -f 3,8`

for p in $lists
do
    host_ip=`echo $p|cut -d \| -f 1`
    deploy_path=`echo $p|cut -d \| -f 2`
    echo $host_ip  $deploy_path
    
    ansible ${host_ip} -m copy -a "src=/app/deploy/scripts/api_perf_analyze.sh dest=${deploy_path} mode=0755"  -i /app/deploy/scripts/hosts
    ansible ${host_ip} -m shell -a "chdir=$deploy_path sh ${deploy_path}/api_perf_analyze.sh"  -i /app/deploy/scripts/hosts
    sleep 60s    
done

　

api_perf_analyze.sh

[appdeploy@DeployServer scripts]$ cat api_perf_analyze.sh
#!/bin/bash

access_file="access.`date -d "-1 day" +"%Y-%m-%d"`.log"

cat logs/${access_file}|grep -v HEAD |awk \
'
{u=split($8,arr,"?");uri=arr[1];
 m=substr($2,1,5);
 muri=m","uri;
if(max[muri] + 0 < $12 +0 )
  {max[muri]=$12};
if(min[muri] + 0 == 0 || min[muri] + 0 > $12)
  {min[muri] = $12} ;
num[muri]+=1;
sum[muri]+=$12}
END{for(i in sum) printf "%d,%d,%d,%.2f,%-s\n",num[i],min[i],max[i],sum[i]/num[i],i}'|sort -t ',' -k 5 |tee /tmp/t_api_perf_analyze.out

MYSQL="mysql -ufemonitor  -h10.200.130.4 -pMpriv#7821@thi  --default-character-set=utf8 -A -N"

log_file="`pwd`/logs/"
service_ip=`/sbin/ip addr|grep inet|grep -v 127.0.0.1|awk -F "/" '{print $1}'|awk '{print $2}'`
service_name=`/bin/ls *.jar|head -n 1|cut -d - -f 1`

while read line;
do
   t_time=`echo $line|awk -F ',' '{print $5}'`
   access_time="`echo $access_file|awk -F '.' '{print $2}'` $t_time"
   uri=`echo $line|awk -F ',' '{print $6}'`
   access_num=`echo $line|awk -F ',' '{print $1}'`
   max_resp=`echo $line|awk -F ',' '{print $3}'`
   min_resp=`echo $line|awk -F ',' '{print $2}'`
   avg_resp=`echo $line|awk -F ',' '{print $4}'`

   sql="
   insert into t_api_perf_analyze (access_time,service_name,uri,access_num,max_resp,min_resp,avg_resp,service_ip,log_file) values('$access_time','$service_name','$uri','$access_num','$max_resp','$min_resp','$avg_resp','$service_ip','$log_file');
 "
#   echo $sql
#   echo "################"
   $MYSQL fe_monitor -e "$sql"
done</tmp/t_api_perf_analyze.out