逐行扫描的日志收集LogCollector

#! /bin/sh
# The script is used for test log collection.
    #usage step:
    #1 configure db's info. (./logcollect.sh db)
    #2 run script to copy DB log to current server
    #3 start to collect log for each feature/testcase


#used for copy db log to current server
dbIP="192.168.107.35"
dbNodeUser="admin"
dbPassword="!QAZ2wsx3edc4rfv"
dbLog="/var/postgres/log/postgresql*.csv"

#cp application log
cpAccessLog="/var/log/access.log"
cpEventLog="/var/event.log"
cpPerformanceLog="/var/log/performance.log"

#up application log
upSessionLog="/var/log/session.log"
upTrafficLog="/var/log/traffic.log"
upMonitorLog="/var/log/event.log"

#copy db log to here for collecting
dbTmpLog="/tmp/dbLog"

#vcs log
vcsLog="/var/VRTSvcs/log/engine_A.log"

#system log
messageLog="/var/log/messages"
cronLog="/var/log/cron"
secureLog="/var/log/secure"

CPAppLog=($cpAccessLog $cpEventLog $cpPerformanceLog )
UPAppLog=($upSessionLog $upTrafficLog )
sysLog=($messageLog $cronLog $secureLog)

ARRAY=
CURRENT_LINE=0
START_TIME=
STOP_TIME=
TIME=0

createDirectory()
{
    logDirectory=$1"-"`date +%Y%m%d%H%M`
    mkdir $logDirectory
    if [ $? -ne 0 ]; then
        echo "create log directory $logDirectory failed, exit!"
        exit
    else
        echo "the logs are stored in ./$logDirectory"
    fi
}

timeConvert()
{
    tmpTIME=`date -d "$1" +%s 2>/dev/null`
    if [[ ! -z tmpTIME ]] ; then
        TIME=$tmpTIME
    fi
}

timeConvert "$3" 
target_start_time=$TIME
timeConvert "$4"
target_stop_time=$TIME

getTime()
{
        logFileName=`echo $2|awk -F '/' '{print $NF}'`

        echo "############## reading $2 #############################"
        tmp_start_time=0
        tmp_stop_time=
        FOUND_START_TIME=0
        LETTER="[a-z\|A-Z]"

        while read LINE
        do
            if [ -z "$LINE" ] || [ "${LINE:0:1}" == "#" ] ; then
                continue
            fi
            time_string=`expr substr "$LINE" 1 $1`
            #echo $time_string
            #if [[ $time_string =~ $LETTER ]]; then
                        #        continue
                        #fi
            timeConvert "$time_string"
            time_in_line=$TIME
            if [ $FOUND_START_TIME -eq 0 ];then

                if [ $time_in_line -lt $target_start_time ]; then
                    continue
                elif [ $time_in_line -eq $target_start_time ]; then
                    FOUND_START_TIME=1
                    tmp_start_time=`expr substr "$LINE" 1 23`
                elif [ $time_in_line -gt $target_start_time ] && [ $time_in_line -le $target_stop_time ]; then
                    FOUND_START_TIME=1
                    tmp_start_time=`expr substr "$LINE" 1 23`
                elif [ -z $time_in_line ]; then
                    continue
                else 
                    #echo "can't find log in "$2" for the required time perid"
                    break 1
                fi
            else
                if [ $time_in_line -eq $target_stop_time ]; then
                    tmp_stop_time=`expr substr "$LINE" 1 23`
                elif [ $time_in_line -gt $target_stop_time ]; then
                    tmp_stop_time=`expr substr "$LINE" 1 23`
                    break 1
                elif [ -z $time_in_line ]; then
                                        continue
                fi
            fi
        done <$2

        if [ $FOUND_START_TIME -eq 1 ];then
            sed -n "/^$tmp_start_time/,/^$tmp_stop_time/p" $2 >> $logDirectory"/"$logFileName
            echo "Found required log in $2"
        fi

}

copyDBLog()
{
    rm -rf "$dbTmpLog"
    mkdir -p "$dbTmpLog"

    expect -c "
    spawn scp  -o StrictHostKeyChecking=no  $dbNodeUser@$dbIP:$dbLog $dbTmpLog

    expect \"assword\"
    send \"$dbPassword\r\"
    expect eof
    "
}

case "$1" in
    CP| cp| MDFCP| mdfcp) 
    createDirectory $2
    for i in ${CPAppLog[@]}
    do
        getTime 19 $i
    done    
    for i in `ls "$dbTmpLog"/*""`
    do
        getTime 19 $i
    done
        for i in ${sysLog[@]}
    do
        getTime 15 $i
    done 
    ;;


    UP| up| MDFUP| mdfup)
    createDirectory $2

        for i in ${UPAppLog[@]}
        do
                getTime 19 $i
        done

        for i in ${sysLog[@]}
        do
                getTime 15 $i
    done
        ;;


    DB| db| MDFDB| mdfdb)
    echo "copy DB log to current server $dbTmpLog"
    copyDBLog
        ;;


    *)
            echo "        Usage: $0 NodeType caseNumber StartTime StopTime"
            echo "        Example: $0 cp createDeliverySession \"2017-09-06 11:26:06\" \"2017-09-06 11:36:06\" "
        echo "        NOTE: please execute command '$0 db' first to copy DB log to current server before collecting CP/UP log, please configure DB ip/usr/passwd in this script."
esac