1 #!bin/bash
2
3 DEPLOY_PATH=$(cd "$(dirname "$0")"; pwd)
4 INI_FILE=$DEPLOY_PATH'/cnedmp_etl.ini'
5 . $INI_FILE
6 BASE_PATH=$(cd "$(dirname "$DEPLOY_PATH")"; pwd)
7 MAIN_PATH=$(cd "$(dirname "$BASE_PATH")"; pwd)
8 CURRRENT_DATE=$(date "+%Y-%m-%d")
9 CURRENT_TIME=`date -u +\%Y\%m\%d`
10 JAR_FILE=$DEPLOY_PATH'/dailydata-1.0-SNAPSHOT.jar'
11 LOG_PATH=$BASE_PATH'/log/'
12 LOAD_CHECK_LOG=$BASE_PATH'/log/Load_Check.'$CURRENT_TIME'.log'
13 PASSWORD=`yarn jar $JAR_FILE com.hypers.etl.file.specialFileProcess.BlowFish`
14
15 logging(){
16 logTime=`date "+%y/%m/%d %T"`
17 echo "$logTime $1 $2" >> $LOAD_CHECK_LOG
18 }
19
20 #Check whether this is the first time running
21 test -e $LOAD_CHECK_LOG || touch $LOAD_CHECK_LOG
22
23 #mysql parameters
24 mysql_user=$USERNAME
25 mysql_password=$PASSWORD
26 mysql_host=$HOSTNAME
27 mysql_port=$PORT
28 mysql_database=$DBNAME
29
30 #Get WARNING LIST
31 sql="select File_Name,Transfer_Status,Load_Status,Job_Status \
32 from $JOBNAME \
33 where File_Date='$CURRRENT_DATE' \
34 and Hive_Partition != 'none' \
35 and (Transfer_Status!=2 or Load_Status!=2 or Job_Status!=2)
36 and Load_Status!=3
37 and Job_Status!=3;"
38
39 WARN_list=$(printf "%s\n" \
40 "[client]" \
41 "user=${mysql_user}" \
42 "password=${mysql_password}" \
43 "host=${mysql_host}" \
44 "port=${mysql_port}" \
45 "database=${mysql_database}" \
46 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}")
47
48 #Use to store email cotent
49 #email_message="File_Name, Transfer_Status, Load_Status, Job_Status"
50
51 while read line
52 do
53 test -z "${line}"
54 if [ $? -eq 0 ]
55 then
56 continue
57 fi
58 exist=`cat $LOAD_CHECK_LOG | grep "Job doesn't finish." | grep "$line"`
59 test -z "${exist}"
60 if [ $? -eq 0 ]
61 then
62 # echo $line >> $LOAD_CHECK_LOG
63 # email_message=$email_message"\\n"$line
64 logging WARN "Job doesn't finish. | Status: File_Name, Transfer_Status, Load_Status, Job_Status | $line"
65 fi
66 done <<EOF
67 $WARN_list
68 EOF
69
70 #Get ERROR LIST
71 sql="select File_Name,Transfer_Status,Load_Status,Job_Status \
72 from $JOBNAME \
73 where File_Date='$CURRRENT_DATE' \
74 and (Load_Status=3 or Job_Status=3);"
75
76 ERROR_list=$(printf "%s\n" \
77 "[client]" \
78 "user=${mysql_user}" \
79 "password=${mysql_password}" \
80 "host=${mysql_host}" \
81 "port=${mysql_port}" \
82 "database=${mysql_database}" \
83 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}")
84
85 while read line
86 do
87 test -z "${line}"
88 if [ $? -eq 0 ]
89 then
90 continue
91 fi
92 exist=`cat $LOAD_CHECK_LOG | grep "Job run ERROR." | grep "$line"`
93 test -z "${exist}"
94 if [ $? -eq 0 ]
95 then
96 # echo $line >> $LOAD_CHECK_LOG
97 # email_message=$email_message"\\n"$line
98 logging FATAL "Job run ERROR. | Status: File_Name, Transfer_Status, Load_Status, Job_Status | $line"
99 fi
100 done <<EOF
101 $ERROR_list
102 EOF
103
104 #Get Row Fail list
105 sql="select File_Name,Transfer_Status,Load_Status,Job_Status, Rows_In_Total, Rows_Succeed, Rows_Fail \
106 from $JOBNAME \
107 where File_Date='$CURRRENT_DATE' \
108 and Rows_Fail!=0 \
109 and Transfer_Status=2 and Load_Status=2 and Job_Status=2;"
110
111 RF_list=$(printf "%s\n" \
112 "[client]" \
113 "user=${mysql_user}" \
114 "password=${mysql_password}" \
115 "host=${mysql_host}" \
116 "port=${mysql_port}" \
117 "database=${mysql_database}" \
118 | HOME="/sys" mysql --defaults-file=/dev/stdin -s -e "${sql}")
119
120 while read line
121 do
122 test -z "${line}"
123 if [ $? -eq 0 ]
124 then
125 continue
126 fi
127 exist=`cat $LOAD_CHECK_LOG | grep "Job finished with Row Fail." | grep "$line"`
128 test -z "${exist}"
129 if [ $? -eq 0 ]
130 then
131 # echo $line >> $LOAD_CHECK_LOG
132 # email_message=$email_message"\\n"$line
133 logging WARN "Job finished with Row Fail. | Status: File_Name, Transfer_Status, Load_Status, Job_Status, Total Row, Succeed Row, Fail Row | $line"
134 fi
135 done <<EOF
136 $RF_list
137 EOF
138 #echo -e $email_message >> testoutput.txt;
139 #$echo -e "hello jason \\n jason" | wc -l
