在外实习，除了工作，自己还可以做一些感兴趣的事情，这段时间一致在做话单比对的工作，最近又在数据中心的比对，哎呀除了核对还是核对，真是单调！
为什么了不做一个全自动的核对机了，就像全自动的洗衣机你把洗衣粉放好了，水龙头打开，设定一个模式，他自己就会洗、漂、脱一体服务。
嘿嘿，我这两天断断续续做的这个解决方案就是基于这个想法哦，是不是很帅哦....
##################### Uniform Solution for Bill Comparison ################################
##################### Date: 2009-3-6 #####################################################
##################### Author: landry yi ##################################################
##################### Email: yijiyong100@163.com #########################################
##################### Copyright: Harbin Normal University@copyright2009 ###################
##################### Version: 1.0.0#######################################################
##################### Histroy: if modified ,please add here and release new veresion#######
############## Config Part ################################################################
SystemBak="/transfer/yijy/cmp_shell/bak"
SystemTmp="/transfer/yijy/cmp_shell/tmp"
SystemLog="/transfer/yijy/cmp_shell/log"
SystemErr="/transfer/yijy/cmp_shell/err"
NewDataSourceDir="/transfer/yijy/cmp_shell/bin/vtof/bak"
NewDataSourceExp="*20090101*"
OldDataSourceDir="/transfer/yijy/cmp_shell/bin/vtof/bak"
OldDataSourceExp="*20090101*"
NewConvertFlag="on"
NewConvertBinDir="/transfer/yijy/cmp_shell/bin/vtof/new/vtof1"
NewConvertBinCfg="/transfer/yijy/cmp_shell/bin/vtof/cfg/new/vtof.cfg"
NewConvertBinIn="/transfer/yijy/cmp_shell/bin/vtof/new/in"
NewConvertBinOut="/transfer/yijy/cmp_shell/bin/vtof/new/out"
NewConvertSplit="|"
OldConvertFlag="on"
OldConvertBinDir="/transfer/yijy/cmp_shell/bin/vtof/old/vtof1"
OldConvertBinCfg="/transfer/yijy/cmp_shell/bin/vtof/cfg/old/vtof.cfg"
OldConvertBinIn="/transfer/yijy/cmp_shell/bin/vtof/old/in"
OldConvertBinOut="/transfer/yijy/cmp_shell/bin/vtof/old/out"
OldConvertSplit="|"
NewCmpExtractFlag="on"
NewCmpExtractKey="1,3,6,8,10"
NewCmpExtractSem="msisdn:2,chat_type:11,fee_type:16,fee1:14,old_fee1:2"
OldCmpExtractFlag="on"
OldCmpExtractKey="1,3,6,8,10"
OldCmpExtractSem="msisdn:2,chat_type:11,fee_type:16,fee1:14,old_fee1:2"
NewCmpFilePrex="new."
OldCmpFilePrex="old."
CmpFileNewIn="/transfer/yijy/cmp_shell/cmp/new/new_in"
CmpFileOldIn="/transfer/yijy/cmp_shell/cmp/new/old_in"
CmpBinDir="/transfer/yijy/cmp_shell/cmp/bin/o2n_cmp"
CmpBinCfg="/transfer/yijy/cmp_shell/cmp/cfg/wa.cfg"
CheckFile()
{
if [ ! -d ${SystemBak} -o ! -d ${SystemTmp} -o ! -d ${CmpFileNewIn} -o ! -d ${CmpFileOldIn} ]
then
echo 'APP:ERROR:Config error -- DIR'
exit
fi
if [ ! -d ${SystemLog} -o ! -d ${SystemErr} ]
then
echo 'APP:ERROR:Config error -- System Dir'
exit
fi
if [ ! -d ${NewConvertBinIn} -o ! -d ${NewConvertBinOut} -o ! -d ${OldConvertBinIn} -o ! -d ${OldConvertBinOut} ]
then
echo 'APP:ERROR:Convert error -- DIR'
exit
fi
if [ ! -f $CmpBinDir -o ! -f $CmpBinCfg ]
then
echo 'APP:ERROR:Config error -- FILE'
exit
fi
if [ ! -d $NewDataSourceDir -o ! -d $OldDataSourceDir ]
then
echo 'APP:ERROR:Config error -- Datasource'
exit
fi
if [ ! ${NewConvertSplit} -o ! ${OldConvertSplit} ]
then
echo 'APP:ERROR:Config error -- ConvertSplitchar can not be NULL'
exit
fi
if [ ! $NewDataSourceExp -o ! $OldDataSourceExp ]
then
echo 'APP:ERROR:Config error -- DataSourceExp can not be NULL'
exit
fi
if [ $NewConvertFlag = "on" ]
then
if [ ! -d ${NewConvertBinIn} -o ! -d ${NewConvertBinOut} -o ! -f ${NewConvertBinDir} -o ! -f ${NewConvertBinCfg} ]
then
echo 'APP:ERROR:Config error -- NewConvert'
exit
fi
fi
if [ $OldConvertFlag = "on" ]
then
if [ ! -d ${OldConvertBinIn} -o ! -d ${OldConvertBinOut} -o ! -f ${OldConvertBinDir} -o ! -f ${OldConvertBinCfg} ]
then
echo 'APP:ERROR:Config error -- OldConvert'
exit
fi
fi
if [ ! $NewCmpFilePrex -o ! $OldCmpFilePrex ]
then
echo 'APP:ERROR:Config error -- prex'
exit
fi
if [ -n ${NewCmpExtractKey} -a -n ${NewCmpExtractSem} -a -n ${OldCmpExtractKey} -a -n ${OldCmpExtractSem} ]
then
echo ${NewCmpExtractKey} | awk -F ',' '{
printf("APP:MSG:NewExkey number is [%d] -- and the New key are [%s]\n",NF,$0);
}'
echo ${NewCmpExtractSem} | awk -F ',' '{
NewExsemNum=NF
printf("APP:MSG:NewExsem number is [%d] -- and the New sem are [%s]\n",NF,$0);
}'
echo ${OldCmpExtractKey} | awk -F ',' '{
OldExkeyNum=NF
printf("APP:MSG:OldExkey number is [%d] -- and the Old key are [%s]\n",NF,$0);
}'
echo ${OldCmpExtractSem} | awk -F ',' '{
OldExsemNum=NF
printf("APP:MSG:OldExsem number is [%d] -- and the Old sem are [%s]\n",NF,$0);
}'
newExtractKeyNumber=`echo ${NewCmpExtractKey} | awk -F ',' '{printf("%d",NF)}'`
newExtractSemNumber=`echo ${NewCmpExtractSem} | awk -F ',' '{printf("%d",NF)}'`
if [ ${newExtractKeyNumber} -ne ${newExtractSemNumber} ]
then
echo 'APP:ERROR:The new extract key number not equal to sem number'
exit
fi
oldExtractKeyNumber=`echo ${OldCmpExtractKey} | awk -F ',' '{printf("%d",NF)}'`
oldExtractSemNumber=`echo ${OldCmpExtractSem} | awk -F ',' '{printf("%d",NF)}'`
if [ ${oldExtractKeyNumber} -ne ${oldExtractSemNumber} ]
then
echo 'APP:ERROR:The old extract key number not equal to sem number'
exit
fi
else
echo 'APP:ERROR:Config error -- Extract'
exit
fi
echo 'APP:MSG:Config part check successful!'
}
############## Clean Enviroment ###########################################################
CleanEnviroment()
{
echo 'APP:MSG:Start clean the system enviroment...'
########### <--add your own script here to clean the enviroment if you like--> ##########
########### <--add your own script here to clean the enviroment if you like--> ##########
echo 'APP:MSG:Clean the system enviroment over...'
}
############## Data Prepare ###############################################################
totalNewConvertIn=0
totalOldConvertIn=0
totalNewExtractIn=0
totalOldExtractIn=0
DataPrepare()
{
if [ ! -d ${NewDataSourceDir} -o ! -d ${OldDataSourceDir} ]
then
echo 'APP:ERROR:Data source error'
exit
fi
echo 'APP:MSG:Data start prepare...'
if [ ${NewConvertFlag} = "on" ]
then
cp -r ${NewDataSourceDir}/${NewDataSourceExp} ${NewConvertBinIn}
let totalNewConvertIn=`ls -1 ${NewConvertBinIn} | wc -l`
echo 'APP:MSG:The total number of new file need convert in ['${NewConvertBinIn}'] is ['${totalNewConvertIn}']'
else
cp -r ${NewDataSourceDir}/${NewDataSourceExp} ${NewConvertBinOut}
let totalNewExtractIn=`ls -1 ${NewConvertBinOut} | wc -l`
echo 'APP:MSG:The total number of new file need extract in ['${NewConvertBinOut}'] is ['${totalNewExtractIn}']'
fi
if [ ${OldConvertFlag} = "on" ]
then
cp -r ${OldDataSourceDir}/${OldDataSourceExp} ${OldConvertBinIn}
let totalOldConvertIn=`ls -1 ${OldConvertBinIn} | wc -l`
echo 'APP:MSG:The total number of new file need convert in ['${OldConvertBinIn}'] is ['${totalOldConvertIn}']'
else
cp -r ${OldDataSourceDir}/${OldDataSourceExp} ${OldConvertBinOut}
let totalOldExtractIn=`ls -1 ${OldConvertBinOut} | wc -l`
echo 'APP:MSG:The total number of new file need extract in ['${OldConvertBinOut}'] is ['${totalOldExtractIn}']'
fi
echo 'APP:MSG:Data prepare over...'
}
############## Convert Part ###############################################################
ConvertFile()
{
if [ ${NewConvertFlag} = "on" ]
then
if [ 0 -eq `ls -1 ${NewConvertBinIn} | wc -l` ]
then
echo 'APP:MSG:No file exist in '${NewConvertBinIn}
exit
else
let totalNewConvertIn=`ls -1 ${NewConvertBinIn} | wc -l`
newCurPercent=0
newTmpPercent=0
echo 'APP:MSG:Start convert new file:'
echo 'APP:MSG:The total file number in the ['${NewConvertBinIn}'] is ['${totalNewConvertIn}']'
echo 'APP:MSG:\c'
nohup ${NewConvertBinDir} ${NewConvertBinCfg} &
sleep 1
echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'
echo 'APP:MSG:Convert Pro:\c'
while [ 0 -lt `ls -1 ${NewConvertBinIn} | wc -l` ]
do
newExtCurFile=`ls -1 ${NewConvertBinIn} | wc -l`
let NewDValue=totalNewConvertIn-newExtCurFile
let newCurPercent=${NewDValue}\*100/${totalNewConvertIn}
while [ ${newTmpPercent} -lt ${newCurPercent} ]
do
echo '#\c'
let newTmpPercent=newTmpPercent+1
done
done
while [ $newTmpPercent -lt 100 ]
do
echo '#\c'
let newTmpPercent=newTmpPercent+1
done
newConvertId=`ps -ef | grep ${NewConvertBinDir} | grep ${NewConvertBinCfg} | awk '{printf("%s",$2);}'`
echo '\nAPP:MSG:New convert program id is ['${newConvertId}']'
kill ${newConvertId}
echo 'APP:MSG:New convert program was killed!'
echo 'APP:MSG:New file Convert over!'
fi
fi
if [ ${OldConvertFlag} = "on" ]
then
if [ 0 -eq `ls -1 ${OldConvertBinIn} | wc -l` ]
then
echo 'APP:MSG:No file exist in '${OldConvertBinIn}
exit
else
let totalOldConvertIn=`ls -1 ${OldConvertBinIn} | wc -l`
oldCurPercent=0
oldTmpPercent=0
echo 'APP:MSG:Start convert old file:'
echo 'APP:MSG:The total file number in the ['${OldConvertBinIn}'] is ['${totalOldConvertIn}']'
echo 'APP:MSG:\c'
nohup ${OldConvertBinDir} ${OldConvertBinCfg} &
sleep 1
echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'
echo 'APP:MSG:Convert Pro:\c'
while [ 0 -lt `ls -1 ${OldConvertBinIn} | wc -l` ]
do
oldExtCurFile=`ls -1 ${OldConvertBinIn} | wc -l`
let OldDValue=totalOldConvertIn-oldExtCurFile
let oldCurPercent=${OldDValue}\*100/${totalOldConvertIn}
while [ ${oldTmpPercent} -lt ${oldCurPercent} ]
do
echo '#\c'
let oldTmpPercent=oldTmpPercent+1
done
done
while [ $oldTmpPercent -lt 100 ]
do
echo '#\c'
let oldTmpPercent=oldTmpPercent+1
done
oldConvertId=`ps -ef | grep ${OldConvertBinDir} | grep ${OldConvertBinCfg} | awk '{printf("%s",$2);}'`
echo '\nAPP:MSG:Old convert program id is ['${oldConvertId}']'
kill ${oldConvertId}
echo 'APP:MSG:Old convert program was killed!'
echo 'APP:MSG:Old file Convert over!'
fi
fi
}
############## Extract Part ###############################################################
ExtractFile()
{
if [ ${NewCmpExtractFlag} = "on" ]
then
if [ 0 -eq `find ${NewConvertBinOut} -type f | wc -l` ]
then
echo 'APP:MSG:No file exist in '${NewConvertBinOut}
else
echo 'APP:MSG:Start Extract new file,and copy the extract file to compare program in dir,then move it to SystemBak dir'
let totalNewExtractIn=`ls -1 ${NewConvertBinOut} | wc -l`
echo 'APP:MSG:The total file in ['${NewConvertBinOut}'] is ['${totalNewExtractIn}']'
BakDate=`date +%Y%m%d`
if [ ! -d ${SystemBak}/${BakDate} ]
then
mkdir ${SystemBak}/${BakDate}
fi
if [ 0 -lt `find ${NewConvertBinOut} -type f | wc -l` ]
then
echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'
echo 'APP:MSG:Extract Pro:\c'
newExtractTmpPercent=0
newExtractCurPercent=0
newExtractTotal=0
find ${NewConvertBinOut} -type f | while read newConvertFile
do
newConvertTmp=`basename ${newConvertFile}`
newExtractTmpFile=`echo ${NewCmpFilePrex}${newConvertTmp}`
cat ${newConvertFile} | awk -F ${NewConvertSplit} '{
newExStr=""
newExKeyStr="'$NewCmpExtractKey'"
newExSemStr="'$NewCmpExtractSem'"
while(length(newExKeyStr) || length(newExSemStr))
{
newKeyTmpCur=index(newExKeyStr,",")
newSemTmpCur=index(newExSemStr,",")
if (newKeyTmpCur == 0)
{
newKeyTmpCur=length(newExKeyStr)+1;
}
if (newSemTmpCur == 0)
{
newSemTmpCur=length(newExSemStr)+1;
}
newSemTmp=substr(newExSemStr,1,newSemTmpCur-1)
newSemTmpIndex=index(newSemTmp,":")
newSemTmpLen=substr(newSemTmp,newSemTmpIndex+1)
newKeyTmp=substr(newExKeyStr,1,newKeyTmpCur-1)
newExStrTmp=$newKeyTmp
newExStrFormat=formatStr(newExStrTmp,newSemTmpLen)
if (newExStr == "")
{
newExStr=newExStrFormat;
}
else
{
newExStr=sprintf("%s %s",newExStr,newExStrFormat)
}
if(index(newExKeyStr,",") == 0)
{
newExKeyStr=""
}
newExKeyStr=substr(newExKeyStr,newKeyTmpCur+1)
if(index(newExSemStr,",") == 0)
{
newExSemStr=""
}
newExSemStr=substr(newExSemStr,newSemTmpCur+1)
}
printf("%s\n",newExStr);
}
function formatStr (str,len)
{
formatstr=""
strLen=length(str)
sprintf("len %d,strlen%d",len,strLen)
if(strLen>len)
{
formatstr=substr(str,1,len);
}
else
{
formatstr=str
dValue=len-strLen
while(dValue)
{
formatstr=sprintf("%s ",formatstr)
dValue--
}
}
return formatstr
}
'>${SystemTmp}/${newExtractTmpFile}
let newExtractTotal=newExtractTotal+1
let newExtractCurPercent=${newExtractTotal}\*100/${totalNewExtractIn}
while [ ${newExtractTmpPercent} -lt ${newExtractCurPercent} ]
do
echo '#\c'
let newExtractTmpPercent=newExtractTmpPercent+1
done
cp ${SystemTmp}/${newExtractTmpFile} ${CmpFileNewIn}
mv ${SystemTmp}/${newExtractTmpFile} ${SystemBak}/${BakDate}
done
fi
echo '\nAPP:MSG:Extract new file over,copy and move the file sucessfully!'
fi
fi
if [ ${OldCmpExtractFlag} = "on" ]
then
if [ 0 -eq `find ${OldConvertBinOut} -type f | wc -l` ]
then
echo 'APP:MSG:No file exist in '${OldConvertBinOut}
else
echo 'APP:MSG:Start Extract old file,and copy the extract file to compare program in dir,then move it to SystemBak dir'
let totalOldExtractIn=`ls -1 ${OldConvertBinOut} | wc -l`
echo 'APP:MSG:The total file in ['${OldConvertBinOut}'] is ['${totalOldExtractIn}']'
BakDate=`date +%Y%m%d`
if [ ! -d ${SystemBak}/${BakDate} ]
then
mkdir ${SystemBak}/${BakDate}
fi
if [ 0 -lt `find ${OldConvertBinOut} -type f | wc -l` ]
then
echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'
echo 'APP:MSG:Extract Pro:\c'
oldExtractTmpPercent=0
oldExtractCurPercent=0
oldExtractTotal=0
find ${OldConvertBinOut} -type f | while read oldConvertFile
do
oldConvertTmp=`basename ${oldConvertFile}`
oldExtractTmpFile=`echo ${OldCmpFilePrex}${oldConvertTmp}`
cat ${oldConvertFile} | awk -F ${OldConvertSplit} '{
oldExStr=""
oldExKeyStr="'$OldCmpExtractKey'"
oldExSemStr="'$OldCmpExtractSem'"
while(length(oldExKeyStr) || length(oldExSemStr))
{
oldKeyTmpCur=index(oldExKeyStr,",")
oldSemTmpCur=index(oldExSemStr,",")
if (oldKeyTmpCur == 0)
{
oldKeyTmpCur=length(oldExKeyStr)+1;
}
if (oldSemTmpCur == 0)
{
oldSemTmpCur=length(oldExSemStr)+1;
}
oldSemTmp=substr(oldExSemStr,1,oldSemTmpCur-1)
oldSemTmpIndex=index(oldSemTmp,":")
oldSemTmpLen=substr(oldSemTmp,oldSemTmpIndex+1)
oldKeyTmp=substr(oldExKeyStr,1,oldKeyTmpCur-1)
oldExStrTmp=$oldKeyTmp
oldExStrFormat=formatStr(oldExStrTmp,oldSemTmpLen)
if (oldExStr == "")
{
oldExStr=oldExStrFormat;
}
else
{
oldExStr=sprintf("%s %s",oldExStr,oldExStrFormat)
}
if(index(oldExKeyStr,",") == 0)
{
oldExKeyStr=""
}
oldExKeyStr=substr(oldExKeyStr,oldKeyTmpCur+1)
if(index(oldExSemStr,",") == 0)
{
oldExSemStr=""
}
oldExSemStr=substr(oldExSemStr,oldSemTmpCur+1)
}
printf("%s\n",oldExStr);
}
function formatStr (str,len)
{
formatstr=""
strLen=length(str)
sprintf("len %d,strlen%d",len,strLen)
if(strLen>len)
{
formatstr=substr(str,1,len);
}
else
{
formatstr=str
dValue=len-strLen
while(dValue)
{
formatstr=sprintf("%s ",formatstr)
dValue--
}
}
return formatstr
}
'>${SystemTmp}/${oldExtractTmpFile}
let oldExtractTotal=oldExtractTotal+1
let oldExtractCurPercent=${oldExtractTotal}\*100/${totalOldExtractIn}
while [ ${oldExtractTmpPercent} -lt ${oldExtractCurPercent} ]
do
echo '#\c'
let oldExtractTmpPercent=oldExtractTmpPercent+1
done
cp ${SystemTmp}/${oldExtractTmpFile} ${CmpFileOldIn}
mv ${SystemTmp}/${oldExtractTmpFile} ${SystemBak}/${BakDate}
done
fi
echo '\nAPP:MSG:Extract old file over,copy and move the file sucessfully!'
fi
fi
}
############## Compare Part ###############################################################
CompareFile()
{
if [ 0 -eq `find ${CmpFileNewIn} -type f | wc -l` ]
then
echo 'APP:MSG:There is no file exist in ['${CmpFileNewIn}']'
exit
fi
if [ 0 -eq `find ${CmpFileOldIn} -type f | wc -l` ]
then
echo 'APP:MSG:There is no file exist in ['${CmpFileOldIn}']'
exit
fi
echo 'APP:MSG:Start comprae file...'
totalCmpNewFileIn=`ls -1 ${CmpFileNewIn} | wc -l`
totalCmpOldFileIn=`ls -1 ${CmpFileOldIn} | wc -l`
echo 'APP:MSG:The total number of file in ['${CmpFileNewIn}'] is ['${totalCmpNewFileIn}']'
echo 'APP:MSG:The total number of file in ['${CmpFileOldIn}'] is ['${totalCmpOldFileIn}']'
cmpCurPercent=0
cmpTmpPercent=0
echo 'APP:MSG:\c'
nohup ${CmpBinDir} ${CmpBinCfg} &
sleep 1
echo 'APP:MSG:Process Bar:____________________________________________50%_________________________________________________100%'
echo 'APP:MSG:Compare Pro:\c'
while [ 0 -lt `ls -1 ${CmpFileNewIn} | wc -l` ]
do
totalCurCmpFileIn=`ls -1 ${CmpFileNewIn} | wc -l`
let fileDvalue=totalCmpNewFileIn-totalCurCmpFileIn
let cmpCurPercent=${fileDvalue}\*100/${totalCmpNewFileIn}
while [ ${cmpTmpPercent} -lt ${cmpCurPercent} ]
do
echo '#\c'
let cmpTmpPercent=cmpTmpPercent+1
done
done
while [ ${cmpTmpPercent} -lt 100 ]
do
echo '#\c'
let cmpTmpPercent=cmpTmpPercent+1
done
sleep 5
theCompareId=`ps -ef | grep ${CmpBinDir} | grep ${CmpBinCfg} | awk '{printf("%s",$2);}'`
if [ -n "$theCompareId" ]
then
kill $theCompareId
fi
echo '\nAPP:MSG:Compare Result:'
tail nohup.out
echo 'APP:MSG:Compare file over...'
}
############## Main Program ###############################################################
CheckFile
#CleanEnviroment
DataPrepare
ConvertFile
ExtractFile
CompareFile
浙公网安备 33010602011771号