kettle设置linux定时任务

kettle每日同步作业，需要在linux上设置定时任务。

 

 

其中/home/CHN/wujunchao/schedule_member.sh的文件内容如下：

 

#!/bin/sh
export JAVA_HOME=/home/CHN/wujunchao/java/jdk1.8.0_202
export CLASSPATH=$:CLASSPATH:$JAVA_HOME/lib/
export PATH=$PATH:$JAVA_HOME/bin
PATH=$PATH:$HOME/.local/bin:$HOME/bin

sh /home/CHN/wujunchao/data-integration/kitchen.sh -norep -file=/home/CHN/wujunchao/calendar_schedule/schedule_member_job.kjb > /home/CHN/wujunchao/tmp_log/`date "+%Y%m%d%H%M%S"`.log

如果定时任务跑的是ktr文件，就用如下命令：

sh /home/CHN/wujunchao/data-integration/pan.sh -file=/home/CHN/wujunchao/Backup/sqlserver-mysql_tag.ktr

 

 

每天获取增量文件：
#!/bin/sh
export JAVA_HOME=/home/CHN/wujunchao/java/jdk1.8.0_202
export CLASSPATH=$:CLASSPATH:$JAVA_HOME/lib/
export PATH=$PATH:$JAVA_HOME/bin
PATH=$PATH:$HOME/.local/bin:$HOME/bin


sh /home/CHN/wujunchao/checkFile.sh
sh /home/CHN/wujunchao/syncS3.sh


for i in `ls /home/CHN/wujunchao/download`;
do   
mv /home/CHN/wujunchao/download/$i /home/CHN/wujunchao/DailyFile
#excute kettle
sh /home/CHN/wujunchao/data-integration/kitchen.sh -norep -file=/home/CHN/wujunchao/ETLinc_JOB.kjb
# -level=Detailed >> /home/CHN/wujunchao/log/$(date '+%Y%m%d_%H%M%S.log')

mv /home/CHN/wujunchao/DailyFile/$i /home/CHN/wujunchao/Backup
done 

 

checkFile:

#!/bin/sh
listFile="*.txt"
cd ~/download
echo start
while [ 1 = 1 ]
do
    s3cmd get s3://CRM_CompletedJobID/$listFile
    if [ -f ./${listFile} ];
    then
     echo `date` data ok
      rm -f ./${listFile}
          break;
    elif [ $(date '+%H') = 23 ];
    then
      echo nofile until 23:00
      break;  
    fi
   echo `date` no data
    sleep 300;
done

 

 syncS3:

#!/bin/sh
#path=/home/arvato
#nowDate=$(date '+%Y%m%d_%H%M%S')
#listFile="downloadList"${nowDate}".txt"
listFile="*.txt"
#echo ${listFile}
#if [ ! -d ./Backup/${nowDate} ];
#then
#    mkdir ./Backup/$nowDate
#fi
#if [ -f "$listFile" ];
#then
#    echo "file exists"
#    mv ./download/* ./Backup/${nowDate}/
#fi
cd download
s3cmd get s3://mkcorp-s3-cn-north-1-arvato-bcmgt-prod-1/production/output/CRM_CompletedJobID/$listFile
jobId=""
cat $listFile |while read line
do
  jobId=${line%%;*}
  if [ ! -d "$jobId" ];
  then
    mkdir $jobId
  fi
    echo ${jobId}
    s3cmd ls s3://mkcorp-s3-cn-north-1-arvato-bcmgt-prod-1/production/output/CRM_Unload2S3/crm/${jobId}/ >${jobId}.log
  cat ${jobId}.log |while read line1
  do
    l=${line1##*crm/}
    echo $l
    echo ${l##*/}
    if [ ! -d ${l} ];
    then
        mkdir $l
    fi
    if [ $l != "437639/OrderItem/" ]
    then
    s3cmd get s3://CRM_Unload2S3/crm/${l}* $l
    fi
  done
  s3cmd del s3://CRM_CompletedJobID/${jobId}.txt
  mv ${jobId}.* ${jobId}
done