python检索日志内容，生成json格式数据，存入kafak，及消费

需求：

    1.检索日志里的mediaType:151，mediaType:2738，mediaType:4814三种信息。

    2.日志检索出来的内容“### storeUCTweets at : 2018-05-15 11:20:00|campaignId:25096|docId:2018050800006514832|bolgId:1944393495885542_2087707064887517|mediaType:151”
    3.根据docId的前8为。例如20180508.找出7天前的日志。
    4.把数据格式转为{"docId": "2018050800006514196", "campaignId": "22101"}存到kafka里面

#!/usr/bin/python 
#-*- coding:utf-8 -*-
#
import json
from os.path import getsize
import re
from sys import exit
from re import compile, IGNORECASE
import sys, getpass
import datetime
from kafka import KafkaProducer

tomcat_log = '/data/server/tomcats/tomcat7_8480_storeService/logs/catalina.out'
# 该文件是用于记录上次读取日志文件的位置
last_position_logfile = '/data/server/tomcats/tomcat7_8480_storeService/last_position.txt'

fb = compile('mediaType:151', IGNORECASE)
yt = compile('mediaType:2738', IGNORECASE)
ig = compile('mediaType:4814', IGNORECASE)

producer = KafkaProducer(bootstrap_servers=['smm-info05.wisers.com:9092','smm-info06.wisers.com:9092'])
#定义检索日志函数
def search_log(mes):
  send_kafka={}
  ret=re.match('.*campaignId:(\d+)\|docId:(\d{8})(\d+)\|.*',mes)
  write_time=ret.groups()[1]
  #print(write_time)
  write_time_date=datetime.datetime.strptime(write_time, "%Y%m%d").date()
  today = datetime.date.today()
  three_day_ago=today - datetime.timedelta(days=7)
  if write_time_date <= three_day_ago:
    c_id=ret.groups()[0]
    d_id=ret.groups()[1] + ret.groups()[2]
    send_kafka['campaignId']= c_id
    send_kafka['docId']= d_id
    j=json.dumps(send_kafka)
    #print(j)
    producer.send('WISERS_CT_HISTORY_DOC', j)
#读取上一次日志文件的读取位置
def get_last_position(file):
  try:
    data = open(file, 'r')
    last_position = data.readline()
    if last_position:
      last_position = int(last_position)
    else:
      last_position = 0
  except:
      last_position = 0
  return last_position

#写入本次日志文件的读取到的本次位置
def write_this_position(file, last_position):
  try:
    data = open(file, 'w')
    data.write(str(last_position))
    data.write('\n' + "Don't Delete This File,It is Very important for Looking Tomcat Error Log !! \n")
    data.close()
  except:
    print "Can't Create File !" + file
    exit()

#分析文件找出Outofmemory
def analysis_log(file):
  try:
    data = open(file, 'r')
  except:
    exit()
  last_position = get_last_position(last_position_logfile)
  this_position = getsize(tomcat_log)
  if this_position < last_position:
    data.seek(0)
  elif this_position == last_position:
    exit()
  elif this_position > last_position:
    data.seek(last_position)
  for line in data:
    if fb.search(line):
      search_log(line)
    elif yt.search(line):
      search_log(line)
    elif ig.search(line):
      search_log(line)
  write_this_position(last_position_logfile, data.tell())
  data.close()
analysis_log(tomcat_log)
producer.close()

2.需求，消费kafka数据

#!/usr/bin/python 
#-*- coding:utf-8 -*-
#
from kafka import KafkaConsumer

consumer = KafkaConsumer('WISERS_CT_HISTORY_DOC',
                         auto_offset_reset='earliest',
                         bootstrap_servers=['smm-info05.wisers.com:9092'])

for message in consumer:
    print ("%s:%d:%d: key=%s value=%s" % (message.topic, message.partition,
                                          message.offset, message.key,
                                          message.value))