nginx访问日志获取访问前10的url

在ELK里面获取top10的url在日志量非常大的情况下是非常消耗内存的，所以写了一个脚本用来快速获取。

配置文件 log.conf

[log]
log_file = /data/logs/nginx/access_all.log

[id_rsa]
id_rsa = /root/.ssh/id_rsa

程序log.py

#!/usr/bin/env python
#coding=utf-8
#author hexm@2016-12-07


import sys
import datetime
import paramiko
import configparser


class Command(object):

    def __init__(self, host, domain, startime, endtime):
        self.getconf()
        self.host = host
        self.domain = domain
        self.command = '''awk \'/%s/, /%s/\' %s | grep \'^%s\' | awk \'{print $8}\' |  awk -F\"?\" \'{print $1}\' | awk \'{access[$1]++} END {for (A in access){print access[A], A}}\' | sort -nr | head ''' % (startime, endtime, self.logfile, self.domain)
        self.username = 'root'
        self.port = 22
        self.private_key = paramiko.RSAKey.from_private_key_file(self.id_rsa)

    def mostrequests(self):
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key)
        stdin, stdout, stderr = ssh.exec_command(self.command)
        ret, err = stdout.read(), stderr.read()
        result = ret if ret else err 
        ssh.close()
        return result.decode()

    def getconf(self):
        config = configparser.ConfigParser()
        config.read('log.conf', encoding='utf-8')
        self.logfile = config.get('log', 'log_file') + ' ' + config.get('log', 'log_file') + '-' + datetime.datetime.now().strftime('%Y%m%d')
        self.id_rsa = config.get('id_rsa', 'id_rsa')


if __name__ == '__main__':

    h_dict = { 'passport.51.com':'10.80.2.173', 'www.51.com':'10.80.2.174', 'gameapi.51.com':'10.80.2.138', 'game.51.com':'10.80.2.139', 'home.51.com':'10.80.2.170'}
    domain = sys.argv[1]
    host = h_dict[domain]
    startime, endtime = sys.argv[2:]
    log = Command(host, domain, startime, endtime)

    ret = log.mostrequests()
    print ret

使用例子：

# python log.py game.51.com 2016:09:01 2016:10:1158702 /rolelevel/api
36558 /act/cps_api/ns
36226 /act/cps_api/kfyg
28344 /webgame/index/tx/s2400
28200 /newserver/index/dtx
26082 /webgameleft/button/
22602 /webgameleft/updlevel
13769 /box/heartbeat/h
8629 /website/ajaxload/
5478 /dss/stat51/