在ELK里面获取top10的url在日志量非常大的情况下是非常消耗内存的,所以写了一个脚本用来快速获取。
配置文件 log.conf
[log] log_file = /data/logs/nginx/access_all.log [id_rsa] id_rsa = /root/.ssh/id_rsa
程序log.py
#!/usr/bin/env python #coding=utf-8 #author hexm@2016-12-07 import sys import datetime import paramiko import configparser class Command(object): def __init__(self, host, domain, startime, endtime): self.getconf() self.host = host self.domain = domain self.command = '''awk \'/%s/, /%s/\' %s | grep \'^%s\' | awk \'{print $8}\' | awk -F\"?\" \'{print $1}\' | awk \'{access[$1]++} END {for (A in access){print access[A], A}}\' | sort -nr | head ''' % (startime, endtime, self.logfile, self.domain) self.username = 'root' self.port = 22 self.private_key = paramiko.RSAKey.from_private_key_file(self.id_rsa) def mostrequests(self): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(hostname=self.host, port=self.port, username=self.username, pkey=self.private_key) stdin, stdout, stderr = ssh.exec_command(self.command) ret, err = stdout.read(), stderr.read() result = ret if ret else err ssh.close() return result.decode() def getconf(self): config = configparser.ConfigParser() config.read('log.conf', encoding='utf-8') self.logfile = config.get('log', 'log_file') + ' ' + config.get('log', 'log_file') + '-' + datetime.datetime.now().strftime('%Y%m%d') self.id_rsa = config.get('id_rsa', 'id_rsa') if __name__ == '__main__': h_dict = { 'passport.51.com':'10.80.2.173', 'www.51.com':'10.80.2.174', 'gameapi.51.com':'10.80.2.138', 'game.51.com':'10.80.2.139', 'home.51.com':'10.80.2.170'} domain = sys.argv[1] host = h_dict[domain] startime, endtime = sys.argv[2:] log = Command(host, domain, startime, endtime) ret = log.mostrequests() print ret
使用例子:
# python log.py game.51.com 2016:09:01 2016:10:1158702 /rolelevel/api 36558 /act/cps_api/ns 36226 /act/cps_api/kfyg 28344 /webgame/index/tx/s2400 28200 /newserver/index/dtx 26082 /webgameleft/button/ 22602 /webgameleft/updlevel 13769 /box/heartbeat/h 8629 /website/ajaxload/ 5478 /dss/stat51/
本站文章如无特殊说明,均为本站原创,如若转载,请注明出处:nginx访问日志获取访问前10的url - Python技术站