python分析nignx访问日志脚本分享

#!/usr/bin/env python/' target='_blank'>python # coding=utf-8  #------------------------------------------------------ # Name:     nginx 日志分析脚本 # Purpose:   此脚本只用来分析nginx的访问日志 # Version:   1.0 # Author:    LEO # Created:   2013-05-07 # Modified:   2013-05-07 # Copyright:  (c) LEO 2013 #------------------------------------------------------  import sys import time  #该类是用来打印格式 class displayFormat(object):    def format_size(self,size):     '''''格式化流量单位'''     KB = 1024      #KB -> B B是字节     MB = 1048576    #MB -> B     GB = 1073741824   #GB -> B     TB = 1099511627776 #TB -> B     if size >= TB :       size = str(size / TB) + 'T'     elif size < KB :       size = str(size) + 'B'     elif size >= GB and size < TB:       size = str(size / GB) + 'G'     elif size >= MB and size < GB :       size = str(size / MB) + 'M'     else :       size = str(size / KB) + 'K'     return size    #定义字符串格式化   formatstring = '%-15s %-10s %-12s %8s %10s %10s %10s %10s %10s %10s %10s'    def transverse_line(self) :     '''''输出横线'''     print self.formatstring % ('-'*15,'-'*10,'-'*12,'-'*12,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10,'-'*10)    def head(self):     '''''输出头部信息'''     print self.formatstring % ('IP','Traffic','Times','Times%','200','404','500','403','302','304','503')    def error_print(self) :     '''''输出错误信息'''     print     print 'Usage : ' + sys.argv[0] + ' NginxLogFilePath [Number]'     print     sys.exit(1)    def execut_time(self):     '''''输出脚本执行的时间'''     print     print "Script Execution Time: %.3f second" % time.clock()     print  #该类是用来生成主机信息的字典 class hostInfo(object):   host_info = ['200','404','500','302','304','503','403','times','size']    def __init__(self,host):     self.host = host = {}.fromkeys(self.host_info,0)    def increment(self,status_times_size,is_size):     '''''该方法是用来给host_info中的各个值加1'''     if status_times_size == 'times':       self.host['times'] += 1     elif is_size:       self.host['size'] = self.host['size'] + status_times_size     else:       self.host[status_times_size] += 1    def get_value(self,value):     '''''该方法是取到各个主机信息中对应的值'''     return self.host[value]  #该类是用来分析文件 class fileAnalysis(object):   def __init__(self):     '''''初始化一个空字典'''     self.report_dict = {}     self.total_request_times,self.total_traffic,self.total_200,     self.total_404,self.total_500,self.total_403,self.total_302,     self.total_304,self.total_503 = 0,0,0,0,0,0,0,0,0    def split_eachline_todict(self,line):     '''''分割文件中的每一行，并返回一个字典'''     split_line = line.split()     split_dict = {'remote_host':split_line[0],'status':split_line[8],            'bytes_sent':split_line[9],}     return split_dict    def generate_log_report(self,logfile):     '''''读取文件，分析split_eachline_todict方法生成的字典'''     for line in logfile:       try:         line_dict = self.split_eachline_todict(line)         host = line_dict['remote_host']         status = line_dict['status']       except ValueError :         continue       except IndexError :         continue        if host not in self.report_dict :         host_info_obj = hostInfo(host)         self.report_dict[host] = host_info_obj       else :         host_info_obj = self.report_dict[host]        host_info_obj.increment('times',False)       if status in host_info_obj.host_info :         host_info_obj.increment(status,False)       try:         bytes_sent = int(line_dict['bytes_sent'])       except ValueError:         bytes_sent = 0       host_info_obj.increment(bytes_sent,True)     return self.report_dict    def return_sorted_list(self,true_dict):     '''''计算各个状态次数、流量总量，请求的总次数，并且计算各个状态的总量 并生成一个正真的字典，方便排序'''     for host_key in true_dict :       host_value = true_dict[host_key]       times = host_value.get_value('times')                  self.total_request_times = self.total_request_times + times       size = host_value.get_value('size')                  self.total_traffic = self.total_traffic + size         o200 = host_value.get_value('200')       o404 = host_value.get_value('404')       o500 = host_value.get_value('500')       o403 = host_value.get_value('403')       o302 = host_value.get_value('302')       o304 = host_value.get_value('304')       o503 = host_value.get_value('503')        true_dict[host_key] = {'200':o200,'404':o404,'500':o500,                   '403':o403,'302':o302,'304':o304,                   '503':o503,'times':times,'size':size}        self.total_200 = self.total_200 + o200       self.total_404 = self.total_404 + o404       self.total_500 = self.total_500 + o500       self.total_302 = self.total_302 + o302       self.total_304 = self.total_304 + o304       self.total_503 = self.total_503 + o503      sorted_list = sorted(true_dict.items(),key=lambda t:(t[1]['times'],                               t[1]['size']),reverse=True)      return sorted_list  class Main(object):   def main(self) :     '''''主调函数'''     display_format = displayFormat()     arg_length = len(sys.argv)     if arg_length == 1 :       display_format.error_print()     elif arg_length == 2 or arg_length == 3:       infile_name = sys.argv[1]       try :         infile = open(infile_name,'r')         if arg_length == 3 :           lines = int(sys.argv[2])         else :           lines = 0       except IOError,e :         print         print e         display_format.error_print()       except ValueError :         print         print "Please Enter A Volid Number !!"         display_format.error_print()     else :       display_format.error_print()      fileAnalysis_obj = fileAnalysis()     not_true_dict = fileAnalysis_obj.generate_log_report(infile)     log_report = fileAnalysis_obj.return_sorted_list(not_true_dict)     total_ip = len(log_report)     if lines :       log_report = log_report[0:lines]     infile.close()      print     total_traffic = display_format.format_size(fileAnalysis_obj.total_traffic)     total_request_times = fileAnalysis_obj.total_request_times     print 'Total IP: %s  Total Traffic: %s  Total Request Times: %d'        % (total_ip,total_traffic,total_request_times)     print     display_format.head()     display_format.transverse_line()      for host in log_report :       times = host[1]['times']       times_percent = (float(times) / float(fileAnalysis_obj.total_request_times)) * 100       print display_format.formatstring % (host[0],                         display_format.format_size(host[1]['size']),                         times,str(times_percent)[0:5],                         host[1]['200'],host[1]['404'],                         host[1]['500'],host[1]['403'],                         host[1]['302'],host[1]['304'],host[1]['503'])                              if (not lines) or total_ip == lines :       display_format.transverse_line()       print display_format.formatstring % (total_ip,total_traffic,                          total_request_times,'100%',                         fileAnalysis_obj.total_200,                         fileAnalysis_obj.total_404,                         fileAnalysis_obj.total_500,                          fileAnalysis_obj.total_403,                         fileAnalysis_obj.total_302,                          fileAnalysis_obj.total_304,                         fileAnalysis_obj.total_503)      display_format.execut_time()  if __name__ == '__main__':   main_obj = Main()   main_obj.main()