简单的日志统计是不需要使用重量级的Hadoop，我用python实现了日志的统计。原理是用fabric登录到远程linux，组合使用grep、uniq、sort、awk对日志进行操作，可以根据正则表达式指定规则抽取符合规则的日志，做查询，计数，分类统计。
注意：要安装fabric库
主文件：LogQuery.py
1#encoding=utf-8 2 3from fabric.api import run,env,local,cd 4from fabric.tasks import execute,abort 5from fabric.contrib.console import confirm 6import logging 7 8logging.basicConfig(format='[%(levelname)s]: %(message)s', level=logging.DEBUG) 9logger = logging.getLogger(__name__) 10logging.getLogger('paramiko.transport').setLevel(logging.ERROR) 11logger.setLevel(logging.DEBUG) 12 13EXECUTE_RESULT = {} 14 15def hosts(hostarr): 16 ''' 17 set hosts 18 hostarr:[(hostname,password),(hostname,password)...] 19 ''' 20 env.hosts = [x[0] for x in hostarr] 21 env.passwords = dict(x for x in hostarr) 22 23def query(expression,hostname,logfile,unique=True,sort=None,output=None,pattern=None,path=None): 24 ''' 25 expression: regex rule 26 hostname: hostname as specified hosts() 27 logfile: log file name, wildcard supported, eg:*.log 28 unique: whether result is unique 29 sort: 1(ASC) or -1(DESC) ,default None 30 output:None or file name, default None imply print stream 31 pattern: group pattern , default None imply '1' 32 path: cd to path before execution 33 ''' 34 35 if not path: 36 path = r'.' 37 cmd_str = generate_cmd(expression,logfile,unique,sort,output,pattern) 38 execute(executor,hostname,cmd_str,path,host=hostname) 39 result = EXECUTE_RESULT[hostname] 40 return result 41 42def aggregate(expression,hostname,logfile,output=None,pattern=None,path=None): 43 ''' 44 expression: regex rule 45 hostname: hostname as specified hosts() 46 logfile: log file name, wildcard supported, eg:*.log 47 output:None or file name, default None imply print stream 48 pattern: group pattern , default None imply '1' 49 path: cd to path before execution 50 ''' 51 if not path: 52 path = r'.' 53 cmd_str = generate_cmd(expression,logfile,False,None,output,pattern,True,True) 54 execute(executor,hostname,cmd_str,path,host=hostname) 55 result = EXECUTE_RESULT[hostname] 56 return result 57 58def count(expression,hostname,logfile,unique=True,sort=None,output=None,pattern=None,path=None): 59 ''' 60 expression: regex rule 61 hostname: hostname as specified hosts() 62 logfile: log file name, wildcard supported, eg:*.log 63 unique: whether result is unique 64 sort: 1(ASC) or -1(DESC) ,default None 65 output:None or file name, default None imply print stream 66 pattern: group pattern , default None imply '1' 67 path: cd to path before execution 68 ''' 69 70 if not path: 71 path = r'.' 72 cmd_str = generate_cmd(expression,logfile,unique,sort,output,pattern,True) 73 execute(executor,hostname,cmd_str,path,host=hostname) 74 result = EXECUTE_RESULT[hostname] 75 if result: 76 result = int(result[0]) 77 return result 78 79 80def executor(hostname,cmd_str,path=None): 81 ''' 82 executor , called by execute 83 ''' 84 if not path: 85 path = r'.' 86 with cd(path): 87 res = run(cmd_str,quiet=True) 88 logger.debug('Command: %s:%s > %s'%(hostname,path,cmd_str)) 89 logger.debug('Command Execute Successful:%s, Failure:%s'%(res.succeeded,res.failed)) 90 EXECUTE_RESULT[hostname] = res.splitlines() 91 92def generate_cmd(expression,logfile,unique=True,sort=None,output=None,pattern=None,count=False,aggregate=False): 93 ''' 94 generate command 95 ''' 96 if not pattern: 97 pattern = r'\1' 98 99 if aggregate: 100 aggregate = '''| awk '{a[$1]++}END{for (j in a) print j","a[j]}' ''' 101 unique = False 102 sort = False 103 count = False 104 else: 105 aggregate = '' 106 107 if not unique: 108 unique = '' 109 else: 110 unique = '| uniq' 111 112 if sort: 113 if sort==1: 114 sort = '| sort' 115 elif sort==-1: 116 sort = '| sort -r' 117 else: 118 sort = '' 119 else: 120 sort = '' 121 122 if count: 123 count = '| wc -l' 124 else: 125 count = '' 126 127 if output: 128 output = '>%s'%output 129 else: 130 output = '' 131 132 cmd_str = '''cat %s | grep "%s" | sed 's/%s/%s/g' %s %s %s %s %s'''%(logfile,expression,expression,pattern,unique,sort,count,output,aggregate) 133 return cmd_str
假设你的日志是这样的：
1spider.A crawled http://www.163.com/abc.html 2spider.A crawled http://www.yahoo.com/xyz.html 3spider.B crawled http://www.baidu.com/mnq.html 4other log no crawing infomation involved 5spider.C crawled http://www.sina.com.cn/yyy.html
使用案例：test.py
1#encoding=utf-8 2 3import LogQuery 4 5#定义多个主机，用户名@主机,登录密码 6myhosts = [('rootman@192.168.2.228','123'),('rootman@192.168.2.229','123'),('rootman@192.168.2.219','123')] 7LogQuery.hosts(myhosts) 8 9''' 10案例1： 11查询有哪些域名被抓取过，使用query方法，会返回所有符合规则的数据 12预期返回： 13www.163.com 14www.yahoo.com 15... 16''' 17res = LogQuery.query('\(.*crawled http:\/\/\)\([^\/]*\)\(\/.*\)',myhosts[0][0],'gcrawler.*.log',unique=True,sort=None,output=None,pattern=r'\2',path='/home/workspace/Case/trunk/src/gcrawler/log') 18''' 19上一行代码解读： 20第一个参数指定了表示抓取的日志正则表达式，并且将其分组（为了提取域名），分组的括号用\(，\)表示，第二组是域名的提取。 21第二个参数指定了要查询那一台主机上的日志 22第三个参数指定了要分析的日志文件名，*表示任何字符 23第四个参数unique，是否对返回的条目进行排重，例如：日志中发现多个www.163.com,只算一个 24第五个参数sort，是否需要对抽取的条目进行排序，1：正序、-1：倒序，这里为None，即不需要排序 25第六个参数output，可以指定运行结果输出到某个文件，在这里不需要输出，为None 26第七个参数pattern，是指从正则表达式中抽取哪个分组，默认是第一组，这里用r'\2'指定第二组 27第八个参数path指定了日志在操作系统上所在的目录 28以下的count、aggregate方法使用的参数和query都是一样的意义 29''' 30 31''' 32案例2： 33统计被抓取过的域名有几个，使用count方法，会返回所有符合规则的统计总数 34预期返回：4 35... 36''' 37res = LogQuery.count('\(.*crawled http:\/\/\)\([^\/]*\)\(\/.*\)',myhosts[1][0],'gcrawler.*.log',unique=True,sort=None,output=None,pattern=r'\2',path='/home/workspace/Case/trunk/src/gcrawler/log') 38 39''' 40案例3： 41分别统计每个域名被抓取的数量 42返回的结果： 43域名1,统计数字 44域名2,统计数字 45... 46''' 47res = LogQuery.aggregate('\(.*crawled http:\/\/\)\([^\/]*\)\(\/.*\)',myhosts[2][0],'gcrawler.*.log',output=None,pattern=r'\2',path='/home/workspace/Case/trunk/src/gcrawler/log')#这里是分类统计就没必要指定unique和sort了。 48#打印分组统计的情况 49for i in res: 50 domain,count = i.split(',') 51 total += int(count) 52 print domain,'=>',count