#!/usr/bin/env python
# -* - coding: UTF-8 -* -
#name : by demon
#time : 2017-05-24
import sys
import re
import time
import os
import datetime
import socket,json
import urllib
import traceback
from struct import pack, unpack
reload(sys)
sys.setdefaultencoding('utf-8')
class clearurl():
	#@staticmethod
	def __init__(self, re_path):
		self.re_path = re_path
		self.re_mess = os.popen('hadoop fs -cat %s' %re_path).read()
		self.dict_re=eval(self.re_mess)

	def clear_api(self,requrlstr,reqParam):
		dict_re = self.dict_re
		#定义变量
		action="-999"
		action_detail="-999"
		bookid="-999"
		chapterNum="-999"
		querymessage="-999"
		requrl = requrlstr.split("?")[0]
		
		for i in dict_re.keys():
			if 're' not in dict_re[i]:
				continue
			patterni = re.compile(r'^%s' % dict_re[i]['re'])
			break_flag = False
			if patterni.match(requrl):
				for n in dict_re[i]['detail'].keys():
					patternn = re.compile(r'^%s' % dict_re[i]['detail'][n]['detail_re'])
					if patternn.match(requrl):
						break_flag = True
						action = dict_re[i]['action']
						action_detail = n
						action_mark = dict_re[i]['detail'][n]['action_mark']
						try:
							if action == 'search':
								querystr = requrlstr.split('query=')[1].split('&')[0]
								querymessage = urllib.unquote(querystr)
							else:
								querymessage = '-999'
						except:
							querymessage='-999'
						#print action_mark,reqParam
						try:
							if action_mark == "0":
								bookid = "-999"
								chapterNum = "-999"
							elif action_mark == "1":
								reqjson=json.loads(reqParam)
								if 'bookIds' in reqParam:
										bookid=reqjson['bookIds']
								elif 'bookId' in reqParam:
										bookid=reqjson['bookId']
								else:
										bookid='-999'
								chapterNum="-999"
							elif action_mark == "2":
								if reqParam !='':
									reqjson = json.loads(reqParam)
									if 'bookIds' in reqParam:
										bookid=reqjson['bookIds']
									elif 'bookId' in reqParam:
										bookid=reqjson['bookId']
									else:
										bookid='-999'	
									if 'chapterNum' in reqParam:  
										chapterNum = reqjson['chapterNum']
									else:
										chapterNum="-999"
							else:
								bookid = "-999"
								chapterNum = "-999"
						except:
							bookid = "-999"
							chapterNum = "-999"
						break
			if break_flag:
				break
		return action, action_detail, bookid, chapterNum,querymessage
	def clear_refer(self,refer,reqParam,platform):
		# print refer
		try:

			refer_com = refer.split("/")[2]
			pattern_com = re.compile(r'[a-z]+.yc.ifeng.com')
			if pattern_com.match(refer_com):
				refer_url = refer.split(".com")[1].split("?")[0]
				#print refer_url
				try:
					if platform == "api":
						referaction, referaction_detail, referbookid, referchapterNum,querymessage = self.clear_api(refer_url,reqParam)
				except:
					referaction = "-999"
			else:
				referaction = "-999"
		except:
			refer_com = "-999"
			referaction = "-999"
		return refer_com, referaction


class IPInfo():
	'''QQWry.Dat数据库查询功能集合
	'''

	def __init__(self, dbname):
		''' 初始化类，读取数据库内容为一个字符串，
		通过开始8字节确定数据库的索引信息'''

		self.dbname = dbname
		# f = file(dbname, 'r')

		# Demon注：在Windows下用'r'会有问题，会把\r\n转换成\n
		# 详见http://demon.tw/programming/python-open-mode.html
		# 还有Python文档中不提倡用file函数来打开文件，推荐用open
		f = open(dbname, 'rb')

		self.img = f.read()
		f.close()

		# QQWry.Dat文件的开始8字节是索引信息,前4字节是开始索引的偏移值，
		# 后4字节是结束索引的偏移值。
		# (self.firstIndex, self.lastIndex) = unpack('II', self.img[:8])

		# Demon注：unpack默认使用的endian是和机器有关的
		# Intel x86和AMD64(x86-64)是little-endian
		# Motorola 68000和PowerPC G5是big-endian
		# 而纯真数据库全部采用了little-endian字节序
		# 所以在某些big-endian的机器上原代码会出错
		(self.firstIndex, self.lastIndex) = unpack('<II', self.img[:8])

		# 每条索引长7字节，这里得到索引总个数
		self.indexCount = (self.lastIndex - self.firstIndex) / 7 + 1

	def getString(self, offset=0):
		''' 读取字符串信息，包括"国家"信息和"地区"信息

		QQWry.Dat的记录区每条信息都是一个以'\0'结尾的字符串'''

		o2 = self.img.find('\0', offset)
		# return self.img[offset:o2]
		# 有可能只有国家信息没有地区信息，
		gb2312_str = self.img[offset:o2]
		try:
			utf8_str = unicode(gb2312_str, 'gb2312').encode('utf-8')
		except:
			return '未知'
		return utf8_str

	def getLong3(self, offset=0):
		'''QQWry.Dat中的偏移记录都是3字节，本函数取得3字节的偏移量的常规表示
		QQWry.Dat使用"字符串"存储这些值'''
		s = self.img[offset: offset + 3]
		s += '\0'
		# unpack用一个'I'作为format，后面的字符串必须是4字节
		# return unpack('I', s)[0]

		# Demon注：和上面一样，强制使用little-endian
		return unpack('<I', s)[0]

	def getAreaAddr(self, offset=0):
		''' 通过给出偏移值，取得区域信息字符串，'''

		byte = ord(self.img[offset])
		if byte == 1 or byte == 2:
			# 第一个字节为1或者2时，取得2-4字节作为一个偏移量调用自己
			p = self.getLong3(offset + 1)
			return self.getAreaAddr(p)
		else:
			return self.getString(offset)

	def getAddr(self, offset, ip=0):
		img = self.img
		o = offset
		byte = ord(img[o])

		if byte == 1:
			# 重定向模式1
			# [IP][0x01][国家和地区信息的绝对偏移地址]
			# 使用接下来的3字节作为偏移量调用字节取得信息
			return self.getAddr(self.getLong3(o + 1))

		if byte == 2:
			# 重定向模式2
			# [IP][0x02][国家信息的绝对偏移][地区信息字符串]
			# 使用国家信息偏移量调用自己取得字符串信息
			cArea = self.getAreaAddr(self.getLong3(o + 1))
			o += 4
			# 跳过前4字节取字符串作为地区信息
			aArea = self.getAreaAddr(o)
			return (cArea, aArea)

		if byte != 1 and byte != 2:
			# 最简单的IP记录形式，[IP][国家信息][地区信息]
			# 重定向模式1有种情况就是偏移量指向包含国家和地区信息两个字符串
			# 即偏移量指向的第一个字节不是1或2,就使用这里的分支
			# 简单地说：取连续取两个字符串！

			cArea = self.getString(o)
			# o += len(cArea) + 1
			# 我们已经修改cArea为utf-8字符编码了，len取得的长度会有变，
			# 用下面方法得到offset
			o = self.img.find('\0', o) + 1
			aArea = self.getString(o)
			return (cArea, aArea)

	def find(self, ip, l, r):
		''' 使用二分法查找网络字节编码的IP地址的索引记录'''
		if r - l <= 1:
			return l

		m = (l + r) / 2
		o = self.firstIndex + m * 7
		# new_ip = unpack('I', self.img[o: o+4])[0]

		# Demon注：和上面一样，强制使用little-endian
		new_ip = unpack('<I', self.img[o: o + 4])[0]

		if ip <= new_ip:
			return self.find(ip, l, m)
		else:
			return self.find(ip, m, r)

	def getIPAddr(self, ip):
		''' 调用其他函数，取得信息！'''
		# 使用网络字节编码IP地址
		ip = unpack('!I', socket.inet_aton(ip))[0]
		# 使用 self.find 函数查找ip的索引偏移
		i = self.find(ip, 0, self.indexCount - 1)
		# 得到索引记录
		o = self.firstIndex + i * 7
		# 索引记录格式是： 前4字节IP信息+3字节指向IP记录信息的偏移量
		# 这里就是使用后3字节作为偏移量得到其常规表示（QQWry.Dat用字符串表示值）
		o2 = self.getLong3(o + 4)
		# IP记录偏移值+4可以丢弃前4字节的IP地址信息。
		(c, a) = self.getAddr(o2 + 4)
		return (c, a)

	def output(self, first, last):
		for i in range(first, last):
			o = self.firstIndex + i * 7
			ip = socket.inet_ntoa(pack('!I', unpack('I', self.img[o:o + 4])[0]))
			offset = self.getLong3(o + 4)
			(c, a) = self.getAddr(offset + 4)
			print "%s %d %s/%s" % (ip, offset, c, a)

		# def getIP(self,address):
		#		i = IPInfo(self.dbname)
		#		(c, a) = i.getIPAddr(address)
		# Demon注：如果是在Windows命令行中运行把编码转回gb2312以避免乱码
		#		if sys.platform == 'win32':
		#			c = unicode(c, 'utf-8').encode('gb2312')
		#			a = unicode(a, 'utf-8').encode('gb2312')
		return c
def read_input(file):
	for line in file:
		yield line.strip()

def main():
	platform = "api"
	#加载清洗url脚本
	clear = clearurl('/newread/tool/clean/dictre/%s.re' %platform)
	#加载IPSeek
	if os.path.isfile('/tmp/QQWry.Dat'):
		pass
	else:
		os.system('hadoop fs -get /newread/tool/clean/QQWry.Dat /tmp/')
	#加载IPSeek
	P = IPInfo('/tmp/QQWry.Dat')
	#清洗程序开始
	data = read_input(sys.stdin)
	for line in data:
		try:
			list = line.split("] [")
			#切割日志各字段
			logversion = list[0].split("[")[1]
			projectname = list[1]
			serverip = list[2]
			reqbegintime = list[3]
			reqendtime = list[4]
			reqspendtime = list[5]
			reqip = list[6]
			requrlstr = list[7]
			#过滤下载日志 add by zhaopf 2018-12-06
			if(('/api/download/' in requrlstr) or ('/api/other/jrttBack' in requrlstr) or ('/kuaishou/notify' in requrlstr) or ('/api/getPublicKey' in requrlstr)):
				continue
			refer = list[8]
			method = list[9]
			httpcode = list[10]
			userid = list[11]
			sessionid = list[12]
			platformid = list[13]
			if( 'type=bbs' in requrlstr):
				if platformid == '1': 
					platformid=100
				elif platformid == '2':
					platformid=200
			appversion = list[14]
			channelid = list[15]
			bookrecommend= list[16]
			payType = list[17]
			paynum = list[18]
			rechargetype = list[19]
			payamount = list[20]
			ssid = list[21]
			isnewuser = list[22]
			ischannelnewuser = list[23]
			readwordstype = list[24]
			readwords = list[25]
			mobile = list[26]
			uaplatform = list[27]
			uastat = list[28]
			apicode = list[29]
			xClient = list[30]
			ua = list[31]
			if('JianKongBao' in ua ):
				continue
			#print(list[33])
            reqParam = list[32]
			processParam = list[33]

			#clear refer 定义来源网址以及referaction_detail
			refer_com, referaction_detail = clear.clear_refer(refer,reqParam,platform)
			#将用户IP转化为省份以及地区
			try:
				UserIp = reqip.split(',')[0]
				(country, detail_area) = P.getIPAddr(UserIp)
			except:
				country="-999"
				detail_area="-999"

			#调用clear方法，将URL转化为action等
			#定义变量
			action="-999"
			action_detail="-999"
			bookid="-999"
			chapterNum="-999"
			querymessage="-999"
			#try:
			if platform =="api":
				action, action_detail, bookid, chapterNum ,querymessage = clear.clear_api(requrlstr, reqParam)
			#except:
			#	action = "-999"
			#	action_detail = "-999"
			#	bookid = "-999"
			#	chapterNum = "-999"
			#	querymessage = '-999'
			#切分Xclient取出相应的数据
			try:
				sdk = xClient.split('sv=')[1].split(';')[0]
			except:
				sdk = "-999"
			try:
				screensize = xClient.split('ss=')[1].split(';')[0]
			except:
				screensize = "-999"
			try:
				type = xClient.split('pm=')[1].split(';')[0]
				type=urllib.unquote(type)
			except:
				type = "-999"
			try:
				imsi = xClient.split('imsi=')[1].split(';')[0]
			except:
				imsi = "-999"
			try:
				version = xClient.split('version=')[1].split(';')[0]
				if version is not None and version.find('.') > 0:
					#print(version)
					first_index = version.find('.') + 1
					#print(first_index)
					last_index = version.rfind('.')
					#print(last_index)
					appversion2 = version[first_index:last_index]
					#print(appversion2)
			except:
				version = "-999"
				appversion2 = "-999"
			try:
				mac = xClient.split('mac=')[1].split(';')[0]
			except:
				mac = "-999"
			try:
				device_id = xClient.split('dID=')[1].split(';')[0]
			except:
				device_id = "-999"
				rn = "-999"
				tdcn = "-999"
			try:
				imei = xClient.split('imei=')[1].split(';')[0]
			except:
				imei = '-999'
			try:
				idfa = xClient.split('idfa=')[1].split(';')[0]
			except:
				idfa = '-999'
			#定义浏览器
			endbrand = "-999"
			browserbrand = "-999"
			browserversion = "-999"
			primaryurlpath = "-999"
			others = querymessage
			cellid = "-999"
			projectType="-999"
			
			print '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t' %(logversion,projectname,serverip,reqbegintime,reqendtime,reqspendtime,reqip,country,detail_area,requrlstr,action,action_detail,bookid,chapterNum,refer,refer_com,referaction_detail,method,httpcode,userid,sessionid,platformid,appversion,channelid,bookrecommend,payType,paynum,rechargetype,payamount,ssid,isnewuser,ischannelnewuser,readwordstype,readwords,mobile,uaplatform,uastat,apicode,imei,sdk,screensize,type,imsi,cellid,version,device_id,mac,endbrand,browserbrand,browserversion,ua,primaryurlpath,reqParam,others,processParam,projectType,idfa,appversion2)
		except Exception, e:
			exc_type, exc_obj, exc_tb = sys.exc_info();
			fname = os.path.split(exc_tb.tb_frame.f_code.co_filename)[1]
			#print("line="+line)
			#print(exc_type, fname, exc_tb.tb_lineno)
			#traceback.print_exc()
			continue
		
if __name__ == "__main__":
	main()
