#!/bin/sh
#time 2017-06-07
################# 初始化变量#########################
source /etc/bi.profile
hadoop="/usr/bin/hadoop";
hdfs="/usr/bin/hdfs";
cleanpath="/home/hdfs/hadoop-hdfs/personas/clean";
if [ "$1" != "" ];then
  date=`date -d "$1" +%Y-%m-%d`;
  logdate=`date -d "$1" +%Y%m%d`;
else
  date=`date -d "-1 day" +%Y-%m-%d`;
  logdate=`date -d "-1 day" +%Y%m%d`;
fi
year=${date:0:4}
month=${date:4:2}
export MYSQL_BI_IP=10.90.8.32
export MYSQL_BI_USER=readtongji
export MYSQL_BI_PWD=uZpvSuDuEiEOwCtk21on
export MYSQL_BI_DB=db_bi
export MYSQL_BI_PORT=3306


function Delete(){
	hadoop fs -rm -r -f /newread/read/log_access_txt/platform=android/day=${date}/
}

function Clean(){
	cd ${cleanpath}
	mark=`hadoop fs -ls /newread/logflag/android/${logdate}/*.lock | wc -l`
        title="android日志检查"
        content="${date}android日志上传成功"
        content1="${date}android日志上传失败"
        flag=0
        if [ $mark -eq 1 ];then
           /bin/python ${alert_path}/alert_hive.py $title $content "android"
           flag=1
        else 
           /bin/python ${alert_path}/alert_hive.py $title $content1 "android"
        fi
        
        while [ $mark -eq 0 ];do
            sleep 10
            mark=`hadoop fs -ls /newread/logflag/android/${logdate}/*.lock | wc -l`
        done
	echo ${mark}
        
        if [ $mark -eq 1 -a $flag -eq 0 ];then
            /bin/python ${alert_path}/alert_hive.py $title $content "android"
        elif [ $mark -eq 0 -a $flag -eq 0 ];then  
            /bin/python ${alert_path}/alert_hive.py $title $content1 "android"
        fi 
	hadoop jar /usr/hdp/2.6.5.0-292/hadoop-mapreduce/hadoop-streaming.jar -input /newread/logbak/android/${logdate}/*-access-*.log -output /newread/read/log_access_txt/platform=android/day=${date}  -file /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py  -mapper "/home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py"   -jobconf mapred.job.name="job_acclog_last_and"
#hadoop jar /usr/hdp/2.6.5.0-292/hadoop-mapreduce/hadoop-streaming.jar -inputformat com.hadoop.mapred.DeprecatedLzoTextInputFormat  -input /newread/logbak/android/${logdate}/*-access-*.log.lzo -output /newread/read/log_access_txt/platform=android/day=${date}  -file /home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py  -mapper "/home/hdfs/hadoop-hdfs/personas/clean/clean_api/clean_api.py"  -jobconf mapred.job.name="job_acclog_last_and" -jobconf mapred.output.compress=true -jobconf mapred.output.compression.codec=org.apache.hadoop.io.compress.SnappyCodec

	#check_result "clean_acclog_android"

}

function LoadHive(){
hive <<EOF
use read;
ALTER TABLE log_access_txt  DROP IF EXISTS partition (platform='android',day='$date');

alter table log_access_txt  add IF NOT EXISTS  partition  (platform='android',day='$date') location '/newread/read/log_access_txt/platform=android/day=$date';

exit;
EOF
exit;

}

function main(){
#hadoop fs -test -e /newread/logbak/novel/$logdate/access/
#while [ $? -ne 0 ];do
  #      sleep 3
 #       hadoop fs -test -e /newread/logbak/novel/$logdate/access/
#done
Delete
Clean
LoadHive
# 插入压缩表read.log_access
#sh /home/hdfs/hadoop-hdfs/personas/clean/read_log_access.sh ${date}
}

main

