#!/usr/bin/perl -w
##
##
##

use 5.012;
$| = 1;
use POSIX qw(strftime);

chdir '/home/odps/';
my $date = strftime("%Y%m%d", localtime(time()-86400*1));
my $partition = strftime("%Y-%m-%d", localtime(time()-86400*1));
my $sql = "tunnel download cbmds.mds_user_attribute_original_big_data_table/requestdate=${date}  -fd  '\x01' ./data/user_attribute_${date}.txt ";
say $sql;

say "===  开始下载odps  ===";

say qx{ /opt/odps/bin/odpscmd -e "$sql"};
my $downloadfile = "./data/user_attribute_${date}.txt";
die("file download error") unless -s $downloadfile;

#__----- 开始导入
#
say "===  开始导入clickhouse  ===";

say qx{ cat $downloadfile |sed -e "s/['\\"{}\\r\\t]//g" -e 's/\$/\\x01${partition}/' | clickhouse-client --format_csv_delimiter='\x01' --query="INSERT INTO user_attribute FORMAT CSV" --input_format_allow_errors_num=10000000};


#__----- 开始清理旧数据
my $old_date =  strftime("%Y%m%d", localtime(time()-86400*9));
my $old_partition = strftime("%Y-%m-%d", localtime(time()-86400*9));
say "=== drop table partition '$old_partition' ===";
qx{rm ./data/user_attribute_${old_date}.txt };
say qx{ clickhouse-client --query=" alter table user_attribute drop partition '$old_partition' "};


say "=== Done ===\n,", `date`;