例子1：编写sqoop的自动shell脚本完成对hive中ods层的增量数据导入

#!/bin/bash

if [ $# -eq 1 ]; then
  date_str=`date -d "$1" +"%Y-%m-%d"`
  year_month=`date -d "$1" +"%Y_%m"`
else
  date_str=`date -d "-1 day" +"%Y-%m-%d"`
  year_month=`date -d "-1 day" +"%Y_%m"`
fi

# echo ${date_str}
# echo ${year_month}

SQOOP_HOME="/usr/bin/sqoop"
JDBCURL="jdbc:mysql://node151:3306/nev?useUnicode=true&characterEncoding=utf-8"
USERNAME="root"
PASSWORD="Heima@123456"

# 1 处理 主表 增量数据
${SQOOP_HOME} import \
--connect "${JDBCURL}" \
--username ${USERNAME} \
--password ${PASSWORD} \
--query "select
id,
create_date_time,
session_id,
sid,
create_time,
seo_source,
seo_keywords,
ip,
area,
country,
province,
city,
origin_channel,
user as user_match,
manual_time,
begin_time,
end_time,
last_customer_msg_time_stamp,
last_agent_msg_time_stamp,
reply_msg_count,
msg_count,
browser_name,
os_info,
'${date_str}' as starts_time
from nev.web_chat_ems_${year_month}
where create_time between '${date_str} 00:00:00' and '${date_str} 23:59:59' and \$CONDITIONS" \
--fields-terminated-by '\t' \
--hcatalog-database itcast_ods \
--hcatalog-table web_chat_ems \
-m 1

# 等前一个执行完, 再执行下一个
wait

# 2 处理 副表 增量数据 
${SQOOP_HOME} import \
--connect "${JDBCURL}" \
--username ${USERNAME} \
--password ${PASSWORD} \
--query "select
*,
'${date_str}' as start_time
from nev.web_chat_text_ems_${year_month} t2
where exists(select * from web_chat_ems_${year_month} t1
             where t1.id=t2.id
               and t1.create_time between '${date_str} 00:00:00' and '${date_str} 23:59:59'
            ) and \$CONDITIONS" \
--fields-terminated-by '\t' \
--hcatalog-database itcast_ods \
--hcatalog-table web_chat_text_ems \
-m 1