wdp中pkg和install整合JKS
# wdp-pkg项目
# 打包脚本命令
待处理密码加解密处理，接入JKS；还是定制统一版本参数；
#现在server agent等相关库；TODO： 版本对应上；变成参数传递
sh download-wdp-server-agent.sh 工号 密码
#在容器中构建前端相关资源；TODO： 版本对应上；变成参数传递
sh build-wdp-web.sh 工号 密码
#重新生成打包生成rpm包；
sh build-wdp-server.sh
#最后生成包的路径：./sh/package/wdp-package/wdp-server/target/rpm/wdp-server/RPMS/x86_64/wdp-server-1.0-1.x86_64.rpm
1
2
3
4
5
6
7
8
2
3
4
5
6
7
8
# wdp-install项目
# 安装脚本
installServer.sh
cd /srv && rm -rf installServer.sh && wget http://10.45.xxx.116:58888/files/wdp/jks/server/installServer.sh && sh installServer.sh
#!/bin/bash
rm -rf /var/lib/wdp-server/resources/template-for-installing-wdp.txt
yum remove -y wdp-server && yum install -y wdp-server
#安装指定测试版本的server;
yum install http://10.45.xxx.116:58888/files/wdp/test/wdp-server-1.0-1.x86_64.rpm
mysql -uroot -pPass-xxx-xx -e "
CREATE DATABASE IF NOT EXISTS wdp;
use wdp;
source /var/lib/wdp-server/resources/Wdp-DDL-MySQL-CREATE.sql;
"
#这个要安装，要不然会报错，mysql-java-8.x的驱动找不到；
yum install -y postgresql-server
templateCfg=/var/lib/wdp-server/resources/template-for-installing-wdp.txt
localIp=$(ifconfig | grep '\<inet\>'| grep -v '127.0.0.1' | awk '{ print $2}' | awk 'NR==1')
sed -i "s/mysql_hostname_ip=10.45.47x.68/mysql_hostname_ip=$localIp/g" $templateCfg
sed -i 's/mysql_database_name=wdp_data/mysql_database_name=wdp/g' $templateCfg
sed -i 's/mysql_password=Pass-xxx-root/mysql_password=Pass-xxx-xx/g' $templateCfg
wdp-install.sh $templateCfg
baseHttp=http://10.45.xxx.116:58888/files/wdp/jks
#更新license
cd /etc/wdp-server/conf && mv -f public.key public.key.old && mv -f wdp.license wdp.license.old
wget $baseHttp/license/public.key && wget $baseHttp/license/wdp.license
wdp-server restart
#更新前端
#cd /srv/ && rm -rf installWeb.sh && wget $baseHttp/web/installWeb.sh && sh installWeb.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
最后完整的重装命令：
wget http://10.45.xxx.116:58888/files/wdp/jks/server/uninstallWDP.sh
wget http://10.45.xxx.116:58888/files/wdp/jks/server/installMysql.sh
wget http://10.45.xxx.116:58888/files/wdp/jks/server/installServer.sh
sh uninstallWDP.sh && sh installMysql.sh && sh installServer.sh
1
2
3
4
2
3
4
# 完全卸载脚本
完全卸载后，再要安装wdp-server先提前安装配置mysql;
uninstallWDP.sh
#!/bin/sh
function killproc(){
awk '{printf "%s %s\n",$2,$8}' |
while read psId psCmd
do
printf "Kill(%s) %s\n" $psId $psCmd
kill -9 $psId &
done
}
function get_wdp_agents(){
curl -u xxx:xxx -H "X-Requested-By: wdp" -X GET http://${WDP_MASTER}:8080/api/v1/hosts
}
#-------------------------------------------------------------------
# replace(source, which, to)
# In POSIX sh, string replacement ${VB//,/ } is not supported .
# Returns replaced string.
#-------------------------------------------------------------------
replace() {
string="$1"
which="$2"
to="$3"
result=$(echo $string | sed -e "s/${which}/${to}/g")
echo ${result}
}
function uninstall_wdp_server(){
wdp-server stop
yum erase -y wdp-server
rm -rf /opt/ibm/wdp
rm -rf /etc/wdp-server
rm -rf /var/log/wdp-server
rm -rf /var/lib/pgsql*
rm -rf /var/lib/wdp-server
rm -rf /usr/lib/wdp-server
rm -rf /var/run/wdp-server
unlink /usr/lib/python2.6/site-packages/wdp_commons
unlink /usr/lib/python2.6/site-packages/resource_management
unlink /usr/lib/python2.6/site-packages/wdp_jinja2
unlink /usr/lib/python2.6/site-packages/wdp_simplejson
rm -rf /usr/lib/python2.6/site-packages/wdp_agent
rm -rf /usr/lib/python2.6/site-packages/wdp_server
rm -rf /usr/lib/python2.6/site-packages/resource_monitoring
}
function uninstall_wdp_agent(){
wdp-agent stop
ps -ef | grep Wdp | killproc
ps -ef | grep Wdp | killproc
ps -ef | grep /usr/wdp | killproc
ps -ef | grep resource_monitoring/main.py | killproc
#python /usr/lib/python2.6/site-packages/wdp_agent/HostCleanup.py --silent --skip=users -f /etc/wdp-agent/conf/HostCleanup.ini,/etc/wdp-agent/conf/HostCleanup_Custom_Actions.ini -o /tmp/cleanup.log
python /usr/lib/python2.6/site-packages/wdp_agent/HostCleanup.py --silent --skip=users
#yum erase -y wdp-metrics* bigtop* extjs* flume* hadoop* hbase* hive* iop-select* knox* oozie* perl-Crypt-DES* perl-Net-SNMP* perl-rrdtool* python-rrdtool* pig* rrdtool* slider* solr* spark* sqoop* storm* zookeeper* kafka* fping* libconfuse* --setopt=tsflags=noscripts
rpms=wdp,hadoop,hbase,hive,spark,zookeeper,pig,sqoop,oozie,kafka,slider,knox,flume,solr,falcon,accumulo,mahout,storm,tez,atlas,ranger
#for _rpm in ${rpms//,/ };
for _rpm in $(replace "${rpms}" "," " ")
do
yum erase -y ${_rpm}*
done
RPM_NAME="wdp|hadoop|hbase|hive|spark|zookeeper|pig|sqoop|oozie|kafka|slider|knox|flume|solr|falcon|accumulo|mahout|storm|tez|atlas"
#call rpm erase to ensure the package will be removed
RPMLIST=$(rpm -qa|grep -E "$RPM_NAME"|awk '{ print $1 }')
if [[ -z "$RPMLIST" ]];then
echo "No $RPM_NAME regex installed"
else
for RPML in $RPMLIST
do
sudo rpm -e $RPML --noscripts > /dev/null 2>&1
done
fi
#utils
utils_rpms=phoenix,ranger,bigtop,extjs,snappy,wdp-select,postgresql
for _rpm in $(replace "${utils_rpms}" "," " ")
do
yum erase -y ${_rpm}*
done
yum erase -y mysql* mysql-devl mysql-server
yum erase -y wdp-agent
yum remove -y smartsense-hst
rm -fr /var/lib/mysql
#rm
servers=wdp-metrics,wdp-agent,wdp,hadoop,hadoop-hdfs,hadoop-mapreduce,hadoop-yarn,hbase,spark,zookeeper,hive,oozie,pig,sqoop,kafka,slider,knox,flume,falcon,solr,accumulo,accumulo,mahout,tez,ranger,ranger-admin,ranger-tagsync,ranger-usersync
#for _server in ${servers//,/ };
for _server in $(replace "${servers}" "," " ")
do
#eval "rm -rf /var/lib/${_server}*"
rm -rf /var/lib/${_server}*
rm -rf /var/log/${_server}*
rm -rf /var/run/${_server}*
rm -rf /etc/${_server}*
done
rm -rf /tmp/*
#full clean
rm -rf /etc/rc.d/init.d/wdp-agent
rm -rf /usr/lib/wdp-agent
rm -rf /usr/lib/python2.6/site-packages/wdp_agent
rm -rf /usr/lib/python2.6/site-packages/resource_monitoring
rm -rf /usr/sbin/wdp-agent
rm -rf /usr/lib/wdp*
rm -rf /usr/wdp
#####################klh################
rm -rf /var/lib/smartsense/
rm -rf /var/log/hst
rm -fr /etc/hst/
rm -fr /usr/lib/flume
rm -fr /usr/lib/storm
rm -rf /data1/hadoop
rm -rf /data2/hadoop
rm -f /usr/bin/ranger-*
unlink /usr/lib/python2.6/site-packages/wdp_commons
unlink /usr/lib/python2.6/site-packages/resource_management
unlink /usr/lib/python2.6/site-packages/wdp_jinja2
unlink /usr/lib/python2.6/site-packages/wdp_simplejson
#user bin
for lns in $(ls -all /usr/bin | grep "/usr/iop" | awk '{print $9}') ;
do
if [ ! -z ${lns} ]; then
echo "rm -rf /usr/bin/$lns";
rm -rf /usr/bin/$lns
fi
done
}
function uninstall_wdp(){
wdp-server stop
#for agent in ${WDP_AGENTS//,/ } ;
#for agent in $(replace "${WDP_AGENTS}" "," " ")
#do
## gLog 3 Info "uninstall_wdp_agent on agent $agent ..."
# ssh $agent "${TEST_WORKDIR}/framework/wdp/${CLUSTER_TYPE}/main.s uninstall_wdp_agent"
#)&
#done
#wait
uninstall_wdp_agent
uninstall_wdp_server
removeServiceAccounts
rm -fr /hadoop/
}
function removeServiceAccounts(){
users=hbase,wdp-qa,ams,hcat,hdfs,hive,kafka,mapred,oozie,spark,sqoop,tez,yarn,zookeeper,accumulo,atlas,falcon,flume,knox,mahout,storm,livy,yarn-ats
for _user in $(replace "${users}" "," " ")
do
userdel -r -f ${_user}
rm -fr /etc/security/limits.d/${_user}.conf
done
}
function removeRepo(){
#rm repo
rm -rf /etc/yum.repos.d/WDP*
rm -rf /etc/yum.repos.d/wdp*
yum clean metadata
yum clean all
}
uninstall_wdp
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
installMysql.sh
#!/usr/bin/bash
echo "----卸载后再重新安装mysql---"
yum remove -y mysql-community-server && yum install -y mysql-community-server
sleep 5
pwdWdp="Passxxx-xxx"
pwdHive="Passxxx-xx"
#sed -i ‘/mysqld/a\skip-grant-tables’ /etc/my.cnf
#echo "skip-grant-tables=true" >> /etc/my.cnf
echo "----设置my.cnf---"
echo "
character_set_server=utf8
init_connect='SET NAMES utf8'
lower_case_table_names=1
skip-grant-tables=true
" >> /etc/my.cnf
systemctl enable mysqld
systemctl restart mysqld
sleep 5
echo "----设置密码----"
mysql -u root -e "update mysql.user set authentication_string=password('$pwdWdp') where user='root';"
systemctl stop mysqld
sleep 5
echo "----修改skip重启----"
sed -i 's/skip-grant-tables/#skip-grant-tables/g' /etc/my.cnf
systemctl restart mysqld
sleep 5
echo "----授权----"
mysql -uroot -p$pwdWdp --connect-expired-password -e "
SET PASSWORD FOR 'root'@'localhost' = PASSWORD('$pwdWdp');
grant all privileges on *.* to 'root'@'%' identified by '$pwdWdp' with grant option;
grant all privileges on *.* to 'root'@'localhost' identified by '$pwdWdp' with grant option;
flush privileges;
"
mysql -uroot -p$pwdWdp --connect-expired-password -e "
CREATE USER 'wdp'@'%' IDENTIFIED BY '$pwdWdp';
GRANT ALL PRIVILEGES ON *.* TO 'wdp'@'%';
CREATE USER 'wdp'@'localhost' IDENTIFIED BY '$pwdWdp';
GRANT ALL PRIVILEGES ON *.* TO 'wdp'@'localhost';
FLUSH PRIVILEGES;
"
mysql -uroot -p$pwdWdp --connect-expired-password -e "
CREATE USER 'hive'@'%' IDENTIFIED BY '$pwdHive';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'%';
CREATE USER 'hive'@'localhost' IDENTIFIED BY '$pwdHive';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'localhost';
FLUSH PRIVILEGES;
"
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# 整合JKS
上次更新: 2023/11/17, 05:08:19