CDH安装

官方文档

本次部署的版本为 6.3.x

准备

导入密钥

ssh-copy-id user@host

设置 hosts

cat > /etc/hosts <<EOF

127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
10.25.55.101 k8s-master1
10.25.55.102 k8s-master2
10.25.55.103 k8s-master3
10.25.55.104 k8s-worker1
10.25.55.105 s1.cdh.com k8s-worker2
10.25.55.106 s2.cdh.com k8s-worker3
10.25.55.107 s3.cdh.com k8s-worker4
10.25.55.108 k8s-worker5
10.25.55.109 k8s-worker6
10.25.55.110 k8s-worker7
10.25.55.111 k8s-worker8
10.25.55.112 k8s-worker9
10.25.55.113 k8s-worker10
10.25.55.114 k8s-worker11
10.25.55.115 k8s-worker12
10.25.55.116 k8s-worker13
10.25.55.117 k8s-worker14
10.25.55.118 k8s-worker15
10.25.55.119 k8s-worker16

EOF

设置 hostname

hostnamectl set-hostname s1.cdh.com
hostnamectl set-hostname s2.cdh.com
hostnamectl set-hostname s3.cdh.com

设置 network

cat > /etc/sysconfig/network <<EOF

HOSTNAME=s1.cdh.cn

EOF

cat > /etc/sysconfig/network <<EOF

HOSTNAME=s2.cdh.cn

EOF

cat > /etc/sysconfig/network <<EOF

HOSTNAME=s3.cdh.cn

EOF

禁用防火墙

sudo systemctl disable firewalld sudo systemctl stop firewalld

设置 SELinux 模式

getenforce 如果为 enforcing，则需要编辑 /etc/selinux/config，将 SELINUX=enforcing 改为 SELINUX=permissive

NTP 服务配置

sudo yum install ntp -y

sudo cat > /etc/ntp.conf <<EOF

server 0.pool.ntp.org
server 1.pool.ntp.org
server 2.pool.ntp.org

EOF

sudo systemctl start ntpd
sudo systemctl enable ntpd

同步系统时间

ntpdate -u 0.pool.ntp.org

同步硬件时间

hwclock --systohc

安装

配置仓库

cat > /etc/yum.repos.d/cloudera-manager.repo <<EOF

[cloudera-manager]
name=Cloudera Manager 6.3.1
baseurl=http://10.25.55.105/cloudera-repos/cm6/6.3.1/
gpgcheck=0
enabled=1
autorefresh=0
type=rpm-md

EOF

安装 JDK

sudo yum install oracle-j2sdk1.8 -y

安装 Cloudera Manager Server

sudo yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server -y

开启 Auto-TLS

sudo JAVA_HOME=/usr/java/jdk1.8.0_181-cloudera /opt/cloudera/cm-agent/bin/certmanager setup --configure-services

配置 MySQL

卸载旧版本

yum list installed | grep mysql 列出一安装的版本，使用 yum remove 一一卸载

安装

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15

wget http://repo.mysql.com/mysql-community-release-el7-5.noarch.rpm

sudo rpm -ivh mysql-community-release-el7-5.noarch.rpm

sudo yum update -y

sudo yum install mysql-server -y

sudo systemctl start mysqld


yum localinstall -y mysql-community-common-8.0.21-1.el7.x86_64.rpm
yum localinstall -y mysql-community-libs-8.0.21-1.el7.x86_64.rpm
yum localinstall -y mysql-community-client-8.0.21-1.el7.x86_64.rpm
yum localinstall -y mysql-community-server-8.0.21-1.el7.x86_64.rpm
  • 必须使用 InnoDB 引擎
  • 推荐将 innodb_flush_method 设置为 O_DIRECT
  • 小于50台机器，推荐将 max_connections 配置为 100 x (机器数量) + 50，其他情况请参考文档

推荐配置

[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0

key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1

max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M

#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log

#In later versions of MySQL, if you enable the binary log and do not set
#a server_id, MySQL will not start. The server_id must be unique within
#the replicating group.
server_id=1

binlog_format = mixed

read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

[mysqld_safe]
log-error=/var/log/mysqld.log
pid-file=/var/run/mysqld/mysqld.pid

sql_mode=STRICT_ALL_TABLES

启动

sudo systemctl enable mysqld
sudo systemctl start mysqld

初始化

查看MySQL 安装后默认密码

cat /var/log/mysqld.log | grep "A temporary password"

运行 sudo /usr/bin/mysql_secure_installation 进行初始化，过程如下所示：

[...]
Enter current password for root (enter for none):
OK, successfully used password, moving on...
[...]
Set root password? [Y/n] Y
New password:
Re-enter new password:
Remove anonymous users? [Y/n] Y
[...]
Disallow root login remotely? [Y/n] N
[...]
Remove test database and access to it [Y/n] Y
[...]
Reload privilege tables now? [Y/n] Y
All done!

安装 MySQL JDBC Driver

wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz

tar zxvf mysql-connector-java-5.1.46.tar.gz

sudo mkdir -p /usr/share/java/
cd mysql-connector-java-5.1.46
sudo cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar

创建数据库

  • 运行 mysql -u root -p，输入密码

  • 创建数据库 CREATE DATABASE IF NOT EXISTS scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci; CREATE USER ‘scm’@’%’ IDENTIFIED BY ‘Onemap.2020’; GRANT ALL ON scm.* TO ‘scm’@’%’;

    CREATE DATABASE IF NOT EXISTS amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'amon'@'%' IDENTIFIED BY 'Onemap.2020';
GRANT ALL ON amon.* TO 'amon'@'%';

CREATE DATABASE IF NOT EXISTS rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'rman'@'%' IDENTIFIED BY 'Onemap.2020';
GRANT ALL ON rman.* TO 'rman'@'%';

CREATE DATABASE IF NOT EXISTS hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'hue'@'%' IDENTIFIED BY 'Onemap.2020';
GRANT ALL ON hue.* TO 'hue'@'%';

CREATE DATABASE IF NOT EXISTS metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'hive'@'%' IDENTIFIED BY 'Onemap.2020';
GRANT ALL ON metastore.* TO 'hive'@'%';

CREATE DATABASE IF NOT EXISTS sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'sentry'@'%' IDENTIFIED BY 'Onemap.2020';
GRANT ALL ON sentry.* TO 'sentry'@'%';

CREATE DATABASE IF NOT EXISTS nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'nav'@'%' IDENTIFIED BY 'Onemap.2020';
GRANT ALL ON nav.* TO 'nav'@'%';

CREATE DATABASE IF NOT EXISTS navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'navms'@'%' IDENTIFIED BY 'Onemap.2020';
GRANT ALL ON navms.* TO 'navms'@'%';

CREATE DATABASE IF NOT EXISTS oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'oozie'@'%' IDENTIFIED BY 'Onemap.2020';
GRANT ALL ON oozie.* TO 'oozie'@'%';

flush privileges;

  • 初始化数据库 sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm Onemap.2020 sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql amon amon Onemap.2020 sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql rman rman Onemap.2020 sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql hue hue Onemap.2020 sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql metastore hive Onemap.2020 sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql sentry sentry Onemap.2020 sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql nav nav Onemap.2020 sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql navms navms Onemap.2020 sudo /opt/cloudera/cm/schema/scm_prepare_database.sh mysql oozie oozie Onemap.2020

启动 CDH 及其他软件

运行命令 sudo systemctl start cloudera-scm-server，启动 Cloudera Manager Server

运行命令 sudo tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log，查看启动日志

当看到 INFO WebServerImpl:com.cloudera.server.cmf.WebServerImpl: Started Jetty server. 代表服务启动成功

在浏览器中访问 http://<server_host>:7180，用户名和密码同为 admin

comments powered by Disqus