CentOS7.9 DolphinScheduler3.1.9集群部署

环境准备

Python 2.0+
Java 1.8.0+
Mysql
ZooKeeper
Datax

文件准备

cd /opt
ll
# apache-dolphinscheduler-3.1.9-bin.tar.gz

配置节点信息

echo '' >> /etc/host
echo '192.168.100.10 node0' >> /etc/host
echo '192.168.100.11 node1' >> /etc/host
echo '192.168.100.12 node2' >> /etc/host
echo '192.168.100.13 node3' >> /etc/host
echo '' >> /etc/host

创建用户

useradd dolphinscheduler
echo "dolphinscheduler" | passwd --stdin dolphinscheduler

设置dolphinscheduler用户免密sudo

sed -i '$adolphinscheduler ALL=(ALL) NOPASSWD: NOPASSWD: ALL' /etc/sudoers
sed -i 's/Defaults requirett/#Defaults    requirett/g' /etc/sudoers

设置dolphinscheduler用户免密SSH登录

使用dolphinscheduler用户可以免密ssh登录集群中所有节点，所以每个节点都要执行下述操作

su dolphinscheduler
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
ssh-copy-id dolphinscheduler@node0
ssh-copy-id dolphinscheduler@node1
ssh-copy-id dolphinscheduler@node2
ssh-copy-id dolphinscheduler@node3

解压安装包

tar -zxvf /opt/apache-dolphinscheduler-3.1.9-bin.tar.gz

添加Mysql-Conector驱动

rm -rf /opt/apache-dolphinscheduler-3.1.9-bin/standalone-server/libs/mysql-connector-java-8.0.16.jar
rm -rf /opt/apache-dolphinscheduler-3.1.9-bin/alert-server/libs/mysql-connector-java-8.0.16.jar
rm -rf /opt/apache-dolphinscheduler-3.1.9-bin/api-server/libs/mysql-connector-java-8.0.16.jar
rm -rf /opt/apache-dolphinscheduler-3.1.9-bin/master-server/libs/mysql-connector-java-8.0.16.jar
rm -rf /opt/apache-dolphinscheduler-3.1.9-bin/standalone-server/libs/mysql-connector-java-8.0.16.jar
rm -rf /opt/apache-dolphinscheduler-3.1.9-bin/tools/libs/mysql-connector-java-8.0.16.jar
rm -rf /opt/apache-dolphinscheduler-3.1.9-bin/worker-server/libs/mysql-connector-java-8.0.16.jar
cp /opt/mysql-connector-java-8.0.16.jar /opt/apache-dolphinscheduler-3.1.9-bin/alert-server/libs/mysql-connector-java-8.0.16.jar
cp /opt/mysql-connector-java-8.0.16.jar /opt/apache-dolphinscheduler-3.1.9-bin/api-server/libs/mysql-connector-java-8.0.16.jar
cp /opt/mysql-connector-java-8.0.16.jar /opt/apache-dolphinscheduler-3.1.9-bin/master-server/libs/mysql-connector-java-8.0.16.jar
cp /opt/mysql-connector-java-8.0.16.jar /opt/apache-dolphinscheduler-3.1.9-bin/standalone-server/libs/mysql-connector-java-8.0.16.jar
cp /opt/mysql-connector-java-8.0.16.jar /opt/apache-dolphinscheduler-3.1.9-bin/tools/libs/mysql-connector-java-8.0.16.jar
cp /opt/mysql-connector-java-8.0.16.jar /opt/apache-dolphinscheduler-3.1.9-bin/worker-server/libs/mysql-connector-java-8.0.16.jar

配置文件修改

dolphinscheduler_env.sh

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# JAVA_HOME, will use it to start DolphinScheduler server
# export JAVA_HOME=${JAVA_HOME:-/opt/java/openjdk}  原模板内容
export JAVA_HOME=/opt/pahtUserDefinition/java/jre

# Database related configuration, set database type, username and password
# export DATABASE=${DATABASE:-postgresql} 原模板内容
export DATABASE=${DATABASE:-mysql}
export SPRING_PROFILES_ACTIVE=${DATABASE}
# export SPRING_DATASOURCE_URL 原模板内容
export SPRING_DATASOURCE_URL="jdbc:mysql://node0:3306/dolphinscheduler?serverTimezone=UTC&useTimezone=true&useUnicode=true&characterEncoding=UTF-8&serverTimezone=Asia/Shanghai"
# export SPRING_DATASOURCE_USERNAME 原模板内容
export SPRING_DATASOURCE_USERNAME=dolphinscheduler
# export SPRING_DATASOURCE_PASSWORD 原模板内容
export SPRING_DATASOURCE_PASSWORD=dolphinscheduler

# DolphinScheduler server related configuration
export SPRING_CACHE_TYPE=${SPRING_CACHE_TYPE:-none}
export SPRING_JACKSON_TIME_ZONE=${SPRING_JACKSON_TIME_ZONE:-UTC}
export MASTER_FETCH_COMMAND_NUM=${MASTER_FETCH_COMMAND_NUM:-10}

# Registry center configuration, determines the type and link of the registry center
export REGISTRY_TYPE=${REGISTRY_TYPE:-zookeeper}
export REGISTRY_ZOOKEEPER_CONNECT_STRING=${REGISTRY_ZOOKEEPER_CONNECT_STRING:-localhost:2181}

# Tasks related configurations, need to change the configuration if you use the related tasks.
export HADOOP_HOME=${HADOOP_HOME:-/opt/soft/hadoop}
export HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-/opt/soft/hadoop/etc/hadoop}
export SPARK_HOME1=${SPARK_HOME1:-/opt/soft/spark1}
export SPARK_HOME2=${SPARK_HOME2:-/opt/soft/spark2}
# export PYTHON_HOME=${PYTHON_HOME:-/opt/soft/python} 原模板内容
export PYTHON_HOME=${PYTHON_HOME:-/usr/bin/python}
export HIVE_HOME=${HIVE_HOME:-/opt/soft/hive}
export FLINK_HOME=${FLINK_HOME:-/opt/soft/flink}
# export DATAX_HOME=${DATAX_HOME:-/opt/soft/datax} 原模板内容
export DATAX_HOME=${DATAX_HOME:-/opt/datax}
export SEATUNNEL_HOME=${SEATUNNEL_HOME:-/opt/soft/seatunnel}
export CHUNJUN_HOME=${CHUNJUN_HOME:-/opt/soft/chunjun}

export PATH=$HADOOP_HOME/bin:$SPARK_HOME1/bin:$SPARK_HOME2/bin:$PYTHON_HOME/bin:$JAVA_HOME/bin:$HIVE_HOME/bin:$FLINK_HOME/bin:$DATAX_HOME/bin:$SEATUNNEL_HOME/bin:$CHUNJUN_HOME/bin:$PATH

install_env.sh

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

# ---------------------------------------------------------
# INSTALL MACHINE
# ---------------------------------------------------------
# A comma separated list of machine hostname or IP would be installed DolphinScheduler,
# including master, worker, api, alert. If you want to deploy in pseudo-distributed
# mode, just write a pseudo-distributed hostname
# Example for hostnames: ips="ds1,ds2,ds3,ds4,ds5", Example for IPs: ips="192.168.8.1,192.168.8.2,192.168.8.3,192.168.8.4,192.168.8.5"
ips=${ips:-"node0,node1,node2,node3"}

# Port of SSH protocol, default value is 22. For now we only support same port in all `ips` machine
# modify it if you use different ssh port
sshPort=${sshPort:-"22"}

# A comma separated list of machine hostname or IP would be installed Master server, it
# must be a subset of configuration `ips`.
# Example for hostnames: masters="ds1,ds2", Example for IPs: masters="192.168.8.1,192.168.8.2"
masters=${masters:-"node0"}

# A comma separated list of machine <hostname>:<workerGroup> or <IP>:<workerGroup>.All hostname or IP must be a
# subset of configuration `ips`, And workerGroup have default value as `default`, but we recommend you declare behind the hosts
# Example for hostnames: workers="ds1:default,ds2:default,ds3:default", Example for IPs: workers="192.168.8.1:default,192.168.8.2:default,192.168.8.3:default"
workers=${workers:-"node0:default,node1:default,node2:default,node3:default"}

# A comma separated list of machine hostname or IP would be installed Alert server, it
# must be a subset of configuration `ips`.
# Example for hostname: alertServer="ds3", Example for IP: alertServer="192.168.8.3"
alertServer=${alertServer:-"node0"}

# A comma separated list of machine hostname or IP would be installed API server, it
# must be a subset of configuration `ips`.
# Example for hostname: apiServers="ds1", Example for IP: apiServers="192.168.8.1"
apiServers=${apiServers:-"node0"}

# The directory to install DolphinScheduler for all machine we config above. It will automatically be created by `install.sh` script if not exists.
# Do not set this configuration same as the current path (pwd). Do not add quotes to it if you using related path.
installPath=${installPath:-"/opt/dolphinscheduler"}

# The user to deploy DolphinScheduler for all machine we config above. For now user must create by yourself before running `install.sh`
# script. The user needs to have sudo privileges and permissions to operate hdfs. If hdfs is enabled than the root directory needs
# to be created by this user
deployUser=${deployUser:-"dolphinscheduler"}

# The root of zookeeper, for now DolphinScheduler default registry server is zookeeper.
zkRoot=${zkRoot:-"/dolphinscheduler"}

刷新安装包文件权限

chown -R dolphinscheduler:dolphinscheduler /opt/apache-dolphinscheduler-3.1.9-bin

初始化数据库

登录mysql数据库

mysql -uroot -proot2024

创建Database并为相关用户赋权

CREATE DATABASE dolphinscheduler DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
CREATE USER 'dolphinscheduler'@'%' IDENTIFIED BY 'dolphinscheduler';
GRANT ALL PRIVILEGES ON dolphinscheduler.* TO 'dolphinscheduler'@'%';
FLUSH PRIVILEGES;
exit;

运行预设脚本初始化数据库

bash /opt/apache-dolphinscheduler-3.1.9-bin/tools/bin/upgrade-schema.sh

放行端口

firewall-cmd --list-ports
firewall-cmd --zone=public --add-port=3306/tcp --permanent
firewall-cmd --zone=public --add-port=1234/tcp --permanent
firewall-cmd --zone=public --add-port=12345/tcp --permanent
service firewalld restart

执行install.sh 安装dolphinscheduler

su dolphinscheduler
sh /opt/apache-dolphinscheduler-3.1.9-bin/bin/install.sh

添加Oracle连接驱动

rm -rf /dolphinscheduler/standalone-server/libs/standalone-server/bin/ojdbc5.jar
rm -rf /dolphinscheduler/alert-server/bin/ojdbc5.jar
rm -rf /dolphinscheduler/api-server/bin/ojdbc5.jar
rm -rf /dolphinscheduler/master-server/bin/ojdbc5.jar
rm -rf /dolphinscheduler/standalone-server/bin/ojdbc5.jar
rm -rf /dolphinscheduler/tools/bin/ojdbc5.jar
rm -rf /dolphinscheduler/worker-server/bin/ojdbc5.jar
cp /opt/ojdbc5.jar /dolphinscheduler/alert-server/bin/ojdbc5.jar
cp /opt/ojdbc5.jar /dolphinscheduler/api-server/bin/ojdbc5.jar
cp /opt/ojdbc5.jar /dolphinscheduler/master-server/bin/ojdbc5.jar
cp /opt/ojdbc5.jar /dolphinscheduler/standalone-server/bin/ojdbc5.jar
cp /opt/ojdbc5.jar /dolphinscheduler/tools/bin/ojdbc5.jar
cp /opt/ojdbc5.jar /dolphinscheduler/worker-server/bin/ojdbc5.jar
 
rm -rf /dolphinscheduler/standalone-server/libs/standalone-server/bin/ojdbc8.jar
rm -rf /dolphinscheduler/alert-server/bin/ojdbc8.jar
rm -rf /dolphinscheduler/api-server/bin/ojdbc8.jar
rm -rf /dolphinscheduler/master-server/bin/ojdbc8.jar
rm -rf /dolphinscheduler/standalone-server/bin/ojdbc8.jar
rm -rf /dolphinscheduler/tools/bin/ojdbc8.jar
rm -rf /dolphinscheduler/worker-server/bin/ojdbc8.jar
cp /opt/ojdbc8.jar /dolphinscheduler/alert-server/bin/ojdbc8.jar
cp /opt/ojdbc8.jar /dolphinscheduler/api-server/bin/ojdbc8.jar
cp /opt/ojdbc8.jar /dolphinscheduler/master-server/bin/ojdbc8.jar
cp /opt/ojdbc8.jar /dolphinscheduler/standalone-server/bin/ojdbc8.jar
cp /opt/ojdbc8.jar /dolphinscheduler/tools/bin/ojdbc8.jar
cp /opt/ojdbc8.jar /dolphinscheduler/worker-server/bin/ojdbc8.jar
 
chown -R dolphinscheduler:dolphinscheduler /opt/dolphinscheduler

设置并发及内存告警

sed -i 's/exec-threads: 100/exec-threads: 20/g' /opt/dolphinscheduler/alert-server/conf/application.yaml
sed -i 's/exec-threads: 100/exec-threads: 20/g' /opt/dolphinscheduler/api-server/conf/application.yaml
sed -i 's/exec-threads: 100/exec-threads: 20/g' /opt/dolphinscheduler/master-server/conf/application.yaml
sed -i 's/exec-threads: 100/exec-threads: 20/g' /opt/dolphinscheduler/worker-server/conf/application.yaml
sed -i 's/reserved-memory: 0.3/reserved-memory: 1/g' /opt/dolphinscheduler/alert-server/conf/application.yaml
sed -i 's/reserved-memory: 0.3/reserved-memory: 1/g' /opt/dolphinscheduler/api-server/conf/application.yaml
sed -i 's/reserved-memory: 0.3/reserved-memory: 1/g' /opt/dolphinscheduler/master-server/conf/application.yaml
sed -i 's/reserved-memory: 0.3/reserved-memory: 1/g' /opt/dolphinscheduler/worker-server/conf/application.yaml

Sping漏洞修复

sed -i 'management:/cmanagement:\
  server:\
    port: -1' /opt/dolphinscheduler/alert-server/conf/application.yaml
sed -i 'management:/cmanagement:\
  server:\
    port: -1' /opt/dolphinscheduler/api-server/conf/application.yaml
sed -i 'management:/cmanagement:\
  server:\
    port: -1' /opt/dolphinscheduler/master-server/conf/application.yaml
sed -i 'management:/cmanagement:\
  server:\
    port: -1' opt/dolphinscheduler/worker-server/conf/application.yaml

指定Zookeeper用户密码

sed -i 's/~/username:passwd/g' /opt/dolphinscheduler/alert-server/conf/application.yaml
sed -i 's/~/username:passwd/g' /opt/dolphinscheduler/api-server/conf/application.yaml
sed -i 's/~/username:passwd/g' /opt/dolphinscheduler/master-server/conf/application.yaml
sed -i 's/~/username:passwd/g' /opt/dolphinscheduler/worker-server/conf/application.yaml

posted @ 2024-08-15 17:18 Mjp110211 阅读(30) 评论(0) 收藏举报

刷新页面返回顶部

备忘录

CentOS7.9 DolphinScheduler3.1.9集群部署

环境准备

文件准备

配置节点信息

创建用户

设置dolphinscheduler用户免密sudo

设置dolphinscheduler用户免密SSH登录

解压安装包

添加Mysql-Conector驱动

配置文件修改

dolphinscheduler_env.sh

install_env.sh

刷新安装包文件权限

初始化数据库

登录mysql数据库

创建Database并为相关用户赋权

运行预设脚本初始化数据库

放行端口

执行install.sh 安装dolphinscheduler

添加Oracle连接驱动

设置并发及内存告警

Sping漏洞修复

指定Zookeeper用户密码