一、规划 StarRocks 集群:
hwuat-starrocks-be-10.9.75.195 10.9.75.195 BE
hwuat-starrocks-be-10.9.80.220 10.9.80.220 BE
hwuat-starrocks-be-10.9.64.158 10.9.64.158 BE
hwuat-starrocks-fe-10.9.65.189 10.9.65.189 FE
hwuat-starrocks-fe-10.9.93.132 10.9.93.132 FE
hwuat-starrocks-fe-10.9.65.206 10.9.65.206 FE
FE 节点：3个
8030：FE HTTP Server 端口（http_port）
9020：FE Thrift Server 端口（rpc_port）
9030：FE MySQL Server 端口（query_port）
9010：FE 内部通讯端口（edit_log_port）
BE 节点：3个
9060：BE Thrift Server 端口（be_port）
8040：BE HTTP Server 端口（be_http_port）
9050：BE 心跳服务端口（heartbeat_service_port）
8060：BE BRPC 端口（brpc_port）
二、下载镜像
FE节点镜像下载：
docker pull starrocks/fe-ubuntu:3.1.11
BE节点镜像下载：
docker pull starrocks/be-ubuntu:3.1.11
三、be 节点
## 10.9.75.195/10.9.80.220/10.9.64.158
mkdir /data/sr/be/{conf,storage,log} -p
cd /data/sr/be
cat > docker-compose.yaml << EOF
services:
be:
image: starrocks/be-ubuntu:3.1.11
container_name: be
restart: always
network_mode: host
command:
/opt/starrocks/be/bin/start_be.sh
volumes:
- /data/sr/be/conf/be.conf:/opt/starrocks/be/conf/be.conf
- /data/sr/be/storage:/opt/starrocks/be/storage
- /data/sr/be/log:/opt/starrocks/be/log
- /etc/localtime:/etc/localtime
healthcheck:
test: ["CMD-SHELL","curl -s -w '%{http_code}' -o /dev/null http://127.0.0.1:8040/api/health || bash -c 'kill -s 15 -1 && (sleep 10; kill -s 9 -1)'"]
interval: 30s
timeout: 20s
retries: 3
start_period: 3m
logging:
driver: "json-file"
options:
tag: "{{.Name}}"
max-size: "10m"
EOF
cat > conf/be.conf << EOF
# INFO, WARNING, ERROR, FATAL
sys_log_level = INFO
#JAVA_HOME=/usr/local/jdk
# ports for admin, web, heartbeat service
be_port = 9060
webserver_port = 8040
heartbeat_service_port = 9050
brpc_port = 8060
# Choose one if there are more than one ip except loopback address.
# Note that there should at most one ip match this list.
# If no ip match this rule, will choose one randomly.
# use CIDR format, e.g. 10.10.10.0/24
# Default value is empty.
# priority_networks = 10.10.10.0/24;192.168.0.0/16
priority_networks = 10.9.75.195/32
# data root path, separate by ';'
# you can specify the storage medium of each root path, HDD or SSD, seperate by ','
# eg:
# storage_root_path = /data1,medium:HDD;/data2,medium:SSD;/data3
# /data1, HDD;
# /data2, SSD;
# /data3, HDD(default);
#
# Default value is ${STARROCKS_HOME}/storage, you should create it by hand.
# storage_root_path = ${STARROCKS_HOME}/storage
# Advanced configurations
# sys_log_dir = ${STARROCKS_HOME}/log
# sys_log_roll_mode = SIZE-MB-1024
# sys_log_roll_num = 10
# sys_log_verbose_modules = *
# log_buffer_level = -1
default_rowset_type = beta
mem_limit = 65%
cumulative_compaction_num_threads_per_disk = 4
base_compaction_num_threads_per_disk = 2
cumulative_compaction_check_interval_seconds = 2
routine_load_thread_pool_size = 40
brpc_max_body_size = 4294967296
trash_file_expire_time_sec=600
EOF
修改配置文件
在 10.9.80.220 上：
sed -i 's/10.9.75.195/10.9.80.220/g' conf/be.conf
在 10.9.64.158 上：
sed -i 's/10.9.75.195/10.9.64.158/g' conf/be.conf
先启动BE节点
docker-compose up -d
查看 BE 日志，检查 BE 节点是否启动成功。
cat log/be.INFO | grep heartbeat
如果日志打印以下内容，则说明该 BE 节点启动成功：
"I0614 17:41:39.782819 3717531 thrift_server.cpp:388] heartbeat has started listening port on 9050"
## fe 节点 10.9.65.189/10.9.93.132/10.9.65.206
mkdir /data/sr/fe/{conf,meta,log} -p
cd /data/sr/fe
cat > docker-compose.yaml << EOF
services:
fe:
image: starrocks/fe-ubuntu:3.1.11
container_name: fe
restart: always
network_mode: host
command:
/opt/starrocks/fe/bin/start_fe.sh
volumes:
- /data/sr/fe/conf/fe.conf:/opt/starrocks/fe/conf/fe.conf
- /data/sr/fe/meta:/opt/starrocks/fe/meta
- /data/sr/fe/log:/opt/starrocks/fe/log
- /etc/localtime:/etc/localtime
healthcheck:
test: ["CMD-SHELL","curl -s -w '%{http_code}' -o /dev/null http://127.0.0.1:8030/api/health || bash -c 'kill -s 15 -1 && (sleep 10; kill -s 9 -1)'"]
interval: 30s
timeout: 20s
retries: 3
start_period: 3m
logging:
driver: "json-file"
options:
tag: "{{.Name}}"
max-size: "10m"
EOF
cat > conf/fe.conf << EOF
# the output dir of stderr and stdout
LOG_DIR = ${STARROCKS_HOME}/log
#JAVA_HOME=/usr/local/jdk
DATE = "$(date +%Y%m%d-%H%M%S)"
JAVA_OPTS="-Dlog4j2.formatMsgNoLookups=true -Duser.timezone=GMT+8 -Xmx16g -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:${LOG_DIR}/fe.gc.log.$DATE -XX:+PrintConcurrentLocks"
# For jdk 9+, this JAVA_OPTS will be used as default JVM options
JAVA_OPTS_FOR_JDK_9="-Dlog4j2.formatMsgNoLookups=true -Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$STARROCKS_HOME/log/fe.gc.log.$DATE:time"
##
## the lowercase properties are read by main program.
##
# INFO, WARN, ERROR, FATAL
sys_log_level = INFO
# store metadata, create it if it is not exist.
# Default value is ${STARROCKS_HOME}/meta
# meta_dir = ${STARROCKS_HOME}/meta
http_port = 8030
rpc_port = 9020
query_port = 9030
edit_log_port = 9010
mysql_service_nio_enabled = true
# Choose one if there are more than one ip except loopback address.
# Note that there should at most one ip match this list.
# If no ip match this rule, will choose one randomly.
# use CIDR format, e.g. 10.10.10.0/24
# Default value is empty.
# priority_networks = 10.10.10.0/24;192.168.0.0/16
priority_networks = 10.9.65.189/32
# Advanced configurations
# log_roll_size_mb = 1024
# sys_log_dir = ${STARROCKS_HOME}/log
# sys_log_roll_num = 10
# sys_log_verbose_modules =
# audit_log_dir = ${STARROCKS_HOME}/log
# audit_log_modules = slow_query, query
# audit_log_roll_num = 10
# meta_delay_toleration_second = 10
# qe_max_connection = 1024
# max_conn_per_user = 100
# qe_query_timeout_second = 300
# qe_slow_log_ms = 5000
max_create_table_timeout_second = 120
report_queue_size = 2048
max_routine_load_task_num_per_be = 40
task_runs_ttl_second=3600
label_keep_max_second=3600
# 存算分离集群
run_mode = shared_data
cloud_native_meta_port = 6090
cloud_native_storage_type = S3
# 如 testbucket/subpath
aws_s3_path = "bnddsj-noprod-starrocks/uat/starrocks"
# 例如：cn-north-4
aws_s3_region = "cn-south-1"
# 例如：https://obs.cn-north-4.myhuaweicloud.com
aws_s3_endpoint = "https://obs.cn-south-1.myhuaweicloud.com"
aws_s3_access_key = "H0FNNE3FT3D5LHCBJBX2"
aws_s3_secret_key = "OSJg7zjt4eJUHpEkEDwzstpMGCou3Qi583aN86w2"
EOF
修改配置文件
在 10.9.93.132 上：
sed -i 's/10.9.65.189/10.9.93.132/g' conf/fe.conf
在 10.9.65.206 上：
sed -i 's/10.9.65.189/10.9.65.206/g' conf/fe.conf
启动 Leader FE 节点，在这里指定为： 10.9.65.189
docker-compose up -d
查看 FE 日志，检查 FE 节点是否启动成功。
cat log/fe.log | grep thrift
如果日志打印以下内容，则说明该 FE 节点启动成功：
"2022-08-10 16:12:29,911 INFO (UNKNOWN x.x.x.x_9010_1660119137253(-1)|1) [FeServer.start():52] thrift server started with port 9020."
添加 BE 节点至集群
-- 将 <be_address> 替换为 BE 节点的 IP 地址（priority_networks）或 FQDN，
-- 并将 <heartbeat_service_port>（默认：9050）替换为您在 be.conf 中指定的 heartbeat_service_port。
ALTER SYSTEM ADD BACKEND "<be_address>:<heartbeat_service_port>", "<be2_address>:<heartbeat_service_port>", "<be3_address>:<heartbeat_service_port>";
mysql -h10.9.65.189 -P9030 -uroot -p
ALTER SYSTEM ADD BACKEND "10.9.75.195:9050", "10.9.80.220:9050", "10.9.64.158:9050";
查看 BE 节点状态
SHOW backends\G
如果字段 Alive 为 true，说明该 BE 节点正常启动并加入集群。
向集群中添加新的 Follower FE 节点时，必须在首次启动新 FE 节点时为其分配一个 helper 节点
在第一次启动FE节点时指定参数 --helper
# 将 <helper_fe_ip> 替换为 Leader FE 节点的 IP 地址（priority_networks），
# 并将 <helper_edit_log_port>（默认：9010）替换为 Leader FE 节点的 edit_log_port。
# ./fe/bin/start_fe.sh --helper <helper_fe_ip>:<helper_edit_log_port> --daemon
docker run --rm \
--network host \
--privileged=true -it \
-v /data/sr/fe/log:/opt/starrocks/fe/log \
-v /data/sr/fe/meta:/opt/starrocks/fe/meta \
-v /data/sr/fe/conf:/opt/starrocks/fe/conf \
starrocks/fe-ubuntu:3.1.11 bash
进入容器之后，指定参数 --helper启动
/opt/starrocks/fe/bin/start_fe.sh --helper 10.9.65.189:9010 --daemon
注意：必须先指定参数 --helper启动后，再将额外的 FE 节点添加至集群，最后才是使用 docker-compose 启动
启动后，将额外的 FE 节点添加至集群，再退出临时容器。
-- 将 <new_fe_address> 替换为您需要添加的新 FE 节点的 IP 地址（priority_networks）或 FQDN，
-- 并将 <edit_log_port>（默认：9010）替换为您在新 FE 节点的 fe.conf 中指定的 edit_log_port。
ALTER SYSTEM ADD FOLLOWER "<new_fe_address>:<edit_log_port>";
ALTER SYSTEM ADD FOLLOWER "10.9.93.132:9010";
ALTER SYSTEM ADD FOLLOWER "10.9.65.206:9010";
查看 Leader FE 节点状态
SHOW frontends\G
如果字段 Alive 为 true，说明该 FE 节点正常启动并加入集群。
如果字段 Role 为 FOLLOWER，说明该 FE 节点有资格被选为 Leader FE 节点。
如果字段 Role 为 LEADER，说明该 FE 节点为 Leader FE 节点。
退出容器，使用 docker-compose 启动
docker-compose up -d
## nginx 节点 10.9.65.189/10.9.93.132/10.9.65.206
mkdir /data/sr/nginx/{nginx,logs} -p
cd /data/sr/nginx
cat > docker-compose.yaml << EOF
services:
nginx:
image: harbor.uddun.com/ops/tengine:2.3.3.1
container_name: nginx
restart: always
network_mode: host
volumes:
- /data/sr/nginx/nginx.conf:/usr/local/nginx/conf/nginx.conf
- /data/sr/nginx/logs/:/usr/local/nginx/logs/
healthcheck:
test: ["CMD-SHELL","curl -k -sIL -w '%{http_code}' -o /dev/null http://localhost:18030 || bash -c 'kill -s 15 -1 && (sleep 10; kill -s 9 -1)'"]
interval: 2m
timeout: 10s
retries: 3
start_period: 3m
logging:
options:
tag: "{{.Name}}"
EOF
cat > nginx.conf << EOF
user www www;
worker_processes 4;
worker_rlimit_nofile 20480;
pid log/nginx.pid;
events {
worker_connections 20480;
accept_mutex off;
multi_accept on;
use epoll;
}
stream {
log_format proxy '$remote_addr [$time_local] '
'$protocol $status $bytes_sent $bytes_received '
'$session_time "$upstream_addr" '
'"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
access_log logs/tcp-access.log proxy;
open_log_file_cache off;
upstream fe_9030 {
#least_conn;
server 10.9.65.189:9030 max_fails=3 fail_timeout=30s;
server 10.9.93.132:9030 max_fails=3 fail_timeout=30s;
server 10.9.65.206:9030 max_fails=3 fail_timeout=30s;
}
upstream fe_8030 {
#least_conn;
server 10.9.65.189:8030 max_fails=3 fail_timeout=30s;
server 10.9.93.132:8030 max_fails=3 fail_timeout=30s;
server 10.9.65.206:8030 max_fails=3 fail_timeout=30s;
}
server {
listen 19030;
proxy_pass fe_9030;
proxy_timeout 60s;
proxy_connect_timeout 30s;
}
server {
listen 18030;
proxy_pass fe_8030;
proxy_timeout 60s;
proxy_connect_timeout 30s;
}
}
EOF
docker-compose up -d
重置root密码：
ALTER USER root@'%' IDENTIFIED BY 'root.COM2020';
浙公网安备 33010602011771号