FE和BE节点扩容、缩容测试

1、FE节点扩容测试

10.9.75.195 节点上

FE节点镜像下载：
docker pull starrocks/fe-ubuntu:3.1.11


mkdir /data/sr/fe/{conf,meta,log} -p
cd /data/sr/fe


cat > docker-compose.yaml << EOF
services:
  fe:
    image: starrocks/fe-ubuntu:3.1.11 
    container_name: fe
    restart: always
    network_mode: host
    command:
      /opt/starrocks/fe/bin/start_fe.sh
    volumes:
      - /data/sr/fe/conf/fe.conf:/opt/starrocks/fe/conf/fe.conf
      - /data/sr/fe/meta:/opt/starrocks/fe/meta
      - /data/sr/fe/log:/opt/starrocks/fe/log
      - /etc/localtime:/etc/localtime
    healthcheck:
      test: ["CMD-SHELL","curl -s -w '%{http_code}' -o /dev/null http://127.0.0.1:8030/api/health || bash -c 'kill -s 15 -1 && (sleep 10; kill -s 9 -1)'"] 
      interval: 30s
      timeout: 20s
      retries: 3
      start_period: 3m
    logging:
      driver: "json-file"
      options:
        tag: "{{.Name}}"
        max-size: "10m"
EOF


cat  > conf/fe.conf << EOF
# the output dir of stderr and stdout 
LOG_DIR = ${STARROCKS_HOME}/log
#JAVA_HOME=/usr/local/jdk
DATE = "$(date +%Y%m%d-%H%M%S)"
JAVA_OPTS="-Dlog4j2.formatMsgNoLookups=true -Duser.timezone=GMT+8 -Xmx16g -XX:+UseMembar -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xloggc:${LOG_DIR}/fe.gc.log.$DATE -XX:+PrintConcurrentLocks"

# For jdk 9+, this JAVA_OPTS will be used as default JVM options
JAVA_OPTS_FOR_JDK_9="-Dlog4j2.formatMsgNoLookups=true -Xmx8192m -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=7 -XX:+CMSClassUnloadingEnabled -XX:-CMSParallelRemarkEnabled -XX:CMSInitiatingOccupancyFraction=80 -XX:SoftRefLRUPolicyMSPerMB=0 -Xlog:gc*:$STARROCKS_HOME/log/fe.gc.log.$DATE:time"

##
## the lowercase properties are read by main program.
##

# INFO, WARN, ERROR, FATAL
sys_log_level = INFO

# store metadata, create it if it is not exist.
# Default value is ${STARROCKS_HOME}/meta
# meta_dir = ${STARROCKS_HOME}/meta

http_port = 8030
rpc_port = 9020
query_port = 9030
edit_log_port = 9010
mysql_service_nio_enabled = true

# Choose one if there are more than one ip except loopback address. 
# Note that there should at most one ip match this list.
# If no ip match this rule, will choose one randomly.
# use CIDR format, e.g. 10.10.10.0/24
# Default value is empty.
# priority_networks = 10.10.10.0/24;192.168.0.0/16
priority_networks = 10.9.75.195/32

# Advanced configurations 
# log_roll_size_mb = 1024
# sys_log_dir = ${STARROCKS_HOME}/log
# sys_log_roll_num = 10
# sys_log_verbose_modules = 
# audit_log_dir = ${STARROCKS_HOME}/log
# audit_log_modules = slow_query, query
# audit_log_roll_num = 10
# meta_delay_toleration_second = 10
# qe_max_connection = 1024
# max_conn_per_user = 100
# qe_query_timeout_second = 300
# qe_slow_log_ms = 5000
max_create_table_timeout_second = 120
report_queue_size = 2048
max_routine_load_task_num_per_be = 40
task_runs_ttl_second=3600
label_keep_max_second=3600
# 存算分离集群
run_mode = shared_data
cloud_native_meta_port = 6090
cloud_native_storage_type = S3

# 如 testbucket/subpath
aws_s3_path = "bnddsj-noprod-starrocks/uat/starrocks"

# 例如：cn-north-4
aws_s3_region = "cn-south-1"

# 例如：https://obs.cn-north-4.myhuaweicloud.com
aws_s3_endpoint = "https://obs.cn-south-1.myhuaweicloud.com"

aws_s3_access_key = "H0FNNE3FT3D5LHCBJBX2"
aws_s3_secret_key = "OSJg7zjt4eJUHpEkEDwzstpMGCou3Qi583aN86w2"
EOF



启动新 FE 节点时为其分配一个 helper 节点


docker run --rm  \
--network host  \
--privileged=true -it \
-v /data/sr/fe/log:/opt/starrocks/fe/log \
-v /data/sr/fe/meta:/opt/starrocks/fe/meta \
-v /data/sr/fe/conf:/opt/starrocks/fe/conf \
starrocks/fe-ubuntu:3.1.11  bash


进入容器之后，指定参数 --helper启动
/opt/starrocks/fe/bin/start_fe.sh --helper 10.9.65.189:9010 --daemon


注意：必须先指定参数 --helper启动后，再将额外的 FE 节点添加至集群，最后才是使用 docker-compose 启动


启动后，将 FE 节点添加至集群，再退出临时容器。

mysql -h10.9.65.189 -P9030 -uroot  -p'root.COM2020'

ALTER SYSTEM ADD FOLLOWER "10.9.75.195:9010";



查看 FE 节点状态
SHOW  frontends\G


退出容器，使用 docker-compose 启动
docker-compose up -d

查看 FE 节点状态
SHOW  frontends\G


2、FE节点缩容测试
mysql -h10.9.65.189 -P9030 -uroot  -p'root.COM2020'

ALTER SYSTEM DROP FOLLOWER "10.9.75.195:9010";


docker-compose down


查看 FE 节点状态
SHOW  frontends\G


3、BE节点扩容测试

10.9.65.189 节点上

BE节点镜像下载：
docker pull starrocks/be-ubuntu:3.1.11



mkdir /data/sr/be/{conf,storage,log} -p
cd /data/sr/be


cat > docker-compose.yaml << EOF
services:
  be:
    image: starrocks/be-ubuntu:3.1.11
    container_name: be
    restart: always
    network_mode: host
    command:
      /opt/starrocks/be/bin/start_be.sh
    volumes:
      - /data/sr/be/conf/be.conf:/opt/starrocks/be/conf/be.conf
      - /data/sr/be/storage:/opt/starrocks/be/storage
      - /data/sr/be/log:/opt/starrocks/be/log
      - /etc/localtime:/etc/localtime
    healthcheck:
      test: ["CMD-SHELL","curl -s -w '%{http_code}' -o /dev/null http://127.0.0.1:8040/api/health || bash -c 'kill -s 15 -1 && (sleep 10; kill -s 9 -1)'"] 
      interval: 30s
      timeout: 20s
      retries: 3
      start_period: 3m
    logging:
      driver: "json-file"
      options:
        tag: "{{.Name}}"
        max-size: "10m"
EOF


cat  >  conf/be.conf   << EOF
# INFO, WARNING, ERROR, FATAL
sys_log_level = INFO
#JAVA_HOME=/usr/local/jdk

# ports for admin, web, heartbeat service 
be_port = 9060
webserver_port = 8040
heartbeat_service_port = 9050
brpc_port = 8060

# Choose one if there are more than one ip except loopback address. 
# Note that there should at most one ip match this list.
# If no ip match this rule, will choose one randomly.
# use CIDR format, e.g. 10.10.10.0/24
# Default value is empty.
# priority_networks = 10.10.10.0/24;192.168.0.0/16
priority_networks = 10.9.65.189/32

# data root path, separate by ';'
# you can specify the storage medium of each root path, HDD or SSD, seperate by ','
# eg:
# storage_root_path = /data1,medium:HDD;/data2,medium:SSD;/data3
# /data1, HDD;
# /data2, SSD;
# /data3, HDD(default);
#
# Default value is ${STARROCKS_HOME}/storage, you should create it by hand.
# storage_root_path = ${STARROCKS_HOME}/storage

# Advanced configurations
# sys_log_dir = ${STARROCKS_HOME}/log
# sys_log_roll_mode = SIZE-MB-1024
# sys_log_roll_num = 10
# sys_log_verbose_modules = *
# log_buffer_level = -1

default_rowset_type = beta
mem_limit = 65%
cumulative_compaction_num_threads_per_disk = 4
base_compaction_num_threads_per_disk = 2
cumulative_compaction_check_interval_seconds = 2
routine_load_thread_pool_size = 40
brpc_max_body_size = 4294967296
trash_file_expire_time_sec=600
EOF


启动BE节点
docker-compose up -d


添加 BE 节点至集群

mysql -h10.9.65.189 -P9030 -uroot  -p'root.COM2020'

ALTER SYSTEM ADD BACKEND "10.9.65.189:9050";


查看 BE 节点状态
SHOW  backends\G


4、BE节点缩容测试

mysql -h10.9.65.189 -P9030 -uroot  -p'root.COM2020'

ALTER SYSTEM DROP BACKEND "10.9.65.189:9050";


docker-compose down



查看 BE 节点状态
SHOW  backends\G