centos7部署kafka3.7单机版zookeeper+SSL及公网代理访问

1.下载相关软件包

#JDK使用1.8 软件包自行下载
wget https://downloads.apache.org/zookeeper/zookeeper-3.8.4/apache-zookeeper-3.8.4-bin.tar.gz
wget https://archive.apache.org/dist/kafka/3.7.0/kafka_2.13-3.7.0.tgz

2.配置Java环境变量

tar xf jdk-8u201-linux-x64.tar.gz -C /usr/local
vim /etc/profile.d/java.sh 
export JAVA_HOME=/usr/local/jdk1.8.0_201
export CLASSPATH=$JAVA_HOME/lib/tools.jar
export PATH=$JAVA_HOME/bin:$PATH

source  /etc/profile.d/java.sh 
echo $JAVA_HOME

3.安装及配置zookeeper

tar xf apache-zookeeper-3.8.4-bin.tar.gz -C /usr/local
cd  /usr/local
ln -sv apache-zookeeper-3.8.4-bin zookeeper
cd zookeeper/conf/
cp zoo_sample.cfg zoo.cfg

vim zoo.cfg
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/Data/zookeeper #数据目录
clientPort=2181
dmin.enableServer=true
admin.serverPort=8888

4.安装及配置kafka

解压软件

tar xf kafka_2.13-3.7.0.tgz -C /usr/local
cd  /usr/local
ln -sv kafka_2.13-3.7.0  kafka
cd kafka
mkdir ssl && cd ssl

生成证书

vim cert.sh
echo "Step1: init paramters"
BASE_DIR=/usr/local/kafka/ssl
CERT_OUTPUT_PATH="$BASE_DIR"
PASSWORD=javsWNHqCseSXPEj
KEY_STORE="$CERT_OUTPUT_PATH/kafka.keystore"
TRUST_STORE="$CERT_OUTPUT_PATH/kafka.truststore"
KEY_PASSWORD=$PASSWORD
STORE_PASSWORD=$PASSWORD
TRUST_KEY_PASSWORD=$PASSWORD
TRUST_STORE_PASSWORD=$PASSWORD
CLUSTER_NAME=xsky-kafka-client
CERT_AUTH_FILE="$CERT_OUTPUT_PATH/ca-cert"
CLUSTER_CERT_FILE="$CERT_OUTPUT_PATH/${CLUSTER_NAME}-cert"
DAYS_VALID=36500
D_NAME="CN=kafka.test.com, OU=test, O=test, L=China, ST=China, C=sh"
mkdir -p $CERT_OUTPUT_PATH

echo "Step2: Create certificate to keystore"
keytool -keystore $KEY_STORE -alias $CLUSTER_NAME -validity $DAYS_VALID -genkey -keyalg RSA -storepass $STORE_PASSWORD -keypass $KEY_PASSWORD -dname "$D_NAME"

echo "Step3: Create CA"
openssl req -new -x509 -keyout "$CERT_OUTPUT_PATH/ca-key" -out "$CERT_AUTH_FILE" -days "$DAYS_VALID" -passin pass:"$PASSWORD" -passout pass:"$PASSWORD" -subj "/C=CN/ST=XX/L=XX/O=XX/CN=XX"

echo "Step4: Import CA into truststore"
keytool -keystore "$TRUST_STORE" -alias CARoot -import -file "$CERT_AUTH_FILE" -storepass "$TRUST_STORE_PASSWORD" -keypass "$TRUST_KEY_PASS" -noprompt

echo "Step5: Export certificate from keystore"
keytool -keystore "$KEY_STORE" -alias "$CLUSTER_NAME" -certreq -file "$CLUSTER_CERT_FILE" -storepass "$STORE_PASSWORD" -keypass "$KEY_PASSWORD" -noprompt

echo "Step6: Signing the certificate"
openssl x509 -req -CA "$CERT_AUTH_FILE" -CAkey $CERT_OUTPUT_PATH/ca-key -in "$CLUSTER_CERT_FILE" -out "${CLUSTER_CERT_FILE}-signed" -days "$DAYS_VALID" -CAcreateserial -passin pass:"$PASSWORD"

echo "Setp7: Import CA into keystore"
keytool -keystore "$KEY_STORE" -alias CARoot -import -file "$CERT_AUTH_FILE" -storepass "$STORE_PASSWORD" -keypass "$KEY_PASSWORD" -noprompt

echo "Setp8: Import signed certificate into keystore"
keytool -keystore "$KEY_STORE" -alias "${CLUSTER_NAME}" -import -file "${CLUSTER_CERT_FILE}-signed" -storepass "$STORE_PASSWORD" -keypass "$KEY_PASSWORD" -noprompt


sh cert.sh
#生成的文件说明
ca-cert：CA文件，不要把该文件拷贝到别的broker机器上！
test-cluster-cert-signed：CA已签发的Kafka证书文件，不要把该文件拷贝到别的broker机器上！
test-cluster-cert：Kafka认证文件（包含公钥和私钥），不要把该文件拷贝到别的broker机器上！
kafka.keystore：Kafka的keystore文件，所有clients端和broker机器上都需要！
kafka.truststore：Kafka的truststore文件，所有clients端和broker机器上都需要！

配置kafka

broker.id=0
log.dirs=/Data/kafka
num.partitions=1

# 网络配置
listeners=SSL://0.0.0.0:9092
advertised.listeners=SSL://kafka.test.com:9092
ssl.keystore.location=/usr/local/kafka/ssl/kafka.keystore
ssl.keystore.password=javsWNHqCseSXPEj
ssl.key.password=javsWNHqCseSXPEj
ssl.truststore.location=/usr/local/kafka/ssl/kafka.truststore
ssl.truststore.password=javsWNHqCseSXPEj
#这一行官方文档没有，加上，否则可能报错
ssl.endpoint.identification.algorithm=
ssl.client.auth=required         

#inter.broker.listener.name=PLAINTEXT # 这句注释掉，加入下面这句
security.inter.broker.protocol=SSL

# 性能配置
num.network.threads=5
num.io.threads=16
socket.send.buffer.bytes=1048576
socket.receive.buffer.bytes=1048576
socket.request.max.bytes=104857600
num.recovery.threads.per.data.dir=4

# 日志配置
log.segment.bytes=1073741824
log.retention.hours=168
# 增加到10GB
log.retention.bytes=10737418240
log.retention.check.interval.ms=300000
log.flush.interval.messages=50000
log.flush.interval.ms=5000

# 副本配置（单节点可保持1，集群需修改）
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
default.replication.factor=1

# Zookeeper配置
zookeeper.connect=localhost:2181
zookeeper.connection.timeout.ms=18000
zookeeper.session.timeout.ms=18000

# 其他
group.initial.rebalance.delay.ms=0

5.启动服务

useradd avatar 
mkdir -pv  /Data/{kafka,zookeeper}
chown -R avatar:avatar  /Data
chown -R avatar:avatar /usr/local/kafka_2.13-3.7.0/
chown -R avatar:avatar  /usr/local/apache-zookeeper-3.8.4-bin/

useradd avatar 
mkdir -pv  /Data/{kafka,zookeeper}
chown -R avatar:avatar  /Data
chown -R avatar:avatar /usr/local/kafka_2.13-3.7.0/
chown -R avatar:avatar  /usr/local/apache-zookeeper-3.8.4-bin/
#启动ZK
su - avatar 
cd /usr/local/zookeeper/bin/
./zkServer.sh start
./zkServer.sh status
netstat -tnlp
 
#启动Kfaka
cd /usr/local/kafka/bin/
./kafka-server-start.sh  -daemon  /usr/local/kafka/config/server.properties
netstat -tnlp

6.配置客户端证书

vim client-ssl.config
bootstrap.servers=kafka.test.com:9092
security.protocol=SSL
ssl.keystore.location=/usr/local/kafka/ssl/kafka.keystore
ssl.keystore.password=javsWNHqCseSXPEj
ssl.truststore.location=/usr/local/kafka/ssl/kafka.truststore
ssl.truststore.password=javsWNHqCseSXPEj
ssl.endpoint.identification.algorithm=
ssl.key.password=javsWNHqCseSXPEj

7.验证

./kafka-topics.sh --create --topic test-topic-ssl --bootstrap-server kafka.test.com:9092  --command-config client-ssl.config  #创建topic 
./kafka-topics.sh --describe --topic test-topic-ssl --bootstrap-server kafka.test.com:9092 --command-config client-ssl.config
./kafka-topics.sh --list --bootstrap-server kafka.test.com:9092 --command-config client-ssl.config
./kafka-console-producer.sh --broker-list kafka.test.com:9092 --topic test-topic-ssl --producer.config client-ssl.config #创建生产者
./kafka-console-consumer.sh --bootstrap-server kafka.test.com:9092 --topic test-topic-ssl --from-beginning --consumer.config client-ssl.config #另起一个终端，启动消费中，启动后在生产者终端输入内容，消费者终端查看是否收到消息

 抄自于：

https://blog.csdn.net/weixin_39004677/article/details/141923340

https://blog.csdn.net/justry_deng/article/details/88383081

https://blog.csdn.net/justry_deng/article/details/88383707