12.kafka单集群安装部署

一.安装jdk ，kafka需要至少jdk8+

1.安装jdk

yum install java-11-openjdk.x86_64 -y

 

2.查看安装是否正常

[root@localhost bin]# java --version
openjdk 11.0.21 2023-10-17 LTS
OpenJDK Runtime Environment (Red_Hat-11.0.21.0.9-1.el7_9) (build 11.0.21+9-LTS)
OpenJDK 64-Bit Server VM (Red_Hat-11.0.21.0.9-1.el7_9) (build 11.0.21+9-LTS, mixed mode, sharing)

 

二.安装kafka及zookeeper

1.官网:http://kafka.apache.org/downloads

curl -O https://downloads.apache.org/kafka/3.5.1/kafka-3.5.1-src.tgz

2.创建目录

mkdir -p /mnt/kafka            # 创建kafka家目录
mkdir -p /mnt/kafka/kafka_data        # 创建kafka数据存储目录
mkdir -p /mnt/kafka/kafka_data/zookeeper    # 创建zookeeper数据目录
mkdir -p /mnt/kafka/kafka_data/log/kafka    # 创建kafka日志存储目录
mkdir -p /mnt/kafka/kafka_data/log/zookeeper    # 创建zookeeper日志存储目录

3.压缩包解压

tar -xf kafka_2.12-3.5.1.tgz -C /mnt/kafka/

4.配置kafka服务

vi /mnt/kafka/kafka_2.12-3.5.1/config/server.properties

5.添加或修改如下内容

broker.id=0
port=9092                                                #端口号
host.name=192.168.186.20                      #服务器IP地址，修改为自己的服务器IP
log.dirs=/mnt/kafka/kafka_data/log/kafka   #日志存放路径，上面创建的目录
zookeeper.connect=localhost:2181         #zookeeper地址和端口，单机配置部署，localhost:2181

 

[root@localhost bin]# grep -Ev '$^|#' /mnt/kafka/kafka_2.12-3.5.1/config/server.properties
broker.id=0
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/mnt/kafka/kafka_data/log/kafka
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.retention.check.interval.ms=300000
zookeeper.connect=localhost:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
port=9092
host.name = 192.168.186.20 

 kafka 服务端口设置为默认端口9092

6.配置zookeeper服务

vi /mnt/kafka/kafka_2.12-3.5.1/config/zookeeper.properties

添加或修改如下内容

dataDir=/mnt/kafka/kafka_data/zookeeper   #zookeeper数据目录
dataLogDir=/mnt/kafka/kafka_data/log/zookeeper #zookeeper日志目录
clientPort=2181
maxClientCnxns=100
tickTimes=2000
initLimit=10
syncLimit=5

 

[root@localhost ~]# grep -Ev '$^|#' /mnt/kafka/kafka_2.12-3.5.1/config/zookeeper.properties  
dataDir=/mnt/kafka/kafka_data/zookeeper
dataLogDir=/mnt/kafka/kafka_data/log/zookeeper
clientPort=2181
maxClientCnxns=100
admin.enableServer=false
tickTimes=2000
initLimit=10
sysncLimit=5

三.创建kafka和zookeeper开启及关闭脚本

1.创建kafka和zookeeper启动脚本

cd /mnt/kafka/kafka_2.12-3.5.1/
vi kafka_start.sh

添加如下内容

#!/bin/sh
#启动zookeeper
/mnt/kafka/kafka_2.12-3.5.1/bin/zookeeper-server-start.sh /mnt/kafka/kafka_2.12-3.5.1/config/zookeeper.properties &
 
sleep 3 #等3秒后执行
 
#启动kafka
/mnt/kafka/kafka_2.12-3.5.1/bin/kafka-server-start.sh /mnt/kafka/kafka_2.12-3.5.1/config/server.properties &

可以看到kafka启动会加载zookeeper一起启动。

2.创建kafka关闭脚本

vi kafka_stop.sh

 

#!/bin/sh
#关闭zookeeper
/mnt/kafka/kafka_2.12-3.5.1/bin/zookeeper-server-stop.sh /mnt/kafka/kafka_2.12-3.5.1/config/zookeeper.properties &
 
sleep 3 #等3秒后执行
 
#启动kafka
/mnt/kafka/kafka_2.12-3.5.1/bin/kafka-server-stop.sh /mnt/kafka/kafka_2.12-3.5.1/config/server.properties &

3.给脚本授执行权限

chmod +x kafka_start.sh

chmod +x kafka_stop.sh

4.启动或关闭kafka

./kafka_start.sh


./kafka_stop.sh

5.启动脚本，设置开机自启动

vi /etc/rc.d/rc.local            #编辑，在最后添加一行

sh /mnt/kafka/kafka_2.12-3.5.1/kafka_start.sh &

6.查看启动是否成功

ps -ef | grep -E 'kafka|zookeeper' 

或查看启动端口

[root@localhost kafka_2.12-3.5.1]# netstat -lntp 
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name    
tcp        0      0 127.0.0.1:25            0.0.0.0:*               LISTEN      1237/master         
tcp        0      0 0.0.0.0:22              0.0.0.0:*               LISTEN      1011/sshd           
tcp6       0      0 ::1:25                  :::*                    LISTEN      1237/master         
tcp6       0      0 :::46461                :::*                    LISTEN      2713/java           
tcp6       0      0 :::9092                 :::*                    LISTEN      2713/java           
tcp6       0      0 :::2181                 :::*                    LISTEN      1543/java           
tcp6       0      0 :::44332                :::*                    LISTEN      1543/java           
tcp6       0      0 :::22                   :::*                    LISTEN      1011/sshd 

9092 和2181 存在 说明启动成功。

四.生产者及消费者应用示例

1.在kafka启动后（启动不了，可能有部分进程已经启用，需要kill点重新启动），开一个终端负责启动生产者

 cd /mnt/kafka/kafka_2.12-3.5.1/bin/

 

2.生产者

./kafka-console-producer.sh --broker-list 192.168.186.20:9092 --topic wd_test

 

3.在另一终端启动消费者

 

./kafka-console-consumer.sh --bootstrap-server 192.168.186.20:9092 --topic wd_test

 

4.生产者输入，消费者输出

input

 output

 

5.查询当前运行的主题topic

[root@localhost bin]# ./kafka-topics.sh --bootstrap-server 192.168.186.20:9092 --list
__consumer_offsets
wd_test