安装Kafka

前提条件：

• 已安装 jdk 环境

安装zookeeper

下载zookeeper

sudo curl -o /usr/local/src/apache-zookeeper-3.6.4-bin.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.6.4/apache-zookeeper-3.6.4-bin.tar.gz

解压zookeeper

sudo tar xf /usr/local/src/apache-zookeeper-3.6.4-bin.tar.gz -C /usr/local
sudo ln -s /usr/local/apache-zookeeper-3.6.4-bin/ /app/zookeeper

创建相关目录

sudo mkdir -p /app/zookeeper/{data,logs}

修改配置文件

dataDir：zookeeper的数据目录
clientPort：zookeeper客户端连接连接端口
server.num：zookeeper的集群地址。其中num是数字，对应的每个节点的myid。
　　　　　　　值是 IP地址:集群内机器通讯使用(Leader监听此端口):选举leader端口(所有节点都有此端口)

cp /app/zookeeper/conf/{zoo_sample.cfg,zoo.cfg}

egrep -v "^$|^#" /app/zookeeper/conf/zoo.cfg 
tickTime=2000
initLimit=10
syncLimit=5
dataDir=/app/zookeeper/data
clientPort=2181
server.1=192.168.32.187:2888:3888
server.2=192.168.32.188:2888:3888
server.3=192.168.32.189:2888:3888

# 192.168.32.187主机
sudo echo 1 > /app/zookeeper/data/myid
# 192.168.32.188主机
sudo echo 2 > /app/zookeeper/data/myid
# 192.168.32.189主机
sudo echo 3 > /app/zookeeper/data/myid

sudo chown -R ops. /usr/local/apache-zookeeper-3.6.4-bin

启动服务

如果有开 firewalld 服务，请开通端口...

• sudo firewall-cmd --add-port=2181/tcp --add-port=2888/tcp --add-port=3888/tcp --permanent
• sudo firewall-cmd --reload
• sudo firewall-cmd --list-all

cat <<'EOF' | sudo tee /usr/lib/systemd/system/zookeeper.service >> /dev/null
[Unit]
Description=Zookeeper Server
Documentation=https://zookeeper.apache.org/doc/
Wants=network.service
After=network.service

[Service]
# zookeeper启动属于fork进程启动，这里务必配置forking
Type=forking
# 配置java环境，zookeeper运行日志目录，zookeeper服务运行pid号
Environment=JAVA_HOME=/opt/jdk ZOO_LOG_DIR=/app/zookeeper/logs ZOOPIDFILE=/app/zookeeper/zookeeper.pid
# 启动用户为ops
User=ops
# 启动zookeeper命令
ExecStart=/app/zookeeper/bin/zkServer.sh start
# 停止zookeeper命令
ExecStop=/app/zookeeper/bin/zkServer.sh stop
# 服务生成的pid文件
PIDFile=/app/zookeeper/zookeeper.pid
# 仅当服务是fail状态重启
Restart=on-failure
# 成功退出的状态为143
SuccessExitStatus=143
# 自动重启间隔10s
RestartSec=10
# 自动重启的次数
StartLimitInterval=3

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload 
sudo systemctl enable zookeeper --now

验证服务

cd /app/zookeeper/bin

./zkServer.sh status                    # 需要放通选举leader端口，这里设置端口是3888
# 连接方式一
./zkCli.sh -server 192.168.32.187:2181  # 需要放通连接连接端口，这里设置端口是2181
# 连接方式二
./zkCli.sh -server 192.168.32.187:2181/kafka

安装Kafka

下载Kafka

sudo curl -kL -o /usr/local/src/kafka_2.12-2.3.1.tgz https://archive.apache.org/dist/kafka/2.3.1/kafka_2.12-2.3.1.tgz

解压zookeeper

sudo tar xf /usr/local/src/kafka_2.12-2.3.1.tgz -C /usr/local
sudo ln -s /usr/local/kafka_2.12-2.3.1/ /app/kafka

创建数据目录

sudo mkdir -p /app/kafka/{data,logs}
sudo chown -R ops. /usr/local/kafka_2.12-2.3.1

修改配置文件

broker.id：broker的id编号
listeners：监听地址以及端口
log.dirs：数据目录
zookeeper.connect：zookeeper的地址
其他配置均是默认值

egrep -v "^$|^#" /app/kafka/config/server.properties 
broker.id=2
listeners=PLAINTEXT://192.168.32.189:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/app/kafka/data
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.32.187:2181,192.168.32.188:2181,192.168.32.189:2181/kafka
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0

启动服务

如果有开 firewalld 服务，请开通端口...

• sudo firewall-cmd --add-port=9092/tcp --permanent
• sudo firewall-cmd --reload
• sudo firewall-cmd --list-all

cat <<'EOF' | sudo tee /usr/lib/systemd/system/kafka.service >> /dev/null
[Unit]
Description=Kafka Server
Documentation=https://kafka.apache.org/documentation/
Wants=network.service zookeeper.service
After=network.service zookeeper.service

[Service]
Type=simple
Environment=JAVA_HOME=/opt/jdk LOG_DIR=/app/kafka/logs
# 将进程名设置为发送到日志系统或内核日志缓冲区的前缀日志行
SyslogIdentifier=kafka
User=ops
ExecStart=/app/kafka/bin/kafka-server-start.sh /app/kafka/config/server.properties
ExecStop=/app/kafka/bin/kafka-server-stop.sh
SuccessExitStatus=143
Restart=on-failure
RestartSec=10
StartLimitInterval=3

[Install]
WantedBy=multi-user.target
EOF

sudo systemctl daemon-reload 
sudo systemctl enable kafka --now

验证服务

cd /app/kafka/bin

# 消费者【终端一】
$ ./kafka-console-consumer.sh --bootstrap-server 192.168.32.189:9092 --topic test
hello, jiaxzeng

# 生产者【终端二】
# 输入数据，看终端一是否有消费成功
$ ./kafka-console-producer.sh --broker-list 192.168.32.189:9092 --topic test
>hello, jiaxzeng