Kafka - 02部署
一、安装Kafka
1.1 准备JDK、Zookeeper、Kafka
[root@my-node51 ~]# cd /opt/
[root@my-node51 opt]# ls
apache-zookeeper-3.6.2-bin jdk1.8.0_201 kafka_2.13-2.6.0
[root@my-node51 opt]# mv apache-zookeeper-3.6.2-bin/ zookeeper-3.6.2
[root@my-node51 opt]# mv kafka_2.13-2.6.0/ kafka-2.6.0
[root@my-node51 opt]# ls
jdk1.8.0_201 kafka-2.6.0 zookeeper-3.6.2
1.2 环境变量
[root@my-node51 ~]# vi /etc/profile
export JAVA_HOME=/opt/jdk1.8.0_201
export JRE_HOME=/opt/jdk1.8.0_201/jre
export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar:$JRE_HOME/lib
export ZOOKEEPER_HOME=/opt/zookeeper-3.6.2
export KAFKA_HOME=/opt/kafka-2.6.0
export PATH=$PATH:$KAFKA_HOME/bin:$ZOOKEEPER_HOME/bin:$JAVA_HOME/bin:$JRE_HOME/bin
1.3 启动Zookeeper集群
[root@my-node51 ~]# grep -v '^#' /opt/zookeeper-3.6.2/conf/zoo.cfg | grep -v '^$'
### zookeeper内部的基本单位，一个tickTime为2000毫秒。
tickTime=2000
### follower与leader 初始连接 最多心跳数(tickTime的数量)
initLimit=10
### follower与leader 请求和应答 最多心跳数(tickTime的数量)
syncLimit=5
### 数据存放文件夹， 存储：快照数据、事务日志
dataDir=/opt/zookeeper-3.6.2/data
### 客户端访问端口
clientPort=2181
### 集群地址
server.1=192.168.6.51:2888:3888
server.2=192.168.6.52:2888:3888
server.3=192.168.6.53:2888:3888
[root@my-node51 opt]# ./zookeeper-3.6.2/bin/zkServer.sh start
1.4 启动Kafka集群
[root@my-node51 config]# grep -v '^#' server.properties | grep -v '^$'
broker.id=1
listeners=PLAINTEXT://192.168.6.51:9092
num.network.threads=3
num.io.threads=8
socket.send.buffer.bytes=102400
socket.receive.buffer.bytes=102400
socket.request.max.bytes=104857600
log.dirs=/opt/kafka-2.6.0/kafka-logs
num.partitions=1
num.recovery.threads.per.data.dir=1
offsets.topic.replication.factor=1
transaction.state.log.replication.factor=1
transaction.state.log.min.isr=1
log.retention.hours=168
log.segment.bytes=1073741824
log.retention.check.interval.ms=300000
zookeeper.connect=192.168.6.51:2181,192.168.6.52:2181,192.168.6.53:2181
zookeeper.connection.timeout.ms=18000
group.initial.rebalance.delay.ms=0
[root@my-node51 bin]# ./kafka-server-start.sh
USAGE: ./kafka-server-start.sh [-daemon] server.properties [--override property=value]*
[root@my-node51 kafka-2.6.0]# ./bin/kafka-server-start.sh -daemon config/server.properties
### 启动后，多了 kafka-logs 和 logs 两个文件夹
[root@my-node52 kafka-2.6.0]# ls
bin config kafka-logs libs LICENSE logs NOTICE site-docs
1.5 Zookeeper配置报错
[root@my-node52 opt]# ./zookeeper-3.6.2/bin/zkServer.sh
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.2/bin/../conf/zoo.cfg
Usage: ./zookeeper-3.6.2/bin/zkServer.sh [--config <conf-dir>] {start|start-foreground|stop|version|restart|status|print-cmd}
### 未配置 Zookeeper节点 myid 报错
2022-07-03 08:55:28,141 [myid:] - ERROR [main:QuorumPeerMain@98] - Invalid config, exiting abnormally
org.apache.zookeeper.server.quorum.QuorumPeerConfig$ConfigException: Error processing /opt/zookeeper-3.6.2/bin/../conf/zoo.cfg
at org.apache.zookeeper.server.quorum.QuorumPeerConfig.parse(QuorumPeerConfig.java:198)
at org.apache.zookeeper.server.quorum.QuorumPeerMain.initializeAndRun(QuorumPeerMain.java:124)
at org.apache.zookeeper.server.quorum.QuorumPeerMain.main(QuorumPeerMain.java:90)
Caused by: java.lang.IllegalArgumentException: myid file is missing
at org.apache.zookeeper.server.quorum.QuorumPeerConfig.checkValidity(QuorumPeerConfig.java:812)
at org.apache.zookeeper.server.quorum.QuorumPeerConfig.setupQuorumPeerConfig(QuorumPeerConfig.java:683)
at org.apache.zookeeper.server.quorum.QuorumPeerConfig.parseProperties(QuorumPeerConfig.java:507)
at org.apache.zookeeper.server.quorum.QuorumPeerConfig.parse(QuorumPeerConfig.java:194)
... 2 more
Invalid config, exiting abnormally
2022-07-03 08:55:28,143 [myid:] - INFO [main:ZKAuditProvider@42] - ZooKeeper audit is disabled.
1.6 Kakfa数据和日志
[root@my-node51 kafka-logs]# ll
总用量 4
-rw-r--r-- 1 root root 0 7月 3 09:18 cleaner-offset-checkpoint
-rw-r--r-- 1 root root 0 7月 3 09:18 log-start-offset-checkpoint
-rw-r--r-- 1 root root 88 7月 3 09:18 meta.properties
-rw-r--r-- 1 root root 0 7月 3 09:18 recovery-point-offset-checkpoint
-rw-r--r-- 1 root root 0 7月 3 09:18 replication-offset-checkpoint
[root@my-node51 logs]# ll
总用量 80
-rw-r--r-- 1 root root 307 7月 3 09:18 controller.log
-rw-r--r-- 1 root root 0 7月 3 09:16 kafka-authorizer.log
-rw-r--r-- 1 root root 0 7月 3 09:16 kafka-request.log
-rw-r--r-- 1 root root 7683 7月 3 09:18 kafkaServer-gc.log.0.current
-rw-r--r-- 1 root root 30474 7月 3 09:18 kafkaServer.out
-rw-r--r-- 1 root root 172 7月 3 09:18 log-cleaner.log
-rw-r--r-- 1 root root 32468 7月 3 09:18 server.log
-rw-r--r-- 1 root root 0 7月 3 09:16 state-change.log
二、常见操作
### 新建topic
[root@my-node51 bin]# ./kafka-topics.sh --bootstrap-server 192.168.6.51:9092 --create --topic ttopic --partitions 3 --replication-factor 1
Created topic ttopic.
### 查询所有topic
[root@my-node51 bin]# ./kafka-topics.sh --bootstrap-server 192.168.6.51:9092 --list
ttopic
### 查看某个topic的信息
[root@my-node51 bin]# ./kafka-topics.sh --bootstrap-server 192.168.6.51:9092 --describe --topic ttopic
Topic: ttopic PartitionCount: 3 ReplicationFactor: 1 Configs: segment.bytes=1073741824
Topic: ttopic Partition: 0 Leader: 3 Replicas: 3 Isr: 3
Topic: ttopic Partition: 1 Leader: 1 Replicas: 1 Isr: 1
Topic: ttopic Partition: 2 Leader: 2 Replicas: 2 Isr: 2
[root@my-node51 bin]# ./kafka-topics.sh --bootstrap-server 192.168.6.51:9092 --create --topic ttopic2 --partitions 3 --replication-factor 2
Created topic ttopic2.
[root@my-node51 bin]# ./kafka-topics.sh --bootstrap-server 192.168.6.51:9092 --describe --topic ttopic2
Topic: ttopic2 PartitionCount: 3 ReplicationFactor: 2 Configs: segment.bytes=1073741824
Topic: ttopic2 Partition: 0 Leader: 3 Replicas: 3,1 Isr: 3,1
Topic: ttopic2 Partition: 1 Leader: 1 Replicas: 1,2 Isr: 1,2
Topic: ttopic2 Partition: 2 Leader: 2 Replicas: 2,3 Isr: 2,3
[root@my-node51 kafka-logs]# ll
总用量 16
-rw-r--r-- 1 root root 0 7月 3 09:15 cleaner-offset-checkpoint
-rw-r--r-- 1 root root 4 7月 3 09:26 log-start-offset-checkpoint
-rw-r--r-- 1 root root 88 7月 3 12:10 meta.properties
-rw-r--r-- 1 root root 39 7月 3 09:26 recovery-point-offset-checkpoint
-rw-r--r-- 1 root root 39 7月 3 12:10 replication-offset-checkpoint
drwxr-xr-x 2 root root 141 7月 3 12:10 ttopic-1
drwxr-xr-x 2 root root 141 7月 3 09:26 ttopic2-0
drwxr-xr-x 2 root root 141 7月 3 12:10 ttopic2-1
[root@my-node52 kafka-logs]# ll
总用量 16
-rw-r--r-- 1 root root 0 7月 3 09:18 cleaner-offset-checkpoint
-rw-r--r-- 1 root root 4 7月 3 12:10 log-start-offset-checkpoint
-rw-r--r-- 1 root root 88 7月 3 12:10 meta.properties
-rw-r--r-- 1 root root 39 7月 3 12:10 recovery-point-offset-checkpoint
-rw-r--r-- 1 root root 39 7月 3 12:11 replication-offset-checkpoint
drwxr-xr-x 2 root root 141 7月 3 12:10 ttopic-2
drwxr-xr-x 2 root root 141 7月 3 12:10 ttopic2-1
drwxr-xr-x 2 root root 141 7月 3 12:10 ttopic2-2
[root@my-node53 kafka-logs]# ll
总用量 16
-rw-r--r-- 1 root root 0 7月 3 09:18 cleaner-offset-checkpoint
-rw-r--r-- 1 root root 4 7月 3 12:11 log-start-offset-checkpoint
-rw-r--r-- 1 root root 88 7月 3 12:09 meta.properties
-rw-r--r-- 1 root root 39 7月 3 12:11 recovery-point-offset-checkpoint
-rw-r--r-- 1 root root 39 7月 3 12:11 replication-offset-checkpoint
drwxr-xr-x 2 root root 141 7月 3 12:10 ttopic-0
drwxr-xr-x 2 root root 141 7月 3 12:10 ttopic2-0
drwxr-xr-x 2 root root 141 7月 3 12:10 ttopic2-2
[root@my-node51 bin]# ./kafka-topics.sh
Create, delete, describe, or change a topic.
Option Description
------ -----------
--alter Alter the number of partitions, replica assignment, and/or configuration for the topic.
--at-min-isr-partitions if set when describing topics, only show partitions whose isr count is equal to the configured minimum. Not supported with the --zookeeper option.
--bootstrap-server <String: server to REQUIRED: The Kafka server to connect to. In case of providing this, a direct Zookeeper connection won't be required.
connect to>
--command-config <String: command Property file containing configs to be passed to Admin Client. This is used only with --bootstrap-server option for describing and altering
config property file> broker configs.
--config <String: name=value> A topic configuration override for the topic being created or altered.The following is a list of valid configurations:
cleanup.policy
compression.type
delete.retention.ms
file.delete.delay.ms
flush.messages
flush.ms
follower.replication.throttled.replicas
index.interval.bytes
leader.replication.throttled.replicas
max.compaction.lag.ms
max.message.bytes
message.downconversion.enable
message.format.version
message.timestamp.difference.max.ms
message.timestamp.type
min.cleanable.dirty.ratio
min.compaction.lag.ms
min.insync.replicas
preallocate
retention.bytes
retention.ms
segment.bytes
segment.index.bytes
segment.jitter.ms
segment.ms
unclean.leader.election.enable
See the Kafka documentation for full details on the topic configs.It is supported only in combination with --create if --bootstrap-server option is used.
--create Create a new topic.
--delete Delete a topic
--delete-config <String: name> A topic configuration override to be removed for an existing topic (see the list of configurations under the --config option).
Not supported with the --bootstrap-server option.
--describe List details for the given topics.
--disable-rack-aware Disable rack aware replica assignment
--exclude-internal exclude internal topics when running list or describe command. The internal topics will be listed by default
--force Suppress console prompts
--help Print usage information.
--if-exists if set when altering or deleting or describing topics, the action will only execute if the topic exists.
--if-not-exists if set when creating topics, the action will only execute if the topic does not already exist.
--list List all available topics.
--partitions <Integer: # of partitions> The number of partitions for the topic being created or altered (WARNING: If partitions are increased for a topic that has a key,
the partition logic or ordering of the messages will be affected). If not supplied for create, defaults to the cluster default.
--replica-assignment <String: A list of manual partition-to-broker assignments for the topic being created or altered.
broker_id_for_part1_replica1 :
broker_id_for_part1_replica2 ,
broker_id_for_part2_replica1 :
broker_id_for_part2_replica2 , ...>
--replication-factor <Integer: The replication factor for each partition in the topic being created. If not supplied, defaults to the cluster default.
replication factor>
--topic <String: topic> The topic to create, alter, describe or delete. It also accepts a regular expression, except for --create option.
Put topic name in double quotes and use the '\' prefix to escape regular expression symbols; e.g. "test\.topic".
--topics-with-overrides if set when describing topics, only show topics that have overridden configs
--unavailable-partitions if set when describing topics, only show partitions whose leader is not available
--under-min-isr-partitions if set when describing topics, only show partitions whose isr count is less than the configured minimum.
Not supported with the --zookeeper option.
--under-replicated-partitions if set when describing topics, only show under replicated partitions
--version Display Kafka version.
--zookeeper <String: hosts> DEPRECATED, The connection string for the zookeeper connection in the form host:port. Multiple hosts can be given to allow fail-over.
浙公网安备 33010602011771号