golang之操作kafka
安装第三方包:
go get github.com/IBM/sarama
生产者实例:
package main import ( "fmt" "github.com/IBM/sarama" ) func main() { //1.生产者配置 config := sarama.NewConfig() config.Producer.RequiredAcks = sarama.WaitForAll //ACK,发送完数据需要leader和follow都确认 config.Producer.Partitioner = sarama.NewRandomPartitioner //分区,新选出一个分区 config.Producer.Return.Successes = true //确认,成功交付的消息将在success channel返回 //2.连接Kafka client, err := sarama.NewSyncProducer([]string{"127.0.0.1:9092"}, config) if err != nil { fmt.Println("Producer error", err) return } defer client.Close() //3.封装消息 msg := &sarama.ProducerMessage{} msg.Topic = "log" msg.Value = sarama.StringEncoder("this is test log") //4.发送消息 pid, offset, err := client.SendMessage(msg) if err != nil { fmt.Println("send faild", err) } fmt.Printf("pid:%v offset:%v\n", pid, offset) }
消费者:
package main import ( "fmt" // "github.com/Shopify/sarama" // 该包已经转移到了IBM/sarama
"github.com/IBM/sarama"
) // kafka consumer func main() { consumer, err := sarama.NewConsumer([]string{"127.0.0.1:9092"}, nil) if err != nil { fmt.Printf("fail to start consumer, err:%v\n", err) return } partitionList, err := consumer.Partitions("web_log") // 根据topic取到所有的分区 if err != nil { fmt.Printf("fail to get list of partition:err%v\n", err) return } fmt.Println(partitionList) for partition := range partitionList { // 遍历所有的分区 // 针对每个分区创建一个对应的分区消费者 pc, err := consumer.ConsumePartition("web_log", int32(partition), sarama.OffsetNewest) if err != nil { fmt.Printf("failed to start consumer for partition %d,err:%v\n", partition, err) return } defer pc.AsyncClose() // 异步从每个分区消费信息 go func(sarama.PartitionConsumer) { for msg := range pc.Messages() { fmt.Printf("Partition:%d Offset:%d Key:%v Value:%v", msg.Partition, msg.Offset, msg.Key, msg.Value) } }(pc) } }
完整文件内容:
package main import ( "context" "fmt" "github.com/IBM/sarama" "log" "math" "os" "os/signal" "sync" "syscall" ) // 更多参考:https://kpretty.tech/archives/gokafkaclient var addrs = []string{"172.29.97.140:9092"} var topic = "log" func producer() { // 生产者配置 config := sarama.NewConfig() config.Producer.RequiredAcks = sarama.WaitForAll // ACK config.Producer.Partitioner = sarama.NewRandomPartitioner // 分区 // 异步回调(两个channel, 分别是成功和错误) config.Producer.Return.Successes = true // 确认 config.Producer.Return.Errors = true sarama.Logger = log.New(os.Stdout, "[Sarama]", log.LstdFlags) // 连接kafka // 同步 client, err := sarama.NewSyncProducer(addrs, config) // 异步 //client, err := sarama.NewAsyncProducer(addrs, config) if err != nil { fmt.Println("producer error", err) return } defer func() { _ = client.Close() }() // 封装消息 msg := &sarama.ProducerMessage{ Topic: topic, Value: sarama.StringEncoder("this is test log"), } pid, offset, err := client.SendMessage(msg) if err != nil { fmt.Println("send failed", err) return } fmt.Printf("pid:%v offset:%v \n", pid, offset) } func consumer() { consumer, err := sarama.NewConsumer(addrs, nil) if err != nil { fmt.Printf("fail to start consumer, err:%v \n", err) return } partitionList, err := consumer.Partitions(topic) // 通过topic获取所有分区 if err != nil { fmt.Printf("fail to get partition list, err:%v\n", err) return } fmt.Println(partitionList) for partition := range partitionList { // 遍历所有分区 pc, err := consumer.ConsumePartition(topic, int32(partition), sarama.OffsetNewest) if err != nil { fmt.Printf("failed to start consumer for partition %d, err:%v\n", partition, err) return } defer pc.AsyncClose() go func(sarama.PartitionConsumer) { for msg := range pc.Messages() { // 当设置了key的时候,不为空 fmt.Printf("Partition:%d Offset:%d Key:%s Value:%s\n", msg.Partition, msg.Offset, string(msg.Key), string(msg.Value)) } }(pc) } //time.Sleep(5 * time.Second) select {} } func groupConsumer() { groupId := "sarama-consumer" config := sarama.NewConfig() // 关闭自动提交 和 初始化策略(oldest|newest) config.Consumer.Offsets.AutoCommit.Enable = false config.Consumer.Offsets.Initial = sarama.OffsetOldest sarama.NewConsumerGroup(addrs, groupId, config) } func main() { // 生产者 //producer() // 消费者(只能连续读取,中断期间会丢失数据) //consumer() // 消费者 //groupConsumer() SimpleConsumer() } var groupID = "sarama-consumer" var asyncOffset chan struct{} var wg sync.WaitGroup const defaultOffsetChannelSize = math.MaxInt func SimpleConsumer() { brokers := addrs // 消费者配置 config := sarama.NewConfig() // 关闭自动提交 config.Consumer.Offsets.AutoCommit.Enable = false config.Consumer.Offsets.Initial = sarama.OffsetOldest // 开启日志 logger := log.New(os.Stdout, "[Sarama] ", log.LstdFlags) sarama.Logger = logger consumer, err := sarama.NewConsumerGroup(brokers, groupID, config) if err != nil { panic(err) } defer func() { _ = consumer.Close() }() // 搞一个上下文用于终止消费者 ctx, cancelFunc := context.WithCancel(context.Background()) // 监听终止信号 go func() { logger.Println("monitor signal") quit := make(chan os.Signal, 1) signal.Notify(quit, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) <-quit logger.Println("stop consumer") cancelFunc() }() // 消费数据 err = consumer.Consume(ctx, []string{topic}, &Consumer{}) if err != nil { panic(err) } // 等待所有偏移量都提交完毕再退出 logger.Println("当前存在未提交的偏移量") wg.Wait() } type Consumer struct{} func (c *Consumer) Setup(session sarama.ConsumerGroupSession) error { // 初始化异步提交的channel asyncOffset = make(chan struct{}, defaultOffsetChannelSize) wg.Add(1) // 异步提交偏移量 go func() { for range asyncOffset { session.Commit() } wg.Done() }() return nil } func (c *Consumer) Cleanup(_ sarama.ConsumerGroupSession) error { // 关闭通道 close(asyncOffset) return nil } func (c *Consumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { EXIT: for { select { case message := <-claim.Messages(): log.Printf("Message claimed: key= %s, value = %s, timestamp = %v, topic = %s", string(message.Key), string(message.Value), message.Timestamp, message.Topic) // 标记消息，并不是提交偏移量 session.MarkMessage(message, "") // 异步提交 asyncOffset <- struct{}{} case <-session.Context().Done(): log.Println("cancel consumer") break EXIT } } return nil }
更深入使用:
