前言
一、使用Docker-Compose部署Kafka集群
1.部署单节点伪分布式集群
单节点伪分布式集群是指集群由一台Zookeeper服务器和一台Kafka broker服务器组成
（1）编写docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
| version: "3"
services:
zookeeper:
image: zookeeper:3.4
container_name: zookeeper
hostname: zookeeper
restart: always
ports:
- 2181:2181
volumes:
- ./zookeeper/data:/data
networks:
- baseNetwork
kafka:
image: wurstmeister/kafka:2.12-2.1.0
container_name: kafka
hostname: kafka
restart: always
ports:
- 9092:9092
networks:
- baseNetwork
environment:
KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
volumes:
- ./kafka/logs:/kafka
kafka-manager:
image: sheepkiller/kafka-manager
container_name: kafka-manager
hostname: kafka-manager
ports:
- 9000:9000
networks:
- baseNetwork
environment:
ZK_HOSTS: zookeeper:2181
networks:
baseNetwork:
driver: bridge
|
执行如下命令
（2）一个Zookeeper多个kafka的扩容
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
| version: "3"
services:
zookeeper:
image: zookeeper:3.4
container_name: zookeeper
hostname: zookeeper
restart: always
ports:
- 2181:2181
volumes:
- ./zookeeper/data:/data
networks:
- baseNetwork
kafka1:
image: wurstmeister/kafka:2.12-2.1.0
container_name: kafka1
hostname: kafka1
restart: always
ports:
- 9092:9092
networks:
- baseNetwork
environment:
KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
volumes:
- ./kafka/logs:/kafka
kafka2:
image: wurstmeister/kafka:2.12-2.1.0
container_name: kafka2
hostname: kafka2
restart: always
ports:
- 9093:9093
networks:
- baseNetwork
environment:
KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
volumes:
- ./kafka/logs:/kafka
kafka3:
image: wurstmeister/kafka:2.12-2.1.0
container_name: kafka3
hostname: kafka3
restart: always
ports:
- 9094:9094
networks:
- baseNetwork
environment:
KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
volumes:
- ./kafka4/logs:/kafka
kafka-manager:
image: sheepkiller/kafka-manager
container_name: kafka-manager
hostname: kafka-manager
ports:
- 9000:9000
networks:
- baseNetwork
environment:
ZK_HOSTS: zookeeper:2181
networks:
baseNetwork:
driver: bridge
|
3.部署多节点分布式集群
多节点Kafka集群由一套多节点Zookeeper集群和一套多节点Kafka集群组成
以下还有点问题
（1）编写 docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
| version: "3"
services:
zk1:
image: zookeeper:3.4
container_name: zk1
hostname: zk1
restart: always
ports:
- 2181:2181
volumes:
- ./zk1/data:/data
networks:
- baseNetwork
zk2:
image: zookeeper:3.4
restart: always
ports:
- 2182:2182
volumes:
- ./zk2/data:/data
networks:
- baseNetwork
kafka:
image: wurstmeister/kafka:2.12-2.1.0
container_name: kafka
hostname: kafka
restart: always
networks:
- baseNetwork
environment:
KAFKA_ADVERTISED_HOST_NAME: 127.0.0.1
KAFKA_ZOOKEEPER_CONNECT: zk1:2181,zk2:2182
volumes:
- ./kafka/logs:/kafka
kafka-manager:
image: sheepkiller/kafka-manager
ports:
- 9000:9000
networks:
- baseNetwork
environment:
ZK_HOSTS: zk1:2181,zk2:2182
networks:
baseNetwork:
driver: bridge
|
（2）执行命令：
1
| docker-compose up -d --scale kafka=3
|
二、测试Kafka
进入 kafka docker 容器
1
| docker exec -it kafka /bin/bash
|
进入kafka默认目录
1
| cd /opt/kafka_2.12-2.1.0
|
会发现此目录中包含了许多kafka的可执行命令，下面我们将使用这些命令进行kafka的相关测试
1.创建topic
创建一个名为test的topic，该topic只有一个分区（partition），且该partition也只有一个副本（replication）处理消息
1
| bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --topic test --partitions 1 --replication-factor 1
|
2.查看topic状态
1
| bin/kafka-topics.sh --describe --zookeeper zookeeper:2181 --topic test
|
3.发送消息
1
| bin/kafka-console-producer.sh --broker-list kafka:9092 --topic test
|
4.消费消息
1
| bin/kafka-console-consumer.sh --bootstrap-server kafka:9092 --topic test --from-beginning
|