集群规划:
三台服务器各部署一个kafka集群节点、一个zookeeper集群节点。
服务器 | zookeeper角色 | kafka角色 | ip |
node1 | master | broker:0 | 192.168.146.199 |
node2 | follower | broker:1 | 192.168.146.200 |
node3 | broker:2 | 192.168.146.201 |
PS:命令行开头为#表示在所有主机上操作,命令行开后为具体主机名则表示在该主机上操作。
1、三台服务器基础设置
设置主机名:
# hostnamectl --static set-hostname node1/node2/node3
# vi /etc/hosts
192.168.146.199 node1
192.168.146.200 node2
192.168.146.201 node3
时区调整,时间校准:
# date -R
# timedatectl set-timezone Asia/Shanghai
# yum -y install ntp
# ntpdate ntp1.aliyun.com
安装wget:
$ yum install -y wget
关闭selinux:
# vi /etc/sysconfig/selinux
SELINUX=disabled
重启服务器并验证:
# getenforce
Disabled
设置防火墙
开放zookeeper相关端口2181,2888,3888
# firewall-cmd --zone=public --add-port=2181/tcp --permanent
success
# firewall-cmd --zone=public --add-port=2888/tcp --permanent
success
# firewall-cmd --zone=public --add-port=3888/tcp --permanent
success
开放kafka相关端口9092
# firewall-cmd --zone=public --add-port=9092/tcp --permanent
success
reload 防火墙配置使其生效
# firewall-cmd --reload
success
2、基础环境安装
安装JDK环境
# yum install java-1.8.0-openjdk
# java -version
openjdk version "1.8.0_262"
OpenJDK Runtime Environment (build 1.8.0_262-b10)
OpenJDK 64-Bit Server VM (build 25.262-b10, mixed mode)
创建目录与文件
# mkdir -p /var/zookeeper/log
# touch /var/zookeeper/myid
# mkdir -p /var/log/kafka
3、安装zookeeper
# cd /opt
# wget http://archive.apache.org/dist/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz
# tar -zxvf apache-zookeeper-3.6.1-bin.tar.gz
# mv apache-zookeeper-3.6.1-bin zookeeper-3.6.1
复制模板
# mv /opt/zookeeper-3.6.1/conf/zoo_sample.cfg /opt/zookeeper-3.6.1/conf/zoo.cfg
修改配置文件
# vi /opt/zookeeper-3.6.1/conf/zoo.cfg
dataDir=/var/zookeeper
server.1=node1:2888:3888
server.2=node2:2888:3888
server.3=node3:2888:3888
[root@node1 opt]# echo 1 > /var/zookeeper/myid
[root@node2 opt]# echo 2 > /var/zookeeper/myid
[root@node3 opt]# echo 3 > /var/zookeeper/myid
4、安装kafka
# cd /opt
# wget https://mirrors.tuna.tsinghua.edu.cn/apache/kafka/2.5.0/kafka_2.13-2.5.0.tgz
# tar -zxvf kafka_2.13-2.5.0.tgz
[root@node1 opt]# vi /opt/kafka_2.13-2.5.0/config/server.properties
broker.id=0
log.dirs=/var/log/kafka
zookeeper.connect=node1:2181,node2:2181,node3:2181
[root@node2 opt]# vi /opt/kafka_2.13-2.5.0/config/server.properties
broker.id=1
log.dirs=/var/log/kafka
zookeeper.connect=node1:2181,node2:2181,node3:2181
[root@node3 opt]# vi /opt/kafka_2.13-2.5.0/config/server.properties
broker.id=2
log.dirs=/var/log/kafka
zookeeper.connect=node1:2181,node2:2181,node3:2181
5、启动和关闭zookeeper
# /opt/zookeeper-3.6.1/bin/zkServer.sh start
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@node1 opt]# /opt/zookeeper-3.6.1/bin/zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower
[root@node2 opt]# /opt/zookeeper-3.6.1/bin/zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower
[root@node3 opt]# /opt/zookeeper-3.6.1/bin/zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/zookeeper-3.6.1/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: leader
关闭方法
# /opt/zookeeper-3.6.1/bin/zkServer.sh stop
查看日志
# tail -f /opt/zookeeper-3.6.1/logs/zookeeper-root-server-node{myid}.out
6、启动和关闭kafka
# /opt/kafka_2.13-2.5.0/bin/kafka-server-start.sh /opt/kafka_2.13-2.5.0/config/server.properties 1>/dev/null 2>&1 &
# tail -f /opt/kafka_2.13-2.5.0/logs/server.log
关闭方法
# /opt/kafka_2.13-2.5.0/bin/kafka-server-stop.sh
7、测试
# cd /opt/kafka_2.13-2.5.0
创建topic
[root@node1 kafka_2.13-2.5.0]# bin/kafka-topics.sh --create --zookeeper node1:2181,node2:2181,node3:2181 --replication-factor 3 --partitions 3 --topic gxm-test
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
Created topic gxm-test.
查看topic
[root@node2 kafka_2.13-2.5.0]# bin/kafka-topics.sh --describe --zookeeper node1:2181,node2:2181,node3:2181 --topic gxm-test
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
Topic: gxm-test PartitionCount: 3 ReplicationFactor: 3 Configs:
Topic: gxm-test Partition: 0 Leader: 1 Replicas: 1,2,0 Isr: 1,2,0
Topic: gxm-test Partition: 1 Leader: 2 Replicas: 2,0,1 Isr: 2,0,1
Topic: gxm-test Partition: 2 Leader: 0 Replicas: 0,1,2 Isr: 0,1,2
列出topic
[root@node3 kafka_2.13-2.5.0]# bin/kafka-topics.sh --list --zookeeper node1:2181,node2:2181,node3:2181
OpenJDK 64-Bit Server VM warning: If the number of processors is expected to increase from one, then you should configure the number of parallel GC threads appropriately using -XX:ParallelGCThreads=N
gxm-test
在node1创建 producer测试
[root@node1 kafka_2.13-2.5.0]# bin/kafka-console-producer.sh --broker-list node1:9092 -topic gxm-test
在node2创建 consumer测试
[root@node2 kafka_2.13-2.5.0]# bin/kafka-console-consumer.sh --bootstrap-server node2:9092 -topic gxm-test --from-beginning
在node3上创建 consumer测试
[root@node3 kafka_2.13-2.5.0]# bin/kafka-console-consumer.sh --bootstrap-server node2:9092 -topic gxm-test --from-beginning
[root@node1 kafka_2.13-2.5.0]# bin/kafka-topics.sh --delete --zookeeper node1:2181,node2:2181,node3:2181 --topic gxm-test