centos7配置kafka集群和cmak

cd /home/app/

mkdir /home/app/zookeeper
mkdir /home/app/zookeeper/data
mkdir /home/app/zookeeper/log

cd /home/app/zookeeper/

wget https://archive.apache.org/dist/zookeeper/zookeeper-3.8.1/apache-zookeeper-3.8.1-bin.tar.gz
tar -zxvf apache-zookeeper-3.8.1-bin.tar.gz
cd apache-zookeeper-3.8.1-bin/
ll
cd conf/
ll
cp zoo_sample.cfg zoo.cfg

vim zoo.cfg
dataDir=/home/app/zookeeper/data
#dataLogDir=/home/app/zookeeper/log
clientPort=2181
admin.serverPort=28080

cd /home/app/zookeeper/apache-zookeeper-3.8.1-bin/bin
/home/app/zookeeper/apache-zookeeper-3.8.1-bin/bin/zkServer.sh start
# 启动zk服务
./zkServer.sh start
# 查看zk服务状态
./zkServer.sh status
# 停止zk服务
./zkServer.sh stop
# 重启zk服务
./zkServer.sh restart

# 启动zk客户端
 sh ./zkCli.sh

mkdir /home/app/kafka/
cd /home/app/kafka/
wget https://downloads.apache.org/kafka/3.9.0/kafka_2.13-3.9.0.tgz

tar -zxvf kafka_2.13-3.9.0.tgz
cd /home/app/kafka/kafka_2.13-3.9.0/config
vim server.properties

确保以下配置正确：
broker.id=0
log.dirs=/home/app/kafka/data
zookeeper.connect=localhost:2181

/home/app/kafka/kafka_2.13-3.9.0/bin/kafka-server-start.sh /home/app/kafka/kafka_2.13-3.9.0/config/server.properties
/home/app/kafka/kafka_2.13-3.9.0/bin/kafka-server-stop.sh

创建一个测试主题
/home/app/kafka/kafka_2.13-3.9.0/bin/kafka-topics.sh --create --topic test1 --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
/home/app/kafka/kafka_2.13-3.9.0/bin/kafka-topics.sh --create --topic test-topic --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1
验证主题
/home/app/kafka/kafka_2.13-3.9.0/bin/kafka-topics.sh --list --bootstrap-server localhost:9092

启动一个生产者并发送消息
/home/app/kafka/kafka_2.13-3.9.0/bin/kafka-console-producer.sh --topic test1 --bootstrap-server localhost:9092

启动一个消费者并接收消息
/home/app/kafka/kafka_2.13-3.9.0/bin/kafka-console-consumer.sh --topic test1 --from-beginning --bootstrap-server localhost:9092

vim /home/app/kafka/restart_kafka.sh
sudo chmod +x /home/app/kafka/restart_kafka.sh

#!/bin/bash

# 指定Kafka和Zookeeper的目录
KAFKA_HOME=/home/app/kafka/kafka_2.13-3.9.0
ZOOKEEPER_HOME=/home/app/zookeeper/apache-zookeeper-3.8.1-bin

# 指定日志目录
KAFKA_LOG_DIR=/home/app/kafka/logs
ZOOKEEPER_LOG_DIR=/home/app/zookeeper/logs

# 创建日志目录（如果不存在）
mkdir -p $KAFKA_LOG_DIR
mkdir -p $ZOOKEEPER_LOG_DIR

# 停止Kafka
echo "Stopping Kafka..."
sudo  $KAFKA_HOME/bin/kafka-server-stop.sh

# 等待Kafka停止
sleep 5

# 停止Zookeeper
echo "Stopping Zookeeper..."
sudo  $ZOOKEEPER_HOME/bin/zkServer.sh stop

# 等待Zookeeper停止
sleep 5

# 启动Zookeeper
echo "Starting Zookeeper..."
sudo  nohup $ZOOKEEPER_HOME/bin/zkServer.sh start > $ZOOKEEPER_LOG_DIR/zookeeper.log 2>&1 &

# 等待Zookeeper启动
sleep 10

# 启动Kafka
echo "Starting Kafka..."
sudo  nohup $KAFKA_HOME/bin/kafka-server-start.sh $KAFKA_HOME/config/server.properties > $KAFKA_LOG_DIR/kafka.log 2>&1 &

echo "Kafka and Zookeeper restarted."

ZooKeeper 集群搭建
vim /home/app/zookeeper/apache-zookeeper-3.8.1-bin/conf/zoo.cfg
server.4=n4:2888:3888
server.3=n3:2888:3888
server.1=n0:2888:3888
vim /home/app/kafka/kafka_2.13-3.9.0/config/server.properties 
n0:2181,n3:2181,n4:2181

sudo echo "1" | sudo tee /home/app/zookeeper/data/myid  # 在 server.1 上
sudo echo "3" | sudo tee /home/app/zookeeper/data/myid  # 在 server.3 上
sudo echo "4" | sudo tee /home/app/zookeeper/data/myid  # 在 server.4 上

# 启动zk服务
/home/app/zookeeper/apache-zookeeper-3.8.1-bin/bin/zkServer.sh start
# 查看zk服务状态
/home/app/zookeeper/apache-zookeeper-3.8.1-bin/bin/zkServer.sh status
# 停止zk服务
/home/app/zookeeper/apache-zookeeper-3.8.1-bin/bin/zkServer.sh stop
# 重启zk服务
/home/app/zookeeper/apache-zookeeper-3.8.1-bin/bin/zkServer.sh restart

firewall-cmd --zone=public --add-port=9092/tcp --permanent   
firewall-cmd --zone=public --add-port=2181/tcp --permanent   
firewall-cmd --zone=public --add-port=2888/tcp --permanent   
firewall-cmd --zone=public --add-port=3888/tcp --permanent  
firewall-cmd --zone=public --add-port=9000/tcp --permanent    
firewall-cmd --reload 

配置kafka后台管理网站cmak  需要安装jdk-11
wget https://github.com/yahoo/CMAK/releases/download/v3.0.0.6/cmak-3.0.0.6.zip
   unzip cmak-3.0.0.6.zip
   cd cmak-3.0.0.6

vim /home/app/cmak/cmak-3.0.0.6/conf/application.conf

#kafka-manager.zkhosts="kafka-manager-zookeeper:2181"
#kafka-manager.zkhosts=${?ZK_HOSTS}
cmak.zkhosts="n0:2181,n3:2181,n4:2181"
cmak.zkhosts=${?ZK_HOSTS}

# Optional: Kafka Broker List
cmak.broker.list="n0:9092,n3:9092,n4:9092"

basicAuthentication.enabled=true
basicAuthentication.username="admin"
basicAuthentication.username=${?KAFKA_MANAGER_USERNAME}
basicAuthentication.password="admin"
basicAuthentication.password=${?KAFKA_MANAGER_PASSWORD}

创建cmak启动服务
sudo vi /etc/systemd/system/cmak.service

[Unit]
Description=CMAK Service
After=network.target

[Service]
User=root
ExecStart=/home/app/cmak/cmak-3.0.0.6/bin/cmak
Restart=on-failure
StandardOutput=append:/home/app/cmak/cmak-3.0.0.6/logs/cmak.log
StandardError=append:/home/app/cmak/cmak-3.0.0.6/logs/cmak-error.log

[Install]
WantedBy=multi-user.target

sudo mkdir -p /home/app/cmak/cmak-3.0.0.6/logs
sudo chown root:root /home/app/cmak/cmak-3.0.0.6/logs
sudo chmod 755 /home/app/cmak/cmak-3.0.0.6/logs

sudo systemctl daemon-reload
sudo systemctl enable cmak
sudo systemctl start cmak