docker搭建es集群+kinbana+logstash，实现日志收集功能

原创已于 2022-11-23 15:47:38 修改 · 800 阅读

0 ·

CC 4.0 BY-SA版权

文章标签：

#elasticsearch #大数据

于 2022-11-23 15:44:44 首次发布

本文介绍如何使用 Docker Compose 部署 Elasticsearch、Kibana 和 Logstash，形成完整的 Elastic Stack。通过配置多节点集群提升 Elasticsearch 的稳定性和性能，并详细说明了各组件的配置参数。

摘要生成于 C知道，由 DeepSeek-R1 满血版支持，前往体验 >

docker-compose

version: '3.1'
services:
  es1:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0
    container_name: es1
    ports:
      - "9200:9200"
      - "9300:9300"
    networks:
      - es-cluster
    volumes:
      - /data/yymagicer/es/data01:/usr/share/elasticsearch/data
    environment:
      - node.name=es1
      - discovery.seed_hosts=es1
      - cluster.initial_master_nodes=es1
      - cluster.name=es-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    ulimits:
      memlock:
        soft: -1
        hard: -1
  es2:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0
    container_name: es2
    networks:
      - es-cluster
    environment:
      - node.name=es2
      - discovery.seed_hosts=es1
      - cluster.initial_master_nodes=es1
      - cluster.name=es-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    volumes:
      - /data/yymagicer/es/data02:/usr/share/elasticsearch/data
    ulimits:
      memlock:
        soft: -1
        hard: -1
  es3:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.0
    container_name: es3
    networks:
      - es-cluster
    environment:
      - node.name=es3
      - discovery.seed_hosts=es1
      - cluster.initial_master_nodes=es1
      - cluster.name=es-cluster
      - bootstrap.memory_lock=true
      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
    volumes:
      - /data/yymagicer/es/data03:/usr/share/elasticsearch/data
    ulimits:
      memlock:
        soft: -1
        hard: -1
  kibana:
    image: docker.elastic.co/kibana/kibana:7.12.0
    container_name: kibana
    #environment:
      #- I18N_LOCALE=zh-CN #汉化
    ports:
      - "5601:5601"
    links:
      - es1:elasticsearch
    depends_on:
      - es1
      - es2
      - es3
    networks:
      - es-cluster
  logstash:
    image: docker.elastic.co/logstash/logstash:7.3.0
    container_name: logstash
    volumes:
      - /home/deepblue/yymagicer/config/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml
      - /home/deepblue/yymagicer/config/logstash/pipeline:/usr/share/logstash/pipeline
    ports:
      - 5044:5044
    environment:
      LS_JAVA_OPTS: "-Xmx256m -Xms256m"
    networks:
      - es-cluster
    depends_on:
      - es1
      - es2
      - es3
networks:
  es-cluster:
    driver: bridge

logstash/config/logstash.yml

http.host: "0.0.0.0"

logstash/pipeline/logstash.conf

input {
    tcp {       
    mode => "server"    
    host => "0.0.0.0"   
    port => 5044        
    codec => json_lines 
  }
}
output {
    ##正对不同的项目，生成不同index
    if "ai-video-monitor" in [appname]{
       elasticsearch {
         hosts => ["127.0.0.1:9200"]
         index => "ai-video-monitor-%{+yyyy.MM.dd}"
      } 
    }
}