Logstash 在 URL“http://elasticsearch:9200/”联系 Elasticsearch 时收到响应代码“401”

Question

我正在尝试在 docker 环境中设置 ELK 堆栈，但到目前为止我对 Logstash 还没有什么运气。我已经运行 Kibana + Elastic 并相互连接，但无法让 Logstash 工作。我已经尝试过不同的配置和 github 存储库，但似乎没有任何效果。

我收到以下错误：

[logstash.licensechecker.licensereader] 尝试恢复与失效 ES 实例的连接，但出现错误 {:url=>"http://elasticsearch:9200/", :exception=>LogStash::Outputs::ElasticSearch:: HttpClient::Pool::BadResponseCodeError，：message=>“通过 URL 'http://elasticsearch:9200/' 联系 Elasticsearch 时收到响应代码 '401'”}

我得到的另一件事是：

[2022-09-12T13:16:18,558][错误][logstash.monitoring.internalpipelinesource] 无法从 Elasticsearch 获取 X-Pack 信息。这可能是由于无法到达实时 Elasticsearch 集群造成的。

我的 docker-compose 是：

version: '3.8'
services:
  server:
    build:
      context: ./
      target: dev
    volumes:
      - .:/src
    command: npm run dev
    container_name: server
    ports:
      - "${NODE_PORT}:${NODE_PORT}"
    environment:
      NODE_ENV: development
      DEBUG: nodejs-docker-express:*
    networks:
      - network-name
  kibana:
    build:
      context: ./docker/kibana/
      args:
        ELASTIC_PORT: ${ELASTIC_PORT}
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    container_name: kibana
    ports:
      - "${KIBANA_PORT}:${KIBANA_PORT}"
    volumes:
      - ./docker/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z
    environment:
      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD}
      ELASTIC_PORT: ${ELASTIC_PORT}
    networks:
      - network-name
  elasticsearch:
    build:
      context: ./docker/elasticsearch/
      args: 
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    volumes:
      - ./docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z
      - ./docker/elasticsearch/data:/usr/share/elasticsearch/data:z
    container_name: elasticsearch
    ports:
      - "${ELASTIC_PORT}:${ELASTIC_PORT}"
    environment:
      ES_JAVA_OPTS: -Xms512m -Xmx512m
      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD:-}
      discovery.type: single-node
    networks:
      - network-name
  logstash:
    build:
      context: ./docker/logstash/
      args:
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    volumes:
      - ./docker/logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
    container_name: logstash
    environment:
      ELASTIC_VERSION: ${ELASTIC_VERSION}
      ELASTIC_PORT: ${ELASTIC_PORT}
      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD}
    ports:
      - "5044:5044"
      - "50000:50000/tcp"
      - "50000:50000/udp"
      - "9600:9600"
    networks:
      - network-name
networks:
  network-name:
    name: "network-name"
    driver_opts:
      icc: "true"
    driver: bridge

logstash.conf

input {
  tcp {
    port => 5000
    type => syslog
  }
}
## Add your filters / logstash plugins configuration here

output {
    elasticsearch { 
      hosts => ["elasticsearch:9200"]
      user => "elastic"
      password => "dxAq9YLjR23VqcA4oS1I"
      #ssl => true
      ssl_certificate_verification => false
      timeout => 240
      #cacert => '${LS_HOME}/config/ssl/ca.pem'
    }
    stdout { codec => rubydebug }
}

logstash.yml

http.host: "0.0.0.0"
config.reload.automatic: true
log.level: debug

queue.type: persisted
queue.max_bytes: 1gb
queue.checkpoint.writes: 1

ENV 文件

#ELK CONFIG
ELASTIC_PORT=9200
ELASTIC_PASSWORD=dxAq9YLjR23VqcA4oS1I
ELASTIC_VERSION=8.4.0
#KIBANA
KIBANA_PORT=5601
KIBANA_SYSTEM_PASSWORD=1cDUbBrO8XRosNoayRpy
#BEATS

#LOGSTASH
LOGSTASH_INTERNAL_PASSWORD=C9i0SgSq3loKYbdooKTV

我不知道我做错了什么。我已经为所有用户生成了好几次新密码，重建了容器，但似乎对 Logstash 不起作用:(

Answer 1

Tldr；

Elasticsearch 服务不在同一网络上。

elasticsearch

已开启

network-name

。

kibana

和

logstash

已开启

tower-defense

然后在

elasticsearch

服务的环境下，你把elastic的密码引用为

${ELASTIC_PASSWORD:-}

……是不是写错了？

解决方案

version: '3.8'
services:
  server:
    build:
      context: ./
      target: dev
    volumes:
      - .:/src
    command: npm run dev
    container_name: server
    ports:
      - "${NODE_PORT}:${NODE_PORT}"
    environment:
      NODE_ENV: development
      DEBUG: nodejs-docker-express:*
    networks:
      - network-name
  kibana:
    build:
      context: ./docker/kibana/
      args:
        ELASTIC_PORT: ${ELASTIC_PORT}
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    container_name: kibana
    ports:
      - "${KIBANA_PORT}:${KIBANA_PORT}"
    volumes:
      - ./docker/kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml:ro,Z
    environment:
      KIBANA_SYSTEM_PASSWORD: ${KIBANA_SYSTEM_PASSWORD}
      ELASTIC_PORT: ${ELASTIC_PORT}
    networks:
      - tower-defense
  elasticsearch:
    build:
      context: ./docker/elasticsearch/
      args: 
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    volumes:
      - ./docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro,z
      - ./docker/elasticsearch/data:/usr/share/elasticsearch/data:z
    container_name: elasticsearch
    ports:
      - "${ELASTIC_PORT}:${ELASTIC_PORT}"
    environment:
      ES_JAVA_OPTS: -Xms512m -Xmx512m
      ELASTIC_PASSWORD: ${ELASTIC_PASSWORD}
      discovery.type: single-node
    networks:
      - tower-defense
  logstash:
    build:
      context: ./docker/logstash/
      args:
        ELASTIC_VERSION: ${ELASTIC_VERSION}
    volumes:
      - ./docker/logstash/pipeline:/usr/share/logstash/pipeline:ro,Z
    container_name: logstash
    environment:
      ELASTIC_VERSION: ${ELASTIC_VERSION}
      ELASTIC_PORT: ${ELASTIC_PORT}
      LOGSTASH_INTERNAL_PASSWORD: ${LOGSTASH_INTERNAL_PASSWORD}
    ports:
      - "5044:5044"
      - "50000:50000/tcp"
      - "50000:50000/udp"
      - "9600:9600"
    networks:
      - tower-defense
networks:
  tower-defense:
    name: "network-name"
    driver_opts:
      icc: "true"
    driver: bridge

Answer 2

En mi caso el error se presentaba por que los Endpoint del elasticsearch estaban protegidos con autenticacion basica (usuario y contraseña).

Entonces en el pipeline de configuracion le agregue usuario y contraseña y funciono.