ELK Stack高级应用

Elasticsearch高级配置

集群配置

# elasticsearch.yml
cluster.name: my-cluster
node.name: node-1
node.roles: [master, data]

network.host: 0.0.0.0
discovery.seed_hosts: ["node-1", "node-2", "node-3"]
cluster.initial_master_nodes: ["node-1", "node-2", "node-3"]

# 索引配置
index.number_of_shards: 3
index.number_of_replicas: 1

索引管理

# 创建索引
curl -X PUT "localhost:9200/my-index" -H 'Content-Type: application/json' -d'
{
  "settings": {
    "number_of_shards": 3,
    "number_of_replicas": 1
  },
  "mappings": {
    "properties": {
      "timestamp": { "type": "date" },
      "level": { "type": "keyword" },
      "message": { "type": "text" },
      "service": { "type": "keyword" }
    }
  }
}'

# 查看索引
curl -X GET "localhost:9200/_cat/indices?v"

# 删除索引
curl -X DELETE "localhost:9200/my-index"

索引生命周期

# 创建生命周期策略
curl -X PUT "localhost:9200/_ilm/policy/logs-policy" -H 'Content-Type: application/json' -d'
{
  "policy": {
    "phases": {
      "hot": {
        "actions": {
          "rollover": {
            "max_size": "50gb",
            "max_age": "1d"
          }
        }
      },
      "warm": {
        "min_age": "7d",
        "actions": {
          "shrink": { "number_of_shards": 1 },
          "forcemerge": { "max_num_segments": 1 }
        }
      },
      "cold": {
        "min_age": "30d",
        "actions": {
          "freeze": {}
        }
      },
      "delete": {
        "min_age": "90d",
        "actions": {
          "delete": {}
        }
      }
    }
  }
}'

Logstash高级配置

多管道配置

# pipelines.yml
- pipeline.id: app-logs
  path.config: "/etc/logstash/conf.d/app.conf"
  pipeline.workers: 4
  pipeline.batch.size: 125

- pipeline.id: system-logs
  path.config: "/etc/logstash/conf.d/system.conf"
  pipeline.workers: 2

高级过滤

# app.conf
input {
  beats {
    port => 5044
  }
}

filter {
  # JSON解析
  if [message] =~ /^\{/ {
    json {
      source => "message"
    }
  }
  
  # Grok解析
  grok {
    match => {
      "message" => "%{COMBINEDAPACHELOG}"
    }
  }
  
  # 日期解析
  date {
    match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
    target => "@timestamp"
  }
  
  # IP地理定位
  geoip {
    source => "clientip"
    target => "geoip"
  }
  
  # User-Agent解析
  useragent {
    source => "http_user_agent"
    target => "useragent"
  }
  
  # 字段映射
  mutate {
    rename => { "host" => "hostname" }
    remove_field => [ "agent", "ecs" ]
  }
}

output {
  elasticsearch {
    hosts => ["elasticsearch:9200"]
    index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}"
  }
}

Kibana高级功能

创建可视化

{
  "visState": {
    "title": "Error Rate by Service",
    "type": "line",
    "params": {
      "type": "line"
    },
    "aggs": [
      {
        "id": "1",
        "type": "date_histogram",
        "params": {
          "field": "@timestamp",
          "interval": "auto"
        }
      },
      {
        "id": "2",
        "type": "filters",
        "params": {
          "filters": [
            { "query": { "match": { "level": "ERROR" } } }
          ]
        }
      }
    ]
  }
}

创建仪表板

# 导出仪表板
curl -X GET "localhost:5601/api/kibana/dashboards/export?dashboard=dashboard-id" > dashboard.json

# 导入仪表板
curl -X POST "localhost:5601/api/kibana/dashboards/import" \
  -H 'kbn-xsrf: true' \
  -H 'Content-Type: application/json' \
  -d @dashboard.json

实践：日志分析系统

# docker-compose.yml
version: '3.8'

services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.10.0
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - "ES_JAVA_OPTS=-Xms2g -Xmx2g"
    volumes:
      - es_data:/usr/share/elasticsearch/data
    ports:
      - "9200:9200"

  logstash:
    image: docker.elastic.co/logstash/logstash:8.10.0
    volumes:
      - ./logstash/pipeline:/usr/share/logstash/pipeline
    depends_on:
      - elasticsearch

  kibana:
    image: docker.elastic.co/kibana/kibana:8.10.0
    ports:
      - "5601:5601"
    environment:
      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
    depends_on:
      - elasticsearch

  filebeat:
    image: docker.elastic.co/beats/filebeat:8.10.0
    user: root
    volumes:
      - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml
      - /var/log:/var/log:ro
    depends_on:
      - elasticsearch

volumes:
  es_data:

总结

ELK Stack高级应用包括集群配置、索引生命周期管理、高级日志处理和可视化。通过这些技术，可以构建企业级的日志分析平台。