Elysia Tools
导航
DevOps
ELK Stack 日志分析示例
全面的 ELK Stack（Elasticsearch、Logstash、Kibana）示例，用于分布式系统中的日志聚合、处理和可视化
案例
这个案例集合中的条目
文本复杂
Elasticsearch 索引映射配置
包含自定义分析器、字段类型和日志数据优化的高级索引映射
难度
6/10
预计时间
15 min
标签
elasticsearch, index, mapping, schema, analyzers, optimization
plaintext
{
"mappings": {
"properties": {
"@timestamp": { "type": "date" },
"level": {
"type": "keyword",
"fields": {
"text": { "type": "text" }
}
},
"message": {
"type": "text",
"analyzer": "standard",
"fields": {
"keyword": { "type": "keyword" }
}
},
"service": { "type": "keyword" },
"trace_id": { "type": "keyword" },
"span_id": { "type": "keyword" },
"user_id": { "type": "keyword" },
"request_id": { "type": "keyword" },
"response_time": { "type": "float" },
"status_code": { "type": "integer" },
"error": {
"type": "object",
"properties": {
"type": { "type": "keyword" },
"message": { "type": "text" },
"stack_trace": { "type": "text" }
}
},
"tags": { "type": "keyword" },
"host": {
"properties": {
"name": { "type": "keyword" },
"ip": { "type": "ip" },
"os": { "type": "keyword" }
}
},
"geoip": {
"properties": {
"location": { "type": "geo_point" },
"country_name": { "type": "keyword" },
"city_name": { "type": "keyword" }
}
}
}
},
"settings": {
"analysis": {
"analyzer": {
"log_analyzer": {
"type": "custom",
"tokenizer": "standard",
"filter": ["lowercase", "stop"]
}
}
},
"number_of_shards": 3,
"number_of_replicas": 1
}
}- format
- JSON
- version
- 8.x
- components
- ["mapping","analyzer","field-types"]
- complexity
- advanced
文本复杂
Logstash 数据处理管道
完整的 Logstash 管道配置，用于解析、过滤和丰富来自多个源的日志数据
难度
8/10
预计时间
25 min
标签
logstash, pipeline, grok, filter, parse, enrich, transform
plaintext
input {
beats {
port => 5044
}
tcp {
port => 5000
codec => json_lines
}
file {
path => "/var/log/*.log"
start_position => "beginning"
}
}
filter {
# Parse JSON logs
if [message] =~ /^{.*}$/ {
json {
source => "message"
target => "parsed"
}
}
# Parse timestamp
date {
# Parse the timestamp extracted by grok and normalize it into @timestamp
match => [ "timestamp", "ISO8601", "yyyy-MM-dd HH:mm:ss.SSS" ]
target => "@timestamp"
}
# Add GeoIP information
if [client_ip] {
geoip {
source => "client_ip"
target => "geoip"
}
}
# Parse user agent
if [user_agent] {
useragent {
source => "user_agent"
target => "ua"
}
}
# Extract fields from log messages
grok {
match => {
"message" => "%{TIMESTAMP_ISO8601:timestamp} %{LOGLEVEL:level} \[%{DATA:thread}\] %{DATA:logger} - %{GREEDYDATA:log_message}"
}
tag_on_failure => ["_grokparsefailure"]
}
# Mutate and clean up fields
mutate {
add_field => { "service_name" => "%{[fields][service]}" }
add_field => { "environment" => "%{[fields][environment]}" }
remove_field => [ "host", "agent", "ecs", "input" ]
convert => {
"response_time" => "float"
"status_code" => "integer"
}
}
# Add fingerprint for deduplication
fingerprint {
source => ["message", "@timestamp", "service_name"]
target => "fingerprint"
}
# Conditional routing based on log level
if [level] == "ERROR" or [level] == "FATAL" {
mutate {
add_tag => ["error", "alert"]
}
}
}
output {
elasticsearch {
hosts => ["http://elasticsearch:9200"]
index => "logs-%{+YYYY.MM.dd}"
template_name => "logs"
template_pattern => "logs-*"
template => "/usr/share/logstash/templates/logs-template.json"
}
# Send critical errors to separate index
if "alert" in [tags] {
elasticsearch {
hosts => ["http://elasticsearch:9200"]
index => "alerts-%{+YYYY.MM.dd}"
}
}
# Debug output to console
if [@metadata][debug] {
stdout {
codec => rubydebug
}
}
}- format
- CONF
- version
- 8.x
- components
- ["input","filter","output"]
- features
- ["grok-patterns","date-parsing","geoip","mutate"]
文本复杂
Kibana 可视化仪表板
包含多种可视化的综合 Kibana 仪表板，用于日志分析和监控指标
难度
5/10
预计时间
20 min
标签
kibana, dashboard, visualization, charts, metrics, monitoring
plaintext
{
"dashboard": {
"title": "System Log Analysis Dashboard",
"panelsJSON": "[{"gridData":{"x":0,"y":0,"w":24,"h":15,"i":"1"},"panelIndex":"1","embeddableConfig":{},"panelRefName":"panel_1"},{"gridData":{"x":24,"y":0,"w":24,"h":15,"i":"2"},"panelIndex":"2","embeddableConfig":{},"panelRefName":"panel_2"},{"gridData":{"x":0,"y":15,"w":48,"h":15,"i":"3"},"panelIndex":"3","embeddableConfig":{},"panelRefName":"panel_3"}]",
"timeRestore": false,
"timeTo": "now",
"timeFrom": "now-24h",
"refreshInterval": {
"pause": false,
"value": 30000
},
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{"query":{"match_all":{}},"filter":[]}"
},
"description": "Comprehensive dashboard for monitoring system logs, errors, and performance metrics",
"version": "8.0.0"
},
"timeField": "@timestamp",
"panels": [
{
"id": "1",
"type": "metric",
"title": "Total Log Events",
"visState": "{ "type": "metric", "aggs": [{ "id": "1", "type": "count", "schema": "metric", "params": {} }] }",
"description": "Total number of log events in the selected time range"
},
{
"id": "2",
"type": "histogram",
"title": "Log Levels Distribution",
"visState": "{ "type": "histogram", "aggs": [{ "id": "1", "type": "count", "schema": "metric", "params": {} }, { "id": "2", "type": "terms", "schema": "segment", "params": { "field": "level", "size": 10 } }] }",
"description": "Distribution of log levels across all services"
},
{
"id": "3",
"type": "line",
"title": "Logs Timeline",
"visState": "{ "type": "line", "aggs": [{ "id": "1", "type": "count", "schema": "metric", "params": {} }, { "id": "2", "type": "date_histogram", "schema": "segment", "params": { "field": "@timestamp", "interval": "1h" } }] }",
"description": "Timeline of log events over time"
}
]
}- format
- JSON
- version
- 8.x
- components
- ["dashboard","visualization","saved-objects"]
- visualizations
- ["bar-chart","line-chart","pie-chart","data-table","heatmap"]
文本简单
应用程序日志聚合格式
用于应用程序日志的结构化 JSON 格式，包含关联 ID 和分布式追踪元数据
难度
2/10
预计时间
5 min
标签
log-format, json, structured-logging, correlation-id, metadata
plaintext
{
"@timestamp": "2025-12-07T10:30:45.123Z",
"level": "INFO",
"message": "User authentication successful",
"service": "auth-service",
"version": "2.1.3",
"trace_id": "550e8400-e29b-41d4-a716-446655440000",
"span_id": "550e8400-e29b-41d4-a716-446655440001",
"parent_span_id": "550e8400-e29b-41d4-a716-446655440000",
"user_id": "user_12345",
"request_id": "req_67890",
"session_id": "sess_abcde",
"correlation_id": "corr_fghij",
"context": {
"host": "auth-server-01",
"pod": "auth-service-7f8d9c2b-k4l5m",
"namespace": "production",
"environment": "prod",
"region": "us-west-2"
},
"request": {
"method": "POST",
"path": "/api/v1/auth/login",
"user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
"client_ip": "192.168.1.100",
"response_time": 145.67,
"status_code": 200
},
"user": {
"id": "user_12345",
"email": "[email protected]",
"roles": ["user", "premium"],
"tenant": "company_xyz"
},
"security": {
"auth_method": "jwt",
"mfa_enabled": true,
"login_attempts": 2,
"risk_score": "low"
},
"performance": {
"cpu_usage": 12.5,
"memory_usage": 67.8,
"disk_io": 2.1,
"network_io": 15.3
},
"tags": ["authentication", "successful", "mfa"],
"metadata": {
"source": "application",
"format_version": "1.0",
"encoding": "utf-8"
}
}- format
- JSON
- standard
- Structured Logging
- fields
- ["timestamp","level","message","service","trace-id","span-id","metadata"]
文本复杂
高级 Elasticsearch 查询示例
包含聚合、过滤器和全文搜索的复杂 Elasticsearch 查询，用于日志分析
难度
7/10
预计时间
30 min
标签
elasticsearch, query, aggregation, filter, search, dsl
plaintext
{
"query": {
"bool": {
"must": [
{
"range": {
"@timestamp": {
"gte": "now-24h",
"lte": "now"
}
}
},
{
"bool": {
"should": [
{
"term": {
"level": "ERROR"
}
},
{
"term": {
"level": "FATAL"
}
}
}
}
}
],
"filter": [
{
"terms": {
"service": ["auth-service", "api-gateway", "user-service"]
}
}
]
}
},
"aggs": {
"services": {
"terms": {
"field": "service",
"size": 10
},
"aggs": {
"error_types": {
"terms": {
"field": "error.type",
"size": 5
}
},
"hourly_distribution": {
"date_histogram": {
"field": "@timestamp",
"calendar_interval": "1h"
}
},
"avg_response_time": {
"avg": {
"field": "response_time"
}
}
}
},
"geoip_distribution": {
"terms": {
"field": "geoip.country_name",
"size": 20
}
},
"user_analysis": {
"cardinality": {
"field": "user_id"
}
},
"response_time_stats": {
"stats": {
"field": "response_time"
}
},
"error_timeline": {
"date_histogram": {
"field": "@timestamp",
"calendar_interval": "10m"
},
"aggs": {
"top_error_messages": {
"top_hits": {
"size": 3,
"sort": [
{
"@timestamp": {
"order": "desc"
}
}
],
"_source": ["message", "error.message", "service"]
}
}
}
}
},
"sort": [
{
"@timestamp": {
"order": "desc"
}
}
],
"size": 100,
"highlight": {
"fields": {
"message": {},
"error.message": {}
}
}
}- format
- JSON
- queryType
- DSL
- features
- ["term-queries","range-queries","aggregations","bool-queries","nested-queries"]
工具
经常与该案例搭配使用的工具
相关内容
相关案例集合
相关内容