DOCKER安装ELK环境
# DOCKER安装ELK环境
文档介绍了使用docker安装elk环境的步骤,使用java的logback日志框架将日志数据发送到rabbitmq,然后使用logstash将日志再同步到elasticsearch, 最后就是在kibana中可以直观的查到日志信息。
# 安装
这里使用了docker-compose将elk的各个组件安装,文件内容如下:
version: '3'
services:
rabbitmq:
restart: always
image: rabbitmq:3-management
ports:
- 15672:15672
- 5672:5672
environment:
RABBITMQ_DEFAULT_USER: yds
RABBITMQ_DEFAULT_PASS: dadada
volumes:
- ${PWD}/rabbitmq:/var/lib/rabbitmq
elasticsearch:
image: docker.elastic.co/elasticsearch/elasticsearch:7.9.0
container_name: elasticsearch
environment:
- discovery.type=single-node
ports:
- 9200:9200
volumes:
- ${PWD}/elasticsearch:/usr/share/elasticsearch/data
logstash:
image: docker.elastic.co/logstash/logstash:7.9.0
container_name: logstash
volumes:
- ${PWD}/logstash/config/:/usr/share/logstash/config
- ${PWD}/logstash/data/:/usr/share/logstash/data
- ${PWD}/logstash/pipeline/:/usr/share/logstash/pipeline
command: logstash -f /usr/share/logstash/pipeline/logstash.conf
kibana:
image: docker.elastic.co/kibana/kibana:7.9.0
container_name: kibana
ports:
- 5601:5601
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
执行 docker-compose up -d
mq: localhost:15672
es: localhost:9200
bibana: localhost:5601
执行前可能还需要将logstash的配置文件copy出来
docker run -d -P --name logstash docker.elastic.co/logstash/logstash:7.9.0
docker cp logstash:/usr/share/logstash/pipeline logstash/
docker cp logstash:/usr/share/logstash/data logstash/
docker cp logstash:/usr/share/logstash/config logstash/
1
2
3
4
2
3
4
# LOGSTASH配置文件
设置一下logstash.conf
input {
rabbitmq {
host => "192.168.1.6"
port => 5672
user => "yds"
password => "dadada"
queue => "log_queue"
durable => true
codec => "json"
}
}
output {
elasticsearch {
hosts => ["192.168.1.6:9200"]
index => "logstash-rabbitmq"
}
stdout {
codec => rubydebug
}
}
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
这里的queue
需要和rabbitmq里的queue对应起来
# JAVA设置LOGBACK
pom.xml
...
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-amqp</artifactId>
</dependency>
...
1
2
3
4
5
6
7
2
3
4
5
6
7
logback.xml
<configuration>
<property name="LOG_DIR" value="."/>
<property name="LOG_FILE" value="app"/>
<!-- 文件滚动 -->
<appender name="APP_LOG" class="ch.qos.logback.core.rolling.RollingFileAppender">
<file>${LOG_DIR}/${LOG_FILE}.log</file>
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
<charset>utf8</charset>
</encoder>
<rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
<fileNamePattern>${LOG_DIR}/${LOG_FILE}.%d{yyyy-MM-dd}.%i.log</fileNamePattern>
<!--日志最大保存15天-->
<maxHistory>5</maxHistory>
<!--日志最大的文件大小-->
<maxFileSize>100MB</maxFileSize>
<!--日志最大保存1GB-->
<totalSizeCap>1GB</totalSizeCap>
</rollingPolicy>
</appender>
<!-- 异步写入 -->
<appender name ="ASYNC_APP_LOG" class="ch.qos.logback.classic.AsyncAppender">
<discardingThreshold>20</discardingThreshold>
<queueSize>512</queueSize>
<neverBlock>true</neverBlock>
<appender-ref ref="APP_LOG"/>
</appender>
<!-- RabbitMQ -->
<appender name="AMQP_RABBITMQ" class="org.springframework.amqp.rabbit.logback.AmqpAppender">
<layout>
<pattern><![CDATA[ %d %p %t [%c] - <%m>%n ]]></pattern>
</layout>
<addresses>localhost:5672</addresses>
<username>yds</username>
<password>dadada</password>
<declareExchange>true</declareExchange>
<applicationId>java-web-demo</applicationId>
<!-- 需要正确的绑定到queue中 -->
<routingKeyPattern>info</routingKeyPattern>
<generateId>true</generateId>
<charset>UTF-8</charset>
<deliveryMode>NON_PERSISTENT</deliveryMode>
</appender>
<!-- 控制台 -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>%d{yyyy-MM-dd HH:mm:ss} [%thread] %-5level %logger{36} - %msg%n</pattern>
</encoder>
</appender>
<springProfile name="default,local">
<root level="INFO">
<appender-ref ref="STDOUT"/>
<appender-ref ref="AMQP_RABBITMQ"/>
</root>
</springProfile>
<springProfile name="pro">
<root level="INFO">
<appender-ref ref="ASYNC_APP_LOG"/>
</root>
</springProfile>
</configuration>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
上次更新: 2024/08/08, 06:59:58