建站不啰嗦，上手跟我做（三十二）Logstash 安装
安装
jdk 安装
Logstash 安装
安装公共签名组件
[root@localhost opt]# sudo rpm --import https://artifacts.elastic.co/GPG-KEY-elasticsearch
配置 Logstash 的 yum 仓库
baseurl 中指定的是 6.x 或者 7.x
[root@localhost yum.repos.d]# touch /etc/yum.repos.d/logstash.repo
[root@localhost yum.repos.d]# vim logstash.repo
[logstash-6.x]
name=Elastic repository for 6.x packages
baseurl=https://artifacts.elastic.co/packages/6.x/yum
gpgcheck=1
gpgkey=https://artifacts.elastic.co/GPG-KEY-elasticsearch
enabled=1
autorefresh=1
type=rpm-md
安装 Logstash
程序路径：/usr/share/logstash/bin/logstash
配置目录： /etc/logstash
[root@localhost opt]# sudo yum install logstash
Loaded plugins: fastestmirror
Loading mirror speeds from cached hostfile
* base: mirrors.aliyun.com
* extras: mirrors.aliyun.com
* updates: mirrors.aliyun.com
base | 3.6 kB 00:00:00
extras | 2.9 kB 00:00:00
logstash-6.x | 1.3 kB 00:00:00
updates | 2.9 kB 00:00:00
logstash-6.x/primary | 279 kB 00:00:00
logstash-6.x 762/762
Resolving Dependencies
--> Running transaction check
---> Package logstash.noarch 1:6.8.16-1 will be installed
--> Finished Dependency Resolution
Dependencies Resolved
[root@localhost conf.d]# ln -s /etc/logstash /usr/share/logstash/config
配置 java 的路径
[root@localhost bin]# pwd
/usr/share/logstash/bin
[root@localhost bin]# vim logstash.lib.sh
# This script is used to initialize a number of env variables and setup the
# runtime environment of logstash. It sets to following env variables:
# LOGSTASH_HOME & LS_HOME
# SINCEDB_DIR
# JAVACMD
# JAVA_OPTS
# GEM_HOME & GEM_PATH
# DEBUG
#
# These functions are provided for the calling script:
# setup() to setup the environment
# ruby_exec() to execute a ruby script with using the setup runtime environment
#
# The following env var will be used by this script if set:
# LS_GEM_HOME and LS_GEM_PATH to overwrite the path assigned to GEM_HOME and GEM_PATH
# LS_JAVA_OPTS to append extra options to the JVM options provided by logstash
# JAVA_HOME to point to the java home
export JAVA_HOME=/usr/local/java/jdk1.8.0_181
export JRE_HOME=$JAVA_HOME/jre
export PATH=$JAVA_HOME/bin:$JRE_HOME/bin:$PATH
unset CDPATH
# This unwieldy bit of scripting is to try to catch instances where Logstash
# was launched from a symlink, rather than a full path to the Logstash binary
Logstash 启动
[root@localhost bin]# systemctl start logstash.service
Failed to start logstash.service: Unit not found.
[root@localhost bin]# sudo /usr/share/logstash/bin/system-install /etc/logstash/startup.options systemd
Using provided startup.options file: /etc/logstash/startup.options
Manually creating startup for specified platform: systemd
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/pleaserun-0.0.31/lib/pleaserun/platform/base.rb:112: warning: constant ::Fixnum is deprecated
Successfully created system startup script for Logstash
[root@localhost bin]# systemctl start logstash.service
查看 Logstash 状态
[root@localhost bin]# systemctl status logstash.service
● logstash.service - logstash
Loaded: loaded (/etc/systemd/system/logstash.service; disabled; vendor preset: disabled)
Active: active (running) since Fri 2021-06-18 15:33:28 CST; 18s ago
Main PID: 17502 (java)
CGroup: /system.slice/logstash.service
└─17502 /usr/local/java/jdk1.8.0_181/bin/java -Xms1g -Xmx1g -XX:+UseConcMarkSweepGC -XX:CMSInit...
Jun 18 15:33:28 localhost.localdomain systemd[1]: Started logstash.
Logstash 停止服务
[root@localhost bin]# systemctl stop logstash.service
Logstash 本地控制台输出测试
[root@localhost opt]# /usr/share/logstash/bin/logstash -e 'input { stdin{} }output { stdout{codec => rubydebug}}'
Sending Logstash logs to /var/log/logstash which is now configured via log4j2.properties
[2021-06-19T09:47:59,691][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2021-06-19T09:47:59,723][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"6.8.16"}
[2021-06-19T09:48:18,463][INFO ][logstash.pipeline ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[2021-06-19T09:48:18,824][INFO ][logstash.pipeline ] Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x8ada459 run>"}
The stdin plugin is now waiting for input:
[2021-06-19T09:48:18,971][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2021-06-19T09:48:19,722][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
hehe
/usr/share/logstash/vendor/bundle/jruby/2.5.0/gems/awesome_print-1.7.0/lib/awesome_print/formatters/base_formatter.rb:31: warning: constant ::Fixnum is deprecated
{
"@timestamp" => 2021-06-19T01:48:32.329Z,
"host" => "localhost.localdomain",
"@version" => "1",
"message" => "hehe"
}
Logstash Elasticsearch 输出测试
配置 es 输出
[root@bogon conf.d]# pwd
/etc/logstash/conf.d
[root@bogon conf.d]# vim es.conf
input{
stdin {}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "logstash-%{+YYYY.MM.dd}"
}
}
logstash 启动
[root@bogon conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Sending Logstash logs to /var/log/logstash which is now configured via log4j2.properties
[2021-06-21T20:14:29,210][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2021-06-21T20:14:29,249][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"6.8.16"}
[2021-06-21T20:14:46,916][INFO ][logstash.pipeline ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
[2021-06-21T20:14:48,195][INFO ][logstash.outputs.elasticsearch] Elasticsearch pool URLs updated {:changes=>{:removed=>[], :added=>[http://localhost:9200/]}}
[2021-06-21T20:14:48,760][WARN ][logstash.outputs.elasticsearch] Restored connection to ES instance {:url=>"http://localhost:9200/"}
[2021-06-21T20:14:48,897][INFO ][logstash.outputs.elasticsearch] ES Output version determined {:es_version=>6}
[2021-06-21T20:14:48,906][WARN ][logstash.outputs.elasticsearch] Detected a 6.x and above cluster: the `type` event field won't be used to determine the document _type {:es_version=>6}
[2021-06-21T20:14:49,013][INFO ][logstash.outputs.elasticsearch] New Elasticsearch output {:class=>"LogStash::Outputs::ElasticSearch", :hosts=>["//localhost:9200"]}
[2021-06-21T20:14:49,126][INFO ][logstash.outputs.elasticsearch] Using default mapping template
[2021-06-21T20:14:49,187][INFO ][logstash.outputs.elasticsearch] Attempting to install template {:manage_template=>{"template"=>"logstash-*", "version"=>60001, "settings"=>{"index.refresh_interval"=>"5s"}, "mappings"=>{"_default_"=>{"dynamic_templates"=>[{"message_field"=>{"path_match"=>"message", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false}}}, {"string_fields"=>{"match"=>"*", "match_mapping_type"=>"string", "mapping"=>{"type"=>"text", "norms"=>false, "fields"=>{"keyword"=>{"type"=>"keyword", "ignore_above"=>256}}}}}], "properties"=>{"@timestamp"=>{"type"=>"date"}, "@version"=>{"type"=>"keyword"}, "geoip"=>{"dynamic"=>true, "properties"=>{"ip"=>{"type"=>"ip"}, "location"=>{"type"=>"geo_point"}, "latitude"=>{"type"=>"half_float"}, "longitude"=>{"type"=>"half_float"}}}}}}}}
[2021-06-21T20:14:49,380][INFO ][logstash.pipeline ] Pipeline started successfully {:pipeline_id=>"main", :thread=>"#<Thread:0x3fa5c84e run>"}
The stdin plugin is now waiting for input:
[2021-06-21T20:14:49,548][INFO ][logstash.agent ] Pipelines running {:count=>1, :running_pipelines=>[:main], :non_running_pipelines=>[]}
[2021-06-21T20:14:50,211][INFO ][logstash.agent ] Successfully started Logstash API endpoint {:port=>9600}
hehehe
ceshi
cong logstash shu chu
浏览器访问http://192.168.8.102:9100/
springboot 日志输出到 Logstash Elasticsearch 输出测试
springboot 配置
依赖包
<!--logStash-->
<dependency>
<groupId>net.logstash.logback</groupId>
<artifactId>logstash-logback-encoder</artifactId>
<version>5.3</version>
</dependency>
logging:
use-json-format: false # By default, logs are not in Json format
logstash: # Forward logs to logstash over a socket, used by LoggingConfiguration
enabled: true #开启logstash
host: 192.168.8.102 #logstash服务的地址
port: 9021 #logstash服务监听的端口，不是logstash的启动端口
queue-size: 512
logstash 的配置
192.168.8.102 为 logstash 的地址
9021 为 logstash 的和服务建立 tcp 连接的端口，不是本身的启动端口
[root@bogon conf.d]# ls
es.conf
[root@bogon conf.d]# vim es.conf
input{
tcp {
mode => "server"
host =>"192.168.8.102"
port => 9021
codec => json {
charset => "UTF-8"
}
}
stdin {}
}
output {
elasticsearch {
hosts => ["localhost:9200"]
index => "logstash-%{+YYYY.MM.dd}"
}
}
logstash 启动
[root@bogon conf.d]# /usr/share/logstash/bin/logstash -f /etc/logstash/conf.d/es.conf
Sending Logstash logs to /var/log/logstash which is now configured via log4j2.properties
[2021-06-21T22:55:04,968][WARN ][logstash.config.source.multilocal] Ignoring the 'pipelines.yml' file because modules or command line options are specified
[2021-06-21T22:55:05,013][INFO ][logstash.runner ] Starting Logstash {"logstash.version"=>"6.8.16"}
[2021-06-21T22:55:27,510][INFO ][logstash.pipeline ] Starting pipeline {:pipeline_id=>"main", "pipeline.workers"=>1, "pipeline.batch.size"=>125, "pipeline.batch.delay"=>50}
浏览器访问http://192.168.8.102:9100/
配置 Logstash
内存大小修改
[root@localhost logstash]# cd /etc/logstash/
[root@localhost logstash]# ls
conf.d jvm.options log4j2.properties logstash-sample.conf logstash.yml pipelines.yml startup.options
[root@localhost logstash]# vim jvm.options
## JVM configuration
# Xms represents the initial size of total heap space
# Xmx represents the maximum size of total heap space
-Xms128M
-Xmx256M
################################################################
## Expert settings
################################################################
##
## All settings below this section are considered
## expert settings. Don't tamper with them unless
## you understand what you are doing
##
################################################################
## GC configuration
-XX:+UseConcMarkSweepGC
-XX:CMSInitiatingOccupancyFraction=75
-XX:+UseCMSInitiatingOccupancyOnly
## Locale
# Set the locale language
#-Duser.language=en
# Set the locale country
#-Duser.country=US
# Set the locale variant, if any
#-Duser.variant=
## basic
# set the I/O temp directory
#-Djava.io.tmpdir=$HOME
# set to headless, just in case
-Djava.awt.headless=true
# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8
# use our provided JNA always versus the system one
#-Djna.nosys=true
# Turn on JRuby invokedynamic
-Djruby.compile.invokedynamic=true
## JVM configuration
[root@localhost bin]# cd /etc/logstash/
[root@localhost logstash]# ls
conf.d jvm.options log4j2.properties logstash-sample.conf logstash.yml pipelines.yml startup.options
[root@localhost logstash]# vim logstash.yml
# Settings file in YAML
#
# Settings can be specified either in hierarchical form, e.g.:
#
# pipeline:
# batch:
# size: 125
# delay: 5
#
# Or as flat keys:
#
# pipeline.batch.size: 125
上一篇 建站不啰嗦，上手跟我做（三十一）Elasticsearch 安装
目录
下一篇 建站不啰嗦，上手跟我做（三十三）Jenkins 安装