ELK8.x使用详解
ELK8.x使用详解
下载安装文件
wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-8.11.3-linux-x86_64.tar.gz //elasticsearch安装文件
wget https://release.infinilabs.com/analysis-ik/stable/elasticsearch-analysis-ik-8.11.3.zip //ik分词器,解析放在elasticsearch的plugin目录下
创建目录
mkdir -p /home/atc/data/es/data
mkdir -p /home/atc/data/es/logs
修改配置参数
sudo vi /etc/security/limits.conf
soft nofile 65535
hard nofile 65535
soft nproc 32000
hard nproc 32000
sudo vi /etc/sysctl.conf
vm.max_map_count=262144
cluster.name: es
node.name: node-25
node.roles: [master,data]
path.data: /home/atc/data/es/data
path.logs: /home/atc/data/es/logs
network.host: 0.0.0.0
network.publish_host: 192.28.7.25
http.port: 9200
transport.port: 9300
# 集群安装
cluster.initial_master_nodes: ["192.28.7.25:9300","192.28.7.26:9300","192.28.7.27:9300"]
discovery.seed_hosts: ["192.28.7.25:9300","192.28.7.26:9300","192.28.7.27:9300"]
http.cors.enabled: true
http.cors.allow-origin: "*"
action.destructive_requires_name: false
bootstrap.memory_lock: false
#自动创建索引
action.auto_create_index: true
xpack.security.http.ssl.enabled: false
xpack.security.enabled: false
xpack.security.transport.ssl.enabled: false
# 调整JVM参数，编辑config/jvm.properties
-Xms8g
-Xmx8g
到elastic search的bin使用命令./elasticsearch -d 启动，用户不能使用root用户。
ES基础概念
- 索引（Index）：类比关系数据库的表
- 映射（Mappings）:类比关系数据库表的字段类型定义
- 文档（Document）:类比关系数据库表中的一行数据
- 字段(Field):类别关系数据库中的一列
- 分片（Shards）:把索引的数据分块存储在不同的数据块，一个多分片的索引中写入数据时，通过Id的hash值除分区数取余来确定具体写入哪一个分片中，所以在创建索引的时候需要指定分片的数量，并且分片的数量一旦确定就不能修改
- 副本（Replicas）：分片的备份，做数据冗余，最多配置集群的N-1,因为分片与副本放在一起就失去了意义
Kibana Devtools使用
索引管理
- 查询所有索引
GET _cat/indices?v
- 查询单个索引信息
GET /track?pretty
- 删除索引,索引被删除后，文档数据信息也被删除了
delete /track
- 创建索引，主要是settings与mappings设置
mappings常用的数据类型：- keyword通常用于未分词的字符串类型，不进行全文索引， -
- date日期类型，支持时间格式yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis
- text文本类型，会进行分词处理，执行文本搜索，不支持排序和聚合操作
- boolean ，存储布尔值
- Object表示json对象，被嵌套的对象被扁平化索引，Nested,允许你对嵌套对象数组中的每个对象单独进行索引，从而能够执行更精确的查询
- long,integer,short,byte,double,float数值型，支持范围查询与聚合
- ip类型，支持ipv4,ipv6地址
- completion,用户自动补全建议
- geo_point，存储地理位置坐标，支持基于地理位置的查询，比如距离计算、边界框查询等，对象形式（{"lat": 41.12, "lon": -71.34}）、字符串形式（"41.12,-71.34"）、数组形式（[-71.34, 41.12]）、geohash形式（"drm3btev3e86"）
- geo_shape 类型允许你存储复杂形状（如多边形或多线），并执行与这些形状相关的查询，如相交、包含
以下是创建一个飞行器轨迹索引
PUT /track
{
"settings": {
"number_of_shards": 3,
"number_of_replicas": 2
},
"mappings": {
"properties": {
"orderId":{
"type":"keyword"
},
"sn": {
"type": "keyword"
},
"flightStatus":{"type":"keyword"},
"manufacturerID": {"type": "keyword"},
"uasID": {"type": "keyword"},
"timestamp": {
"type": "date",
"format":"epoch_millis"
},
"uasModel": {"type": "text","analyzer": "ik_max_word"},
"coordinate": {"type": "integer"},
"latitude": {"type": "float"},
"heightAltitype": {"type": "integer"},
"height": {"type": "integer"},
"altitude": {"type": "integer"},
"vS": {"type": "integer"},
"gS": {"type": "integer"},
"course": {"type": "integer"},
"uavAuthInfo": {
"type":"nested",
"properties": {
"uavState": {"type": "keyword"},
"uavType": {"type": "keyword"},
"uavCategory": {"type": "keyword"},
"uas": {"type": "keyword"},
"uavName": {"type": "text","analyzer": "ik_max_word"},
"uavModel": {"type": "text","analyzer": "ik_max_word"},
"uavManufacturer": {"type": "text","analyzer": "ik_max_word"},
"uavEmptyWeight": {"type": "float"},
"uavMaxWeight": {"type": "float"},
"uavUserType": {"type": "keyword"},
"uavPerson": {
"properties":{
"name":{"type":"text","analyzer": "ik_max_word"},
"iDType":{"type":"keyword"},
"iDNumber":{"type":"text","analyzer": "ik_max_word"},
"phoneNumber":{"type":"text","analyzer": "ik_max_word"}
}
},
"uavUnit": {
"properties": {
"usccode": {"type": "keyword"},
"unitType": {"type": "keyword"},
"unitName": {"type": "text"},
"phoneNumber": {"type": "text","analyzer": "ik_max_word"}
}
}
}
},
"originTimeStamp": {"type": "date", "format": "yyyyMMddHHmmss"},
"mockClock": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"},
"packetTime": {"type": "date", "format": "yyyy-MM-dd HH:mm:ss"},
"location": {
"type": "geo_point"
}
}
}
}
数据查询
- term精准匹配
GET track/_search
{
"query":{
"term": {
"sn": "1581F6Z9C2443003R69X"
}
}
}
- terms条件为多个
GET track/_search
{
"query":{
"terms": {
"sn": [
"1581F6Z9C2443003R69X","3N34K7E002N0GN"
]
}
}
}
- 嵌套查询
GET track/_search
{
"query":{
"nested": {
"path": "uavAuthInfo",
"query": {
"match": {
"uavAuthInfo.uavManufacturer": "深圳"
}
}
}
}
}
- 分析指定字段分词查询结果
POST /track/_analyze
{
"field":"orderId",
"text":"1581F45TB21AU1AE00AD-20241030-1Z43Cgd4"
}
- match_phrease分词结果全部满足，使用slop调节因子，指定少几个也行
GET track/_search
{
"query":{
"nested": {
"path": "uavAuthInfo",
"query": {
"match_phrase": {
"uavAuthInfo.uavManufacturer": {
"query":"深圳大疆",
"slop":1
}
}
}
}
}
}
- 其中有字段匹配上就行
{
"query":{
"multi_match": {
"query": "大疆",
"fields": ["title","content"]
}
}
}
完全匹配上的评分更高
{
"query": {
"multi_match": {
"query": "大疆",
"type": "best_fields",
"fields": [
"tag",
"content"
],
"tie_breaker": 0.3
}
}
}
匹配字段越多，评分越高
{
"query": {
"multi_match": {
"query": "大疆",
"type": "most_fields",
"fields": [
"tag",
"content"
]
}
}
}
大疆分词结果是分词在不同的字段中，评分越高
{
"query": {
"multi_match": {
"query": "大疆",
"type": "cross_fields",
"fields": [
"tag",
"content"
]
}
}
}
- 联合查询
must: 文档必须完全匹配条件
should: should下面会带一个以上的条件，至少满足一个条件，这个文档就符合should
must_not: 文档必须不匹配条件
filter: 过滤满足条件数据
GET track/_search
{
"query":{
"bool": {
"must":{
"term":{
"orderID":"1581F45TB21AU1AE00AD-20241030-1Z43Cgd4"
}
},
"must_not":
{
"match":{
"uasModel":"UNKONWN"
}
},
"filter": {
"range": {
"height": {
"gte": 20,
"lt": 30
}
}
}
}
}
- 通配符，应用keyword类型字段
GET track/_search
{
"query":{
"wildcard": {
"orderID":{
"value":"1*"
}
}
}
}
- sort排序
GET track/_search
{
"query":{
"wildcard": {
"orderID":{
"value":"1*"
}
}
},
"sort":[
{
"flightStatus":{
"order":"desc"
}
}
]
}
10、 分页查询，默认最多大小不能超过1w
GET track/_search
{
"query":{
"wildcard": {
"orderID":{
"value":"1*"
}
}
},
"sort":[
{
"flightStatus":{
"order":"desc"
}
}
],
"from":0,
"size":1000
}
- GIS点在范围内
GET /poi/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"address":"南京南站"
}
},
{
"geo_bounding_box": {
"location": { // 确保将这里的"location"替换为你的地理点字段名
"top_left": {
"lat": 31.99,
"lon":118.77
},
"bottom_right": {
"lat": 31.96,
"lon": 118.86
}
}
}
}
]
}
}
}
GET /poi/_search
{
"query": {
"bool": {
"must": [
{
"match": {
"address":"南京"
}
},
{
"geo_distance": {
"distance": 1000,
"location": {
"lat": 31.96,
"lon": 118.86
}
}
}
]
}
}
}
logstash使用
运行bin/logstash -f config/logstash.conf，以下示例时抓取kafka数据到elasticsearch中
input {
kafka {
bootstrap_servers => "192.28.7.25:9092,192.28.7.26:9092,192.28.7.27:9092"
topics => ["RECV_UAV_TRACK_FROM_UOM_V1"]
group_id => "g-es-track"
auto_offset_reset => "latest"
decorate_events => false
codec => json
}
}
filter {
mutate {
add_field => {
"[location][lat]" => "%{[latitude]}"
"[location][lon]" => "%{[longitude]}"
}
}
# 将经纬度从整数转换为浮点数
mutate {
convert => {
"location.lat" => "float"
"location.lon" => "float"
}
}
//如果需要的话，可以在这里添加更多的数据处理逻辑
//比如对经纬度进行除以10^6的操作来得到正确的坐标值
ruby {
code => "
event.set('[location][lat]', event.get('[location][lat]').to_f / 10000000)
event.set('[location][lon]', event.get('[location][lon]').to_f / 10000000)
"
}
}
output {
elasticsearch {
hosts => ["http://192.28.7.27:9200"]
index => "track"
manage_template => false #禁用自动索引
#user => "elastic"
#password => "changeme"
}
增量同步Mysql数据到ES中
input {
jdbc {
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_driver_library => "/home/atc/weblib/mysql-connector-java-8.0.11.jar"
jdbc_connection_string => "jdbc:mysql://192.28.7.21:3306/daas?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowLoadLocalInfile=false&autoDeserialize=false"
jdbc_user => "username"
jdbc_password => "password"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
statement => "select id,name,address,province ,city,area,lat,lng,first_catename as firstCatename,second_catename as secondCatename,third_catename as thirdCatename,third_cateid as thirdCateid,create_time,update_time as updateTime,is_deleted as isDeleted from daas.gis_poi where create_time >= :sql_last_value"
schedule => "* * * * *"
record_last_run => true
last_run_metadata_path => "/home/atc/log/web_log/last_run_metadata_update_time.txt"
clean_run => false
tracking_column_type => "timestamp"
use_column_value => true
tracking_column => "create_time"
}
}
filter {
mutate {
add_field => {
"[location][lat]" => "%{[lat]}"
"[location][lon]" => "%{[lng]}"
}
}
date {
match => [ "create_time", "yyyy-MM-dd'T'HH:mm:ss.SSSZ" ]
target => "createTime"
}
# 将经纬度从整数转换为浮点数
mutate {
remove_field => ["create_time"]
convert => {
"location.lat" => "float"
"location.lon" => "float"
}
}
}
output {
elasticsearch {
hosts => "http://192.28.7.27:9200"
index => "poi"
document_id => "%{id}"
}
}
本文链接：
/archives/elk-use
版权声明：
本站所有文章除特别声明外，均采用 CC BY-NC-SA 4.0 许可协议。转载请注明来自
编程之家！
喜欢就支持一下吧
