AWS ES ISM学习应用笔记

Elastic Search 6以上版本推出 ILM，用于管理Index的生命周期，但AWS上的ES是基于OSS版本的ES，所以自己开发了ISM来代替ILM。项目是从logstash往ES写入数据，但碰到无法在Logstash开启ILM_enabled=true的问题，这样就无法直接让logstash应用ISM。通过查询资料，最后发现AWS的 ES OSS关闭了ilm。而是直接在ism里去做所有的设置，Logstash指定index alias写入数据即可。

第一步：编写 State management policies

{
    "policy_id": "Test_ILM_policy",
    "description": "Demonstrate a hot-warm-cold-delete workflow.",
    "last_updated_time": 1640873761350,
    "schema_version": 1,
    "error_notification": null,
    "default_state": "hot",
    "states": [
        {
            "name": "hot",
            "actions": [
                {
                    "rollover": {
                        "min_doc_count": 200,
                        "min_index_age": "2h"
                    }
                },
                {
                    "read_write": {}
                }
            ],
            "transitions": [
                {
                    "state_name": "warm",
                    "conditions": {
                        "min_index_age": "3h"
                    }
                }
            ]
        },
        {
            "name": "warm",
            "actions": [
                {
                    "timeout": "24h",
                    "retry": {
                        "count": 5,
                        "backoff": "exponential",
                        "delay": "15m"
                    },
                    "warm_migration": {}
                }
            ],
            "transitions": [
                {
                    "state_name": "cold",
                    "conditions": {
                        "min_index_age": "4h"
                    }
                }
            ]
        },
        {
            "name": "cold",
            "actions": [
                {
                    "cold_migration": {
                        "start_time": null,
                        "end_time": null,
                        "timestamp_field": "@timestamp",
                        "ignore": "none"
                    }
                }
            ],
            "transitions": [
                {
                    "state_name": "delete",
                    "conditions": {
                        "min_index_age": "30d"
                    }
                }
            ]
        },
        {
            "name": "delete",
            "actions": [
                {
                    "cold_delete": {}
                }
            ],
            "transitions": []
        }
    ],
    "ism_template": [
        {
            "index_patterns": [
                "test-ilm-index-*"
            ],
            "priority": 100,
            "last_updated_time": 1640094369756
        }
    ]
}

 

第二步：编写index template关联 policy

PUT _index_template/test_ilm_index_template
{
"index_patterns": ["test-ilm-index-*"],
"template": {
      "settings": {
        "number_of_shards": 2,
        "number_of_replicas": 0,
      "index": {
      "opendistro": {
        "index_state_management": {
          "rollover_alias": "test-ilm-indexs"
        }
      }
    }
      },
      "mappings": { 
        "properties": {
          "@timestamp": {
            "format": "strict_date_optional_time||epoch_millis", 
            "type": "date"
          }
        }
      }
  }
}

 

第三步：创建index，关联index template rollover alias

PUT test-ilm-index-000001
{

      "aliases": {
        "test-ilm-indexs": {
          "is_write_index": true
    }
  }
}

 

第四步：Logstash指定写入的index=index template rollover alias

hosts => ["]
                    user => ""
                    password => ""
                    index => "test-ilm-indexs"
                    ssl => true

 

 

官方文档：https://opensearch.org/docs/latest/im-plugin/ism/policies/#sample-policy-with-ism-template-for-auto-rollover