[ELK]定期清理 elasticsearch 上的过期数据

 

#!/bin/bash
# @Author: richard
# @Date:   2017-08-11 17:27:49
# @Last Modified by:   richard
# @Last Modified time: 2017-08-11 18:04:58
 
#保留近 N 天
KEEP_DAYS=7
 
# 删除前 N的所有天到 前N+10天==>每天执行
function get_todelete_days()
{
    # declare -A DAY_ARR
    # DAY_ARR=""
    for i in $(seq 1 10);
    do
        THIS_DAY=$(date -d "$(($KEEP_DAYS+$i)) day ago" +%Y.%m.%d)
        DAY_ARR=( "${DAY_ARR[@]}" $THIS_DAY)
    done
 
    echo ${DAY_ARR[*]}
 
}
 
# 返回数组的写法
TO_DELETE_DAYS=(`get_todelete_days`)
 
for day in "${TO_DELETE_DAYS[@]}"
do
    echo "$day will be delete"   
    curl -XDELETE 'http://127.0.0.1:9200/*-'${day}
done

　　

写到 crontab 里面去

crontab -l
1 1 * * * /home/admin/elasticsearch/es_clear.sh

　　

 看看效果 只保留最近 7 天

---------------  更新 ---------------------

#!/bin/bash
# @Author: richard
# @Date:   2017-08-11 17:27:49
# @Last Modified by:   richard
# @Last Modified time: 2017-08-11 18:04:58
# 用于es中日志索引的定时删除

# 保留近 N 天
KEEP_DAYS=7
# 删除最近 M 天，即删除是从 M+N到N
DEL_DAYS=10
# 删除索引名，留空则使用*-日期(*-2021.02.20)
INDEX_NAME=()
# todo 需保留的索引名,留空则使用


# 删除前 N的所有天到 前N天==>每天执行
function get_todelete_days()
{
    # declare -A DAY_ARR
    # DAY_ARR=""
    for i in $(seq $DEL_DAYS $(($KEEP_DAYS+$DEL_DAYS)));
    do
        THIS_DAY=$(date -d "$i day ago" +%Y.%m.%d)
        DAY_ARR=( "${DAY_ARR[@]}" $THIS_DAY)
    done
    echo ${DAY_ARR[*]}
}

# 返回数组的写法
TO_DELETE_DAYS=(`get_todelete_days`)

for day in "${TO_DELETE_DAYS[@]}"
do
    if [ ${#INDEX_NAME[@]} -eq 0 ]; then
      L_INDEX_NAME=( "${L_INDEX_NAME[@]}" "*-"${day} )
    else
      for each in "${INDEX_NAME[@]}"
      do
        L_INDEX_NAME=( "${L_INDEX_NAME[@]}" "${each}-${day}" )
      done
    fi
done

for l_index in "${L_INDEX_NAME[@]}"
do
      echo -n "$l_index will be delete"
      curl -XDELETE -u"elastic:elastic#" 'http://127.0.0.1:9200/'$l_index
      echo ""
done

　　

　　

# sh clean.sh
nginx_access-2021.02.21 will be delete{"acknowledged":true}
nginx_error-2021.02.21 will be delete{"acknowledged":true}
nginx_access-2021.02.20 will be delete{"acknowledged":true}
nginx_error-2021.02.20 will be delete{"acknowledged":true}
nginx_access-2021.02.19 will be delete{"acknowledged":true}
nginx_error-2021.02.19 will be delete{"acknowledged":true}
nginx_access-2021.02.18 will be delete{"acknowledged":true}
nginx_error-2021.02.18 will be delete{"acknowledged":true}