前景:部署skywalking,elk等服務時會用到es存儲資料,運作一段時間會占用磁盤空間,影響伺服器性能。在保證服務正常使用的情況下,我們可以對es的索引進行删除操作,舍棄舊資料。是以我編寫了一個腳本,僅供參考。如果要做定時任務的話,可以對稍作修改。
[[email protected]_es:shell]# vim delete_es_data.sh
#!/bin/bash
#本腳本用于按月份清理ES存儲資料
#擷取本月份與索引中的日期進行比較,本月份前的索引資料将被删除。
#删除前要确認
NOW_DATE=`date +%Y%m`
ALLLINES=`/usr/bin/curl -s -XGET http://127.0.0.1:9200/_cat/indices?v | egrep *-20*`
echo
echo "THIS IS WHAT SHOULD BE DELETED FOR ES:"
echo
echo "${ALLLINES}" | while read LINE
do
FORMATEDLINE=`echo ${LINE} |awk '{ print $3 }' | awk -F'-' '{ print $2 }' | cut -c 1-6 `
#echo ${FORMATEDLINE}
if [ -n "${FORMATEDLINE}" -a "${FORMATEDLINE}" -lt "${NOW_DATE}" ]
then
TODELETE=`echo ${LINE} | awk '{ print $3 }'`
echo "http://127.0.0.1:9200/${TODELETE}"
fi
done
echo
echo -n "if this make sence, Y to continue N to exit [Y/N]:"
read INPUT
if [ "${INPUT}" == "Y" ] || [ "${INPUT}" == "y" ] || [ "${INPUT}" == "yes" ] || [ "${INPUT}" == "YES" ]
then
echo "${ALLLINES}" | while read LINE
do
FORMATEDLINE=`echo ${LINE} | awk '{ print $3 }' | awk -F'-' '{ print $2 }' | cut -c 1-6 `
if [ -n "${FORMATEDLINE}" -a "${FORMATEDLINE}" -lt "${NOW_DATE}" ]
then
TODELETE=`echo ${LINE} | awk '{ print $3 }'`
/usr/bin/curl -XDELETE http://127.0.0.1:9200/$TODELETE
sleep 1
fi
done
else
echo SCRIPT CLOSED BY USER, BYE ...
echo
exit
fi
~
寫進crontab的腳本可以參考:
#!/bin/bash
NOW_DATE=`date +%Y%m`
#NOW_DATE=202010
ALLLINES=`/usr/bin/curl -s -XGET http://127.0.0.1:9200/_cat/indices?v | egrep *-20*`
echo "${ALLLINES}" | while read LINE
do
FORMATEDLINE=`echo ${LINE} |awk '{ print $3 }' | awk -F'-' '{ print $2 }' | cut -c 1-6 `
if [ -n "${FORMATEDLINE}" -a "${FORMATEDLINE}" -lt "${NOW_DATE}" ]
then
TODELETE=`echo ${LINE} | awk '{ print $3 }'`
#echo "http://127.0.0.1:9200/${TODELETE}"
/usr/bin/curl -XDELETE http://127.0.0.1:9200/$TODELETE
sleep 1
fi
done
ELK的索引删除:
#I use a bash script, just change the 30 with the # of days you want to keep
#!/bin/bash
# Zero padded days using %d instead of %e
DAYSAGO=`date --date="30 days ago" +%Y%m%d`
ALLLINES=`/usr/bin/curl -s -XGET http://127.0.0.1:9200/_cat/indices?v | egrep logstash`
echo
echo "THIS IS WHAT SHOULD BE DELETED FOR ELK:"
echo
echo "$ALLLINES" | while read LINE
do
FORMATEDLINE=`echo $LINE | awk '{ print $3 }' | awk -F'-' '{ print $2 }' | sed 's/\.//g' `
if [ "$FORMATEDLINE" -lt "$DAYSAGO" ]
then
TODELETE=`echo $LINE | awk '{ print $3 }'`
echo "http://127.0.0.1:9200/$TODELETE"
fi
done
echo
echo -n "if this make sence, Y to continue N to exit [Y/N]:"
read INPUT
if [ "$INPUT" == "Y" ] || [ "$INPUT" == "y" ] || [ "$INPUT" == "yes" ] || [ "$INPUT" == "YES" ]
then
echo "$ALLLINES" | while read LINE
do
FORMATEDLINE=`echo $LINE | awk '{ print $3 }' | awk -F'-' '{ print $2 }' | sed 's/\.//g' `
if [ "$FORMATEDLINE" -lt "$DAYSAGO" ]
then
TODELETE=`echo $LINE | awk '{ print $3 }'`
/usr/bin/curl -XDELETE http://127.0.0.1:9200/$TODELETE
sleep 1
fi
done
else
echo SCRIPT CLOSED BY USER, BYE ...
echo
exit
fi