Last active
April 7, 2026 20:41
-
-
Save mbreksopuro/df4d8f11f1452608befa0d55e14d30e4 to your computer and use it in GitHub Desktop.
Reindex API from Remote ElasticSearch to Local ElasticSearch Servers (for up to ES 6.x)
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| ################################################### | |
| ## Start ElasticSearch Reindex | |
| ## | |
| ## Created by : Hartfordfive - 2019-02-01 | |
| ## Modified by : Marjono B. Reksopuro - 2019-03-11 | |
| ## | |
| ## | |
| ################################################### | |
| # | |
| # Load the home .bashrc if it hasn't been included. | |
| # Note: BASHRC_LOADED needs to be set in the .bashrc. | |
| # Use this piece of code before any other variables are set since | |
| # the code might override the .bashrc variables | |
| if [ -f ~/.bashrc -a -z "$BASHRC_LOADED" ] | |
| then | |
| . ~/.bashrc | |
| fi | |
| # General variables | |
| export ACCOUNT=`whoami` | |
| export PGM=`basename $0` | |
| export SERVER=`hostname` | |
| export DATE=`date +%Y%m%d%H%M` | |
| export OVERDIR="/opt/elasticsearch/scripts" | |
| export TERM=linux | |
| export http_proxy= | |
| export https_proxy= | |
| export ftp_proxy= | |
| OVERRIDES() | |
| { | |
| # There are two places where you can override the | |
| # variables. Either at the environment test or dev level | |
| # -or- at the server level. This allows us to keep generic | |
| # scripts but will accomodate upgrades. | |
| if [ -f $OVERDIR/overrides.${ENV} ] | |
| then | |
| . $OVERDIR/overrides.${ENV} | |
| fi | |
| if [ -f $OVERDIR/overrides.$SERVER ] | |
| then | |
| . $OVERDIR/overrides.$SERVER | |
| fi | |
| } | |
| # Determine if we have the correct number of input variables | |
| ##if [ "$1" == "" ] || [ "$2" == "" ]; then | |
| if [ $# -lt 5 ] | |
| then | |
| echo "Usage: $PGM <sourcehost> <sourceport> <user> <pwd> <file_indices_list> <desthost> <destport>" | |
| echo " Example: $PGM elasticsearch-dev.local.domain 9200 elastic xxxx to_list_reindex_indices elasticsearch-dev.local.domain 9200" | |
| echo " Example: $PGM elasticsearch-dev.local.domain 9200 elastic xxxx to_list_reindex_indices elasticsearch-sandbox.local.domain 19200" | |
| exit 1 | |
| fi | |
| # Set the variables as appropriate for this product | |
| export REMOTE_HOST=$1:$2 | |
| export ESUSER=$3 | |
| export ESPASSWORD="$4" | |
| export PATTERN=$5 | |
| if [ "$6" == "" ]; then | |
| export LOCAL_HOST=$1:$2 | |
| else | |
| export LOCAL_HOST=$6:$7 | |
| fi | |
| echo "---------------------------- NOTICE ----------------------------------" | |
| echo "You must ensure you have the following setting in your local ES host's:" | |
| echo "elasticsearch.yml config (the one re-indexing to):" | |
| echo " reindex.remote.whitelist: $REMOTE_HOST" | |
| echo "Also, if an index template is necessary for this data, you must create" | |
| echo "locally before you start the re-indexing process" | |
| echo "----------------------------------------------------------------------" | |
| sleep 3 | |
| ##INDICES=$(curl -H'Content-Type: application/json' --silent "$REMOTE_HOST/_cat/indices/$PATTERN?h=index") | |
| export INDICES=`cat $PATTERN` | |
| export TOTAL_INCOMPLETE_INDICES=0 | |
| export TOTAL_INDICES=0 | |
| export TOTAL_DURATION=0 | |
| export INCOMPLETE_INDICES=() | |
| for INDEX in $INDICES; do | |
| TOTAL_DOCS_REMOTE=$(curl -H'Content-Type: application/json' --silent "http://${ESUSER}:${ESPASSWORD}@$REMOTE_HOST/_cat/indices/$INDEX?h=docs.count") | |
| echo "Attempting to re-indexing $INDEX ($TOTAL_DOCS_REMOTE docs total) from remote ES server..." | |
| SECONDS=0 | |
| echo curl -H'Content-Type: application/json' -XPOST "http://${ESUSER}:${ESPASSWORD}@$LOCAL_HOST/_reindex?wait_for_completion=true&pretty=true" -d "{ | |
| \"conflicts\": \"proceed\", | |
| \"source\": { | |
| \"remote\": { | |
| \"host\": \"http://$REMOTE_HOST\", | |
| \"username\": \"${ESUSER}\", | |
| \"password\": \"${ESPASSWORD}\" | |
| }, | |
| \"index\": \"${INDEX}\" | |
| }, | |
| \"dest\": { | |
| \"index\": \"${INDEX}\" | |
| } | |
| }" | |
| curl -H'Content-Type: application/json' -XPOST "http://${ESUSER}:${ESPASSWORD}@$LOCAL_HOST/_reindex?wait_for_completion=true&pretty=true" -d "{ | |
| \"conflicts\": \"proceed\", | |
| \"source\": { | |
| \"remote\": { | |
| \"host\": \"http://$REMOTE_HOST\", | |
| \"username\": \"${ESUSER}\", | |
| \"password\": \"${ESPASSWORD}\" | |
| }, | |
| \"index\": \"${INDEX}\" | |
| }, | |
| \"dest\": { | |
| \"index\": \"${INDEX}\" | |
| } | |
| }" | |
| duration=$SECONDS | |
| LOCAL_INDEX_EXISTS=$(curl -H'Content-Type: application/json' -o /dev/null --silent --head --write-out '%{http_code}' "http://${ESUSER}:${ESPASSWORD}@$LOCAL_HOST/$INDEX") | |
| if [ "$LOCAL_INDEX_EXISTS" == "200" ]; then | |
| TOTAL_DOCS_REINDEXED=$(curl -H'Content-Type: application/json' --silent "http://${ESUSER}:${ESPASSWORD}@$LOCAL_HOST/_cat/indices/$INDEX?h=docs.count") | |
| else | |
| TOTAL_DOCS_REINDEXED=0 | |
| fi | |
| echo " Re-indexing results:" | |
| echo " -> Time taken: $(($duration / 60)) minutes and $(($duration % 60)) seconds" | |
| echo " -> Docs indexed: $TOTAL_DOCS_REINDEXED out of $TOTAL_DOCS_REMOTE" | |
| echo "" | |
| TOTAL_DURATION=$(($TOTAL_DURATION+$duration)) | |
| if [ "$TOTAL_DOCS_REMOTE" -ne "$TOTAL_DOCS_REINDEXED" ]; then | |
| TOTAL_INCOMPLETE_INDICES=$(($TOTAL_INCOMPLETE_INDICES+1)) | |
| INCOMPLETE_INDICES+=($INDEX) | |
| fi | |
| TOTAL_INDICES=$((TOTAL_INDICES+1)) | |
| done | |
| echo "---------------------- STATS --------------------------" | |
| echo "Total Duration of Re-Indexing Process: $((TOTAL_DURATION / 60))m $((TOTAL_DURATION % 60))" | |
| echo "Total Indices: $TOTAL_INDICES" | |
| echo "Total Incomplete Re-Indexed Indices: $TOTAL_INCOMPLETE_INDICES" | |
| if [ "$TOTAL_INCOMPLETE_INDICES" -ne "0" ]; then | |
| printf '%s\n' "${INCOMPLETE_INDICES[@]}" | |
| fi | |
| echo "-------------------------------------------------------" | |
| echo "" |
It is a flat file with index names on each line
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Can You proviede pattern example?
What is the expected value of to_list_reindex_indices