ericwastaken · April 16, 2022 06:07 · Apr 16, 2022 · Apr 16, 2022
diff --git a/dump-query.sh b/dump-query.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# **************************************************************
+############################################################################
 # This is a frontend to the elasticdump utility.
 # Syntax:
 # ./dump-query.sh "INDEX-NAME-OR-PATTERN" /path/to/query.json /path/to/result.json
@@ -13,7 +13,25 @@
 # - Requires Elastic Dump (an NPM package).
 #   https://github.com/elasticsearch-dump/elasticsearch-dump#readme
 #   Install globally on your workstation with `npm i elasticdump -g`
-# **************************************************************
+# 
+# Copyright 2022 Eric A. Soto, eric@issfl.com
+# 
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the 
+# Software is furnished to do so, subject to the following conditions:
+# - The above copyright notice and this permission notice shall be 
+#   included in all copies or substantial portions of the Software.
+# - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 
+#   EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 
+#   MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+#   IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 
+#   CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 
+#   TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 
+#   SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+############################################################################
 
 # Command Line Arguments
 INDEX="$1"

diff --git a/dump-query-template.env b/dump-query-template.env
@@ -0,0 +1,8 @@
+# Pass headers into Elasticsearch:
+# - Authorization can be used to pass BASIC AUTH with a TOKEN. Use
+#   a tool of your choice to convert your username/password into
+#   the proper token for basic auth.
+ED_HEADERS='{"Authorization": "Basic YOUR-BASIC-AUTH-TOKEN-HERE"}'
+# Host must end in "/"
+# Include ":port-number" if necessary (otherwise, 443 is inferred by https)
+ED_HOST="https://your-host.com:port-number/"
diff --git a/dump-query.sh b/dump-query.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+# **************************************************************
+# This is a frontend to the elasticdump utility.
+# Syntax:
+# ./dump-query.sh "INDEX-NAME-OR-PATTERN" /path/to/query.json /path/to/result.json
+# Example:
+# ./dump-query.sh "logstash-*" query-received-message-times.json result-week2.json
+# 
+# Dependencies:
+# - Create an environment file with your host and authorization header.
+#   For the structure of the file, copy dump-query-template.env into dump-query.env.
+# - Requires Elastic Dump (an NPM package).
+#   https://github.com/elasticsearch-dump/elasticsearch-dump#readme
+#   Install globally on your workstation with `npm i elasticdump -g`
+# **************************************************************
+
+# Command Line Arguments
+INDEX="$1"
+INQUERY="$2"
+OUTFILE="$3"
+
+# Verify we received all the stuff
+if [[ -z "${INDEX}" ]] || [[ -z "${INQUERY}" ]] || [[ -z "${OUTFILE}" ]]; then
+  echo "Missing parameter."
+  echo "Syntax: ./dump-query.sh \"INDEX-NAME-OR-PATTERN\" /path/to/query.json /path/to/result.json"
+  exit 1
+fi
+
+# Load environment file
+set -o allexport
+[[ -f dump-query.env ]] && source dump-query.env
+set +o allexport
+
+# Constants
+# From Environment: 
+# - ED_HEADERS
+# - ED_HOST
+# Batch Limit (will pull in batches of this amount). Edit to suit!
+ED_LIMIT=5000
+
+echo "You are about to DUMP records from Elasticsearch."
+echo "Host: $ED_HOST"
+echo "Index: $INDEX"
+echo "Query File: $INQUERY"
+echo "Output File: $OUTFILE"
+echo ""
+# Wait for the user to press any KEY to proceed or allow them to Ctrl+C
+read -n1 -rsp $'Press any key to continue or Ctrl+C to exit...\n'
+
+# Do it!
+elasticdump \
+  --headers "${ED_HEADERS}" \
+  --input="${ED_HOST}${INDEX}" \
+  --output="${OUTFILE}" \
+  --searchBody=@"${INQUERY}" \
+  --limit "$ED_LIMIT" \
+  --concurrency 3
No results found