Skip to content

Instantly share code, notes, and snippets.

@jacmkno
Last active October 3, 2022 18:22
Show Gist options
  • Select an option

  • Save jacmkno/95ce6095d622d788f5bb to your computer and use it in GitHub Desktop.

Select an option

Save jacmkno/95ce6095d622d788f5bb to your computer and use it in GitHub Desktop.

Revisions

  1. jacmkno revised this gist Apr 19, 2016. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -30,7 +30,7 @@ do
    do
    ans=`curl -is -m $CURL_TIMEOUT "$PROTOCOL://$host"|tr -d "\n"|grep -e "$TEST_EXPRESSION"|wc -l`
    echo $host:$ans >> /tmp/healthchecks
    errorsCnt=`grep "$host" /tmp/healthchecks|tail -n $MAX_ERRORS|grep ":0"|wc -l`
    errorsCnt=`tr [:space:] '\n' < /tmp/healthchecks|grep "$host"|tail -n $MAX_ERRORS|grep ":0"|wc -l`
    let serverDown=errorsCnt/MAX_ERRORS
    oldStatus=`cat /tmp/health.$host`
    status="$host is down? $serverDown"
  2. jacmkno revised this gist Apr 19, 2016. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -21,7 +21,7 @@ PROTOCOL='http' # All servers will be tested with HTTP protocol on port 80
    let MIN_LINES=HOSTS_COUNT*MAX_ERRORS*5
    let MAX_TIME=`date +%s`+SCRIPT_DURATION-CURL_TIMEOUT-SLEEP_SECS

    last_lines=`tail -n $MIN_LINES /tmp/healthchecks`
    last_lines=`tr [:space:] '\n' < /tmp/healthchecks|grep \.|tail -n $MIN_LINES`
    echo $last_lines > /tmp/healthchecks

    while [ `date +%s` -lt $MAX_TIME ]
  3. jacmkno revised this gist Feb 26, 2016. 1 changed file with 5 additions and 1 deletion.
    6 changes: 5 additions & 1 deletion healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -12,7 +12,7 @@ NGINX_CONF=/etc/nginx/nginx.conf
    #Expression used in grep to accept a proper response. Server response is stripped from linebreaks and the response headers are included at the begining
    TEST_EXPRESSION='HTTP/1.1 200 OK.*region-footer'

    SCRIPT_DURATION=60 # Keep calling .
    SCRIPT_DURATION=60 # Keep sending proves until this.
    MAX_ERRORS=3 # Flag server as down if the last MAX_ERRORS tests are all errors
    CURL_TIMEOUT=2 # Maximum time culr will wait for an answer
    SLEEP_SECS=2 # Wait this number of seconds after each prove
    @@ -56,6 +56,10 @@ do
    fi
    #echo "DEBUG INFO: $status, errors: $errorsCnt"
    echo $status > /tmp/health.$host
    if [ `date +%s` -gt $MAX_TIME ]
    then
    exit 1;
    fi
    sleep $SLEEP_SECS
    done
    done
  4. jacmkno revised this gist Feb 26, 2016. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -7,7 +7,7 @@ HOSTS=(201.208.65.152 46.56.115.234 42.79.216.211)
    HOSTS_COUNT=3 # Number of hosts in the array
    MAIL_SUBJECT='ACTIVISUAL - Health check status change'
    MAIL_TO='webmaster@website.net'
    NGINX_CONF=nginx.conf
    NGINX_CONF=/etc/nginx/nginx.conf

    #Expression used in grep to accept a proper response. Server response is stripped from linebreaks and the response headers are included at the begining
    TEST_EXPRESSION='HTTP/1.1 200 OK.*region-footer'
  5. jacmkno revised this gist Feb 26, 2016. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -56,6 +56,6 @@ do
    fi
    #echo "DEBUG INFO: $status, errors: $errorsCnt"
    echo $status > /tmp/health.$host
    sleep $EXTRA_SECS
    sleep $SLEEP_SECS
    done
    done
  6. jacmkno revised this gist Feb 26, 2016. 1 changed file with 8 additions and 6 deletions.
    14 changes: 8 additions & 6 deletions healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -12,18 +12,20 @@ NGINX_CONF=nginx.conf
    #Expression used in grep to accept a proper response. Server response is stripped from linebreaks and the response headers are included at the begining
    TEST_EXPRESSION='HTTP/1.1 200 OK.*region-footer'

    SCRIPT_CRONSECS=120 # Calculate repetitions to take this ammount of time. Should equal your crontab setup.
    CURL_TIMEOUT=2 # Maximum time culr will wait for an answer
    EXTRA_SECS=2 # Wait this number of seconds after each prove
    SCRIPT_DURATION=60 # Keep calling .
    MAX_ERRORS=3 # Flag server as down if the last MAX_ERRORS tests are all errors
    CURL_TIMEOUT=2 # Maximum time culr will wait for an answer
    SLEEP_SECS=2 # Wait this number of seconds after each prove
    PROTOCOL='http' # All servers will be tested with HTTP protocol on port 80

    let REPEAT="SCRIPT_CRONSECS/((CURL_TIMEOUT+EXTRA_SECS)*HOSTS_COUNT) - 1"
    let MIN_LINES=HOSTS_COUNT*MAX_ERRORS*5
    let MAX_TIME=`date +%s`+SCRIPT_DURATION-CURL_TIMEOUT-SLEEP_SECS

    last_lines=`tail -n 100 /tmp/healthchecks`
    last_lines=`tail -n $MIN_LINES /tmp/healthchecks`
    echo $last_lines > /tmp/healthchecks

    for i in `seq $REPEAT`; do
    while [ `date +%s` -lt $MAX_TIME ]
    do
    for host in "${HOSTS[@]}"
    do
    ans=`curl -is -m $CURL_TIMEOUT "$PROTOCOL://$host"|tr -d "\n"|grep -e "$TEST_EXPRESSION"|wc -l`
  7. jacmkno revised this gist Feb 25, 2016. No changes.
  8. jacmkno revised this gist Feb 25, 2016. No changes.
  9. jacmkno revised this gist Feb 25, 2016. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -48,7 +48,7 @@ for i in `seq $REPEAT`; do
    echo "status change: $status, errors: $errorsCnt" | mail -s "$MAIL_SUBJECT" $MAIL_TO
    #echo "DEBUG INFO: status change - $status, errors: $errorsCnt"
    mv /tmp/health.newconf $NGINX_CONF
    sudo service nginx restart
    service nginx restart
    fi
    fi
    fi
  10. jacmkno revised this gist Feb 25, 2016. 1 changed file with 2 additions and 2 deletions.
    4 changes: 2 additions & 2 deletions healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -46,13 +46,13 @@ for i in `seq $REPEAT`; do
    sed '/down/! s/'"$host"'\(:[0-9]\+ \| \)/&down /' $NGINX_CONF > /tmp/health.newconf
    fi
    echo "status change: $status, errors: $errorsCnt" | mail -s "$MAIL_SUBJECT" $MAIL_TO
    echo "status change: $status, errors: $errorsCnt"
    #echo "DEBUG INFO: status change - $status, errors: $errorsCnt"
    mv /tmp/health.newconf $NGINX_CONF
    sudo service nginx restart
    fi
    fi
    fi
    echo "$status, errors: $errorsCnt"
    #echo "DEBUG INFO: $status, errors: $errorsCnt"
    echo $status > /tmp/health.$host
    sleep $EXTRA_SECS
    done
  11. jacmkno revised this gist Feb 25, 2016. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -9,7 +9,7 @@ MAIL_SUBJECT='ACTIVISUAL - Health check status change'
    MAIL_TO='webmaster@website.net'
    NGINX_CONF=nginx.conf

    #Expression used in grep to accept a proper response. Server response y stripped from linebreaks and the response headers are included at the begining
    #Expression used in grep to accept a proper response. Server response is stripped from linebreaks and the response headers are included at the begining
    TEST_EXPRESSION='HTTP/1.1 200 OK.*region-footer'

    SCRIPT_CRONSECS=120 # Calculate repetitions to take this ammount of time. Should equal your crontab setup.
  12. jacmkno revised this gist Feb 25, 2016. 1 changed file with 1 addition and 1 deletion.
    2 changes: 1 addition & 1 deletion healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -5,7 +5,7 @@

    HOSTS=(201.208.65.152 46.56.115.234 42.79.216.211)
    HOSTS_COUNT=3 # Number of hosts in the array
    MAIL_SUBJECT='HSBNOTICIAS - Health check status change'
    MAIL_SUBJECT='ACTIVISUAL - Health check status change'
    MAIL_TO='webmaster@website.net'
    NGINX_CONF=nginx.conf

  13. jacmkno created this gist Feb 25, 2016.
    59 changes: 59 additions & 0 deletions healthcheck.sh
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,59 @@
    #!/bin/bash
    # By @jacmkno - Activisual S.A.S. - http://activisual.net
    # Script summary
    # This script runs health cehcks in a set of hosts and updates their status on an nginx server

    HOSTS=(201.208.65.152 46.56.115.234 42.79.216.211)
    HOSTS_COUNT=3 # Number of hosts in the array
    MAIL_SUBJECT='HSBNOTICIAS - Health check status change'
    MAIL_TO='webmaster@website.net'
    NGINX_CONF=nginx.conf

    #Expression used in grep to accept a proper response. Server response y stripped from linebreaks and the response headers are included at the begining
    TEST_EXPRESSION='HTTP/1.1 200 OK.*region-footer'

    SCRIPT_CRONSECS=120 # Calculate repetitions to take this ammount of time. Should equal your crontab setup.
    CURL_TIMEOUT=2 # Maximum time culr will wait for an answer
    EXTRA_SECS=2 # Wait this number of seconds after each prove
    MAX_ERRORS=3 # Flag server as down if the last MAX_ERRORS tests are all errors
    PROTOCOL='http' # All servers will be tested with HTTP protocol on port 80

    let REPEAT="SCRIPT_CRONSECS/((CURL_TIMEOUT+EXTRA_SECS)*HOSTS_COUNT) - 1"

    last_lines=`tail -n 100 /tmp/healthchecks`
    echo $last_lines > /tmp/healthchecks

    for i in `seq $REPEAT`; do
    for host in "${HOSTS[@]}"
    do
    ans=`curl -is -m $CURL_TIMEOUT "$PROTOCOL://$host"|tr -d "\n"|grep -e "$TEST_EXPRESSION"|wc -l`
    echo $host:$ans >> /tmp/healthchecks
    errorsCnt=`grep "$host" /tmp/healthchecks|tail -n $MAX_ERRORS|grep ":0"|wc -l`
    let serverDown=errorsCnt/MAX_ERRORS
    oldStatus=`cat /tmp/health.$host`
    status="$host is down? $serverDown"
    if [ -n "$oldStatus" ]
    then
    if [ "$oldStatus" != "$status" ]
    then
    if [ "$serverDown" -eq 0 ] && [ "$errorsCnt" -gt 0 ]; then
    status="$oldStatus"
    else
    if [ "$serverDown" -eq 0 ]
    then
    sed 's/'"$host"'\(:[0-9]\+ \| \)down /'$host'\1/' $NGINX_CONF > /tmp/health.newconf
    else
    sed '/down/! s/'"$host"'\(:[0-9]\+ \| \)/&down /' $NGINX_CONF > /tmp/health.newconf
    fi
    echo "status change: $status, errors: $errorsCnt" | mail -s "$MAIL_SUBJECT" $MAIL_TO
    echo "status change: $status, errors: $errorsCnt"
    mv /tmp/health.newconf $NGINX_CONF
    sudo service nginx restart
    fi
    fi
    fi
    echo "$status, errors: $errorsCnt"
    echo $status > /tmp/health.$host
    sleep $EXTRA_SECS
    done
    done