Created
May 26, 2020 17:48
-
-
Save shlomi-noach/28986cb30f0e14d51594f0bc741b464c to your computer and use it in GitHub Desktop.
Revisions
-
shlomi-noach created this gist
May 26, 2020 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,126 @@ #!/bin/bash # This playbook assumes you have cloned https://github.com/openark/orchestrator # and ran: ./script/dock system # which landed you in orchestrator's playground environment. # Further information available on the welcome screen once you've ran the docker image. # FYI, orchestrator's config file is at /etc/orchestrator.conf.json orchestrator-client -c topology-tabulated -alias ci orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t' # Refactoring: orchestrator-client -c relocate -i 127.0.0.1:10113 -d 127.0.0.1:10112 orchestrator-client -c relocate -i 127.0.0.1:10113 -d 127.0.0.1:10111 orchestrator-client -c relocate-replicas -i 127.0.0.1:10111 -d 127.0.0.1:10112 orchestrator-client -c which-cluster-instances -alias ci orchestrator-client -c which-cluster-master -i 127.0.0.1:10112 orchestrator-client -c which-master -i 127.0.0.1:10112 orchestrator-client -c which-master -i 127.0.0.1:10113 orchestrator-client -c which-replicas -i 127.0.0.1:10112 # Scripting samples: orchestrator-client -c which-cluster-instances -alias ci | while read i ; do orchestrator-client -c relocate -i "$i" -d 127.0.0.1:10111 done orchestrator-client -c enable-semi-sync-master -i 127.0.0.1:10111 orchestrator-client -c which-replicas -i 127.0.0.1:10111 | while read i ; do orchestrator-client -c enable-semi-sync-replica -i "$i" done orchestrator-client -c which-cluster-instances -alias ci | while read i ; do orchestrator-client -c disable-semi-sync-master -i "$i" orchestrator-client -c disable-semi-sync-replica -i "$i" done # Examples: GTID, Errant GTIS ~/sandboxes/ci/s2 -e "flush slow logs" orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t' orchestrator-client -c which-gtid-errant -i 127.0.0.1:10113 orchestrator-client -c replication-analysis # Digress: use of API orchestrator-client -c api -path all-instances orchestrator-client -c api -path all-instances | jq -C . | less -r orchestrator-client -c api -path all-instances | jq -C '.[] | {Port: .Key.Port, ServerUUID: .ServerUUID, ExecutedGtidSet: .ExecutedGtidSet, GtidErrant:.GtidErrant}' orchestrator-client -c api -path cluster/ci | jq '.[] | select(.GtidErrant!="")' orchestrator-client -c api -path cluster/ci | jq '.[] | select(.GtidErrant!="") | {Port: .Key.Port}' # Digress: further API show off orchestrator-client -c api -path cluster/ci | jq '.[] | select(.SlaveHosts | length == 0) | .Key.Port' orchestrator-client -c api -path cluster/ci | jq '.[] | select(.MasterKey.Port != 0) | .Key.Port' orchestrator-client -c api -path cluster/ci | jq -r '.[] | .Version' | grep -c "5.7.26" # Back to GTID :) # Assuming we've validated the errant transaction didn't change data: orchestrator-client -c gtid-errant-inject-empty -i 127.0.0.1:10113 # or... ~/sandboxes/ci/s2 -e "flush slow logs" sleep 3 orchestrator-client -c gtid-errant-reset-master -i 127.0.0.1:10113 # or... ~/sandboxes/ci/s2 -e "flush slow logs" sleep 3 orchestrator-client -c api -path gtid-errant-reset-master/127.0.0.1/10113 | jq -C . # Demonstrate how HAProxy routes 13306 traffic to current master: mysql -uci -pci -h 127.0.0.1 --port 13306 -e "select @@hostname, @@port" consul kv get -recurse mysql/master/ci # Graceful takeovers: orchestrator-client -c graceful-master-takeover-auto -alias ci -d 127.0.0.1:10112 orchestrator-client -c graceful-master-takeover-auto -alias ci orchestrator-client -c register-candidate -i 127.0.0.1:10113 --promotion-rule prefer # Let orchestrator pick the promoted replica, expect it to be 10113 orchestrator-client -c graceful-master-takeover-auto -alias ci orchestrator-client -c graceful-master-takeover-auto -alias ci -d 127.0.0.1:10111 # Expect the following to fail due to '"PreventCrossRegionMasterFailover": true' config orchestrator-client -c graceful-master-takeover-auto -alias ci -d 127.0.0.1:10114 orchestrator-client -c which-cluster-instances -alias ci | while read i ; do orchestrator-client -c relocate -i "$i" -d 127.0.0.1:10111 done orchestrator-client -c ack-all-recoveries --reason demo mysqladmin -uci -pci -h 127.0.0.1 --port 13306 processlist mysqladmin -uci -pci -h 127.0.0.1 --port 13306 shutdown orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t' # for this environment, orchestrator's config allows failovers 5 seconds apart. # In production the config would be higher. sleep 5 mysqladmin -uci -pci -h 127.0.0.1 --port 13306 processlist mysqladmin -uci -pci -h 127.0.0.1 --port 13306 shutdown while sleep 0.5 ; do orchestrator-client -c replication-analysis ; done redeploy-ci-env # Failover: # A complex scenario. We suggest to orchestrator that 127.0.0.1:10113 is preferred to be promoted in case of failover. # But we set up the topology such that 127.0.0.1:10113 is the most lagging replica, # and such that 127.0.0.1:10114 (which may not be promoted due to PreventCrossRegionMasterFailover) is the most up-to-date replica. orchestrator-client -c register-candidate -i 127.0.0.1:10113 --promotion-rule prefer orchestrator-client -c stop-replica -i 127.0.0.1:10113 sleep 3 orchestrator-client -c stop-replica -i 127.0.0.1:10112 sleep 3 mysqladmin -uci -pci -h 127.0.0.1 --port 13306 processlist mysqladmin -uci -pci -h 127.0.0.1 --port 13306 shutdown while sleep 0.5 ; do orchestrator-client -c topology-tabulated -alias ci | tr '|' '\t' ; echo ""; done