Skip to content

Instantly share code, notes, and snippets.

@justisGipson

justisGipson/heroku_pg_metrics.py

Last active October 7, 2024 21:43
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save justisGipson/2711ea519648f0097adc1a2455ed23c5 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save justisGipson/2711ea519648f0097adc1a2455ed23c5 to your computer and use it in GitHub Desktop.
Download ZIP
heroku postgres metrics
Raw
heroku_pg_metrics.py
import sys
import time
from subprocess import CalledProcessError, check_output
def convert_bytes_to_gb(bytes_value):
return round(bytes_value / 1000000000, 2)
def convert_kb_to_gb(kb_value):
if isinstance(kb_value, str) and kb_value.endswith("kB"):
kb_value = int(kb_value[:-2])
return round(kb_value / 1000000, 2)
def convert_percentage_to_percent(percentage):
return round(percentage * 100, 2)
def get_source_type(source):
if source == "HEROKU_POSTGRESQL_GRAY":
return "PROD-FOLLOWER"
else:
return "PROD-PRIMARY"
def fetch_heroku_logs(app_name):
try:
logs = check_output(
["heroku", "logs", "-p", "heroku-postgres", "-a", app_name],
universal_newlines=True,
)
return logs
except CalledProcessError as e:
print(f"Error fetching Heroku Postgres logs for app '{app_name}': {e}")
return None
def parse_metrics(line):
metrics = {}
source = next(
(s.split("=")[1] for s in line.split() if s.startswith("source=")), None
)
metrics["Source"] = get_source_type(source)
metrics["Active Connections"] = int(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#active-connections=")
),
0,
)
)
metrics["Waiting Connections"] = int(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#waiting-connections=")
),
0,
)
)
metrics["Max Connections"] = int(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#max-connections=")
),
0,
)
)
metrics["Connections Percentage Used"] = convert_percentage_to_percent(
float(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#connections-percentage-used=")
),
0,
)
)
)
metrics["Load Average (1m)"] = float(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#load-avg-1m=")
),
0,
)
)
metrics["Load Average (5m)"] = float(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#load-avg-5m=")
),
0,
)
)
metrics["Load Average (15m)"] = float(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#load-avg-15m=")
),
0,
)
)
metrics["Read IOPS"] = float(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#read-iops=")
),
0,
)
)
metrics["Write IOPS"] = float(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#write-iops=")
),
0,
)
)
metrics["Max IOPS"] = get_max_iops(metrics["Source"])
metrics["IOPS Percentage Used"] = convert_percentage_to_percent(
float(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#iops-percentage-used=")
),
0,
)
)
)
metrics["Temporary Disk Used"] = convert_bytes_to_gb(
int(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#tmp-disk-used=")
),
0,
)
)
)
metrics["Temporary Disk Available"] = convert_bytes_to_gb(
int(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#tmp-disk-available=")
),
0,
)
)
)
metrics["Memory Total"] = convert_kb_to_gb(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#memory-total=")
),
"0kB",
)
)
metrics["Memory Free"] = convert_kb_to_gb(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#memory-free=")
),
"0kB",
)
)
metrics["Memory Percentage Used"] = convert_percentage_to_percent(
float(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#memory-percentage-used=")
),
0,
)
)
)
metrics["Memory Cached"] = convert_kb_to_gb(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#memory-cached=")
),
"0kB",
)
)
metrics["Memory Postgres"] = convert_kb_to_gb(
next(
(
s.split("=")[1]
for s in line.split()
if s.startswith("sample#memory-postgres=")
),
"0kB",
)
)
return metrics
def get_max_iops(source_type):
if source_type == "PROD-PRIMARY":
return 12000
elif source_type == "PROD-FOLLOWER":
return 3000
else:
return 0
def main():
if len(sys.argv) != 3:
print(
"Usage: python heroku_postgres_metrics.py <app_name> <duration_in_minutes>"
)
return
app_name = sys.argv[1]
duration_in_minutes = int(sys.argv[2])
output_file = f"heroku_postgres_metrics_{app_name}_{duration_in_minutes}min.txt"
start_time = time.time()
end_time = start_time + (duration_in_minutes * 60)
primary_metrics_data = []
follower_metrics_data = []
while time.time() < end_time:
logs = fetch_heroku_logs(app_name)
if logs:
for line in logs.splitlines():
if "sample#" in line:
metrics = parse_metrics(line)
if metrics["Source"] == "PROD-PRIMARY":
primary_metrics_data.append(metrics)
elif metrics["Source"] == "PROD-FOLLOWER":
follower_metrics_data.append(metrics)
time.sleep(10)
with open(output_file, "w") as f:
f.write("Heroku Postgres Metrics Summary:\n")
f.write("- Source: MIXED\n\n")
if primary_metrics_data:
f.write("Primary Database:\n")
for metric, values in calculate_metrics(primary_metrics_data).items():
if metric != "Source" and metric != "Max IOPS":
f.write(f" - {metric}:\n")
f.write(f" - Average: {values['average']}\n")
f.write(f" - Minimum: {values['minimum']}\n")
f.write(f" - Maximum: {values['maximum']}\n")
f.write(f" - Max IOPS: {get_max_iops('PROD-PRIMARY')}\n\n")
if follower_metrics_data:
f.write("Follower Database:\n")
for metric, values in calculate_metrics(follower_metrics_data).items():
if metric != "Source" and metric != "Max IOPS":
f.write(f" - {metric}:\n")
f.write(f" - Average: {values['average']}\n")
f.write(f" - Minimum: {values['minimum']}\n")
f.write(f" - Maximum: {values['maximum']}\n")
f.write(f" - Max IOPS: {get_max_iops('PROD-FOLLOWER')}\n\n")
f.write(f"Metrics collected over {duration_in_minutes} minutes.\n")
print(f"Heroku Postgres metrics have been saved to '{output_file}'.")
def calculate_metrics(metrics_data):
result = {}
for metric in metrics_data[0].keys():
if metric != "Source" and metric != "Max IOPS" and metric != "Max Connections":
values = [d[metric] for d in metrics_data]
numeric_values = [v for v in values if isinstance(v, (int, float))]
result[metric] = {
"average": sum(numeric_values) / len(numeric_values),
"minimum": min(numeric_values),
"maximum": max(numeric_values),
}
return result
if __name__ == "__main__":
main()
'''
Returns a doc formatted like:
Heroku Postgres Metrics Summary:
- Source: MIXED
Primary Database:
- Active Connections:
- Average: 53.0
- Minimum: 53
- Maximum: 53
- Waiting Connections:
- Average: 0.0
- Minimum: 0
- Maximum: 0
- Connections Percentage Used:
- Average: 10.6
- Minimum: 10.6
- Maximum: 10.6
- Load Average (1m):
- Average: 0.05140625
- Minimum: 0.05140625
- Maximum: 0.05140625
- Load Average (5m):
- Average: 0.0471875
- Minimum: 0.0471875
- Maximum: 0.0471875
- Load Average (15m):
- Average: 0.0428125
- Minimum: 0.0428125
- Maximum: 0.0428125
- Read IOPS:
- Average: 0.0
- Minimum: 0.0
- Maximum: 0.0
- Write IOPS:
- Average: 2.125
- Minimum: 2.125
- Maximum: 2.125
- IOPS Percentage Used:
- Average: 0.02
- Minimum: 0.02
- Maximum: 0.02
- Temporary Disk Used:
- Average: 0.54
- Minimum: 0.54
- Maximum: 0.54
- Temporary Disk Available:
- Average: 72.44
- Minimum: 72.44
- Maximum: 72.44
- Memory Total:
- Average: 522.58
- Minimum: 522.58
- Maximum: 522.58
- Memory Free:
- Average: 474.81
- Minimum: 474.81
- Maximum: 474.81
- Memory Percentage Used:
- Average: 9.14
- Minimum: 9.14
- Maximum: 9.14
- Memory Cached:
- Average: 42.41
- Minimum: 42.41
- Maximum: 42.41
- Memory Postgres:
- Average: 0.76
- Minimum: 0.76
- Maximum: 0.76
- Max IOPS: 12000
Follower Database:
- Active Connections:
- Average: 10.0
- Minimum: 10
- Maximum: 10
- Waiting Connections:
- Average: 0.0
- Minimum: 0
- Maximum: 0
- Connections Percentage Used:
- Average: 2.0
- Minimum: 2.0
- Maximum: 2.0
- Load Average (1m):
- Average: 0.0
- Minimum: 0.0
- Maximum: 0.0
- Load Average (5m):
- Average: 0.0
- Minimum: 0.0
- Maximum: 0.0
- Load Average (15m):
- Average: 0.0
- Minimum: 0.0
- Maximum: 0.0
- Read IOPS:
- Average: 0.081633
- Minimum: 0.081633
- Maximum: 0.081633
- Write IOPS:
- Average: 9.898
- Minimum: 9.898
- Maximum: 9.898
- IOPS Percentage Used:
- Average: 0.33
- Minimum: 0.33
- Maximum: 0.33
- Temporary Disk Used:
- Average: 0.54
- Minimum: 0.54
- Maximum: 0.54
- Temporary Disk Available:
- Average: 72.44
- Minimum: 72.44
- Maximum: 72.44
- Memory Total:
- Average: 32.39
- Minimum: 32.39
- Maximum: 32.39
- Memory Free:
- Average: 27.24
- Minimum: 27.24
- Maximum: 27.24
- Memory Percentage Used:
- Average: 15.91
- Minimum: 15.91
- Maximum: 15.91
- Memory Cached:
- Average: 4.29
- Minimum: 4.29
- Maximum: 4.29
- Memory Postgres:
- Average: 0.02
- Minimum: 0.02
- Maximum: 0.02
- Max IOPS: 3000
Metrics collected over 1 minutes.
'''
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment