Skip to content

Instantly share code, notes, and snippets.

@snyderra

snyderra/iftopparser.py

Created January 25, 2024 19:56
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save snyderra/e4d06e73b42f0c8f42e1371d830c410f to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save snyderra/e4d06e73b42f0c8f42e1371d830c410f to your computer and use it in GitHub Desktop.
Download ZIP
Raw
iftopparser.py
from pprint import pprint
import re
import json
def parse_records(records: str):
# see if we have any complete entries
beg = records.index(" # Host name")
end = records.rindex("============================================================================================")
if (beg > 0 and end > 0):
# get full entries (could be multiple)
full = records[beg: end]
# trim off front
records = records[end:]
# split full entries
entries = full.split("============================================================================================")
print(f"entry count {len(entries)}")
for ent in entries:
if re.search(r"^\s{3,4}# Host",ent,re.MULTILINE):
p = ent.split("--------------------------------------------------------------------------------------------")
ary = re.split(r"\n \d+",p[1],re.MULTILINE)
obj = {}
obj['flows'] = []
# parse flows, iterate by 2 b/c evens contain line indices
for i in range(1,len(ary)):
flowLine = re.match(r"^[ ]{1,2}([^\s]+)\s+=>\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?B)\s+([^\s]+)\s+<=\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?B)",ary[i],re.MULTILINE)
obj["flows"].append({
"src": flowLine[1],
"src2": flowLine[2],
"src10": flowLine[3],
"src40": flowLine[4],
"srcCum": flowLine[5],
"dst": flowLine[6],
"dst2": flowLine[7],
"dst10": flowLine[8],
"dst40": flowLine[9],
"dstCum": flowLine[10]
})
# parse send and receive rates
rateLine = re.search(r"Total send rate:\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+Total receive rate:\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+Total send and receive rate:\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)",p[2],re.MULTILINE)
obj["send2"] = rateLine[1]
obj["send10"] = rateLine[2]
obj["send40"] = rateLine[3]
obj["receive2"] = rateLine[4]
obj["receive10"] = rateLine[5]
obj["receive40"] = rateLine[6]
obj["both2"] = rateLine[7]
obj["both10"] = rateLine[8]
obj["both40"] = rateLine[9]
# parse peak and cumulative
peakLine = re.search(r"Peak rate \(sent\/received\/total\):\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+([0-9\.]+M?K?b)\s+Cumulative \(sent\/received\/total\):\s+([0-9\.]+M?K?B)\s+([0-9\.]+M?K?B)\s+([0-9\.]+M?K?B)",p[3],re.MULTILINE)
obj["peakSent"] = peakLine[1]
obj["peakReceived"] = peakLine[2]
obj["peakTotal"] = peakLine[3]
obj["cumSent"] = peakLine[4]
obj["cumReceived"] = peakLine[5]
obj["cumTotal"] = peakLine[6]
obj["flows"] = sorted(obj["flows"],key=lambda x: x["dst"])
yield(obj)
else:
print("no match")
# Example usage:
records = """
# Host name (port/service if enabled) last 2s last 10s last 40s cumulative
--------------------------------------------------------------------------------------------
1 raspberrypi:12001 => 1.52Kb 1.42Kb 1.42Kb 726B
IP.1:41848 <= 1.52Kb 882b 882b 441B
2 raspberrypi:12001 => 1.32Kb 1.32Kb 1.32Kb 674B
IP.1:60611 <= 208b 882b 882b 441B
3 raspberrypi:12001 => 1.32Kb 1.32Kb 1.32Kb 674B
IP.1:14465 <= 208b 208b 208b 104B
--------------------------------------------------------------------------------------------
Total send rate: 4.15Kb 4.05Kb 4.05Kb
Total receive rate: 1.93Kb 1.93Kb 1.93Kb
Total send and receive rate: 6.08Kb 5.98Kb 5.98Kb
--------------------------------------------------------------------------------------------
Peak rate (sent/received/total): 4.15Kb 1.92Kb 6.08Kb
Cumulative (sent/received/total): 2.03KB 986B 2.99KB
============================================================================================
# Host name (port/service if enabled) last 2s last 10s last 40s cumulative
--------------------------------------------------------------------------------------------
1 raspberrypi:12001 => 1.54Kb 1.39Kb 1.39Kb 1.04KB
IP.1:14465 <= 1.54Kb 664b 664b 498B
2 raspberrypi:12001 => 1.34Kb 1.39Kb 1.39Kb 1.04KB
IP.1:41848 <= 208b 657b 657b 493B
3 raspberrypi:12001 => 1.34Kb 1.32Kb 1.32Kb 0.99KB
IP.1:60611 <= 208b 657b 657b 493B
4 raspberrypi:12001 => 1.77Kb 605b 605b 454B
LINK.:30472 <= 1.29Kb 440b 440b 330B
--------------------------------------------------------------------------------------------
Total send rate: 5.98Kb 4.70Kb 4.70Kb
Total receive rate: 3.23Kb 2.36Kb 2.36Kb
Total send and receive rate: 9.22Kb 7.06Kb 7.06Kb
--------------------------------------------------------------------------------------------
Peak rate (sent/received/total): 5.98Kb 3.23Kb 9.22Kb
Cumulative (sent/received/total): 3.52KB 1.77KB 5.29KB
============================================================================================
# Host name (port/service if enabled) last 2s last 10s last 40s cumulative
--------------------------------------------------------------------------------------------
1 raspberrypi:12001 => 4.46Kb 2.11Kb 2.11Kb 2.11KB
IP.1:60611 <= 3.48Kb 1.35Kb 1.35Kb 1.35KB
2 raspberrypi:12001 => 1.74Kb 1.48Kb 1.48Kb 1.48KB
IP.1:14465 <= 2.34Kb 1.07Kb 1.07Kb 1.07KB
3 raspberrypi:12001 => 1.73Kb 1.48Kb 1.48Kb 1.48KB
IP.1:41848 <= 2.32Kb 1.06Kb 1.06Kb 1.06KB
4 raspberrypi:12001 => 0b 454b 454b 454B
LINK.:30472 <= 0b 330b 330b 330B
--------------------------------------------------------------------------------------------
Total send rate: 7.93Kb 5.50Kb 5.50Kb
Total receive rate: 8.14Kb 3.81Kb 3.81Kb
Total send and receive rate: 16.1Kb 9.31Kb 9.31Kb
--------------------------------------------------------------------------------------------
Peak rate (sent/received/total): 7.93Kb 8.13Kb 16.1Kb
Cumulative (sent/received/total): 5.50KB 3.81KB 9.31KB
============================================================================================
# Host name (port/service if enabled) last 2s last 10s last 40s cumulative
--------------------------------------------------------------------------------------------
1 raspberrypi:12001 => 2.88Kb 2.26Kb 2.26Kb 2.83KB
IP.1:60611 <= 3.29Kb 1.74Kb 1.74Kb 2.17KB
2 raspberrypi:12001 => 2.88Kb 1.76Kb 1.76Kb 2.20KB
IP.1:14465 <= 416b 961b 961b 1.17KB
3 raspberrypi:12001 => 2.88Kb 1.76Kb 1.76Kb 2.20KB
IP.1:41848 <= 416b 953b 953b 1.16KB
4 raspberrypi:12001 => 2.88Kb 953b 953b 1.16KB
LINK.:30472 <= 416b 347b 347b 434B
--------------------------------------------------------------------------------------------
Total send rate: 11.5Kb 6.71Kb 6.71Kb
Total receive rate: 4.50Kb 3.95Kb 3.95Kb
Total send and receive rate: 16.0Kb 10.7Kb 10.7Kb
--------------------------------------------------------------------------------------------
Peak rate (sent/received/total): 11.5Kb 8.13Kb 16.1Kb
Cumulative (sent/received/total): 8.38KB 4.93KB 13.3KB
============================================================================================
# Host name (port/service if enabled) last 2s last 10s last 40s cumulative
--------------------------------------------------------------------------------------------
1 raspberrypi:12001 => 22.6Kb 6.02Kb 5.24Kb 7.86KB
IP.1:14465 <= 21.6Kb 5.22Kb 4.38Kb 6.57KB
2 raspberrypi:12001 => 16.7Kb 5.35Kb 4.68Kb 7.01KB
IP.1:60611 <= 9.96Kb 3.43Kb 3.11Kb 4.66KB
3 raspberrypi:12001 => 16.7Kb 4.84Kb 4.26Kb 6.38KB
IP.1:41848 <= 11.5Kb 3.19Kb 2.69Kb 4.04KB
4 raspberrypi:12001 => 6.94Kb 2.32Kb 1.93Kb 2.90KB
LINK.:30472 <= 416b 430b 359b 538B
--------------------------------------------------------------------------------------------
Total send rate: 63.1Kb 18.5Kb 16.1Kb
Total receive rate: 43.5Kb 12.3Kb 10.5Kb
Total send and receive rate: 107Kb 30.8Kb 26.6Kb
--------------------------------------------------------------------------------------------
Peak rate (sent/received/total): 63.1Kb 43.5Kb 107Kb
Cumulative (sent/received/total): 24.2KB 15.8KB 40.0KB
============================================================================================
# Host name (port/service if enabled) last 2s last 10s last 40s cumulative
--------------------------------------------------------------------------------------------
1 raspberrypi:12001 => 11.2Kb 8.00Kb 6.09Kb 10.7KB
IP.1:14465 <= 7.02Kb 6.58Kb 4.76Kb 8.33KB
2 raspberrypi:12001 => 12.0Kb 7.48Kb 5.72Kb 10.0KB
"""
# recrods=list(parse_records(open("/Users/rasnyder/flighttestSession1.txt").read()))
# json_data = json.dumps(), indent=4)
# print(len(records))
index = open("charts.html","w")
index.write("<html><body>")
for title,file in [
["Session1","fileone.txt"],
]:
flows=dict()
totals=dict()
for i,r in enumerate(parse_records(open(file).read())):
for f in r["flows"]:
flows.setdefault(f["dst"],[]).append((i,f["dst10"]))
totals.setdefault("From EUD (2 second avg)",[]).append((i,r["receive2"]))
totals.setdefault("To EUDs (2 second avg)",[]).append((i,r["send2"]))
totals.setdefault("From EUD (10 second avg)",[]).append((i,r["receive10"]))
totals.setdefault("To EUDs (10 second avg)",[]).append((i,r["send10"]))
totals.setdefault("From EUD (40 second avg)",[]).append((i,r["receive40"]))
totals.setdefault("To EUDs (40 second avg)",[]).append((i,r["send40"]))
scale = {"Kb": 1024/8, "Mb": 1024*1024/8,"KB": 1024, "MB": 1024*1024, "b": 1/8, "B": 1}
def parse_number(s):
match = re.match(r"([0-9\.]+)([KMbB]*)", s)
if match:
number, unit = match.groups()
return float(number)*scale[unit]/1024.0
else:
return None, None
def fill_missing(pairs):
m=dict(pairs)
for i in range(0,pairs[-1][0]+1):
if i not in m:
m[i]="0b"
return sorted(m.items(),key=lambda x: x[0])
import pygal
chart = pygal.StackedBar(logarithmic=True)
chart.value_formatter = lambda x: "%.2fKB" % x
chart.title = f'{title} - flows'
chart.x_title = "KB/s 10 second averaging window 2 second sampling"
for k,v in sorted(flows.items(),key=lambda x: len(x[1]),reverse=True):
chart.add(k, [parse_number(x[1]) for x in fill_missing(v)])
chart.render_to_file(f'chart-{title}-flows.svg')
index.write(f'<object type="image/svg+xml" data="chart-{title}-flows.svg">Your browser does not support SVG</object>')
import pygal
chart = pygal.Line(logarithmic=True)
chart.value_formatter = lambda x: "%.2fKB" % x
chart.title = f'{title} - totals'
chart.x_title = "KB/s 2 second sampling"
for k,v in totals.items():
chart.add(k, [parse_number(x[1]) for x in fill_missing(v)])
chart.render_to_file(f'chart-{title}-totals.svg')
index.write(f'<object type="image/svg+xml" data="chart-{title}-totals.svg">Your browser does not support SVG</object>')
index.write("</body></html>")
index.close()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment