Skip to content

Instantly share code, notes, and snippets.

@ProBackup-nl

ProBackup-nl/dsm_reader.py

Forked from ius/gist:e59adb64bbe8855cfc5c18297f6d692e
Last active February 6, 2024 13:45
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ProBackup-nl/efbbb30747dad4e2f1e7dfd23506696b to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ProBackup-nl/efbbb30747dad4e2f1e7dfd23506696b to your computer and use it in GitHub Desktop.
Download ZIP
Dutch Smart Meter (DSM) P1 to InfluxDB 2.0 and pvoutput.org
Raw
dsm_reader.py
#!/usr/bin/env python
# Store DSMR telegrams from P1 into influxdb 2.0 and pvoutput.org
# EN-IEC 62056-21, Part 21: direct local data exchange, 2002-05
#- Requires python2+
#- Install deps
# # pacman -S python-requests python-pytz
#- Create script
# # nano /root/dsm_reader.py && chmod +x /root/dsm_reader.py
#- Create config
# # echo "auth_token='abcdefg'" > /root/dsm_reader_config.py
# ToDo: handle influxPi4B socat[137394]: HTTPSConnectionPool(host='pvoutput.org', port=443): Max retries exceeded with url: /service/
import logging
import os
import re
import sys
import threading
import time
import requests
import urllib3
from datetime import datetime
from pytz import timezone, utc
from dsm_reader_config import auth_token, pvoutput_token_6, pvoutput_sid_6
print_output = 0
url = 'http://127.0.0.1:8086/api/v2/write'
urlpv = 'https://pvoutput.org/service/r2/addstatus.jsp'
fp = sys.stdin
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
table = [
0x0000, 0xc0c1, 0xc181, 0x0140, 0xc301, 0x03c0, 0x0280, 0xc241,
0xc601, 0x06c0, 0x0780, 0xc741, 0x0500, 0xc5c1, 0xc481, 0x0440,
0xcc01, 0x0cc0, 0x0d80, 0xcd41, 0x0f00, 0xcfc1, 0xce81, 0x0e40,
0x0a00, 0xcac1, 0xcb81, 0x0b40, 0xc901, 0x09c0, 0x0880, 0xc841,
0xd801, 0x18c0, 0x1980, 0xd941, 0x1b00, 0xdbc1, 0xda81, 0x1a40,
0x1e00, 0xdec1, 0xdf81, 0x1f40, 0xdd01, 0x1dc0, 0x1c80, 0xdc41,
0x1400, 0xd4c1, 0xd581, 0x1540, 0xd701, 0x17c0, 0x1680, 0xd641,
0xd201, 0x12c0, 0x1380, 0xd341, 0x1100, 0xd1c1, 0xd081, 0x1040,
0xf001, 0x30c0, 0x3180, 0xf141, 0x3300, 0xf3c1, 0xf281, 0x3240,
0x3600, 0xf6c1, 0xf781, 0x3740, 0xf501, 0x35c0, 0x3480, 0xf441,
0x3c00, 0xfcc1, 0xfd81, 0x3d40, 0xff01, 0x3fc0, 0x3e80, 0xfe41,
0xfa01, 0x3ac0, 0x3b80, 0xfb41, 0x3900, 0xf9c1, 0xf881, 0x3840,
0x2800, 0xe8c1, 0xe981, 0x2940, 0xeb01, 0x2bc0, 0x2a80, 0xea41,
0xee01, 0x2ec0, 0x2f80, 0xef41, 0x2d00, 0xedc1, 0xec81, 0x2c40,
0xe401, 0x24c0, 0x2580, 0xe541, 0x2700, 0xe7c1, 0xe681, 0x2640,
0x2200, 0xe2c1, 0xe381, 0x2340, 0xe101, 0x21c0, 0x2080, 0xe041,
0xa001, 0x60c0, 0x6180, 0xa141, 0x6300, 0xa3c1, 0xa281, 0x6240,
0x6600, 0xa6c1, 0xa781, 0x6740, 0xa501, 0x65c0, 0x6480, 0xa441,
0x6c00, 0xacc1, 0xad81, 0x6d40, 0xaf01, 0x6fc0, 0x6e80, 0xae41,
0xaa01, 0x6ac0, 0x6b80, 0xab41, 0x6900, 0xa9c1, 0xa881, 0x6840,
0x7800, 0xb8c1, 0xb981, 0x7940, 0xbb01, 0x7bc0, 0x7a80, 0xba41,
0xbe01, 0x7ec0, 0x7f80, 0xbf41, 0x7d00, 0xbdc1, 0xbc81, 0x7c40,
0xb401, 0x74c0, 0x7580, 0xb541, 0x7700, 0xb7c1, 0xb681, 0x7640,
0x7200, 0xb2c1, 0xb381, 0x7340, 0xb101, 0x71c0, 0x7080, 0xb041,
0x5000, 0x90c1, 0x9181, 0x5140, 0x9301, 0x53c0, 0x5280, 0x9241,
0x9601, 0x56c0, 0x5780, 0x9741, 0x5500, 0x95c1, 0x9481, 0x5440,
0x9c01, 0x5cc0, 0x5d80, 0x9d41, 0x5f00, 0x9fc1, 0x9e81, 0x5e40,
0x5a00, 0x9ac1, 0x9b81, 0x5b40, 0x9901, 0x59c0, 0x5880, 0x9841,
0x8801, 0x48c0, 0x4980, 0x8941, 0x4b00, 0x8bc1, 0x8a81, 0x4a40,
0x4e00, 0x8ec1, 0x8f81, 0x4f40, 0x8d01, 0x4dc0, 0x4c80, 0x8c41,
0x4400, 0x84c1, 0x8581, 0x4540, 0x8701, 0x47c0, 0x4680, 0x8641,
0x8201, 0x42c0, 0x4380, 0x8341, 0x4100, 0x81c1, 0x8081, 0x4040
]
obis_objects = {
'1-3:0.2.8': 'version_info',
'0-0:1.0.0': 'datetime_stamp', # '210404113641S'
'0-0:96.1.1': 'equipment_id_electricity', # '4530303331303033303031363939353135' ___tag
'1-0:1.8.1': 'mr_toclient_t1', # '000009.691' uint _________________________________fields
'1-0:1.8.2': 'mr_toclient_t2', # '000014.551' uint
'1-0:2.8.1': 'mr_byclient_t1', # '002502.491' uint
'1-0:2.8.2': 'mr_byclient_t2', # '005920.901' uint
'0-0:96.14.0': 'tariff_indicator', # '0001' uint
'1-0:1.7.0': 'actual_power_delivered', # '00.000' * 1000 uint
'1-0:2.7.0': 'actual_power_received', # '10.806' * 1000 uint
'0-0:96.7.21': 'total_pf', # '00220' uint
'0-0:96.7.9': 'total_long_pf', # '00034' uint
'1-0:99.97.0': 'pf_event_log',
'1-0:32.32.0': 'volt_sag_l1', # '00024' uint
'1-0:52.32.0': 'volt_sag_l2', # '00024' uint
'1-0:72.32.0': 'volt_sag_l3', # '00024' uint
'1-0:32.36.0': 'volt_swell_l1', # '00000' uint
'1-0:52.36.0': 'volt_swell_l2', # '00000' uint
'1-0:72.36.0': 'volt_swell_l3', # '00000' uint
'0-0:96.13.1': 'text_message_codes',
'0-0:96.13.0': 'text_message', # []
'1-0:32.7.0': 'inst_volt_l1', # '231.0' float
'1-0:52.7.0': 'inst_volt_l2', # '232.0' float
'1-0:72.7.0': 'inst_volt_l3', # '231.0' float
'1-0:31.7.0': 'inst_current_l1', # '015' uint
'1-0:51.7.0': 'inst_current_l2', # '015' uint
'1-0:71.7.0': 'inst_current_l3', # '016' uint
'1-0:21.7.0': 'inst_power_l1_p', # '00.000' * 1000 uint
'1-0:41.7.0': 'inst_power_l2_p', # '00.000' * 1000 uint
'1-0:61.7.0': 'inst_power_l3_p', # '00.000' * 1000 uint
'1-0:22.7.0': 'inst_power_l1_m', # '03.543' * 1000 uint
'1-0:42.7.0': 'inst_power_l2_m', # '03.552' * 1000 uint
'1-0:62.7.0': 'inst_power_l3_m', # '03.710' * 1000 uint
'0-1:96.1.0': 'equipment_id_gas',
'0-1:24.2.1': 'gas_toclient',
'0-1:24.1.0': 'device_type',
}
def crc16(s):
crc = 0x00
for ch in s:
crc = (crc >> 8) ^ table[(crc ^ ord(ch)) & 0xff]
return crc
def obis_map(ref, value):
if ref in obis_objects:
ref = obis_objects[ref]
return ref, value
def parse(buf):
values = {}
for line in buf.split():
mo = re.findall('^([0-9.:-]+)(.+)', line.rstrip())
# example mo=[('1-0:62.7.0', '(04.003*kW)')]
for match in mo:
pair = obis_map(*match)
values.update([pair])
if print_output == 2:
print('%s: %s' % pair)
return values
def verify_and_parse(buf):
crc = int(buf[-1][1:5], 16)
data = ''.join(buf[:-1]) + '!'
if crc == crc16(data):
return parse(data)
def parse_value(s):
t = re.findall('\(([0-9.-]+[SW]?).*?\)', s)
if re.search('^[0-9]+[.][0-9][0-9][0-9]$', t[0]):
t[0] = str(int(float(t[0]) * 1000)) + 'u'
elif re.search('^[0-9]{1,33}$', t[0]):
t[0] = str(int(t[0])) + 'u'
return t
def parse_ts(s):
tm, tz = s[:-1], s[-1]
is_dst = (tz == 'S')
#if tm.startswith('23'):
# tm = '16' + tm[2:]
date = datetime.strptime(tm, '%y%m%d%H%M%S')
ams_tz = timezone('Europe/Amsterdam')
date = ams_tz.localize(date, is_dst=is_dst)
#date = date.astimezone(utc)
unix = time.mktime(date.timetuple())
return int(unix)
def handle(values):
global arr_vals, pts, hivolt
pts = parse_value(values['datetime_stamp'])[0]
ts = parse_ts(pts)
equip_id_elec = parse_value(values['equipment_id_electricity'])[0]
keys = ['mr_toclient_t1', 'mr_toclient_t2', 'mr_byclient_t1', 'mr_byclient_t2', 'tariff_indicator', 'actual_power_delivered', 'actual_power_received', 'total_pf', 'total_long_pf', 'volt_sag_l1', 'volt_sag_l2', 'volt_sag_l3', 'volt_swell_l1', 'volt_swell_l2', 'volt_swell_l3', 'inst_volt_l1', 'inst_volt_l2', 'inst_volt_l3', 'inst_current_l1', 'inst_current_l2', 'inst_current_l3', 'inst_power_l1_p', 'inst_power_l2_p', 'inst_power_l3_p', 'inst_power_l1_m', 'inst_power_l2_m', 'inst_power_l3_m']
#gas_ts, gas_value = parse_value(values['gas_toclient'])
#gas_ts = parse_ts(gas_ts)
arr_vals = lambda v: ','.join('%s=%s' % (k, parse_value(values[k])[0]) for k in v)
# remember highest voltage of any phase
for t in range (1, 3):
hivolt = max(hivolt, float(arr_vals(['inst_volt_l' + str(t)])[13:]))
# post selected keys
postdata = '%s %s %d\n' % ('measurement,equipment_id_electricity=' + equip_id_elec, arr_vals(keys), ts)
#postdata += '%s gas_toclient=%s %d\n' % ('gas', gas_value, gas_ts)
if print_output:
print(postdata)
# use InfluxDB 1.x compatibility API - Line protocol // db = bucket, precision = seconds
try:
r = requests.post(url,
params={'org': 'Organization', 'bucket': 'dsm', 'precision': 's'},
data=postdata,
headers={'Authorization': 'Token ' + auth_token},
timeout=0.9)
except requests.exceptions.Timeout:
# Log request to file
filename = 'insert-failed-influx-posts.sh'
filesize = 0
with open(filename, 'r') as file:
file.seek(0, os.SEEK_END)
filesize = file.tell()
with open(filename, 'a') as file:
if filesize == 0:
file.write('#!/bin/sh' + "\n")
file.write('curl -X POST ' + url + '?org=Organization&bucket=dsm&precision=s ' + \
'-H "Authorization: Token ' + auth_token + '" ' + \
'-H "Content-Type: application/x-www-form-urlencoded" ' + \
'-d "' + postdata + '"' + "\n")
except requests.exceptions.ConnectionError:
print('Connection refused')
except requests.exceptions.RequestException as e:
print(str(e))
else:
if print_output:
print(str(r))
def post_pvoutput():
global arr_vals, pts, hivolt
threading.Timer(300.0, post_pvoutput).start()
if pts:
d = 'd=20' + pts[:6]
t = 't=' + pts[6:8] + ':' + pts[8:10]
v1 = 'v1=' + str(int(arr_vals(['mr_byclient_t1'])[15:-1]) + int(arr_vals(['mr_byclient_t2'])[15:-1]))
v3 = 'v3=' + str(int(arr_vals(['mr_toclient_t1'])[15:-1]) + int(arr_vals(['mr_toclient_t2'])[15:-1]))
v6 = 'v6=' + str(hivolt)
hivolt = 0.0
postdata = "&".join([d, t, v1, v3, v6, 'c1=1'])
for i in range(1, 9):
try:
r = requests.post(
urlpv,
data=postdata,
headers={
'Content-Type': 'application/x-www-form-urlencoded',
'X-Pvoutput-Apikey': pvoutput_token_6,
'X-Pvoutput-SystemId': pvoutput_sid_6,
'Accept': 'text/plain'
},
timeout=30)
if print_output:
print(r.text)
#print(postdata)
if r.status_code == 200:
return
if 400 <= r.status_code < 500:
warningMsg = ("Unable to connect to pvoutput.org - Reason: " + r.reason)
logging.warning(warningMsg)
return
except requests.exceptions.RequestException as e:
print(str(e))
else:
if print_output:
print(str(r))
if __name__ == '__main__':
buf = []
hivolt = 0.0
called_once = False
while True:
line = fp.readline().strip('\x00')
if line.startswith('/'):
buf = [line]
else:
buf.append(line)
if line.startswith('!'):
values = verify_and_parse(buf)
if values:
handle(values)
if called_once == False:
called_once = True
post_pvoutput() # threaded
Raw
dsm_reader.service.d
# nano /lib/systemd/system/dsm_reader.service
[Unit]
Description=Read smart meter P1 data and store the numeric values in Influxdb and pvoutput
Requires=influxdb2-bin
After=influxdb2-bin
[Install]
WantedBy=multi-user.target
[Service]
ExecStart=/usr/bin/socat -u FILE:/dev/ttyUSB0,b115200,crtscts,cfmakeraw EXEC:/root/dsm_reader.py
Restart=always
RestartSec=3
Raw
help.txt
EN-IEC 62056-21, Part 21: direct local data exchange, 2002-05
- Requires python2+
- Install deps
# pacman -S python-requests python-pytz
- Create script
# nano /root/dsm_reader.py && chmod +x /root/dsm_reader.py
- Create config
# echo "auth_token='abcdefg'" > /root/dsm_reader_config.py
- Start
# systemctl enable dsm_reader.service && systemctl start dsm_reader
- ToDo:
1. modify service file to a template unit file (name@string.service), where the serial device becomes variable
3. post fluxdb exception handler that stores failing/missed posts to memory/disk and lator on insert the backlog
4. implement retries, with backoff? like
- solution 4 from https://izziswift.com/can-i-set-max_retries-for-requests-request/
- https://github.com/pvl7/emu-to-pvoutput/blob/2edf7fff8d1af180fa203968b2d33acb80485dec/rainforest-to-pvoutput.py#L88
5. fire off influxdb posting as separate thread, limit amount of threads?
6. handle influxdb related error: socat[291]: HTTPConnectionPool(host='127.0.0.1', port=8086): Read timed out. (read timeout=0.9)
Dec 08 08:39:20 influxPi4B socat[320]: HTTPConnectionPool(host='127.0.0.1', port=8086): Read timed out. (read timeout=0.9)
Dec 08 08:39:20 influxPi4B socat[320]: HTTPConnectionPool(host='127.0.0.1', port=8086): Read timed out. (read timeout=0.9)
Dec 08 08:39:20 influxPi4B socat[320]: Traceback (most recent call last):
Dec 08 08:39:20 influxPi4B socat[320]: File "/root/dsm_reader.py", line 259, in <module>
Dec 08 08:39:20 influxPi4B socat[320]: values = verify_and_parse(buf)
Dec 08 08:39:20 influxPi4B socat[320]: File "/root/dsm_reader.py", line 133, in verify_and_parse
Dec 08 08:39:20 influxPi4B socat[320]: crc = int(buf[-1][1:5], 16)
Dec 08 08:39:20 influxPi4B socat[320]: ValueError: invalid literal for int() with base 16: '-0:6'
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment