Community
OpenEnergyMonitor

Community

Python script runs but emoncms.org does not update (Hangs at end of script)

python
Tags: #<Tag:0x00007f1fe4257d48>

(Bo Herrmannsen) #41

after some adjusting i came up with this (complete code):

#!/usr/bin/python
# -*- coding: utf-8 -*-

#SETTINGS START

start_time = '07:00'
end_time = '22:00'
bake_temp = float(25)
target1 = float(22)
target2 = float(17)

# Domain you want to post to: localhost would be an emoncms installation on your own laptop
# this could be changed to emoncms.org to post to emoncms.org
domain = "emoncms.org"

# Location of emoncms in your server, the standard setup is to place it in a folder called emoncms
# To post to emoncms.org change this to blank: ""
emoncmspath = ""

# Write apikey of emoncms account
apikey = "takeaguess"

# Node id youd like the emontx to appear as
nodeid = 1

#SETTINGS END

#IMPORTS START

# imports for thermometer reading test

import os
import glob
import time
import datetime

# import for internet connect check

import socket

# this needed to post to emocms

import sys, string
import httplib

#this one is for the wiringpi lib

import wiringpi

#IMPORTS END

# wiringpi numbers

wiringpi.wiringPiSetup()
wiringpi.pinMode(0, 1)  # sets pin 0 to output (GPIO 17, Actual hardware pin number is 11) (Relay)
wiringpi.pinMode(2, 1)  # sets pin 2 to output (GPIO 27, Actual hardware pin number is 13) (Internet connection LED)
wiringpi.pinMode(3, 0)  # sets pin 3 to input (GPIO 22, Actual hardware pin number is 15) (latching button)

# Find temperature from thermometer

os.system('modprobe w1-gpio')
os.system('modprobe w1-therm')

base_dir = '/sys/bus/w1/devices/'
device_folder = glob.glob(base_dir + '28*')[0]
device_file = device_folder + '/w1_slave'

csv = 0
target = 0
relay = 0
t_loop = 0
t15 = 0
test = 0
next_time = 0

# check if internet - just so we know if time can be trusted

def internet_connected(host='8.8.8.8', port=53):
    """
    Host: 8.8.8.8 (google-public-dns-a.google.com)
    OpenPort: 53/tcp
    Service: domain (DNS/TCP)
    """
    try:
        socket.setdefaulttimeout(20)
        socket.socket(socket.AF_INET,
                      socket.SOCK_STREAM).connect((host, port))
        return True
    except Exception, ex:
        pass

    return False

# temp reading

def read_temp_raw():
    f = open(device_file, 'r')
    lines = f.readlines()
    f.close()
    return lines

def read_temp():
    lines = read_temp_raw()
    while lines[0].strip()[-3:] != 'YES':
        time.sleep(0.2)
        lines = read_temp_raw()
    equals_pos = lines[1].find('t=')
    if equals_pos != -1:
        temp_string = (lines[1])[equals_pos + 2:]
        temp_c = float(temp_string) / 1000.0
        temp_f = temp_c * 9.0 / 5.0 + 32.0
        return temp_c  # , temp_f

# schedule, check if current time is after schedule start and sets target

def schedule():

    global target

    t = time.strftime('%H:%M')
    if is_between(t, (start_time, end_time)) == True:
        print('Running Daytime Schedule')
        target = target1
    else:
        print('Running Nighttime Schedule')
        target = target2
  
def is_between(time, time_range):

    if time_range[1] < time_range[0]:
        return time >= time_range[0] or time <= time_range[1]
    return time_range[0] <= time <= time_range[1]

def emoncms():

    try:
        seq = (read_temp(), target, my_input, relay)
        str_join = ",".join(str(x) for x in seq)
        print 'Preparing Data for emoncms.org'
        conn = httplib.HTTPConnection(domain)
        conn.request("GET", "/"+emoncmspath+"/input/post.json?apikey="+apikey+"&node="+str(nodeid)+"&csv="+str_join)
        
        response = conn.getresponse()
        print response.read()
        print 'Data sent to emoncms'
    except Exception, ex:
        print 'This error occurred: ' + str(ex)
        
while True:
    my_input = wiringpi.digitalRead(3)

    if my_input == 1:
        print 'Switch is ON'
        t15 = 15

    time_now = time.time()

    if time_now >= next_time:

        print 'Start of loop: ' + str(test)

        temp_now = read_temp()
        print 'Current temp: ' + str(temp_now)

        schedule()

        print 'We have an internet connection? ' + str(internet_connected())

        # Control heating
        if t15 > 0:
            target = bake_temp
            t15 -= 1
            if target > temp_now:        # Compare temp read to temp from DS18B20
                wiringpi.digitalWrite(0, 1) # sets port 0 to ON
                relay = 1
                print 'Target temp: ' + str(target)
                print 'HEATING ON (Boost)'
            else:
                wiringpi.digitalWrite(0, 0) # sets port 0 to OFF
                relay = 0
                print 'Target temp: ' + str(target)
                print 'HEATING OFF (Boost)'
        else:
            if target > temp_now:           # Compare temp read to temp from DS18B20
                wiringpi.digitalWrite(0, 1) # sets port 0 to ON
                relay = 1
                print 'Target temp: ' + str(target)
                print 'HEATING ON'
            else:
                wiringpi.digitalWrite(0, 0) # sets port 0 to OFF
                relay = 0
                print 'Target temp: ' + str(target)
                print 'HEATING OFF'

        # Calculate the next loop time by adding interval in secs
        next_time = time_now + 60

        emoncms()
        print "Remaining boost time: " + str(t15) + " mins"
        test += 1
        print 'end of loop'
        print '--------------------'

    # Don't loop too fast
    time.sleep(1)

(Paul) #42

Ahh ok I had no idea that my_input was a button, yes with the current code you would need to hold it down until the next iteration (up to 60secs). If it was moved out to the outer loop it could still be missed as the code is looping once a second, unless you hold it for 1 sec each push. So you would need to remove or decrease the time.sleep(1) but this will significantly increase the cpu load.

The correct way to do it is to set up an interrupt for the button press and an “onButtonPress” function that sets the “my_input” variable, that way the code can remain as it is (calm) and the button push will be registered no matter when it’s pushed or how fast it is pushed.

Take a look at the Pulse monitoring on a Pi and Directly connecting to Optical Pulse Counter with RPi? threads for ideas on how that’s done. Those threads are about pulse counting, but a button press is just a single pulse that instead of counting them you could just toggle 1 and 0 giving you a manual “off” if you choose to turn boost off after it’s only been on a couple of mins.


(Bo Herrmannsen) #43

will do that, but at least its a heck more stable now :smiley:

last Q… are there any “reaonsble” way to detect it it hangs? my thoughts are that if it hangs it rather dificult to do any action ie telling “i’m stuck”


(Paul) #44

Indeed, if a program “hangs” or “crashes” it is no longer able to alert you, take evasive action, ask for help or even be self aware that it’s dead and restart. Therefore you need to write robust code to avoid such instances.

You could install Monit, set something up in node-red to tell you when it stops (and restart it?), write a bash script to check it (but what happens if that crashes) or you could just write a cron job to restart it again every 1,2 or 24 hrs etc etc etc but all of these are a poor substitute for good code. Some of them have their place, but not to keep a piece of bad code alive instead of fixing it.

you could set up a log file so that useful info and error messages are recorded so that you can improve the code, ultimately there is rarely any good reason for a script to crash/hang. If all fails are handled, the code should just keep trying or carry on. Hangs and crashes are just where “loose ends” are not tied off.

In very general terms if you could wrap your whole script in a try/except like this

while True:
    try:
        run_my_script()
    except Exception, ex:
        print str(ex)

it should never “crash” because there are only 2 options, “run_my_script” successfully or tell me why not and try it again, there are no un-handled options for the code to go. In practice this method is unlikely to be useful because there are too many possible errors that can happen in the one try/exception which is why you should use them when ever there is the possibility of an error and the more of them there are the more info you can get, the pythonic way is to put as little as possible in each try/except, not all code needs an escape route so you do not need to put every line in one.

while True:
    try:
        run_my_script_part_A()
    except Exception, ex:
        print "part_A failed with: " + str(ex)
    try:
        run_my_script_part_B()
    except Exception, ex:
        print "part_B failed with: " + str(ex)
    try:
        run_my_script_part_C()
    except Exception, ex:
        print "part_C failed with: " + str(ex)

You have the time module installed so you can easily add a timestamp

while True:
    try:
        run_my_functionA()
    except Exception, ex:
        print str(time.time()) + "\tFunction A Error: " str(ex)

Al you need to do is write those prints to a file instead of the screen and you have a useful logfile you can look back at.

The main thing to do is to get as much info as you can so you can fix the code (and learn how to avoid the same issues in future scripts). Never use “pass” in a try except, it’s the coding equivalent of wearing a blindfold and ear plugs.

As well as error messages you should record some successful messages too to compare against and confirm normal running, if you are worried about filling the logfile with good stuff and missing the errors restric your self to just one or more successful message(s) per iteration but include lots of info in one message.

You could also set a debug log level eg

debug = 1

while True:
    if debug  > 2:
        print str(time.time()) + "\tAnd yet another loop started"
    try:
        run_my_functionA()
        if debug  > 1:
            print str(time.time()) + "\tFunction A is happy, here's the proof: " + info
    except Exception, ex:
        if debug  > 0:
            print str(time.time()) + "\tFunction A Error: " str(ex)

here if debug is “1” only the error messages are printed, 2 success messages are printed, 3 less useful per loop messages are printed and 0 is no messages at all.

once you use that info and get wise to the likely errors and handle them, since not all errors can be fixed but you can “handle” them with additional functions, many functions have their own exception types or you can write your own

while True:
    try:
        run_my_functionA()
    except NetworkConnectionException, ex:
        start_flashing_massive_led_to_alert_me()
        print str(time.time()) + "\tFunction A NetworkConnectionError: " str(ex)
    except Exception, ex:
        print str(time.time()) + "\tFunction A Error: " str(ex)

will log all errors but if it’s specifically a NetworkConnection tagged issue it will flash a beacon to alert you.

keeping the script alive to be able to tell you there’s a problem is possibly more important than it doing the job you’ve written it to do, if it can crash, it will fail to do the job, if it cannot crash, it is possible it can do the job, or at least tell you it why it can’t.

Rather a long answer I know and it isn’t intended as a lecture if it sounds that way. The subject of “how to restart crashed scripts” and “how to check if script is still running” come up pretty frequently so hopefully these notes will help others too as that’s not the best way to do things.


(Bo Herrmannsen) #45

i dont mind the long post, it will just take me some time to get it all in my head

one failure that came to mind that nothing can help with is if the sd card goes bad :smiley:

but i will prob read through the post several times


(Bo Herrmannsen) #46

first time arround i have removed the pass where it checks the internet and added the same print of error as with posting to emoncms.org

a bit in doubt if def read_temp_raw, def read_temp and would benefit from it

but after dinner i will add timestamps, really good idea


(Paul) #47

Just to expand on one point before someone corrects me.

It is also considered pad practice to use just “except Exception” as a catch all because generally you might want to do different things under different circumstances, eg ignore some errors using pass or writing clearer print messages etc etc. (see https://stackoverflow.com/questions/21553327/why-is-except-pass-a-bad-programming-practice)

My encouragement to use “except Exception” is for simplicity and it ensures all errors are caught as opposed to certain errors slipping through the net, even if all other errors are believed to be handled elsewhere, the final catchall will ensure you don’t miss anything.

while True:
    try:
        run_my_functionA()
    except NetworkConnectionException, ex:
        start_flashing_massive_led_to_alert_me()
        print str(time.time()) + "\tFunction A NetworkConnectionError: " str(ex)

will only act on NetworkErrors, everything else is ignored

while True:
    try:
        run_my_functionA()
    except NetworkConnectionException, ex:
        start_flashing_massive_led_to_alert_me()
        print str(time.time()) + "\tFunction A NetworkConnectionError: " str(ex)
    except Exception, ex:
        print str(time.time()) + "\tFunction A Error: " str(ex)

will act differently for NetworkErrors but no errors will be missed

while True:
    try:
        run_my_functionA()
    except NetworkConnectionException, ex:
        start_flashing_massive_led_to_alert_me()
        print str(time.time()) + "\tFunction A NetworkConnectionError: " str(ex)
    except SomeOtherSpecificException, ex:
        pass
    except Exception, ex:
        print str(time.time()) + "\tFunction A Error: " str(ex)

will do something specifically for NetworkConnectionException, ignore all “SomeOtherSpecificException” completely and then also record any exceptions not covered by the 2 previous exception types.


(Paul) #48

If you were to disconnect the temp sensor (eg intermittent bad connection) the file you are reading might cease to exist, therefore that would cause a crash as that error isn’t handled. By using a try/except you could record the error and use a default setting just so you don’t cook or freeze during your sleep if a temp sensor fails.

You do not need to put everything in a try/except for example if you make “read_temp_raw” return a value no matter what, then you can probably get away with not having the read_temp" in a try/except eg

def read_temp_raw():
    try:
        f = open(device_file, 'r')
        lines = f.readlines()
        f.close()
        return lines
    except  Exception, ex:
        print "Failed to read temp sensor, error is: " + str(ex)
        return False

def read_temp():
    if read_temp_raw():
        lines = read_temp_raw()
        blah, blah, blah
    else:
        return 60

if read_temp_raw fails it returns False instead of the temperature file contents. The “if read_temp_raw():” not run if the result is False and the else will supply a value of 60F.

I’m not suggesting you use this, it’s just to demonstrate how to manage errors and exceptions. You would probably want it to reuse the last temp reading rather than use 60, but you get the idea. I’ve used “blah blah blah” there in place of your code because I’m not entirely sure what you are up to there. I’m trying not to correct your code, but to give you the pointers so you can do what ever you want.


(Bo Herrmannsen) #49

yeah… at least it has to be given thoughts, ie for the temp you are right… its only needed for the raw temp read as the other depends on it


(Bo Herrmannsen) #50

added timestamping but in “human friendly” format and added the try to reading the temp sensor

its a bit late so i did not change the return 60 part, will do that tomorrow

#!/usr/bin/python
# -*- coding: utf-8 -*-

#SETTINGS START

start_time = '07:00'
end_time = '22:00'
bake_temp = float(25)
target1 = float(22)
target2 = float(17)

# Domain you want to post to: localhost would be an emoncms installation on your own laptop
# this could be changed to emoncms.org to post to emoncms.org
domain = "emoncms.org"

# Location of emoncms in your server, the standard setup is to place it in a folder called emoncms
# To post to emoncms.org change this to blank: ""
emoncmspath = ""

# Write apikey of emoncms account
apikey = "takeaguess"

# Node id youd like the emontx to appear as
nodeid = 1

#SETTINGS END

#IMPORTS START

# imports for thermometer reading test

import os
import glob
import time
import datetime

# import for internet connect check

import socket

# this needed to post to emocms

import sys, string
import httplib

#this one is for the wiringpi lib

import wiringpi

#IMPORTS END

# wiringpi numbers

wiringpi.wiringPiSetup()
wiringpi.pinMode(0, 1)  # sets pin 0 to output (GPIO 17, Actual hardware pin number is 11) (Relay)
wiringpi.pinMode(2, 1)  # sets pin 2 to output (GPIO 27, Actual hardware pin number is 13) (Internet connection LED)
wiringpi.pinMode(3, 0)  # sets pin 3 to input (GPIO 22, Actual hardware pin number is 15) (latching button)

# Find temperature from thermometer

os.system('modprobe w1-gpio')
os.system('modprobe w1-therm')

base_dir = '/sys/bus/w1/devices/'
device_folder = glob.glob(base_dir + '28*')[0]
device_file = device_folder + '/w1_slave'

csv = 0
target = 0
relay = 0
t_loop = 0
t15 = 0
test = 0
next_time = 0

# check if internet - just so we know if time can be trusted

def internet_connected(host='8.8.8.8', port=53):
    """
    Host: 8.8.8.8 (google-public-dns-a.google.com)
    OpenPort: 53/tcp
    Service: domain (DNS/TCP)
    """
    try:
        socket.setdefaulttimeout(20)
        socket.socket(socket.AF_INET,
                      socket.SOCK_STREAM).connect((host, port))
        return True
    except Exception, ex:
        print time.asctime( time.localtime(time.time()) ) + " This error occurred: " + str(ex)

    return False

# temp reading

def read_temp_raw():
    try:
        f = open(device_file, 'r')
        lines = f.readlines()
        f.close()
        return lines
    except  Exception, ex:
        print time.asctime( time.localtime(time.time()) ),
        print " Failed to read temp sensor, error is: " + str(ex)
        return False

def read_temp():
    if read_temp_raw():
        lines = read_temp_raw()
        while lines[0].strip()[-3:] != 'YES':
            time.sleep(0.2)
            lines = read_temp_raw()
        equals_pos = lines[1].find('t=')
        if equals_pos != -1:
            temp_string = (lines[1])[equals_pos + 2:]
            temp_c = float(temp_string) / 1000.0
            temp_f = temp_c * 9.0 / 5.0 + 32.0
            return temp_c  # , temp_f
    else:
        return 60

# schedule, check if current time is after schedule start and sets target

def schedule():

    global target

    t = time.strftime('%H:%M')
    if is_between(t, (start_time, end_time)) == True:
        print time.asctime( time.localtime(time.time()) ),
        print (' Running Daytime Schedule')
        target = target1
    else:
        print time.asctime( time.localtime(time.time()) ),
        print(' Running Nighttime Schedule')
        target = target2
  
def is_between(time, time_range):

    if time_range[1] < time_range[0]:
        return time >= time_range[0] or time <= time_range[1]
    return time_range[0] <= time <= time_range[1]

def emoncms():

    try:
        seq = (read_temp(), target, relay)
        str_join = ",".join(str(x) for x in seq)
        print time.asctime( time.localtime(time.time()) ),
        print ' Preparing Data for emoncms.org'
        conn = httplib.HTTPConnection(domain)
        conn.request("GET", "/"+emoncmspath+"/input/post.json?apikey="+apikey+"&node="+str(nodeid)+"&csv="+str_join)

        response = conn.getresponse()
        print time.asctime( time.localtime(time.time()) ),
        print ' Response from emoncms.org:',
        print response.read()
        print time.asctime( time.localtime(time.time()) ),
        print ' Data sent to emoncms'
    except Exception, ex:
        print time.asctime( time.localtime(time.time()) ),
        print ' This error occurred: ' + str(ex)
        
while True:
    my_input = wiringpi.digitalRead(3)

    if my_input == 1:
        print time.asctime( time.localtime(time.time()) ),
        print ' Switch is ON'
        t15 = 15

    time_now = time.time()

    if time_now >= next_time:
        print time.asctime( time.localtime(time.time()) ),
        print ' Start of loop: ' + str(test)

        temp_now = read_temp()
        print time.asctime( time.localtime(time.time()) ),
        print ' Current temp: ' + str(temp_now)

        schedule()

        print time.asctime( time.localtime(time.time()) ),
        print ' We have an internet connection? ' + str(internet_connected())
        if internet_connected() == True:
             wiringpi.digitalWrite(2, 0) # sets port 2 to OFF
        else:
             wiringpi.digitalWrite(2, 1) # sets port 2 to ON

        # Control heating
        if t15 > 0:
            target = bake_temp
            t15 -= 1
            if target > temp_now:        # Compare temp read to temp from DS18B20
                wiringpi.digitalWrite(0, 1) # sets port 0 to ON
                relay = 1
                print time.asctime( time.localtime(time.time()) ),
                print ' Target temp: ' + str(target)
                print time.asctime( time.localtime(time.time()) ),
                print ' HEATING ON (Boost)'
            else:
                wiringpi.digitalWrite(0, 0) # sets port 0 to OFF
                relay = 0
                print time.asctime( time.localtime(time.time()) ),
                print ' Target temp: ' + str(target)
                print time.asctime( time.localtime(time.time()) ),
                print ' HEATING OFF (Boost)'
        else:
            if target > temp_now:           # Compare temp read to temp from DS18B20
                wiringpi.digitalWrite(0, 1) # sets port 0 to ON
                relay = 1
                print time.asctime( time.localtime(time.time()) ),
                print ' Target temp: ' + str(target)
                print time.asctime( time.localtime(time.time()) ),
                print ' HEATING ON'
            else:
                wiringpi.digitalWrite(0, 0) # sets port 0 to OFF
                relay = 0
                print time.asctime( time.localtime(time.time()) ),
                print ' Target temp: ' + str(target)
                print time.asctime( time.localtime(time.time()) ),
                print ' HEATING OFF'

        # Calculate the next loop time by adding interval in secs
        next_time = time_now + 60

        emoncms()
        print time.asctime( time.localtime(time.time()) ),
        print " Remaining boost time: " + str(t15) + " mins"
        test += 1
        print time.asctime( time.localtime(time.time()) ),
        print ' End of loop'
        print time.asctime( time.localtime(time.time()) ),
        print ' --------------------'

    # Don't loop too fast
    time.sleep(1)

(Bo Herrmannsen) #51

this has run great for days now

i also have a 2nd pi that just runs speedtest every hour to track my internet connection

i have discovered that even thou the pi sends data to emoncms every 60 sec it can sometimes be 2-3 mins between the input is last updated and at same time internet connections is normal

is 40 degree cpu temp to high for a pi?

and i’m not sure yet how to add a ds18b20 more to read temp at desk level instead of the bottom/input slots for the heating panel

EDIT… the extra ds18b20 is for logging only, not control of any kind

i could of course sort the ouput from the read function so it looks for serial number, but then the scripts needs to be changed if a sensor is replaced

EDIT2… hmm it might be more easy to add serial numbers as a setting in the top… will thinker with that idea for some hours


(Bo Herrmannsen) #52

the button for the boost function… would it be possible in some way to have that on the dashboard instead?


(Paul) #53

It’s a long time since I used a button in emoncms, but if you can get a button and feed working together in emoncms, you can simply query the feed using a emoncms feed api instead of checking if the button is pressed.

The feed can be checked every few seconds since you do not need to catch the button press itself as the feed stores the state.