#!/usr/bin/env python3

# Take dictionaries, fetched from a web API or wherever,

# cache them locally, and allow access that will transparently

# read from cache, if available, or fetch from the API.

# The cache will be a series of CSV files, one per day,

# with the date as one of the fields.

# To use it, subclass from Cachefile and redefine the following:

# fetch_one_day_data(self, day=None):

# fetch one day's worth of data from the web API or wherever.

# Will fetch for whatever day contains the given datetime.

# apply_types(self, row):

# Given a row that was just read in as strings, change the items

# to appropriate types, e.g. int, float, datetime etc.

# self.DATE: the name for the date field (default 'date').

### Some tweaks to make it work with Python2, for web servers without 3:

from __future__ import print_function

try:

PermissionError

except NameError:

PermissionError = OSError

try:

FileNotFoundError

except NameError:

FileNotFoundError = OSError

### end Python2 tweaks

import datetime

import csv

import os

class Cachefile(object):

def __init__(self, cachedir):

self.TIME = 'time'

self.verbose = True

self.fieldnames = None

self.writer = None

if cachedir.startswith('/'):

self.cachedir = cachedir

else:

self.cachedir = None

# cachedir can be in ~/.cache/cachedir, or /var/cache/cachedir

userdir = os.path.expanduser('~/.cache')

for d in [ userdir, '/var/cache' ]:

dpath = os.path.join(d, cachedir)

if os.path.exists(dpath):

self.cachedir = dpath

break

if not self.cachedir:

self.cachedir = os.path.join(userdir, cachedir)

def write_cache_file(self, day_data):

"""Write (or overwrite) the whole cache file for the given day

Otherwise, will overwrite the whole file.

"""

if not self.fieldnames:

self.fieldnames = list(day_data[0].keys())

# Make sure the data doesn't span more than one day.

if day_data[0][self.TIME].day != day_data[-1][self.TIME].day:

print("Can't cache data for multiple days: %s - %s" % (

day_data[0][self.TIME].strftime('%Y-%M-%d'),

day_data[-1][self.TIME].strftime('%Y-%M-%d')))

return

if not os.path.exists(self.cachedir):

os.makedirs(self.cachedir)

cachefile = os.path.join(self.cachedir,

day_data[-1][self.TIME].strftime('%Y-%m-%d')

+ ".csv")

# Write to a temporary copy then move it into place.

# This is deliberately not unique -- it'll be chmodded

# as a locking mechanism, so only one instance of Cachefile

# can make changes at once.

tmpfile = cachefile + ".new"

try:

cachefp = self.open_cache_file(tmpfile)

for item in day_data:

self.write_cache_line(cachefp, item)

cachefp.close()

self.writer = None

if self.verbose:

print("Wrote cache file", cachefile)

os.rename(tmpfile, cachefile)

except PermissionError:

print("Can't update temp file %s, it's locked" % tmpfile)

os.system("ls -la " + self.cachedir)

def open_cache_file(self, cachefile):

# To avoid race conditions, use os.open to create the file

# with a mode so that no one else can write to it.

fd = os.open(cachefile, os.O_CREAT | os.O_WRONLY, 0o444)

# In Python3, open() will take an integer file descriptor.

# In Python2, that fails with

# TypeError: coercing to Unicode: need string or buffer, int found

# but there's os.fdopen.

try:

fp = open(fd, 'w')

except TypeError:

fp = os.fdopen(fd, 'w')

# In Python2, CSV adds \r as part of the line terminator

# even on Unix, which breaks the automated tests.

self.writer = csv.DictWriter(fp, fieldnames=self.fieldnames,

lineterminator='\n')

self.writer.writeheader()

return fp

def write_cache_line(self, cachefp, item):

"""Write one interval (one line) to a cache file.

"""

self.writer.writerow(item)

# Cache files are organized by date, one per day:

# ~/.cache/enphase/YYYY-MM-DD.dat

# Inside each file are CSV lines: time,powr,enwh,devices_reporting

def read_cache_file(self, day=None):

"""Read the cache file for the given day (a datetime)

and return the cache file name plus a list of dictionaries,

one for each line.

"""

if not day:

day = datetime.datetime.now()

cachefile = os.path.join(self.cachedir,

day.strftime('%Y-%m-%d') + ".csv")

if self.verbose:

print("Trying to read cache file", cachefile)

data = []

try:

with open(cachefile) as csvfp:

reader = csv.DictReader(csvfp)

for row in reader:

# csv.DictReader reads everything as strings. Convert back.

self.apply_types(row)

data.append(row)

except (FileNotFoundError, OSError, IOError):

# File isn't there yet, first run of the day?

pass

return cachefile, data

def fetch_one_day_data(self, day):

"""fetch_data, undefined in this base class, should fetch

one day's data using whatever web or other API is appropriate.

Override it in your derived class.

"""

raise NotImplementedError("Don't know how to fetch: "

"override fetch_data in subclasses")

def day_start(self, day):

"""Given a datetime, return the beginning of that day as a datetime.

"""

return day.replace(hour=0, minute=0, second=0, microsecond=0)

def day_end(self, day):

"""Given a datetime, return the beginning of that day as a datetime.

"""

return day.replace(hour=23, minute=59, second=59, microsecond=0)

@staticmethod

def parse_time(timestr):

try:

return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M')

except ValueError:

pass

try:

return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S')

except ValueError:

pass

# If those didn't work, and this doesn't either,

# go ahead and throw a ValueError.

return datetime.datetime.strptime(timestr, '%Y-%m-%d %H:%M:%S.%f')

def time_bounds(self, starttime=None, endtime=None, day=None, now=None):

"""If day is specified, return the beginning and end of that day.

Otherwise, if no starttime, set it to the beginning of today.

If no endtime, set it to the end of today.

In every case, restrict the time range to a single day,

and don't allow anything later than now.

now can be specified because it may need to be set back by some

amount, like 10 minutes, depending on how often the API updates.

"""

if not now:

now = datetime.datetime.now()

if day:

starttime = self.day_start(day)

endtime = self.day_end(day)

elif not starttime and not endtime:

# Set back to the beginning of the day:

starttime = self.day_start(now)

# and end now.

endtime = now

elif not starttime:

starttime = self.day_start(endtime)

elif not endtime:

# If we're starting today, end now:

if starttime.year == now.year and starttime.month == now.month \

and starttime.day == now.day:

endtime = now

# Else end at the end of the day we started:

else:

endtime = self.day_end(starttime)

elif starttime.year != endtime.year \

or starttime.month != endtime.month \

or starttime.day != endtime.day:

raise ValueError("time_bounds: %s and %s must start and end on the same day" % (endtime, starttime))

if starttime > endtime:

raise ValueError("endtime %s can't be earlier than starttime %s"

% (endtime, starttime))

if endtime > now:

endtime = now

return starttime, endtime

def get_data(self, starttime=None, endtime=None):

"""Get a block of data between two datetimes,

reading from cache when possible, otherwise fetching from API

and writing new cache files.

starttime defaults to midnight today.

endtime defaults to now, or the end of the day of starttime.

"""

data = []

if not endtime:

if starttime:

endtime = self.day_start(starttime)

else:

endtime = datetime.datetime.now()

if not starttime:

starttime = self.day_start(endtime)

# Loop over days, fetching one day's data at a time:

while True:

cachefile, cached_data = self.read_cache_file(starttime)

# Do we already have enough cached?

try:

modtime = datetime.datetime.fromtimestamp(os.stat(cachefile).st_mtime)

except:

# Set a very early date:

modtime = datetime.datetime(1, 1, 1, 0, 0)

if cached_data and \

modtime >= (endtime - datetime.timedelta(minutes=10)):

data += cached_data

if self.verbose:

print("We already have enough cached. Hooray!")

else:

new_data = self.fetch_one_day_data(starttime)

if self.verbose:

print("Fetched data from API", cachefile)

# If the data is new, re-write the cache file,

# protecting it with chmod though that still allows

# for race conditions.

# What's considered new?

if cached_data:

lastcache = cached_data[-1][self.TIME]

else:

lastcache = endtime.replace(hour=0, minute=0,

second=0, microsecond=0)

if new_data:

self.write_cache_file(new_data)

data += new_data

# Next day.

starttime += datetime.timedelta(days=1)

if starttime >= endtime:

break

return data

if __name__ == '__main__':

print("No main routine: Cachefile is only useful if you subclass it.")