# By default, the calendar page only shows the current month,

# even when there are meetings scheduled for next month.

# To see anything from the upcoming month you have to set cookies

# in the HTTP request.

# If you do that manually, here are the cookies it sets:

# Setting-69-ASP.meetingdetail_aspx.gridMain.SortExpression=Sequence ASC; Setting-69-Calendar Options=info|; Setting-69-Calendar Year=Next Month; Setting-69-Calendar Body=All; Setting-69-ASP.calendar_aspx.gridCalendar.SortExpression=MeetingStartDate DESC; ASP.NET_SessionId=tmk5pfksowfid2t3nqjmpvac; BIGipServerprod_insite_443=874644234.47873.0000

# but with any luck, 'Next Month' is the only one that's actually needed.

# This has to be done before reading the default page,

# to match the decreasing date order of the meetings on each month's page.

if now.day > 20:

cookiedict = { 'Setting-69-Calendar Year': 'Next Month' }

r = requests.get(MEETING_LIST_URL, cookies=cookiedict)

parse_html_meeting_list(r.text, only_future)

# Get the meetings on the default (this month) page.

# These will be appended to the global list upcoming_meetings.

r = requests.get(MEETING_LIST_URL, timeout=30)

parse_html_meeting_list(r.text, only_future)

# The meeting list is in date/time order, latest first.

# Better to list them in the other order, starting with

# meetings today, then meetings tomorrow, etc.

# That's why we couldn't just write meetings from the earlier loop.

# Could sort by keys, 'Meeting Date' and 'Meeting Time',

# but since it's already sorted, it's easier just to reverse.

"""Parse the page listing meetings, which is HTML generated by pdftohtml.

soup = BeautifulSoup(page_html, 'lxml')

# Remove a bunch of spurious tags

for badtag in [ "font", "span", "div" ]:

badtags = soup.find_all(badtag)

for tag in badtags:

tag.replace_with_children()

caltbl = soup.find("table",

id="ctl00_ContentPlaceHolder1_gridCalendar_ctl00")

# The legend is in the thead

fieldnames = []

for i, field in enumerate(caltbl.thead.find_all("th")):

if field.text:

fieldnames.append(field.text.strip())

else:

fieldnames.append(str(i))

# Loop over meetings, rows in the table:

for row in caltbl.tbody.find_all("tr"):

dic = {}

# Loop over columns describing this meeting:

for i, field in enumerate(row.find_all("td")):

if fieldnames[i].startswith("Agenda"):

# If there's an Agenda URL, make it absolute.

a = field.find("a")

href = a.get("href")

if href:

dic[fieldnames[i]] = urljoin(MEETING_LIST_URL, href)

else:

dic[fieldnames[i]] = None

elif fieldnames[i] == 'Meeting Location':

# The Location field has simple formatting

# such as <br>, so can't just take .text, alas.

dic[fieldnames[i]] = ' '.join([str(c).strip()

for c in field.contents]) \

.strip()

# The little calendar icon somehow comes out with a name of '2'.

# Skip it.

elif fieldnames[i] == '2' or not fieldnames[i]:

continue

# Most fields are simple and won't have any formatting.

# They are full of nbsps '\u00a0', though.

else:

dic[fieldnames[i]] = field.text.replace('\u00a0', ' ').strip()

if "Meeting Date" in dic:

mtg_datetime = meeting_datetime(dic)

if mtg_datetime and only_future and mtg_datetime < utcnow:

continue

"""Parse the meeting date and time and return an aware local datetime.

# The parsed time is in the local time and is unaware,

# because strptime can't create a timezone aware object

# even if the string it's parsing includes a timezone (see above).

if "Meeting Time" not in mtg or not mtg["Meeting Time"]:

mtg["Meeting Time"] = NO_TIME

try:

if mtg["Meeting Time"] != NO_TIME:

unaware = datetime.datetime.strptime(mtg["Meeting Date"] + " "

+ mtg["Meeting Time"],

'%m/%d/%Y %I:%M %p')

else: # No time, so list it at 23:59

unaware = datetime.datetime.strptime(mtg["Meeting Date"],

'%m/%d/%Y')

unaware.replace(hour=23, minute=59, second=0)

# Make it aware in localtime

localtime = unaware.astimezone(localtz)

return localtime

except ValueError:

print("ERROR: Can't parse date on meeting:", mtg)

"""Diffs the two files, and returns an html fragment that wraps

if not title:

title = "Changed Agenda"

# lxml.html.htmldiff only accepts strings, not bytes, but these

# were read in as bytes because that's what comes from requests;

# so translate them.

if type(before_html) is bytes:

before_html = before_html.decode()

if type(after_html) is bytes:

after_html = after_html.decode()

# lxml.html.htmldiff returns fragments, not full documents.

# So add a header that includes a style for ins and del.

diff = '''<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">

''' % (title, title)

diff += htmldiff(before_html, after_html)

diff += "\n</body></html>\n"

# encode to return bytes.

if 'fitz' in sys.modules:

print("Using fitz")

return html_agenda_fitz(agendaloc, meetingtime, save_pdf_filename)

print("No fitz, using pdftohtml")

doc = fitz.open(agendaloc)

def find_indent_levels(pdfdoc):

indents = []

for page in pdfdoc.pages():

for block in page.get_text("blocks"):

indent = round(block[0])

if indent not in indents:

indents.append(indent)

indents.sort()

def group_clusters(lis, max_sep):

"""Reduce a list of numbers removing numbers that are

# clusters is a list of [ (low, high) ]

clusters = []

def add_to_clusters(l):

for c in clusters:

if l >= c[0] and l <= c[1]:

# the value is already represented by this cluster

return

# Okay, l is outside this cluster. But is it close?

# On the low end?

if l <= c[1] and l >= c[1] - max_sep:

c[0] = l

return

# Or on the high end?

if l >= c[0] and l <= c[0] + max_sep:

c[1] = l

return

# It's outside all of the known ranges.

# Add a new range.

clusters.append([l, l])

for l in lis:

add_to_clusters(l)

# Now clusters is a list of ranges. Take the average of each.

return [ int((c[0] + c[1])/2.) for c in clusters ]

return group_clusters(indents, 10)

indent_levels = find_indent_levels(doc)

print("Found indent levels:", indent_levels)

html = """<html>

""" % (meetingtime.strftime("%a %y-%m-%d"))

for page in doc.pages():

# blocks are like paragraphs in a normal PDF. Here, ??

# block is supposedly a tuple,

# (x0, y0, x1, y1, "lines in block", block_type, block_no)

# according to https://pymupdf.readthedocs.io/en/latest/app2.html

# but I think the last two are reversed, it's really no, type.

# flags=0 disables images, but might also disable other

# desirable things, so watch out.

# https://pymupdf.readthedocs.io/en/latest/app2.html#text-extraction-flags

# values include TEXT_PRESERVE_IMAGES, TEXT_PRESERVE_LIGATURES,

# TEXT_PRESERVE_SPANS, TEXT_PRESERVE_WHITESPACE.

blocks = page.get_text("blocks", flags=0)

for b in blocks:

# This clause isn't needed if TEXT_PRESERVE_IMAGES isn't set.

# if b[4].startswith("<image:"):

# print("Skipping an image")

# continue

# Is the indent close to the minimum indent?

# Then it's a header.

# Decide which level of header to use based on content.

if isclose(b[0], indent_levels[0], abs_tol=10):

print("HEADER:", b[4].replace('\n', ' '))

if re.match('[0-9]+\.\n', b[4]):

html += "<h1>%s</h1>\n<p>\n" % b[4]

elif re.match('[A-Z]+\.\n', b[4]):

html += "<h2>%s</h2>\n<p>\n" % b[4]

else:

html += "<p>%s</p>" % b[4]

elif isclose(b[0], indent_levels[1], abs_tol=10):

print(" ", b[4].replace('\n', ' '))

html += "<br>\n%s\n" % b[4]

else:

print(" OTHER INDENT:", b[4].replace('\n', ' '))

html += "<br><blockquote>\n%s</blockquote>\n" % b[4]

html += "</body></html>"

"""Convert a PDF agenda to text and/or HTML using pdftohtml,

if agendaloc.lower().startswith('http') and ':' in agendaloc:

r = requests.get(agendaloc, timeout=30)

with open(save_pdf_filename, "wb") as pdf_fp:

pdf_fp.write(r.content)

agendaloc = save_pdf_filename

htmlfile = save_pdf_filename + ".html"

args = [ "pdftohtml", "-c", "-s", "-i", "-noframes",

# "-enc", "utf-8",

save_pdf_filename, htmlfile ]

print("Calling", ' '.join(args))

subprocess.call(args)

"""Clean up the scary HTML written by pdftohtml,

with open(htmlfile, 'rb') as htmlfp:

# The files produced by pdftohtml contain '\240' characters,

# which are ISO-8859-1 for nbsp.

# Adding "-enc", "utf-8" doesn't change that.

# If they aren't decoded, BeautifulSoup will freak out

# and won't see anything in the file at all.

html_bytes = htmlfp.read().decode('ISO-8859-1')

# Make some changes. Primarily,

# replace the grey background that htmltotext wires in

soup = BeautifulSoup(html_bytes, "lxml")

body = soup.body

# Insert the meeting date at the beginning of the body

h_tag = soup.new_tag("h3")

soup.body.insert(0, h_tag)

datetext = NavigableString(meetingtime.strftime("%a %b %d"))

h_tag.append(datetext)

# Sometimes pdftohtml mysteriously doesn't work, and gives

# a basically empty HTML file: everything is using position:absolute

# and that makes it invisible to BeautifulSoup.

# This seems to be

# https://gitlab.freedesktop.org/poppler/poppler/-/issues/417

# Check for that.

# If all else fails, htmltotext works to extract the text,

# and might produce cleaner output anyway.

# Or there may be some way to get BS to find those

# <p style="position:absolute" tags that it isn't seeing.

bodylen = len(body.text.strip())

if bodylen == 0:

print("** Yikes! Empty HTML from pdftohtml", htmlfile)

return html

elif bodylen < 10:

print(f"Short! Body text is: '{body.text}'")

del body["bgcolor"]

del body["vlink"]

del body["link"]

# Remove all the fixed pixel width styles

for tag in body.find_all('style'):

tag.decompose()

for tag in body.find_all('div'):

del tag["style"]

for tag in body.find_all('p'):

del tag["style"]

# Get rid of the pagination

pagediv_pat = re.compile('page[0-9]*-div')

divs = list(body.find_all(id=pagediv_pat))

for div in divs:

div.replace_with_children()

# There are also anchors like <a name="8">\n</a>

# but they're not really hurting anything.

# Remove hard line breaks. This is a tough decision:

# some line breaks help readability, some hurt it.

# for tag in body.find_all('br'):

# tag.decompose()

# pdftohtml renders each line as a separate paragraph,

# so joining paragraphs helps readability.

# Call join_consecutive_tags starting with outer tags and working inward.

# Do this while the p tags still have classes, so paragraphs of

# different classes don't get merged.

join_consecutive_tags(body, 'p')

# return soup.prettify(encoding='utf-8')

join_consecutive_tags(body, 'i')

join_consecutive_tags(body, 'b')

# Now don't need the class tags any more, so delete them.

for tag in body.find_all('p'):

del tag["class"]

# Try to identify major headers, to highlight them better.

header_pat = re.compile('([0-9]+)\.\s*([A-Z \(\)\/,]+)$', flags=re.DOTALL)

# This doesn't work. I don't know why. find_all(text=pat) works fine

# in simple test cases, but not on the real files.

# for b in soup.find_all('b', text=header_pat):

# Instead, loop over all b tags doing the match explicitly:

for bold in body.find_all('b'):

m = re.match(header_pat, bold.get_text().strip())

if not m:

continue

# Can't change text like this

# b.text = f"## {m.groups(1)}. {m.groups(2)}"

# but we can change the tag name:

bold.name = 'h2'

pretty_html_bytes = soup.prettify(encoding='utf-8')

# Testing: maybe the above changes removed the body contents?

# (I think this bug is long since fixed.)

if not body.text:

print("**Yikes! The changes to", save_pdf_file,

"made the HTML empty. Saving original instead.")

with open(os.path.join(RSS_DIR, save_pdf_filename + "_cleaned.html"),

"wb") as savfp:

savefp.write(pretty_html_bytes)

return html

"""Join consecutive tags of name tag if they have the same attributes.

to_merge = []

tags = list(soup.find_all(tagname))

prev = None

sectionnum_pat = re.compile('[0-9A-Z]{,2}\.')

for tag in tags:

prev = tag.find_previous_sibling()

# If the two tags have the same parent and the same class,

# they should be merged.

if prev and prev.attrs == tag.attrs:

# First merge in the list?

if not to_merge:

to_merge.append([prev, tag])

continue

else:

# Should these be merged with the last merge?

last_group = to_merge[-1]

last_tag_merged = last_group[-1]

prev_sib = prev.find_previous_sibling()

# SPECIAL CASE FOR LEGISTAR:

# Does it look like a section header?

# Don't merge a paragraph that looks like "2."

# with whatever was before it.

if tag.name == 'p' and re.match(sectionnum_pat, tag.text):

continue

elif (prev == last_tag_merged and

tag.attrs == last_tag_merged.attrs):

# Continue a group merge of three or more tags

last_group.append(tag)

else:

# New pair of mergers, make a new group

to_merge.append([prev, tag])

prev = tag

for group in to_merge:

first = group[0]

for tag in group[1:]:

# Iterating directly over tag.children gets crossed up

# when some of the children are moved to another tag.

children = list(tag.children)

for child in children:

first.append(child)

# All of tag's children have been moved to first.

# Delete tag.

"""Take a list meeting dictionaries

active_meetings = []

##############

# Generate index HTML and RSS file headers.

# Open both the RSS and HTML files:

outrssfilename = os.path.join(RSS_DIR, "index.rss")

outhtmlfilename = os.path.join(RSS_DIR, "index.html")

with open(outrssfilename, 'w') as rssfp, \

open(outhtmlfilename, 'w') as htmlfp:

print("\n==== Generating RSS for", len(mtglist), "meetings")

gendate = now.strftime(RSS_DATE_FORMAT)

print(f"""<?xml version="1.0" encoding="iso-8859-1" ?>

""",

file=rssfp)

print(f"""<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">

""", file=htmlfp)

for mtg in mtglist:

# Is the meeting in the future? Don't list past meetings.

meetingtime = meeting_datetime(mtg)

if not meetingtime:

print("Skipping", mtg["Name"], mtg["Meeting Date"],

"because of bad meeting datetime",

mtg["Meeting Date"], mtg["Meeting Date"])

continue

if meetingtime < now:

print("Skipping", mtg["Name"], mtg["Meeting Date"],

"because", meetingtime, "<", now)

continue

lastmod = None

mtg['cleanname'] = mtgdic_to_cleanname(mtg)

cleanname = mtg['cleanname']

print("\n====", cleanname)

if mtg["Agenda"]: # There is an agenda URL listed

print(cleanname, "has an agenda: fetching it")

# XXX TEMPORARY: save the PDF filename, because sometimes

# pdftohtml produces an HTML file with no content even

# though there's content in the PDF.

pdfout = os.path.join(RSS_DIR, cleanname + ".pdf")

try:

agenda_html = agenda_to_html(mtg["Agenda"],

meetingtime,

save_pdf_filename=pdfout)

except ReadTimeoutError:

print("Timed out on " + agendaloc)

agenda_html = NO_AGENDA

agendastatus = "timeout"

else: # No previous agenda

print(cleanname, "has no agenda to fetch")

agenda_html = NO_AGENDA

# Now agenda_html is guaranteed to be set, either way.

# Might need a diff file too:

agenda_diff = None

# Does the agenda file need to be (re)written?

write_agenda_file = False

# See if there was already an agenda file left from previous runs:

agendafile = os.path.join(RSS_DIR, cleanname + ".html")

# print("agenda filename", agendafile)

try:

with open(agendafile, "rb") as oldfp:

oldhtml = oldfp.read()

if NO_AGENDA in oldhtml: # no agenda previously

if Verbose:

print("No agenda previously")

if agenda_html != NO_AGENDA: # but there is now

write_agenda_file = True

agendastatus = "new"

print(cleanname, ": new agenda")

else:

agendastatus = "no"

print(cleanname, ": no agenda")

else: # there was a previous agenda

if Verbose:

print("There was a previous agenda: ===============")

if agenda_html == NO_AGENDA:

print(oldhtml)

print("========= but now, ===============")

print(agenda_html)

if not agendastatus:

agendastatus = "removed"

print(cleanname, ": removed agenda")

# don't write over the old agenda file

write_agenda_file = False

elif agenda_html != oldhtml: # changed agenda

write_agenda_file = True

# XXX TEMPORARY: save the previous file,

# to have them available while debugging diffs.

os.rename(agendafile,

os.path.join(RSS_DIR,

cleanname + "-old.html"))

# End temporary clause

# Since the agenda has changed, make a diff file

# highlighting the parts that changed.

agenda_diff = diffhtml(oldhtml, agenda_html,

title=cleanname)

agenda_diff_file = os.path.join(RSS_DIR,

mtg['cleanname'] + "-diff.html")

with open(agenda_diff_file, 'wb') as difffp:

difffp.write(agenda_diff)

agendastatus = "changed"

print(cleanname, ": changed agenda")

else:

agendastatus = "unchanged"

print(cleanname, ": unchanged agenda")

except FileNotFoundError:

# No agenda file there previously; probably a new meeting

if Verbose:

print("No previous agenda file")

write_agenda_file = True

if agenda_html == NO_AGENDA:

if Verbose:

print("... and no new agenda now")

agendastatus = "no"

else:

if Verbose:

print("but there's an agenda there")

agendastatus = "new"

# Now agenda_html and agendastatus should be set,

# and maybe agenda_diff too.

# Write the agenda file if it's changed:

if write_agenda_file:

with open(agendafile, 'wb') as outfp:

outfp.write(agenda_html)

# Either way, this meeting is still listed:

# note it so it won't be cleaned from the directory.

active_meetings.append(cleanname)

# Get the old JSON

changestr = ""

jsonfile = os.path.join(RSS_DIR, mtg['cleanname'] + ".json")

try:

with open(jsonfile) as jsonfp:

oldmtg = json.loads(jsonfp.read())

# mtg doesn't have lastmod, so to make sure that

# doesn't trigger a change, copy it:

mtg['lastmod'] = oldmtg['lastmod']

changed_keys = {key for key in oldmtg.keys() & mtg

if oldmtg[key] != mtg[key]}

if changed_keys:

lastmod = now

changestr = "<p>Changed: " + ', '.join(changed_keys) \

+ "</p>"

print("Keys changed:", changed_keys, "lastmod is", lastmod)

elif not lastmod:

print("Nothing has changed, keeping lastmod")

lastmod = datetime.datetime.strptime(oldmtg['lastmod'],

RSS_DATE_FORMAT)

except (RuntimeError, OSError) as e:

if os.path.exists(jsonfile):

print("Error reading jsonfile: %s" % e)

changestr += "Error reading jsonfile: %s\n<p>" % e

elif Verbose:

print("No JSON file there previously")

lastmod = now

changestr += "<b>New listing.</b>\n<p>"

# Update to the real lastmod date before writing JSON

mtg['lastmod'] = lastmod.strftime(RSS_DATE_FORMAT)

mtg['GUID'] = cleanname + '.' + lastmod.strftime("%Y%m%d-%H%M")

# If the meeting is new or something has changed,

# (re)write the JSON file..

with open(jsonfile, 'w') as jsonfp:

jsonfp.write(json.dumps(mtg, indent=4))

mtgtitle = f"""{mtg['Name']} on {meetingtime.strftime("%a %b %d")}"""

desc = f"""{mtg['Name']}: {mtg['Meeting Date']} at {mtg['Meeting Time']}<br />

# Set up the change strings for the header and body

if agendastatus == "new":

agenda_hdr = " (AGENDA)"

desc += "<p><b>There is a new agenda.</b>"

elif agendastatus == "removed":

agenda_hdr = " (REMOVED AGENDA)"

desc += "<p><b>The agenda has been removed.</b>"

elif agendastatus == "changed":

agenda_hdr = " (CHANGED AGENDA)"

desc += "<p><b>The agenda has changed.</b>"

elif agendastatus == "unchanged":

agenda_hdr = ""

desc += "<p>The agenda hasn't changed."

elif agendastatus == "no":

agenda_hdr = " (no agenda)"

desc += "<p>No agenda yet."

if mtg['Meeting Location']:

desc += "<p>Location:" + mtg['Meeting Location']

if changestr:

desc += "<p>" + changestr + '\n'

if agenda_diff:

link = f"{RSS_URL}{cleanname}-diff.html"

else:

link = f"{RSS_URL}{cleanname}.html"

if mtg["Agenda"]:

desc = f"""{desc}<p>

if mtg["Agenda Packets"]:

# The agenda packet links tend to have & in them

# and so need to be escaped with CDATA

if 'http' in mtg["Agenda Packets"]:

desc += f"""<p><a href="{mtg["Agenda Packets"]}">Agenda Packet</a></p>\n"""

else:

desc = f"""<p>Agenda packet: {mtg["Agenda Packets"]}</p>\n"""

print("GUID will be", mtg['GUID'], "lastmod is", mtg['lastmod'])

# Add the item to the RSS

print(f"""<item>

</item>""", file=rssfp)

# And add it to the HTML

print(f"<p><h2>{mtgtitle} {agenda_hdr}</h2>", file=htmlfp)

if mtg["Agenda"]:

print(f'<p><b><a href="{link}">Agenda: {mtgtitle}</a></b>',

file=htmlfp)

# else:

# print("<p>No agenda yet", file=htmlfp)

print(f"""<p>

""",

file=htmlfp)

print("</channel>\n</rss>", file=rssfp)

print("</body></html>", file=htmlfp)

print("Wrote", outrssfilename, "and", outhtmlfilename)

# Remove obsolete files for meetings no longer listed.

for f in os.listdir(RSS_DIR):

# Only clean up certain extensions:

rmexts = [ '.json', '.rss', '.html', '.pdf' ]

name, ext = os.path.splitext(f)

if ext not in rmexts:

continue

# Protected files

if f.startswith("index") or f.startswith("about"):

continue

def is_active(f):

for act in active_meetings:

if f.startswith(act):

return True

return False

if not is_active(f):

print("removing", f)

"""A standard way to turn date and committee name into something

mtg_dt = meeting_datetime(mtgdic)

if mtg_dt:

return mtg_dt.strftime("%Y-%m-%d") + '-' \

+ clean_filename(mtgdic["Name"])