import os

import sys

import inspect

currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))

sys.path.insert(0, currentdir)

from sysadmws_common import *

from gsuite_scripts import *

import gitlab

import glob

import psycopg2

import textwrap

from datetime import datetime

from datetime import timedelta

from datetime import time

from dateutil.relativedelta import relativedelta

from num2words import num2words

import re

from zipfile import ZipFile

import subprocess

import woocommerce

import paramiko

WORK_DIR = os.environ.get("ACC_WORKDIR", "/opt/sysadmws/accounting")

LOG_DIR = os.environ.get("ACC_LOGDIR", "/opt/sysadmws/accounting/log")

LOG_FILE = "accounting.log"

TARIFFS_SUBDIR = "tariffs"

CLIENTS_SUBDIR = "clients"

YAML_GLOB = "*.yaml"

YAML_EXT = "yaml"

DB_STRUCTURE_FILE = "accounting_db_structure.sql"

ACC_YAML = "accounting.yaml"

INVOICE_TYPES = ["Hourly", "Monthly", "Storage"]

def calculate_range_size(range_id):

return range_size

def download_pdf_package(acc_yaml_dict, client_dict, client_folder_files, invoice_type, invoice_number, invoice_needed, act_needed, pack_to_archive, double_act):

raise

if __name__ == "__main__":

# Set parser and parse args

parser = argparse.ArgumentParser(description='{LOGO} functions.'.format(LOGO=LOGO))

parser.add_argument("--debug", dest="debug", help="enable debug", action="store_true")

parser.add_argument("--debug-gitlab", dest="debug_gitlab", help="enable gitlab api debug", action="store_true")

parser.add_argument("--no-exceptions-on-label-errors", dest="no_exceptions_on_label_errors", help="use with dry-runs to bulk check label errors", action="store_true")

parser.add_argument("--dry-run-db", dest="dry_run_db", help="do not commit to database", action="store_true")

parser.add_argument("--dry-run-gitlab", dest="dry_run_gitlab", help="no new objects created in gitlab", action="store_true")

parser.add_argument("--dry-run-gsuite", dest="dry_run_gsuite", help="no new objects created in gsuite", action="store_true")

parser.add_argument("--dry-run-print", dest="dry_run_print", help="no print commands executed", action="store_true")

parser.add_argument("--dry-run-woocommerce", dest="dry_run_woocommerce", help="no woocommerce api commands executed", action="store_true")

parser.add_argument("--timelogs-spent-before-date", dest="timelogs_spent_before_date", help="select unchecked timelogs for hourly invoices spent before date DATE", nargs=1, metavar=("DATE"))

parser.add_argument("--at-date", dest="at_date", help="use DATETIME instead of now for tariff", nargs=1, metavar=("DATETIME"))

group = parser.add_mutually_exclusive_group(required=False)

group.add_argument("--exclude-clients", dest="exclude_clients", help="exclude clients defined by JSON_LIST from all-clients operations", nargs=1, metavar=("JSON_LIST"))

group.add_argument("--include-clients", dest="include_clients", help="include only clients defined by JSON_LIST for all-clients operations", nargs=1, metavar=("JSON_LIST"))

group = parser.add_mutually_exclusive_group(required=True)

group.add_argument("--db-structure", dest="db_structure", help="create database structure", action="store_true")

group.add_argument("--yaml-check", dest="yaml_check", help="check yaml structure", action="store_true")

group.add_argument("--asset-labels", dest="asset_labels", help="sync asset labels", action="store_true")

group.add_argument("--issues-check", dest="issues_check", help="report issue activities as new issue in accounting project", action="store_true")

group.add_argument("--merge-requests-check", dest="merge_requests_check", help="report MR activities as new issue in accounting project", action="store_true")

group.add_argument("--storage-usage", dest="storage_usage", help="save all clients billable storage usage to database, excluding --exclude-clients or only for --include-clients", action="store_true")

group.add_argument("--report-hourly-employee-timelogs", dest="report_hourly_employee_timelogs", help="check new timelogs for EMPLOYEE_EMAIL and report them as new issue", nargs=1, metavar=("EMPLOYEE_EMAIL"))

group.add_argument("--update-envelopes-for-client", dest="update_envelopes_for_client", help="update envelope pdfs in envelopes folder for client CLIENT", nargs=1, metavar=("CLIENT"))

group.add_argument("--update-envelopes-for-all-clients", dest="update_envelopes_for_all_clients", help="update envelope pdfs in envelopes folder for all clients excluding --exclude-clients or only for --include-clients", action="store_true")

group.add_argument("--make-pdfs-for-client", dest="make_pdfs_for_client", help="make pdfs for client CLIENT folder for docs that do not have pdf copy yet", nargs=1, metavar=("CLIENT"))

group.add_argument("--make-pdfs-for-all-clients", dest="make_pdfs_for_all_clients", help="make pdfs for every client folder for docs that do not have pdf copy yet", action="store_true")

group.add_argument("--make-gmail-drafts-for-client", dest="make_gmail_drafts_for_client", help="make GMail drafts with PDFs of invoices with status Prepared/Sent for CLIENT", nargs=1, metavar=("CLIENT"))

group.add_argument("--make-gmail-drafts-for-all-clients", dest="make_gmail_drafts_for_all_clients", help="make GMail drafts with PDFs of invoices with status Prepared/Sent for all clients", action="store_true")

group.add_argument("--print-papers-for-client", dest="print_papers_for_client", help="print invoice papers with status not Printed for CLIENT", nargs=1, metavar=("CLIENT"))

group.add_argument("--print-papers-for-all-clients", dest="print_papers_for_all_clients", help="print invoice papers with status not Printed for all clients", action="store_true")

group.add_argument("--make-hourly-invoice-for-client", dest="make_hourly_invoice_for_client", help="check new timelogs for hourly issues or MRs in projects of CLIENT and make invoice", nargs=1, metavar=("CLIENT"))

group.add_argument("--make-hourly-invoice-for-all-clients", dest="make_hourly_invoice_for_all_clients", help="check new timelogs for hourly issues or MRs in projects of all clients and make invoice for each", action="store_true")

group.add_argument("--make-monthly-invoice-for-client", dest="make_monthly_invoice_for_client", help="make monthly invoice for month +MONTH by current month for client CLIENT", nargs=2, metavar=("CLIENT", "MONTH"))

group.add_argument("--make-monthly-invoice-for-all-clients", dest="make_monthly_invoice_for_all_clients", help="make monthly invoice for month +MONTH by current month for all clients excluding --exclude-clients or only for --include-clients", nargs=1, metavar=("MONTH"))

group.add_argument("--make-storage-invoice-for-client", dest="make_storage_invoice_for_client", help="compute monthly storage usage for month -MONTH of CLIENT and make invoice", nargs=2, metavar=("CLIENT", "MONTH"))

group.add_argument("--make-storage-invoice-for-all-clients", dest="make_storage_invoice_for_all_clients", help="compute monthly storage usage for month -MONTH of all clients and make invoice for each", nargs=1, metavar=("MONTH"))

group.add_argument("--list-assets-for-client", dest="list_assets_for_client", help="list assets for CLIENT", nargs=1, metavar=("CLIENT"))

group.add_argument("--list-assets-for-all-clients", dest="list_assets_for_all_clients", help="list assets for all clients", action="store_true")

group.add_argument("--count-assets", dest="count_assets", help="add a new record to the asset_count table", action="store_true")

group.add_argument("--count-timelog-stats", dest="count_timelog_stats", help="add a new record to the timelogs_stats table", action="store_true")

if len(sys.argv) > 1:

args = parser.parse_args()

else:

parser.print_help()

sys.exit(1)

# Set logger and console debug

if args.debug:

logger = set_logger(logging.DEBUG, LOG_DIR, LOG_FILE)

else:

logger = set_logger(logging.ERROR, LOG_DIR, LOG_FILE)

# Skip vars check where not needed

if not (args.yaml_check or args.list_assets_for_client is not None or args.list_assets_for_all_clients):

PG_DB_HOST = os.environ.get("PG_DB_HOST")

if PG_DB_HOST is None:

raise Exception("Env var PG_DB_HOST missing")

PG_DB_NAME = os.environ.get("PG_DB_NAME")

if PG_DB_NAME is None:

raise Exception("Env var PG_DB_NAME missing")

PG_DB_USER = os.environ.get("PG_DB_USER")

if PG_DB_USER is None:

raise Exception("Env var PG_DB_USER missing")

PG_DB_PASS = os.environ.get("PG_DB_PASS")

if PG_DB_PASS is None:

raise Exception("Env var PG_DB_PASS missing")

if not (args.yaml_check or args.list_assets_for_client is not None or args.list_assets_for_all_clients or args.db_structure):

GL_ADMIN_PRIVATE_TOKEN = os.environ.get("GL_ADMIN_PRIVATE_TOKEN")

if GL_ADMIN_PRIVATE_TOKEN is None:

raise Exception("Env var GL_ADMIN_PRIVATE_TOKEN missing")

GL_BOT_PRIVATE_TOKEN = os.environ.get("GL_BOT_PRIVATE_TOKEN")

if GL_BOT_PRIVATE_TOKEN is None:

raise Exception("Env var GL_BOT_PRIVATE_TOKEN missing")

GL_PG_DB_HOST = os.environ.get("GL_PG_DB_HOST")

if GL_PG_DB_HOST is None:

raise Exception("Env var GL_PG_DB_HOST missing")

GL_PG_DB_NAME = os.environ.get("GL_PG_DB_NAME")

if GL_PG_DB_NAME is None:

raise Exception("Env var GL_PG_DB_NAME missing")

GL_PG_DB_USER = os.environ.get("GL_PG_DB_USER")

if GL_PG_DB_USER is None:

raise Exception("Env var GL_PG_DB_USER missing")

GL_PG_DB_PASS = os.environ.get("GL_PG_DB_PASS")

if GL_PG_DB_PASS is None:

raise Exception("Env var GL_PG_DB_PASS missing")

SA_SECRETS_FILE = os.environ.get("SA_SECRETS_FILE")

if SA_SECRETS_FILE is None:

raise Exception("Env var SA_SECRETS_FILE missing")

SSH_DU_S_M_KEYFILE = os.environ.get("SSH_DU_S_M_KEYFILE")

if SSH_DU_S_M_KEYFILE is None:

raise Exception("Env var SSH_DU_S_M_KEYFILE missing")

SSH_DU_S_M_USER = os.environ.get("SSH_DU_S_M_USER")

if SSH_DU_S_M_USER is None:

raise Exception("Env var SSH_DU_S_M_USER missing")

# Catch exception to logger

try:

logger.info("Starting {LOGO}".format(LOGO=LOGO))

# Chdir to work dir

os.chdir(WORK_DIR)

# Skip pgconnect where not needed

if not (args.yaml_check or args.list_assets_for_client is not None or args.list_assets_for_all_clients):

# Connect to PG

dsn = "host={} dbname={} user={} password={}".format(PG_DB_HOST, PG_DB_NAME, PG_DB_USER, PG_DB_PASS)

conn = psycopg2.connect(dsn)

# Read ACC_YAML

acc_yaml_dict = load_yaml("{0}/{1}".format(WORK_DIR, ACC_YAML), logger)

if acc_yaml_dict is None:

raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, ACC_YAML))

# Do tasks

# If --at-date is set use it instead of today and now

if args.at_date is not None:

used_today = datetime.strptime(args.at_date[0], "%Y-%m-%d")

used_now = datetime.strptime(args.at_date[0], "%Y-%m-%d")

else:

used_today = datetime.today()

used_now = datetime.now()

if args.exclude_clients is not None:

json_str, = args.exclude_clients

exclude_clients_list = json.loads(json_str)

else:

exclude_clients_list = []

if args.include_clients is not None:

json_str, = args.include_clients

include_clients_list = json.loads(json_str)

else:

include_clients_list = []

if args.db_structure:

# New cursor

cur = conn.cursor()

# Queries

sql = load_file_string("{0}/{1}".format(WORK_DIR, DB_STRUCTURE_FILE), logger)

try:

cur.execute(sql)

logger.info("Query execution status:")

logger.info(cur.statusmessage)

conn.commit()

except Exception as e:

raise Exception("Caught exception on query execution")

# Close cursor

cur.close()

if args.storage_usage:

errors = False

# For *.yaml in client dir

for client_file in sorted(glob.glob("{0}/{1}".format(CLIENTS_SUBDIR, YAML_GLOB))):

logger.info("Found client file: {0}".format(client_file))

# Load client YAML

client_dict = load_client_yaml(WORK_DIR, client_file, CLIENTS_SUBDIR, YAML_GLOB, logger)

if client_dict is None:

raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, client_file))

# Check if client is active

if client_dict["active"] and (

(

args.exclude_clients is not None

and

client_dict["name"].lower() not in exclude_clients_list

)

or

(

args.include_clients is not None

and

client_dict["name"].lower() in include_clients_list

)

or

(

args.exclude_clients is None

and

args.include_clients is None

)

):

asset_list = get_asset_list(client_dict, WORK_DIR, TARIFFS_SUBDIR, logger, used_now)

# If there are assets

if len(asset_list) > 0:

# Iterate over assets in client

for asset in asset_list:

if asset["active"]:

logger.info("Active asset: {0}".format(asset["fqdn"]))

if "storage" in asset:

for storage_item in asset["storage"]:

for storage_asset, storage_paths in storage_item.items():

for storage_path in storage_paths:

# Compute path usage

# As we have ssh cmd restrictions we need only to supply path as command

cmd = "{folder}".format(folder=storage_path)

logger.info("SSH cmd: {storage_asset}:{cmd}".format(storage_asset=storage_asset, cmd=cmd))

# Clear value

mb_used = None

try:

private_key = paramiko.Ed25519Key.from_private_key_file(SSH_DU_S_M_KEYFILE)

ssh_client = paramiko.SSHClient()

ssh_client.load_system_host_keys()

ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())

ssh_client.connect(hostname=storage_asset, username=SSH_DU_S_M_USER, pkey=private_key)

stdin, stdout, stderr = ssh_client.exec_command(cmd)

if stdout.channel.recv_exit_status() != 0:

logger.error("SSH exit code is not 0")

logger.error("SSH stderr:")

for line in iter(stderr.readline, ""):

logger.error(line)

ssh_client.close()

errors = True

else:

logger.info("SSH value received via stdout:")

mb_used = int("".join(stdout.readlines()))

logger.info(mb_used)

# Save usage to db

# New cursor

cur = conn.cursor()

# Queries

sql = """

""".format(client_asset_fqdn=asset["fqdn"], storage_asset_fqdn=storage_asset, storage_asset_path=storage_path, mb_used=mb_used)

logger.info("Query:")

logger.info(sql)

try:

cur.execute(sql)

logger.info("Query execution status:")

logger.info(cur.statusmessage)

conn.commit()

except Exception as e:

raise Exception("Caught exception on query execution")

# Close cursor

cur.close()

ssh_client.close()

except Exception as e:

logger.error("Caught exception on SSH execution")

logger.exception(e)

errors = True

# Exit with error if there were errors

if errors:

raise Exception("There were errors within SSH execution")

if args.update_envelopes_for_all_clients or args.update_envelopes_for_client is not None:

# List all files in envelopes folder

try:

envelopes_folder_files = drive_ls(SA_SECRETS_FILE, acc_yaml_dict["envelopes"], acc_yaml_dict["gsuite"]["drive_user"])

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# For *.yaml in client dir

for client_file in sorted(glob.glob("{0}/{1}".format(CLIENTS_SUBDIR, YAML_GLOB))):

logger.info("Found client file: {0}".format(client_file))

# Load client YAML

client_dict = load_client_yaml(WORK_DIR, client_file, CLIENTS_SUBDIR, YAML_GLOB, logger)

if client_dict is None:

raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, client_file))

# Check specific client

if args.update_envelopes_for_client is not None:

client, = args.update_envelopes_for_client

if client_dict["name"].lower() != client:

continue

# If client yaml has envelope address and not excluded

if (

"papers" in client_dict

and

(

(

args.exclude_clients is not None

and

client_dict["name"].lower() not in exclude_clients_list

)

or

(

args.include_clients is not None

and

client_dict["name"].lower() in include_clients_list

)

or

(

args.exclude_clients is None

and

args.include_clients is None

)

)

and

(

(

"envelope_address" in client_dict["billing"]["papers"] and "envelope" in acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]]

)

or

(

"envelope_address_no_recipient" in client_dict["billing"]["papers"] and "envelope_no_recipient" in acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]]

)

)

):

# Envelope file name client - merchant - template

envelope_file_name = client_dict["name"] + " - " + client_dict["billing"]["merchant"] + " - " + client_dict["billing"]["template"]

# Remove file in envelopes folder

if not args.dry_run_gsuite:

for item in envelopes_folder_files:

if item["name"] == envelope_file_name:

try:

response = drive_rm(SA_SECRETS_FILE, item["id"], acc_yaml_dict["gsuite"]["drive_user"])

logger.info("Envelope drive_rm response: {0}".format(response))

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# Copy envelope from template

if not args.dry_run_gsuite:

try:

client_doc_envelope = drive_cp(SA_SECRETS_FILE,

acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]]["envelope"] if "envelope_address" in client_dict["billing"]["papers"] else acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]]["envelope_no_recipient"],

acc_yaml_dict["envelopes"],

envelope_file_name, acc_yaml_dict["gsuite"]["drive_user"])

logger.info("New client envelope id: {0}".format(client_doc_envelope))

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# Templates

envelope_data = {

"__CONTRACT_RECIPIENT__": client_dict["billing"]["contract"]["recipient"],

"__ENVELOPE_ADDRESS__": client_dict["billing"]["papers"]["envelope_address"] if "envelope_address" in client_dict["billing"]["papers"] else client_dict["billing"]["papers"]["envelope_address_no_recipient"]

}

if not args.dry_run_gsuite:

try:

response = docs_replace_all_text(SA_SECRETS_FILE, client_doc_envelope, json.dumps(envelope_data))

logger.info("Envelope docs_replace_all_text response: {0}".format(response))

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# PDF

# Change dir to tmp

os.chdir("tmp")

# Remove tmp file

if os.path.exists(envelope_file_name + ".pdf"):

os.remove(envelope_file_name + ".pdf")

# Download as pdf to tmp file

try:

response = drive_pdf(SA_SECRETS_FILE, client_doc_envelope, envelope_file_name + ".pdf", acc_yaml_dict["gsuite"]["drive_user"])

logger.info("Envelope drive_pdf response: {0}".format(response))

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# Upload pdf back to drive

if not args.dry_run_gsuite:

try:

response = drive_upload(SA_SECRETS_FILE, envelope_file_name + ".pdf", acc_yaml_dict["envelopes"], envelope_file_name + ".pdf", acc_yaml_dict["gsuite"]["drive_user"])

logger.info("Envelope drive_upload response: {0}".format(response))

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# Remove original

if not args.dry_run_gsuite:

try:

response = drive_rm(SA_SECRETS_FILE, client_doc_envelope, acc_yaml_dict["gsuite"]["drive_user"])

logger.info("Envelope drive_rm response: {0}".format(response))

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# Remove tmp file

if os.path.exists(envelope_file_name + ".pdf"):

os.remove(envelope_file_name + ".pdf")

# Change dir back to work dir

os.chdir("..")

if args.make_pdfs_for_all_clients or args.make_pdfs_for_client is not None:

# Init empty list of new pdfs

uploaded_pdfs = []

# For *.yaml in client dir

for client_file in sorted(glob.glob("{0}/{1}".format(CLIENTS_SUBDIR, YAML_GLOB))):

logger.info("Found client file: {0}".format(client_file))

# Load client YAML

client_dict = load_client_yaml(WORK_DIR, client_file, CLIENTS_SUBDIR, YAML_GLOB, logger)

if client_dict is None:

raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, client_file))

# Check specific client

if args.make_pdfs_for_client is not None:

client, = args.make_pdfs_for_client

if client_dict["name"].lower() != client:

continue

# If client yaml has gsuite folder

if "gsuite" in client_dict and "folder" in client_dict["gsuite"]:

# List all files in client folder

try:

client_folder_files = drive_ls(SA_SECRETS_FILE, client_dict["gsuite"]["folder"], acc_yaml_dict["gsuite"]["drive_user"])

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# For every file that is not pdf and do not have pdf copy and has mimeType = application/vnd.google-apps.document

for item in client_folder_files:

if not re.match(r"^.*\.pdf$", item["name"]) and not any(subitem["name"] == item["name"] + ".pdf" for subitem in client_folder_files) and item["mimeType"] == "application/vnd.google-apps.document":

# Change dir to tmp

os.chdir("tmp")

# Remove tmp file

if os.path.exists(item["name"] + ".pdf"):

os.remove(item["name"] + ".pdf")

# Download as pdf to tmp file

try:

response = drive_pdf(SA_SECRETS_FILE, item["id"], item["name"] + ".pdf", acc_yaml_dict["gsuite"]["drive_user"])

logger.info("Doc drive_pdf response: {0}".format(response))

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# Upload pdf back to drive

if not args.dry_run_gsuite:

try:

response = drive_upload(SA_SECRETS_FILE, item["name"] + ".pdf", client_dict["gsuite"]["folder"], item["name"] + ".pdf", acc_yaml_dict["gsuite"]["drive_user"])

logger.info("Pdf drive_upload response: {0}".format(response))

if response is not None:

uploaded_pdfs.append(response)

else:

logger.error("Cannot upload document {0} to folder {1} - may be duplicate".format(item["name"] + ".pdf", client_dict["gsuite"]["folder"]))

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# Remove tmp file

if os.path.exists(item["name"] + ".pdf"):

os.remove(item["name"] + ".pdf")

# Change dir back to work dir

os.chdir("..")

print("New PDFs to check:")

for item in uploaded_pdfs:

print("https://drive.google.com/file/d/" + item + "/view")

# Create issue to check new PDFs

# Prepare issue header

issue_text = textwrap.dedent("""

""")

# Add report rows

for item in uploaded_pdfs:

issue_text = "{}\n- https://drive.google.com/file/d/{}/view".format(issue_text, item)

# Connect to GitLab as Bot

gl = gitlab.Gitlab(acc_yaml_dict["gitlab"]["url"], private_token=GL_BOT_PRIVATE_TOKEN)

gl.auth()

if args.debug_gitlab:

gl.enable_debug()

# Post report as an issue in GitLab

logger.info("Going to create new issue:")

logger.info("Title: New PDFs Check Report")

logger.info("Body:")

logger.info(issue_text)

project = gl.projects.get(acc_yaml_dict["accounting"]["project"])

if not args.dry_run_gitlab:

issue = project.issues.create({"title": "New PDFs Check Report", "description": issue_text})

# Add assignee

issue.assignee_ids = [acc_yaml_dict["accounting"]["manager_id"]]

issue.save()

if args.make_gmail_drafts_for_client is not None or args.make_gmail_drafts_for_all_clients or args.print_papers_for_client is not None or args.print_papers_for_all_clients:

# Get Invoices raw data

try:

invoices_raw_dict = sheets_get_as_json(SA_SECRETS_FILE, acc_yaml_dict["invoices"]["spreadsheet"], acc_yaml_dict["invoices"]["invoices"]["sheet"], acc_yaml_dict["invoices"]["invoices"]["range"], 'ROWS', 'FORMATTED_VALUE', 'FORMATTED_STRING')

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# Prepare structured Invoices dict per client

invoices_dict = {}

for invoices_line in invoices_raw_dict:

invoices_order_dict = acc_yaml_dict["invoices"]["invoices"]["columns"]["order"]

invoices_line_client = invoices_line[invoices_order_dict['client'] - 1]

if not invoices_line_client in invoices_dict:

invoices_dict[invoices_line_client] = []

invoices_dict[invoices_line_client].append(

{

'date_created': invoices_line[invoices_order_dict['date_created'] - 1],

'type': invoices_line[invoices_order_dict['type'] - 1],

'period': invoices_line[invoices_order_dict['period'] - 1],

'merchant': invoices_line[invoices_order_dict['merchant'] - 1],

'ext_order_number': invoices_line[invoices_order_dict['ext_order_number'] - 1],

'invoice_number': invoices_line[invoices_order_dict['invoice_number'] - 1],

'invoice_currency': invoices_line[invoices_order_dict['invoice_currency'] - 1],

'invoice_sum': invoices_line[invoices_order_dict['invoice_sum'] - 1],

'status': invoices_line[invoices_order_dict['status'] - 1],

'sum_processed': invoices_line[invoices_order_dict['sum_processed'] - 1],

'sum_received': invoices_line[invoices_order_dict['sum_received'] - 1],

'papers': invoices_line[invoices_order_dict['papers'] - 1]

}

)

# Read all or specific clients

clients_dict = {}

if args.make_gmail_drafts_for_all_clients or args.print_papers_for_all_clients:

# For *.yaml in client dir

for client_file in sorted(glob.glob("{0}/{1}".format(CLIENTS_SUBDIR, YAML_GLOB))):

logger.info("Found client file: {0}".format(client_file))

# Load client YAML

client_dict = load_client_yaml(WORK_DIR, client_file, CLIENTS_SUBDIR, YAML_GLOB, logger)

if client_dict is None:

raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, client_file))

# Check client active and exclude/include

if client_dict["active"] and (

(

args.exclude_clients is not None

and

client_dict["name"].lower() not in exclude_clients_list

)

or

(

args.include_clients is not None

and

client_dict["name"].lower() in include_clients_list

)

or

(

args.exclude_clients is None

and

args.include_clients is None

)

):

clients_dict[client_dict["name"]] = client_dict

else:

# Read specific client yaml

if args.make_gmail_drafts_for_client is not None:

client_in_arg, = args.make_gmail_drafts_for_client

elif args.print_papers_for_client is not None:

client_in_arg, = args.print_papers_for_client

else:

raise Exception("Impossible became possible")

client_dict = load_client_yaml(WORK_DIR, "{0}/{1}.{2}".format(CLIENTS_SUBDIR, client_in_arg.lower(), YAML_EXT), CLIENTS_SUBDIR, YAML_GLOB, logger)

if client_dict is None:

raise Exception("Config file error or missing: {0}/{1}".format(WORK_DIR, client_file))

clients_dict[client_dict["name"]] = client_dict

# List all files in envelopes folder for further usage

try:

envelopes_folder_files = drive_ls(SA_SECRETS_FILE, acc_yaml_dict["envelopes"], acc_yaml_dict["gsuite"]["drive_user"])

except Exception as e:

raise Exception("Caught exception on gsuite execution")

# Iterate over clients and check Invoices data per client

for client in clients_dict:

client_dict = clients_dict[client]

# If client has Invoices and client active

if client in invoices_dict and client_dict["active"]:

# Make Drafts

if args.make_gmail_drafts_for_client is not None or args.make_gmail_drafts_for_all_clients:

# Iterate over each type of Invoice (needed for template file names)

for invoice_type in INVOICE_TYPES:

# Find all unique merchants for client

client_merchants = set([invoice["merchant"] for invoice in invoices_dict[client] if invoice["type"] == invoice_type])

# For each merchant within client

for client_merchant in client_merchants:

# Throw error if more than 1 Prepared Invoice of each type for the same merchant

# We shouldn't accumulate Prepared Invoices - we need to send them after each preparation as email templates has specific Invoice number

if sum(invoice["merchant"] == client_merchant and invoice["status"] == "Prepared" and invoice["type"] == invoice_type for invoice in invoices_dict[client]) == 1:

# Find Prepared Invoice (first item of one item list)

client_prepared_invoice = [invoice for invoice in invoices_dict[client] if invoice["merchant"] == client_merchant and invoice["status"] == "Prepared" and invoice["type"] == invoice_type][0]

# Prepare draft debt text template (any invoices with Status == Sent)

if any(invoice["merchant"] == client_merchant and invoice["status"] == "Sent" and invoice["type"] == invoice_type for invoice in invoices_dict[client]):

if "pack_to_archive" in client_dict["billing"]["papers"]["email"] and client_dict["billing"]["papers"]["email"]["pack_to_archive"] == False:

client_gmail_draft_text_debt = \

acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]][invoice_type.lower()]["invoice"]["email"]["text"]["debt"].format(

debt_list="\n".join([

"- "

+ acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]]["number_symbol"]

+ " "

+ invoice["invoice_number"]

for invoice in invoices_dict[client] if invoice["merchant"] == client_merchant and invoice["status"] == "Sent" and invoice["type"] == invoice_type

])

)

else:

client_gmail_draft_text_debt = \

acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]][invoice_type.lower()]["invoice"]["email"]["text"]["debt"].format(

debt_list="\n".join([

"- "

+ invoice["invoice_number"]

+ ".zip"

for invoice in invoices_dict[client] if invoice["merchant"] == client_merchant and invoice["status"] == "Sent" and invoice["type"] == invoice_type

])

)

else:

client_gmail_draft_text_debt = ""

# Prepare draft partially received text template (any invoices with Status == Partially Received)

if any(invoice["merchant"] == client_merchant and invoice["status"] == "Partially Received" and invoice["type"] == invoice_type for invoice in invoices_dict[client]):

if "pack_to_archive" in client_dict["billing"]["papers"]["email"] and client_dict["billing"]["papers"]["email"]["pack_to_archive"] == False:

client_gmail_draft_text_part = \

acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]][invoice_type.lower()]["invoice"]["email"]["text"]["part"].format(

part_list="\n".join([

"- "

+ acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]]["number_symbol"]

+ " "

+ invoice["invoice_number"]

+ " "

+ "("

+ (invoice["sum_processed"] if (invoice["sum_processed"] is not None and invoice["sum_processed"] != "") else invoice["sum_received"])

+ "/"

+ invoice["invoice_sum"]

+ " "

+ invoice["invoice_currency"]

+ ")"

for invoice in invoices_dict[client] if invoice["merchant"] == client_merchant and invoice["status"] == "Partially Received" and invoice["type"] == invoice_type

])

)

else:

client_gmail_draft_text_part = \

acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]][invoice_type.lower()]["invoice"]["email"]["text"]["part"].format(

part_list="\n".join([

"- "

+ invoice["invoice_number"]

+ ".zip"

+ " "

+ "("

+ (invoice["sum_processed"] if (invoice["sum_processed"] is not None and invoice["sum_processed"] != "") else invoice["sum_received"])

+ "/"

+ invoice["invoice_sum"]

+ " "

+ invoice["invoice_currency"]

+ ")"

for invoice in invoices_dict[client] if invoice["merchant"] == client_merchant and invoice["status"] == "Partially Received" and invoice["type"] == invoice_type

])

)

else:

client_gmail_draft_text_part = ""

# Prepare draft over received text template (any invoices with Status == Over Received)

if any(invoice["merchant"] == client_merchant and invoice["status"] == "Over Received" and invoice["type"] == invoice_type for invoice in invoices_dict[client]):

if "pack_to_archive" in client_dict["billing"]["papers"]["email"] and client_dict["billing"]["papers"]["email"]["pack_to_archive"] == False:

client_gmail_draft_text_over = \

acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]][invoice_type.lower()]["invoice"]["email"]["text"]["over"].format(

over_list="\n".join([

"- "

+ acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]]["number_symbol"]

+ " "

+ invoice["invoice_number"]

+ " "

+ "("

+ (invoice["sum_processed"] if (invoice["sum_processed"] is not None and invoice["sum_processed"] != "") else invoice["sum_received"])

+ "/"

+ invoice["invoice_sum"]

+ " "

+ invoice["invoice_currency"]

+ ")"

for invoice in invoices_dict[client] if invoice["merchant"] == client_merchant and invoice["status"] == "Over Received" and invoice["type"] == invoice_type

])

)

else:

client_gmail_draft_text_over = \

acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]][invoice_type.lower()]["invoice"]["email"]["text"]["over"].format(

over_list="\n".join([

"- "

+ invoice["invoice_number"]

+ ".zip"

+ " "

+ "("

+ (invoice["sum_processed"] if (invoice["sum_processed"] is not None and invoice["sum_processed"] != "") else invoice["sum_received"])

+ "/"

+ invoice["invoice_sum"]

+ " "

+ invoice["invoice_currency"]

+ ")"

for invoice in invoices_dict[client] if invoice["merchant"] == client_merchant and invoice["status"] == "Over Received" and invoice["type"] == invoice_type

])

)

else:

client_gmail_draft_text_over = ""

# Prepare draft act text template if act needed

if client_dict["billing"]["papers"]["act"]["email"]:

client_gmail_draft_text_act = \

acc_yaml_dict["merchants"][client_dict["billing"]["merchant"]]["templates"][client_dict["billing"]["template"]][invoice_type.lower()]["invoice"]["email"]["text"]["act"]

else:

client_gmail_draft_text_act = ""