#!/usr/bin/env python

# -*- coding: utf-8 -*-

'''

This is the main PythonTeX script. It should be launched via pythontex.py.

Two versions of this script are provided. One, with name ending in "2", runs

under Python 2.7. The other, with name ending in "3", runs under Python 3.2+.

This script needs to be able to import pythontex_engines.py; in general it

should be in the same directory.

Licensed under the BSD 3-Clause License:

Copyright (c) 2012-2013, Geoffrey M. Poore

All rights reserved.

Redistribution and use in source and binary forms, with or without

modification, are permitted provided that the following conditions are met:

* Redistributions of source code must retain the above copyright

notice, this list of conditions and the following disclaimer.

* Redistributions in binary form must reproduce the above copyright

notice, this list of conditions and the following disclaimer in the

documentation and/or other materials provided with the distribution.

* Neither the name of the <organization> nor the

names of its contributors may be used to endorse or promote products

derived from this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND

ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE

DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY

DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES

(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;

LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND

ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT

(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS

SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

'''

# Imports

#// Python 2

#from __future__ import absolute_import

#from __future__ import division

#from __future__ import print_function

#from __future__ import unicode_literals

#\\ End Python 2

import sys

import os

import argparse

import codecs

import time

from hashlib import sha1

from collections import defaultdict, OrderedDict, namedtuple

from re import match, sub, search

import subprocess

import multiprocessing

from pygments.styles import get_all_styles

from pythontex_engines import *

import textwrap

if sys.version_info[0] == 2:

try:

import cPickle as pickle

except:

import pickle

from io import open

else:

import pickle

# Script parameters

# Version

version = 'v0.12'

class Pytxcode(object):

def __init__(self, data, gobble):

self.delims, self.code = data.split('#\n', 1)

self.input_family, self.input_session, self.input_restart, self.input_instance, self.input_command, self.input_context, self.input_args_run, self.input_args_prettyprint, self.input_file, self.input_line = self.delims.split('#')

self.input_instance_int = int(self.input_instance)

self.input_line_int = int(self.input_line)

self.key_run = self.input_family + '#' + self.input_session + '#' + self.input_restart

self.key_typeset = self.key_run + '#' + self.input_instance

self.hashable_delims_run = self.key_typeset + '#' + self.input_command + '#' + self.input_context + '#' + self.input_args_run

self.hashable_delims_typeset = self.key_typeset + '#' + self.input_command + '#' + self.input_context + '#' + self.input_args_run

if len(self.input_command) > 1:

self.is_inline = False

# Environments start on the next line

self.input_line_int += 1

self.input_line = str(self.input_line_int)

else:

self.is_inline = True

self.is_extfile = True if self.input_session.startswith('EXT:') else False

if self.is_extfile:

self.extfile = os.path.expanduser(os.path.normcase(self.input_session.replace('EXT:', '', 1)))

self.is_cc = True if self.input_family.startswith('CC:') else False

self.is_pyg = True if self.input_family.startswith('PYG') else False

self.is_verb = True if self.input_restart.endswith('verb') else False

if self.is_cc:

self.input_instance += 'CC'

self.cc_type, self.cc_pos = self.input_family.split(':')[1:]

if self.is_verb or self.is_pyg or self.is_cc:

self.is_cons = False

else:

self.is_cons = engine_dict[self.input_family].console

self.is_code = False if self.is_verb or self.is_pyg or self.is_cc or self.is_cons else True

if self.input_command in ('c', 'code') or (self.input_command == 'i' and not self.is_cons):

self.is_typeset = False

else:

self.is_typeset = True

if gobble == 'auto':

self.code = textwrap.dedent(self.code)

def process_argv(data, temp_data):

'''

Process command line options using the argparse module.

Most options are passed via the file of code, rather than via the command

line.

'''

# Create a command line argument parser

parser = argparse.ArgumentParser()

parser.add_argument('TEXNAME',

help='LaTeX file, with or without .tex extension')

parser.add_argument('--version', action='version',

version='PythonTeX {0}'.format(data['version']))

parser.add_argument('--encoding', default='UTF-8',

help='encoding for all text files (see codecs module for encodings)')

parser.add_argument('--error-exit-code', default='true',

choices=('true', 'false'),

help='return exit code of 1 if there are errors (not desirable with some TeX editors and workflows)')

group = parser.add_mutually_exclusive_group()

group.add_argument('--runall', nargs='?', default='false',

const='true', choices=('true', 'false'),

help='run ALL code; equivalent to package option')

group.add_argument('--rerun', default='errors',

choices=('never', 'modified', 'errors', 'warnings', 'always'),

help='set conditions for rerunning code; equivalent to package option')

parser.add_argument('--hashdependencies', nargs='?', default='false',

const='true', choices=('true', 'false'),

help='hash dependencies (such as external data) to check for modification, rather than using mtime; equivalent to package option')

parser.add_argument('-v', '--verbose', default=False, action='store_true',

help='verbose output')

parser.add_argument('--interpreter', default=None, help='set a custom interpreter; argument should be in the form "<interpreter>:<command>, <interp>:<cmd>, ..." where <interpreter> is "python", "ruby", etc., and <command> is the command for invoking the interpreter; argument may also be in the form of a Python dictionary')

args = parser.parse_args()

# Store the parsed argv in data and temp_data

data['encoding'] = args.encoding

if args.error_exit_code == 'true':

temp_data['error_exit_code'] = True

else:

temp_data['error_exit_code'] = False

# runall can be mapped onto rerun, so both are stored under rerun

if args.runall == 'true':

temp_data['rerun'] = 'always'

else:

temp_data['rerun'] = args.rerun

# hashdependencies need only be in temp_data, since changing it would

# change hashes (hashes of mtime vs. file contents)

if args.hashdependencies == 'true':

temp_data['hashdependencies'] = True

else:

temp_data['hashdependencies'] = False

temp_data['verbose'] = args.verbose

# Update interpreter_dict based on interpreter

if args.interpreter is not None:

interp_list = args.interpreter.lstrip('{').rstrip('}').split(',')

for interp in interp_list:

if interp:

try:

k, v = interp.split(':')

k = k.strip(' \'"')

v = v.strip(' \'"')

interpreter_dict[k] = v

except:

print('Invalid --interpreter argument')

return sys.exit(2)

if args.TEXNAME is not None:

# Determine if we a dealing with just a filename, or a name plus

# path. If there's a path, we need to make the document directory

# the current working directory.

dir, raw_jobname = os.path.split(args.TEXNAME)

dir = os.path.expanduser(os.path.normcase(dir))

if dir:

os.chdir(dir)

sys.path.append(dir)

# If necessary, strip off an extension to find the raw jobname that

# corresponds to the .pytxcode.

if not os.path.exists(raw_jobname + '.pytxcode'):

raw_jobname = raw_jobname.rsplit('.', 1)[0]

if not os.path.exists(raw_jobname + '.pytxcode'):

print('* PythonTeX error')

print(' Code file ' + raw_jobname + '.pytxcode does not exist.')

print(' Run LaTeX to create it.')

return sys.exit(1)

# We need a "sanitized" version of the jobname, with spaces and

# asterisks replaced with hyphens. This is done to avoid TeX issues

# with spaces in file names, paralleling the approach taken in

# pythontex.sty. From now on, we will use the sanitized version every

# time we create a file that contains the jobname string. The raw

# version will only be used in reference to pre-existing files created

# on the TeX side, such as the .pytxcode file.

jobname = raw_jobname.replace(' ', '-').replace('"', '').replace('*', '-')

# Store the results in data

data['raw_jobname'] = raw_jobname

data['jobname'] = jobname

# We need to check to make sure that the "sanitized" jobname doesn't

# lead to a collision with a file that already has that name, so that

# two files attempt to use the same PythonTeX folder.

#

# If <jobname>.<ext> and <raw_jobname>.<ext> both exist, where <ext>

# is a common LaTeX extension, we exit. We operate under the

# assumption that there should be only a single file <jobname> in the

# document root directory that has a common LaTeX extension. That

# could be false, but if so, the user probably has worse things to

# worry about than a potential PythonTeX output collision.

# If <jobname>* and <raw_jobname>* both exist, we issue a warning but

# attempt to proceed.

if jobname != raw_jobname:

resolved = False

for ext in ('.tex', '.ltx', '.dtx'):

if os.path.isfile(raw_jobname + ext):

if os.path.isfile(jobname + ext):

print('* PythonTeX error')

print(' Directory naming collision between the following files:')

print(' ' + raw_jobname + ext)

print(' ' + jobname + ext)

return sys.exit(1)

else:

resolved = True

break

if not resolved:

ls = os.listdir('.')

for file in ls:

if file.startswith(jobname):

print('* PythonTeX warning')

print(' Potential directory naming collision between the following names:')

print(' ' + raw_jobname)

print(' ' + jobname + '*')

print(' Attempting to proceed.')

temp_data['warnings'] += 1

break

def load_code_get_settings(data, temp_data):

'''

Load the code file, preprocess the code, and extract the settings.

'''

# Bring in the .pytxcode file as a single string

raw_jobname = data['raw_jobname']

encoding = data['encoding']

# The error checking here is a little redundant

if os.path.isfile(raw_jobname + '.pytxcode'):

f = open(raw_jobname + '.pytxcode', 'r', encoding=encoding)

pytxcode = f.read()

f.close()

else:

print('* PythonTeX error')

print(' Code file ' + raw_jobname + '.pytxcode does not exist.')

print(' Run LaTeX to create it.')

return sys.exit(1)

# Split code and settings

try:

pytxcode, pytxsettings = pytxcode.rsplit('=>PYTHONTEX:SETTINGS#', 1)

except:

print('The .pytxcode file appears to have an outdated format or be invalid')

print('Run LaTeX to make sure the file is current')

return sys.exit(1)

# Prepare to process settings

#

# Create a dict for storing settings.

settings = {}

# Create a dict for storing Pygments settings.

# Each dict entry will itself be a dict.

pygments_settings = defaultdict(dict)

# Create a dict of processing functions, and generic processing functions

settings_func = dict()

def set_kv_data(k, v):

if v == 'true':

settings[k] = True

elif v == 'false':

settings[k] = False

else:

settings[k] = v

# Need a function for when assignment is only needed if not default value

def set_kv_temp_data_if_not_default(k, v):

if v != 'default':

if v == 'true':

temp_data[k] = True

elif v == 'false':

temp_data[k] = False

else:

temp_data[k] = v

def set_kv_data_fvextfile(k, v):

# Error checking on TeX side should be enough, but be careful anyway

try:

v = int(v)

except ValueError:

print('* PythonTeX error')

print(' Unable to parse package option fvextfile.')

return sys.exit(1)

if v < 0:

settings[k] = sys.maxsize

elif v == 0:

settings[k] = 1

print('* PythonTeX warning')

print(' Invalid value for package option fvextfile.')

temp_data['warnings'] += 1

else:

settings[k] = v

def set_kv_pygments(k, v):

input_family, lexer_opts, options = v.replace(' ','').split('|')

lexer = None

lex_dict = {}

opt_dict = {}

if lexer_opts:

for l in lexer_opts.split(','):

if '=' in l:

k, v = l.split('=', 1)

if k == 'lexer':

lexer = l

else:

lex_dict[k] = v

else:

lexer = l

if options:

for o in options.split(','):

if '=' in o:

k, v = o.split('=', 1)

if v in ('true', 'True'):

v = True

elif v in ('false', 'False'):

v = False

else:

k = option

v = True

opt_dict[k] = v

if input_family != ':GLOBAL':

if 'lexer' in pygments_settings[':GLOBAL']:

lexer = pygments_settings[':GLOBAL']['lexer']

lex_dict.update(pygments_settings[':GLOBAL']['lexer_options'])

opt_dict.update(pygments_settings[':GLOBAL']['formatter_options'])

if 'style' not in opt_dict:

opt_dict['style'] = 'default'

opt_dict['commandprefix'] = 'PYG' + opt_dict['style']

if lexer is not None:

pygments_settings[input_family]['lexer'] = lexer

pygments_settings[input_family]['lexer_options'] = lex_dict

pygments_settings[input_family]['formatter_options'] = opt_dict

settings_func['version'] = set_kv_data

settings_func['outputdir'] = set_kv_data

settings_func['workingdir'] = set_kv_data

settings_func['gobble'] = set_kv_data

settings_func['rerun'] = set_kv_temp_data_if_not_default

settings_func['hashdependencies'] = set_kv_temp_data_if_not_default

settings_func['makestderr'] = set_kv_data

settings_func['stderrfilename'] = set_kv_data

settings_func['keeptemps'] = set_kv_data

settings_func['pyfuture'] = set_kv_data

settings_func['pyconfuture'] = set_kv_data

settings_func['pygments'] = set_kv_data

settings_func['fvextfile'] = set_kv_data_fvextfile

settings_func['pygglobal'] = set_kv_pygments

settings_func['pygfamily'] = set_kv_pygments

settings_func['pyconbanner'] = set_kv_data

settings_func['pyconfilename'] = set_kv_data

settings_func['depythontex'] = set_kv_data

# Process settings

for line in pytxsettings.split('\n'):

if line:

key, val = line.split('=', 1)

try:

settings_func[key](key, val)

except KeyError:

print('* PythonTeX warning')

print(' Unknown option "' + key + '"')

temp_data['warnings'] += 1

# Check for compatility between the .pytxcode and the script

if 'version' not in settings or settings['version'] != data['version']:

print('* PythonTeX warning')

print(' The version of the PythonTeX scripts does not match')

print(' the last code saved by the document--run LaTeX to create')

print(' an updated version. Attempting to proceed.')

sys.stdout.flush()

# Store all results that haven't already been stored.

data['settings'] = settings

data['pygments_settings'] = pygments_settings

# Create a tuple of vital quantities that invalidate old saved data

# Don't need to include outputdir, because if that changes, no old output

# fvextfile could be checked on a case-by-case basis, which would result

# in faster output, but that would involve a good bit of additional

# logic, which probably isn't worth it for a feature that will rarely be

# changed.

data['vitals'] = (data['version'], data['encoding'],

settings['gobble'], settings['fvextfile'])

# Create tuples of vital quantities

data['code_vitals'] = (settings['workingdir'], settings['keeptemps'],

settings['makestderr'], settings['stderrfilename'])

data['cons_vitals'] = (settings['workingdir'])

data['typeset_vitals'] = ()

# Pass any customizations to types

for k in engine_dict:

engine_dict[k].customize(pyfuture=settings['pyfuture'],

pyconfuture=settings['pyconfuture'],

pyconbanner=settings['pyconbanner'],

pyconfilename=settings['pyconfilename'])

# Store code

# Do this last, so that Pygments settings are available

if pytxcode.startswith('=>PYTHONTEX#'):

gobble = settings['gobble']

temp_data['pytxcode'] = [Pytxcode(c, gobble) for c in pytxcode.split('=>PYTHONTEX#')[1:]]

else:

temp_data['pytxcode'] = []

def get_old_data(data, old_data, temp_data):

'''

Load data from the last run, if it exists, into the dict old_data.

Determine the path to the PythonTeX scripts, either by using a previously

found, saved path or via kpsewhich.

The old data is used for determining when PythonTeX has been upgraded,

when any settings have changed, when code has changed (via hashes), and

what files may need to be cleaned up. The location of the PythonTeX

scripts is needed so that they can be imported by the scripts created by

PythonTeX. The location of the scripts is confirmed even if they were

previously located, to make sure that the path is still valid. Finding

the scripts depends on having a TeX installation that includes the

Kpathsea library (TeX Live and MiKTeX, possibly others).

All code that relies on old_data is written based on the assumption that

if old_data exists and has the current PythonTeX version, then it

contains all needed information. Thus, all code relying on old_data must

check that it was loaded and that it has the current version. If not,

code should adapt gracefully.

'''

# Create a string containing the name of the data file

pythontex_data_file = os.path.join(data['settings']['outputdir'], 'pythontex_data.pkl')

# Load the old data if it exists (read as binary pickle)

if os.path.isfile(pythontex_data_file):

f = open(pythontex_data_file, 'rb')

old = pickle.load(f)

f.close()

# Check for compabilility

if 'vitals' in old and data['vitals'] == old['vitals']:

temp_data['loaded_old_data'] = True

old_data.update(old)

else:

temp_data['loaded_old_data'] = False

# Clean up all old files

if 'files' in old:

for key in old['files']:

for f in old['files'][key]:

f = os.path.expanduser(os.path.normcase(f))

if os.path.isfile(f):

os.remove(f)

if 'pygments_files' in old:

for key in old['pygments_files']:

for f in old['pygments_files'][key]:

f = os.path.expanduser(os.path.normcase(f))

if os.path.isfile(f):

os.remove(f)

else:

temp_data['loaded_old_data'] = False

# Set the utilspath

if os.path.isfile(os.path.join(sys.path[0], 'pythontex_utils.py')):

# Need the path with forward slashes, so escaping isn't necessary

data['utilspath'] = sys.path[0].replace('\\', '/')

else:

print('* PythonTeX error')

print(' Could not determine the utils path from sys.path[0]')

print(' The file "pythontex_utils.py" may be missing')

return sys.exit(1)

def modified_dependencies(key, data, old_data, temp_data):

hashdependencies = temp_data['hashdependencies']

if key not in old_data['dependencies']:

return False

else:

old_dep_hash_dict = old_data['dependencies'][key]

workingdir = data['settings']['workingdir']

for dep in old_dep_hash_dict.keys():

# We need to know if the path is relative (based off the

# working directory) or absolute. We can't use

# os.path.isabs() alone for determining the distinction,

# because we must take into account the possibility of an

# initial ~ (tilde) standing for the home directory.

dep_file = os.path.expanduser(os.path.normcase(dep))

if not os.path.isabs(dep_file):

dep_file = os.path.join(workingdir, dep_file)

if not os.path.isfile(dep_file):

print('* PythonTeX error')

print(' Cannot find dependency "' + dep + '"')

print(' It belongs to ' + key.replace('#', ':'))

print(' Relative paths to dependencies must be specified from the working directory.')

temp_data['errors'] += 1

# A removed dependency should trigger an error, but it

# shouldn't cause code to execute. Running the code

# again would just give more errors when it can't find

# the dependency. (There won't be issues when a

# dependency is added or removed, because that would

# involve modifying code, which would trigger

# re-execution.)

elif hashdependencies:

# Read and hash the file in binary. Opening in text mode

# would require an unnecessary decoding and encoding cycle.

f = open(dep_file, 'rb')

hasher = sha1()

hash = hasher(f.read()).hexdigest()

f.close()

if hash != old_dep_hash_dict[dep][1]:

return True

else:

mtime = os.path.getmtime(dep_file)

if mtime != old_dep_hash_dict[dep][0]:

return True

return False

def should_rerun(hash, old_hash, old_exit_status, key, rerun, data, old_data, temp_data):

# #### Need to clean up arg passing here

if rerun == 'never':

if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data)):

print('* PythonTeX warning')

print(' Session ' + key.replace('#', ':') + ' has rerun=never')

print(' But its code or dependencies have been modified')

temp_data['warnings'] += 1

return False

elif rerun == 'modified':

if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data)):

return True

else:

return False

elif rerun == 'errors':

if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data) or

old_exit_status[0] != 0):

return True

else:

return False

elif rerun == 'warnings':

if (hash != old_hash or modified_dependencies(key, data, old_data, temp_data) or

old_exit_status != (0, 0)):

return True

else:

return False

elif rerun == 'always':

return True

def hash_all(data, temp_data, old_data, engine_dict):

'''

Hash the code to see what has changed and needs to be updated.

Save the hashes in hashdict. Create update_code, a list of bools

regarding whether code should be executed. Create update_pygments, a

list of bools determining what needs updated Pygments highlighting.

Update pygments_settings to account for Pygments (as opposed to PythonTeX)

commands and environments.

'''

# Note that the PythonTeX information that accompanies code must be

# hashed in addition to the code itself; the code could stay the same,

# but its context or args could change, which might require that code be

# executed. All of the PythonTeX information is hashed except for the

# input line number. Context-dependent code is going too far if

# it depends on that.

# Create variables to more easily access parts of data

pytxcode = temp_data['pytxcode']

encoding = data['encoding']

loaded_old_data = temp_data['loaded_old_data']

rerun = temp_data['rerun']

pygments_settings = data['pygments_settings']

# Calculate cumulative hashes for all code that is executed

# Calculate individual hashes for all code that will be typeset

code_hasher = defaultdict(sha1)

cons_hasher = defaultdict(sha1)

cc_hasher = defaultdict(sha1)

typeset_hasher = defaultdict(sha1)

for c in pytxcode:

if c.is_code:

code_hasher[c.key_run].update(c.hashable_delims_run.encode(encoding))

code_encoded = c.code.encode(encoding)

code_hasher[c.key_run].update(code_encoded)

if c.is_typeset:

typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))

typeset_hasher[c.key_typeset].update(code_encoded)

elif c.is_cons:

cons_hasher[c.key_run].update(c.hashable_delims_run.encode(encoding))

code_encoded = c.code.encode(encoding)

cons_hasher[c.key_run].update(code_encoded)

if c.is_typeset:

typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))

typeset_hasher[c.key_typeset].update(code_encoded)

elif c.is_cc:

cc_hasher[c.cc_type].update(c.hashable_delims_run.encode(encoding))

cc_hasher[c.cc_type].update(c.code.encode(encoding))

elif c.is_typeset:

typeset_hasher[c.key_typeset].update(c.hashable_delims_typeset.encode(encoding))

typeset_hasher[c.key_typeset].update(c.code.encode(encoding))

# Store hashes

code_hash_dict = {}

for key in code_hasher:

input_family = key.split('#', 1)[0]

code_hash_dict[key] = (code_hasher[key].hexdigest(),

cc_hasher[input_family].hexdigest(),

engine_dict[input_family].get_hash())

data['code_hash_dict'] = code_hash_dict

cons_hash_dict = {}

for key in cons_hasher:

input_family = key.split('#', 1)[0]

cons_hash_dict[key] = (cons_hasher[key].hexdigest(),

cc_hasher[input_family].hexdigest(),

engine_dict[input_family].get_hash())

data['cons_hash_dict'] = cons_hash_dict

typeset_hash_dict = {}

for key in typeset_hasher:

typeset_hash_dict[key] = typeset_hasher[key].hexdigest()

data['typeset_hash_dict'] = typeset_hash_dict

# See what needs to be updated.

# In the process, copy over macros and files that may be reused.

code_update = {}

cons_update = {}

pygments_update = {}

macros = defaultdict(list)

files = defaultdict(list)

pygments_macros = {}

pygments_files = {}

typeset_cache = {}

dependencies = defaultdict(dict)

exit_status = {}

pygments_settings_changed = {}

if loaded_old_data:

old_macros = old_data['macros']

old_files = old_data['files']

old_pygments_macros = old_data['pygments_macros']

old_pygments_files = old_data['pygments_files']

old_typeset_cache = old_data['typeset_cache']

old_dependencies = old_data['dependencies']

old_exit_status = old_data['exit_status']

old_code_hash_dict = old_data['code_hash_dict']

old_cons_hash_dict = old_data['cons_hash_dict']

old_typeset_hash_dict = old_data['typeset_hash_dict']

old_pygments_settings = old_data['pygments_settings']

for s in pygments_settings:

if (s in old_pygments_settings and

pygments_settings[s] == old_pygments_settings[s]):

pygments_settings_changed[s] = False

else:

pygments_settings_changed[s] = True

# If old data was loaded (and thus is compatible) determine what has

# changed so that only

# modified code may be executed. Otherwise, execute everything.

# We don't have to worry about checking for changes in pyfuture, because

# custom code and default code are hashed. The treatment of keeptemps

# could be made more efficient (if changed to 'none', just delete old temp

# files rather than running everything again), but given that it is

# intended as a debugging aid, that probable isn't worth it.

# We don't have to worry about hashdependencies changing, because if it

# does the hashes won't match (file contents vs. mtime) and thus code will

# be re-executed.

if loaded_old_data and data['code_vitals'] == old_data['code_vitals']:

# Compare the hash values, and set which code needs to be run

for key in code_hash_dict:

if (key in old_code_hash_dict and

not should_rerun(code_hash_dict[key], old_code_hash_dict[key], old_exit_status[key], key, rerun, data, old_data, temp_data)):

code_update[key] = False

macros[key] = old_macros[key]

files[key] = old_files[key]

dependencies[key] = old_dependencies[key]

exit_status[key] = old_exit_status[key]

else:

code_update[key] = True

else:

for key in code_hash_dict:

code_update[key] = True

if loaded_old_data and data['cons_vitals'] == old_data['cons_vitals']:

# Compare the hash values, and set which code needs to be run

for key in cons_hash_dict:

if (key in old_cons_hash_dict and

not should_rerun(cons_hash_dict[key], old_cons_hash_dict[key], old_exit_status[key], key, rerun, data, old_data, temp_data)):

cons_update[key] = False

macros[key] = old_macros[key]

files[key] = old_files[key]

typeset_cache[key] = old_typeset_cache[key]

dependencies[key] = old_dependencies[key]

exit_status[key] = old_exit_status[key]

else:

cons_update[key] = True

else:

for key in cons_hash_dict:

cons_update[key] = True

if loaded_old_data and data['typeset_vitals'] == old_data['typeset_vitals']:

for key in typeset_hash_dict:

input_family = key.split('#', 1)[0]

if input_family in pygments_settings:

if (not pygments_settings_changed[input_family] and

key in old_typeset_hash_dict and

typeset_hash_dict[key] == old_typeset_hash_dict[key]):

pygments_update[key] = False

if key in old_pygments_macros:

pygments_macros[key] = old_pygments_macros[key]

if key in old_pygments_files:

pygments_files[key] = old_pygments_files[key]

else:

pygments_update[key] = True

else:

pygments_update[key] = False

# Make sure Pygments styles are up-to-date

pygments_style_list = list(get_all_styles())

if pygments_style_list != old_data['pygments_style_list']:

pygments_style_defs = {}

# Lazy import

from pygments.formatters import LatexFormatter

for s in pygments_style_list:

formatter = LatexFormatter(style=s, commandprefix='PYG'+s)

pygments_style_defs[s] = formatter.get_style_defs()

else:

pygments_style_defs = old_data['pygments_style_defs']

else:

for key in typeset_hash_dict:

input_family = key.split('#', 1)[0]

if input_family in pygments_settings:

pygments_update[key] = True

else:

pygments_update[key] = False

# Create Pygments styles

pygments_style_list = list(get_all_styles())

pygments_style_defs = {}

# Lazy import

from pygments.formatters import LatexFormatter

for s in pygments_style_list:

formatter = LatexFormatter(style=s, commandprefix='PYG'+s)

pygments_style_defs[s] = formatter.get_style_defs()

# Save to data

temp_data['code_update'] = code_update

temp_data['cons_update'] = cons_update

temp_data['pygments_update'] = pygments_update

data['macros'] = macros

data['files'] = files

data['pygments_macros'] = pygments_macros

data['pygments_style_list'] = pygments_style_list

data['pygments_style_defs'] = pygments_style_defs

data['pygments_files'] = pygments_files

data['typeset_cache'] = typeset_cache

data['dependencies'] = dependencies

data['exit_status'] = exit_status

# Clean up for code that will be run again, and for code that no longer

# exists.

if loaded_old_data:

# Take care of code files

for key in code_hash_dict:

if code_update[key] and key in old_files:

for f in old_files[key]:

f = os.path.expanduser(os.path.normcase(f))

if os.path.isfile(f):

os.remove(f)

for key in old_code_hash_dict:

if key not in code_hash_dict:

for f in old_files[key]:

f = os.path.expanduser(os.path.normcase(f))

if os.path.isfile(f):

os.remove(f)

# Take care of old console files

for key in cons_hash_dict:

if cons_update[key] and key in old_files:

for f in old_files[key]:

f = os.path.expanduser(os.path.normcase(f))

if os.path.isfile(f):

os.remove(f)

for key in old_cons_hash_dict:

if key not in cons_hash_dict:

for f in old_files[key]:

f = os.path.expanduser(os.path.normcase(f))

if os.path.isfile(f):

os.remove(f)

# Take care of old Pygments files

# The approach here is a little different since there isn't a

# Pygments-specific hash dict, but there is a Pygments-specific

# dict of lists of files.

for key in pygments_update:

if pygments_update[key] and key in old_pygments_files:

for f in old_pygments_files[key]:

f = os.path.expanduser(os.path.normcase(f))

if os.path.isfile(f):

os.remove(f)

for key in old_pygments_files:

if key not in pygments_update:

for f in old_pygments_files[key]:

f = os.path.expanduser(os.path.normcase(f))

if os.path.isfile(f):

os.remove(f)

def parse_code_write_scripts(data, temp_data, engine_dict):

'''

Parse the code file into separate scripts, and write them to file.

'''

code_dict = defaultdict(list)

cc_dict_begin = defaultdict(list)

cc_dict_end = defaultdict(list)

cons_dict = defaultdict(list)

pygments_list = []

# Create variables to ease data access

encoding = data['encoding']

utilspath = data['utilspath']

outputdir = data['settings']['outputdir']

workingdir = data['settings']['workingdir']

pytxcode = temp_data['pytxcode']

code_update = temp_data['code_update']

cons_update = temp_data['cons_update']

pygments_update = temp_data['pygments_update']

files = data['files']

# We need to keep track of the last instance for each session, so

# that duplicates can be eliminated. Some LaTeX environments process

# their content multiple times and thus will create duplicates. We

# need to initialize everything at -1, since instances begin at zero.

def negative_one():

return -1

last_instance = defaultdict(negative_one)

for c in pytxcode:

if c.input_instance_int > last_instance[c.key_run]:

last_instance[c.key_run] = c.input_instance_int

if c.is_code:

if code_update[c.key_run]:

code_dict[c.key_run].append(c)

if c.is_typeset and pygments_update[c.key_typeset]:

pygments_list.append(c)

elif c.is_cons:

# Only append to Pygments if not run, since Pygments is

# automatically taken care of during run for console content

if cons_update[c.key_run]:

cons_dict[c.key_run].append(c)

elif c.is_typeset and pygments_update[c.key_typeset]:

pygments_list.append(c)

elif (c.is_pyg or c.is_verb) and pygments_update[c.key_typeset]:

pygments_list.append(c)

elif c.is_cc:

if c.cc_pos == 'begin':

cc_dict_begin[c.cc_type].append(c)

else:

cc_dict_end[c.cc_type].append(c)

# Save

temp_data['code_dict'] = code_dict

temp_data['cc_dict_begin'] = cc_dict_begin

temp_data['cc_dict_end'] = cc_dict_end

temp_data['cons_dict'] = cons_dict

temp_data['pygments_list'] = pygments_list

# Save the code sessions that need to be updated

# Keep track of the files that are created

# Also accumulate error indices for handling stderr

code_index_dict = {}

for key in code_dict:

input_family, input_session, input_restart = key.split('#')

fname = os.path.join(outputdir, input_family + '_' + input_session + '_' + input_restart + '.' + engine_dict[input_family].extension)

files[key].append(fname)

sessionfile = open(fname, 'w', encoding=encoding)

script, code_index = engine_dict[input_family].get_script(encoding,

utilspath,

workingdir,

cc_dict_begin[input_family],

code_dict[key],

cc_dict_end[input_family])

for lines in script:

sessionfile.write(lines)

sessionfile.close()

code_index_dict[key] = code_index

temp_data['code_index_dict'] = code_index_dict

def do_multiprocessing(data, temp_data, old_data, engine_dict):

jobname = data['jobname']

encoding = data['encoding']

outputdir = data['settings']['outputdir']

workingdir = data['settings']['workingdir']

keeptemps = data['settings']['keeptemps']

fvextfile = data['settings']['fvextfile']

pygments_settings = data['pygments_settings']

verbose = temp_data['verbose']

code_dict = temp_data['code_dict']

cons_dict = temp_data['cons_dict']

cc_dict_begin = temp_data['cc_dict_begin']

cc_dict_end = temp_data['cc_dict_end']

pygments_list = temp_data['pygments_list']

pygments_style_defs = data['pygments_style_defs']

files = data['files']

macros = data['macros']

pygments_files = data['pygments_files']

pygments_macros = data['pygments_macros']

typeset_cache = data['typeset_cache']

errors = temp_data['errors']

warnings = temp_data['warnings']

makestderr = data['settings']['makestderr']

stderrfilename = data['settings']['stderrfilename']

code_index_dict = temp_data['code_index_dict']

hashdependencies = temp_data['hashdependencies']

dependencies = data['dependencies']

exit_status = data['exit_status']

start_time = data['start_time']

# Set maximum number of concurrent processes for multiprocessing

# Accoding to the docs, cpu_count() may raise an error

try:

max_processes = multiprocessing.cpu_count()

except NotImplementedError:

max_processes = 1

pool = multiprocessing.Pool(max_processes)

tasks = []

# If verbose, print a list of processes

if verbose:

print('\n* PythonTeX will run the following processes:')

# Add code processes. Note that everything placed in the codedict

# needs to be executed, based on previous testing, except for custom code.

for key in code_dict:

input_family = key.split('#')[0]

# Uncomment the following for debugging, and comment out what follows

'''run_code(encoding, outputdir, workingdir, code_dict[key],

engine_dict[input_family].language,

engine_dict[input_family].command,

engine_dict[input_family].created,

engine_dict[input_family].extension,

makestderr, stderrfilename,

code_index_dict[key],

engine_dict[input_family].errors,

engine_dict[input_family].warnings,

engine_dict[input_family].linenumbers,

engine_dict[input_family].lookbehind,

keeptemps, hashdependencies)'''

tasks.append(pool.apply_async(run_code, [encoding, outputdir,

workingdir, code_dict[key],

engine_dict[input_family].language,

engine_dict[input_family].command,

engine_dict[input_family].created,