#

# ----------------------------------------------------------------------------------------------------

#

# Copyright (c) 2007, 2019, Oracle and/or its affiliates. All rights reserved.

# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.

#

# This code is free software; you can redistribute it and/or modify it

# under the terms of the GNU General Public License version 2 only, as

# published by the Free Software Foundation.

#

# This code is distributed in the hope that it will be useful, but WITHOUT

# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or

# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License

# version 2 for more details (a copy is included in the LICENSE file that

# accompanied this code).

#

# You should have received a copy of the GNU General Public License version

# 2 along with this work; if not, write to the Free Software Foundation,

# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.

#

# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA

# or visit www.oracle.com if you need additional information or have any

# questions.

#

# ----------------------------------------------------------------------------------------------------

#

r"""

mx is a command line tool for managing the development of Java code organized as suites of projects.

"""

from __future__ import print_function

import sys

if sys.version_info < (2, 7):

major, minor, micro, _, _ = sys.version_info

raise SystemExit('mx requires python 2.7+, not {0}.{1}.{2}'.format(major, minor, micro))

from abc import ABCMeta, abstractmethod, abstractproperty

if __name__ == '__main__':

# Rename this module as 'mx' so it is not re-executed when imported by other modules.

sys.modules['mx'] = sys.modules.pop('__main__')

try:

import defusedxml #pylint: disable=unused-import

from defusedxml.ElementTree import parse as etreeParse

except ImportError:

from xml.etree.ElementTree import parse as etreeParse

import os, errno, time, subprocess, shlex, zipfile, signal, tempfile, platform

from platform import system

import textwrap

import socket

import tarfile, gzip

import hashlib

import itertools

from functools import cmp_to_key

# TODO use defusedexpat?

import xml.parsers.expat, xml.sax.saxutils, xml.dom.minidom

from xml.dom.minidom import parseString as minidomParseString

import shutil, re

import pipes

import difflib

import glob

import filecmp

import json

from collections import OrderedDict, namedtuple, deque

from datetime import datetime, timedelta

from threading import Thread

from argparse import ArgumentParser, PARSER, REMAINDER, Namespace, HelpFormatter, ArgumentTypeError, RawTextHelpFormatter, FileType

from os.path import join, basename, dirname, exists, lexists, isabs, expandvars, isdir, islink, normpath, realpath, splitext

from tempfile import mkdtemp, mkstemp

from io import BytesIO

import fnmatch

import operator

import calendar

import multiprocessing

from stat import S_IWRITE

from mx_commands import MxCommands, MxCommand

from copy import copy, deepcopy

_mx_commands = MxCommands("mx")

# Temporary imports and (re)definitions while porting mx from Python 2 to Python 3

if sys.version_info[0] < 3:

filter = itertools.ifilter # pylint: disable=redefined-builtin,invalid-name

def input(prompt=None): # pylint: disable=redefined-builtin

return raw_input(prompt) # pylint: disable=undefined-variable

import __builtin__ as builtins

import urllib2 # pylint: disable=unused-import

_urllib_request = urllib2

_urllib_error = urllib2

del urllib2

import urlparse as _urllib_parse

def _decode(x):

return x

def _encode(x):

return x

_unicode = unicode # pylint: disable=undefined-variable

else:

import builtins # pylint: disable=unused-import,no-name-in-module

import urllib.request as _urllib_request # pylint: disable=unused-import,no-name-in-module

import urllib.error as _urllib_error # pylint: disable=unused-import,no-name-in-module

import urllib.parse as _urllib_parse # pylint: disable=unused-import,no-name-in-module

def _decode(x):

return x.decode()

def _encode(x):

return x.encode()

_unicode = str

### ~~~~~~~~~~~~~ _private

def _hashFromUrl(url):

logvv('Retrieving SHA1 from {}'.format(url))

hashFile = _urllib_request.urlopen(url)

try:

return hashFile.read()

except _urllib_error.URLError as e:

_suggest_http_proxy_error(e)

abort('Error while retrieving sha1 {}: {}'.format(url, str(e)))

finally:

if hashFile:

hashFile.close()

def _merge_file_contents(input_files, output_file):

for file_name in input_files:

with open(file_name, 'r') as input_file:

shutil.copyfileobj(input_file, output_file)

output_file.flush()

def _make_absolute(path, prefix):

"""

If 'path' is not absolute prefix it with 'prefix'

"""

return join(prefix, path)

def _cache_dir():

return _cygpathW2U(get_env('MX_CACHE_DIR', join(dot_mx_dir(), 'cache')))

def _global_env_file():

return _cygpathW2U(get_env('MX_GLOBAL_ENV', join(dot_mx_dir(), 'env')))

def _get_path_in_cache(name, sha1, urls, ext=None, sources=False, oldPath=False):

"""

Gets the path an artifact has (or would have) in the download cache.

"""

assert sha1 != 'NOCHECK', 'artifact for ' + name + ' cannot be cached since its sha1 is NOCHECK'

if ext is None:

for url in urls:

# Use extension of first URL whose path component ends with a non-empty extension

o = _urllib_parse.urlparse(url)

if o.path == "/remotecontent" and o.query.startswith("filepath"):

path = o.query

else:

path = o.path

ext = get_file_extension(path)

if ext:

ext = '.' + ext

break

if not ext:

abort('Could not determine a file extension from URL(s):\n ' + '\n '.join(urls))

assert os.sep not in name, name + ' cannot contain ' + os.sep

assert os.pathsep not in name, name + ' cannot contain ' + os.pathsep

if oldPath:

return join(_cache_dir(), name + ('.sources' if sources else '') + '_' + sha1 + ext) # mx < 5.176.0

filename = _map_to_maven_dist_name(name) + ('.sources' if sources else '') + ext

return join(_cache_dir(), name + '_' + sha1 + ('.dir' if not ext else ''), filename)

def _urlopen(*args, **kwargs):

timeout_attempts = [0]

timeout_retries = kwargs.pop('timeout_retries', 3)

def on_timeout():

if timeout_attempts[0] <= timeout_retries:

timeout_attempts[0] += 1

kwargs['timeout'] = kwargs.get('timeout', 5) * 2

warn("urlopen() timed out! Retrying with timeout of {}s.".format(kwargs['timeout']))

return True

return False

error500_attempts = 0

error500_limit = 5

while True:

try:

return _urllib_request.urlopen(*args, **kwargs)

except (_urllib_error.HTTPError) as e:

if e.code == 500:

if error500_attempts < error500_limit:

error500_attempts += 1

url = '?' if len(args) == 0 else args[0]

warn("Retrying after error reading from " + url + ": " + str(e))

time.sleep(0.2)

continue

raise

except _urllib_error.URLError as e:

if isinstance(e.reason, socket.error):

if e.reason.errno == errno.EINTR and 'timeout' in kwargs and is_interactive():

warn("urlopen() failed with EINTR. Retrying without timeout.")

del kwargs['timeout']

return _urllib_request.urlopen(*args, **kwargs)

if e.reason.errno == errno.EINPROGRESS:

if on_timeout():

continue

if isinstance(e.reason, socket.timeout):

if on_timeout():

continue

raise

except socket.timeout:

if on_timeout():

continue

raise

abort("should not reach here")

def _check_file_with_sha1(path, sha1, sha1path, mustExist=True, newFile=False, logErrors=False):

"""

Checks if a file exists and is up to date according to the sha1.

Returns False if the file is not there or does not have the right checksum.

"""

sha1Check = sha1 and sha1 != 'NOCHECK'

def _sha1CachedValid():

if not exists(sha1path):

return False

if TimeStampFile(path, followSymlinks=True).isNewerThan(sha1path):

return False

return True

def _sha1Cached():

with open(sha1path, 'r') as f:

return f.read()[0:40]

def _writeSha1Cached(value=None):

with SafeFileCreation(sha1path) as sfc, open(sfc.tmpPath, 'w') as f:

f.write(value or sha1OfFile(path))

if exists(path):

if sha1Check and sha1:

if not _sha1CachedValid() or (newFile and sha1 != _sha1Cached()):

logv('Create/update SHA1 cache file ' + sha1path)

_writeSha1Cached()

if sha1 != _sha1Cached():

computedSha1 = sha1OfFile(path)

if sha1 == computedSha1:

warn('Fixing corrupt SHA1 cache file ' + sha1path)

_writeSha1Cached(computedSha1)

return True

if logErrors:

size = os.path.getsize(path)

log_error('SHA1 of {} [size: {}] ({}) does not match expected value ({})'.format(TimeStampFile(path), size, computedSha1, sha1))

return False

elif mustExist:

if logErrors:

log_error("'{}' does not exist".format(path))

return False

return True

def _needsUpdate(newestInput, path):

"""

Determines if the file denoted by `path` does not exist or `newestInput` is not None

and `path`'s latest modification time is older than the `newestInput` TimeStampFile.

Returns a string describing why `path` needs updating or None if it does not need updating.

"""

if not exists(path):

return path + ' does not exist'

if newestInput:

ts = TimeStampFile(path, followSymlinks=False)

if ts.isOlderThan(newestInput):

return '{} is older than {}'.format(ts, newestInput)

return None

def _function_code(f):

if hasattr(f, 'func_code'):

# Python 2

return f.func_code

# Python 3

return f.__code__

def _check_output_str(*args, **kwargs):

try:

return _decode(subprocess.check_output(*args, **kwargs))

except subprocess.CalledProcessError as e:

if e.output:

e.output = _decode(e.output)

if hasattr(e, 'stderr') and e.stderr:

e.stderr = _decode(e.stderr)

raise e

def _with_metaclass(meta, *bases):

"""Create a base class with a metaclass."""

# Copyright (c) 2010-2018 Benjamin Peterson

# Taken from https://github.com/benjaminp/six/blob/8da94b8a153ceb0d6417d76729ba75e80eaa75c1/six.py#L820

# MIT license

# This requires a bit of explanation: the basic idea is to make a dummy

# metaclass for one level of class instantiation that replaces itself with

# the actual metaclass.

class MetaClass(type):

def __new__(mcs, name, this_bases, d):

return meta(name, bases, d)

@classmethod

def __prepare__(mcs, name, this_bases):

return meta.__prepare__(name, bases)

return type.__new__(MetaClass, '_with_metaclass({}, {})'.format(meta, bases), (), {}) #pylint: disable=unused-variable

def _validate_abolute_url(urlstr, acceptNone=False):

if urlstr is None:

return acceptNone

url = _urllib_parse.urlsplit(urlstr)

return url.scheme and (url.netloc or url.path)

def _safe_path(path):

"""

If not on Windows, this function returns `path`.

Otherwise, it return a potentially transformed path that is safe for file operations.

This works around the MAX_PATH limit on Windows:

https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath

"""

if is_windows():

if _opts.verbose and '/' in path:

warn("Forward slash in path on windows: {}".format(path))

import traceback

traceback.print_stack()

path = normpath(path)

MAX_PATH = 260 # pylint: disable=invalid-name

path_len = len(path) + 1 # account for trailing NUL

if isabs(path) and path_len >= MAX_PATH:

if path.startswith('\\\\'):

if path[2:].startswith('?\\'):

# if it already has a \\?\ don't do the prefix

pass

else:

# Only a UNC path has a double slash prefix.

# Replace it with `\\?\UNC\'. For example:

#

# \\Mac\Home\mydir

#

# becomes:

#

# \\?\UNC\Mac\Home\mydir

#

path = '\\\\?\\UNC' + path[1:]

else:

path = '\\\\?\\' + path

path = _unicode(path)

return path

def atomic_file_move_with_fallback(source_path, destination_path):

is_directory = isdir(source_path) and not islink(source_path)

copy_function = copytree if is_directory else shutil.copyfile

remove_function = rmtree if is_directory else os.remove

temp_function = mkdtemp if is_directory else mkstemp

try:

# This can fail if we move across file systems.

os.rename(source_path, destination_path)

except:

destination_temp_path = temp_function(prefix=basename(destination_path), dir=dirname(destination_path))

# We are only interested in a path, not a file itself. For directories, using copytree on an existing directory can fail.

remove_function(destination_temp_path)

# This can get interrupted mid-copy. Since we cannot guarantee the atomicity of copytree,

# we copy to a .tmp folder first and then atomically rename.

copy_function(source_path, destination_temp_path)

os.rename(destination_temp_path, destination_path)

remove_function(source_path)

def _tarfile_chown(tf, tarinfo, targetpath):

if sys.version_info < (3, 5):

tf.chown(tarinfo, targetpath)

else:

tf.chown(tarinfo, targetpath, False) # extra argument in Python 3.5, False gives previous behavior

### ~~~~~~~~~~~~~ command

def command_function(name, fatalIfMissing=True):

"""

Return the function for the (possibly overridden) command named `name`.

If no such command, abort if `fatalIsMissing` is True, else return None

"""

return _mx_commands.command_function(name, fatalIfMissing)

def update_commands(suite, new_commands):

"""

Using the decorator mx_command is preferred over this function.

:param suite: for which the command is added.

:param new_commands: keys are command names, value are lists: [<function>, <usage msg>, <format doc function>]

if any of the format args are instances of callable, then they are called with an 'env' are before being

used in the call to str.format().

"""

suite_name = suite if isinstance(suite, str) else suite.name

_length_of_command = 4

for command_name, command_list in new_commands.items():

assert len(command_list) > 0 and command_list[0] is not None

args = [suite_name, command_name] + command_list[1:_length_of_command]

command_decorator = command(*args)

# apply the decorator so all functions are tracked

command_list[0] = command_decorator(command_list[0])

def command(suite_name, command_name, usage_msg='', doc_function=None, props=None, auto_add=True):

"""

Decorator for making a function an mx shell command.

The annotated function should have a single parameter typed List[String].

:param suite_name: suite to which the command belongs to.

:param command_name: the command name. Will be used in the shell command.

:param usage_msg: message to display usage.

:param doc_function: function to render the documentation for this feature.

:param props: a dictionary of properties attributed to this command.

:param auto_add: automatically it to the commands.

:return: the decorator factory for the function.

"""

def mx_command_decorator_factory(command_func):

mx_command = MxCommand(_mx_commands, command_func, suite_name, command_name, usage_msg, doc_function, props)

if auto_add:

_mx_commands.add_commands([mx_command])

return mx_command

return mx_command_decorator_factory

### ~~~~~~~~~~~~~ Language support

# Support for comparing objects given removal of `cmp` function in Python 3.

# https://portingguide.readthedocs.io/en/latest/comparisons.html

def compare(a, b):

return (a > b) - (a < b)

class Comparable(object):

def _checked_cmp(self, other, f):

compar = self.__cmp__(other) #pylint: disable=assignment-from-no-return

return f(compar, 0) if compar is not NotImplemented else compare(id(self), id(other))

def __lt__(self, other):

return self._checked_cmp(other, lambda a, b: a < b)

def __gt__(self, other):

return self._checked_cmp(other, lambda a, b: a > b)

def __eq__(self, other):

return self._checked_cmp(other, lambda a, b: a == b)

def __le__(self, other):

return self._checked_cmp(other, lambda a, b: a <= b)

def __ge__(self, other):

return self._checked_cmp(other, lambda a, b: a >= b)

def __ne__(self, other):

return self._checked_cmp(other, lambda a, b: a != b)

def __cmp__(self, other): # to override

raise TypeError("No override for compare")

from mx_javacompliance import JavaCompliance

class DynamicVar(object):

def __init__(self, initial_value):

self.value = initial_value

def get(self):

return self.value

def set_scoped(self, newvalue):

return DynamicVarScope(self, newvalue)

class DynamicVarScope(object):

def __init__(self, dynvar, newvalue):

self.dynvar = dynvar

self.newvalue = newvalue

def __enter__(self):

assert not hasattr(self, "oldvalue")

self.oldvalue = self.dynvar.value

self.dynvar.value = self.newvalue

def __exit__(self, tpe, value, traceback):

self.dynvar.value = self.oldvalue

self.oldvalue = None

self.newvalue = None

class ArgParser(ArgumentParser):

# Override parent to append the list of available commands

def format_help(self):

return ArgumentParser.format_help(self) + """

environment variables:

JAVA_HOME Default value for primary JDK directory. Can be overridden with --java-home option.

EXTRA_JAVA_HOMES Secondary JDK directories. Can be overridden with --extra-java-homes option.

MX_BUILD_EXPLODED Create and use jar distributions as extracted directories.

MX_ALT_OUTPUT_ROOT Alternate directory for generated content. Instead of <suite>/mxbuild, generated

content will be placed under $MX_ALT_OUTPUT_ROOT/<suite>. A suite can override

this with the suite level "outputRoot" attribute in suite.py.

MX_EXEC_LOG Specifies default value for --exec-log option.

MX_CACHE_DIR Override the default location of the mx download cache. Defaults to `~/.mx/cache`.

MX_GLOBAL_ENV Override the default location of the global env file that is always loaded at startup.

Defaults to `~/.mx/env`. Can be disabled by setting it to an empty string.

MX_GIT_CACHE Use a cache for git objects during clones.

* Setting it to `reference` will clone repositories using the cache and let them

reference the cache (if the cache gets deleted these repositories will be

incomplete).

* Setting it to `dissociated` will clone using the cache but then dissociate the

repository from the cache.

* Setting it to `refcache` will synchronize with server only if a branch is

requested or if a specific revision is requested which does not exist in the

local cache. Hence, remote references will be synchronized occasionally. This

allows cloning without even contacting the git server.

The cache is located at `~/.mx/git-cache`.

""" + _format_commands()

def __init__(self, parents=None):

self.parsed = False

if not parents:

parents = []

ArgumentParser.__init__(self, prog='mx', parents=parents, add_help=len(parents) != 0, formatter_class=lambda prog: HelpFormatter(prog, max_help_position=32, width=120))

if len(parents) != 0:

# Arguments are inherited from the parents

return

self.add_argument('-v', action='store_true', dest='verbose', help='enable verbose output')

self.add_argument('-V', action='store_true', dest='very_verbose', help='enable very verbose output')

self.add_argument('--no-warning', action='store_false', dest='warn', help='disable warning messages')

self.add_argument('--quiet', action='store_true', help='disable log messages')

self.add_argument('-y', action='store_const', const='y', dest='answer', help='answer \'y\' to all questions asked')

self.add_argument('-n', action='store_const', const='n', dest='answer', help='answer \'n\' to all questions asked')

self.add_argument('-p', '--primary-suite-path', help='set the primary suite directory', metavar='<path>')

self.add_argument('--dbg', dest='java_dbg_port', help='make Java processes wait on [<host>:]<port> for a debugger', metavar='<address>') # metavar=[<host>:]<port> https://bugs.python.org/issue11874

self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"')

self.add_argument('--attach', dest='attach', help='Connect to existing server running at [<host>:]<port>', metavar='<address>') # metavar=[<host>:]<port> https://bugs.python.org/issue11874

self.add_argument('--backup-modified', action='store_true', help='backup generated files if they pre-existed and are modified')

self.add_argument('--exec-log', help='A file to which the environment and command line for each subprocess executed by mx is appended', metavar='<path>', default=get_env("MX_EXEC_LOG"))

self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='<arg>')

self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='<arg>')

jargs = self.add_mutually_exclusive_group()

jargs.add_argument('-J', dest='java_args', help='Java VM arguments (e.g. "-J-dsa")', metavar='<arg>')

jargs.add_argument('--J', dest='java_args_legacy', help='Java VM arguments (e.g. "--J @-dsa")', metavar='@<args>')

jpargs = self.add_mutually_exclusive_group()

jpargs.add_argument('-P', action='append', dest='java_args_pfx', help='prefix Java VM arguments (e.g. "-P-dsa")', metavar='<arg>', default=[])

jpargs.add_argument('--Jp', action='append', dest='java_args_pfx_legacy', help='prefix Java VM arguments (e.g. --Jp @-dsa)', metavar='@<args>', default=[])

jaargs = self.add_mutually_exclusive_group()

jaargs.add_argument('-A', action='append', dest='java_args_sfx', help='suffix Java VM arguments (e.g. "-A-dsa")', metavar='<arg>', default=[])

jaargs.add_argument('--Ja', action='append', dest='java_args_sfx_legacy', help='suffix Java VM arguments (e.g. --Ja @-dsa)', metavar='@<args>', default=[])

self.add_argument('--user-home', help='users home directory', metavar='<path>', default=os.path.expanduser('~'))

self.add_argument('--java-home', help='primary JDK directory (must be JDK 7 or later)', metavar='<path>')

self.add_argument('--jacoco', help='instruments selected classes using JaCoCo', default='off', choices=['off', 'on', 'append'])

self.add_argument('--jacoco-whitelist-package', help='only include classes in the specified package', metavar='<package>', action='append', default=[])

self.add_argument('--jacoco-exclude-annotation', help='exclude classes with annotation from JaCoCo instrumentation', metavar='<annotation>', action='append', default=[])

self.add_argument('--jacoco-dest-file', help='path of the JaCoCo dest file, which contains the execution data', metavar='<path>', action='store', default='jacoco.exec')

self.add_argument('--extra-java-homes', help='secondary JDK directories separated by "' + os.pathsep + '"', metavar='<path>')

self.add_argument('--strict-compliance', action='store_true', dest='strict_compliance', help='use JDK matching a project\'s Java compliance when compiling (legacy - this is the only supported mode)', default=True)

self.add_argument('--ignore-project', action='append', dest='ignored_projects', help='name of project to ignore', metavar='<name>', default=[])

self.add_argument('--kill-with-sigquit', action='store_true', dest='killwithsigquit', help='send sigquit first before killing child processes')

self.add_argument('--suite', action='append', dest='specific_suites', help='limit command to the given suite', metavar='<name>', default=[])

self.add_argument('--suitemodel', help='mechanism for locating imported suites', metavar='<arg>')

self.add_argument('--primary', action='store_true', help='limit command to primary suite')

self.add_argument('--dynamicimports', action='append', dest='dynamic_imports', help='dynamically import suite <name>', metavar='<name>', default=[])

self.add_argument('--no-download-progress', action='store_true', help='disable download progress meter')

self.add_argument('--version', action='store_true', help='print version and exit')

self.add_argument('--mx-tests', action='store_true', help='load mxtests suite (mx debugging)')

self.add_argument('--jdk', action='store', help='JDK to use for the "java" command', metavar='<tag:compliance>')

self.add_argument('--jmods-dir', action='store', help='path to built jmods (default JAVA_HOME/jmods)', metavar='<path>')

self.add_argument('--version-conflict-resolution', dest='version_conflict_resolution', action='store', help='resolution mechanism used when a suite is imported with different versions', default='suite', choices=['suite', 'none', 'latest', 'latest_all', 'ignore'])

self.add_argument('-c', '--max-cpus', action='store', type=int, dest='cpu_count', help='the maximum number of cpus to use during build', metavar='<cpus>', default=None)

self.add_argument('--proguard-cp', action='store', help='class path containing ProGuard jars to be used instead of default versions')

self.add_argument('--strip-jars', action='store_true', help='produce and use stripped jars in all mx commands.')

self.add_argument('--env', dest='additional_env', help='load an additional env file in the mx dir of the primary suite', metavar='<name>')

self.add_argument('--trust-http', action='store_true', help='Suppress warning about downloading from non-https sources')

self.add_argument('--multiarch', action='store_true', help='enable all architectures of native multiarch projects (not just the host architecture)')

self.add_argument('--dump-task-stats', help='Dump CSV formatted start/end timestamps for each build task. If set to \'-\' it will print it to stdout, otherwise the CSV will be written to <path>', metavar='<path>', default=None)

self.add_argument('--compdb', action='store', metavar='<file>', help="generate a JSON compilation database for native "

"projects and store it in the given <file>. If <file> is 'default', the compilation database will "

"be stored in the parent directory of the repository containing the primary suite. This option "

"can also be configured using the MX_COMPDB environment variable. Use --compdb none to disable.")

if not is_windows():

# Time outs are (currently) implemented with Unix specific functionality

self.add_argument('--timeout', help='timeout (in seconds) for command', type=int, default=0, metavar='<secs>')

self.add_argument('--ptimeout', help='timeout (in seconds) for subprocesses', type=int, default=0, metavar='<secs>')

def _parse_cmd_line(self, opts, firstParse):

if firstParse:

parser = ArgParser(parents=[self])

parser.add_argument('initialCommandAndArgs', nargs=REMAINDER, metavar='command args...')

# Legacy support - these options are recognized during first parse and

# appended to the unknown options to be reparsed in the second parse

parser.add_argument('--vm', action='store', dest='vm', help='the VM type to build/run')

parser.add_argument('--vmbuild', action='store', dest='vmbuild', help='the VM build to build/run')

# Parse the known mx global options and preserve the unknown args, command and

# command args for the second parse.

_, self.unknown = parser.parse_known_args(namespace=opts)

for deferrable in _opts_parsed_deferrables:

deferrable()

if opts.version:

print('mx version ' + str(version))

sys.exit(0)

if opts.vm: self.unknown += ['--vm=' + opts.vm]

if opts.vmbuild: self.unknown += ['--vmbuild=' + opts.vmbuild]

self.initialCommandAndArgs = opts.__dict__.pop('initialCommandAndArgs')

# For some reason, argparse considers an unknown argument starting with '-'

# and containing a space as a positional argument instead of an optional

# argument. Subsequent arguments starting with '-' are also considered as

# positional. We need to treat all of these as unknown optional arguments.

while len(self.initialCommandAndArgs) > 0:

arg = self.initialCommandAndArgs[0]

if arg.startswith('-'):

self.unknown.append(arg)

del self.initialCommandAndArgs[0]

else:

break

# Give the timeout options a default value to avoid the need for hasattr() tests

opts.__dict__.setdefault('timeout', 0)

opts.__dict__.setdefault('ptimeout', 0)

if opts.java_args_legacy:

opts.java_args = opts.java_args_legacy.lstrip('@')

if opts.java_args_pfx_legacy:

opts.java_args_pfx = [s.lstrip('@') for s in opts.java_args_pfx_legacy]

if opts.java_args_sfx_legacy:

opts.java_args_sfx = [s.lstrip('@') for s in opts.java_args_sfx_legacy]

if opts.very_verbose:

opts.verbose = True

if opts.user_home is None or opts.user_home == '':

abort('Could not find user home. Use --user-home option or ensure HOME environment variable is set.')

if opts.primary and primary_suite():

opts.specific_suites.append(primary_suite().name)

if opts.java_home is not None:

os.environ['JAVA_HOME'] = opts.java_home

if opts.extra_java_homes is not None:

os.environ['EXTRA_JAVA_HOMES'] = opts.extra_java_homes

os.environ['HOME'] = opts.user_home

global _primary_suite_path

_primary_suite_path = opts.primary_suite_path or os.environ.get('MX_PRIMARY_SUITE_PATH')

if _primary_suite_path:

_primary_suite_path = os.path.abspath(_primary_suite_path)

global _suitemodel

_suitemodel = SuiteModel.create_suitemodel(opts)

# Communicate primary suite path to mx subprocesses

if _primary_suite_path:

os.environ['MX_PRIMARY_SUITE_PATH'] = _primary_suite_path

opts.ignored_projects += os.environ.get('IGNORED_PROJECTS', '').split(',')

mx_gate._jacoco = opts.jacoco

mx_gate._jacoco_whitelisted_packages.extend(opts.jacoco_whitelist_package)

mx_gate.add_jacoco_excluded_annotations(opts.jacoco_exclude_annotation)

mx_gate.Task.verbose = opts.verbose

if opts.exec_log:

try:

ensure_dir_exists(dirname(opts.exec_log))

with open(opts.exec_log, 'a'):

pass

except IOError as e:

abort('Error opening {} specified by --exec-log: {}'.format(opts.exec_log, e))

else:

parser = ArgParser(parents=[self])

parser.add_argument('commandAndArgs', nargs=REMAINDER, metavar='command args...')

args = self.unknown + self.initialCommandAndArgs

parser.parse_args(args=args, namespace=opts)

commandAndArgs = opts.__dict__.pop('commandAndArgs')

if self.initialCommandAndArgs != commandAndArgs:

abort('Suite specific global options must use name=value format: {0}={1}'.format(self.unknown[-1], self.initialCommandAndArgs[0]))

self.parsed = True

return commandAndArgs

def add_argument(*args, **kwargs):

"""

Defines a single command-line argument.

"""

assert _argParser is not None

_argParser.add_argument(*args, **kwargs)

def remove_doubledash(args):

if '--' in args:

args.remove('--')

def ask_question(question, options, default=None, answer=None):

""""""

assert not default or default in options

questionMark = '? ' + options + ': '

if default:

questionMark = questionMark.replace(default, default.upper())

if answer:

answer = str(answer)

print(question + questionMark + answer)

else:

if is_interactive():

answer = input(question + questionMark) or default

while not answer:

answer = input(question + questionMark)

else:

if default:

answer = default

else:

abort("Can not answer '" + question + "?' if stdin is not a tty")

return answer.lower()

def ask_yes_no(question, default=None):

""""""

return ask_question(question, '[yn]', default, _opts.answer).startswith('y')

def warn(msg, context=None):

if _opts.warn and not _opts.quiet:

if context is not None:

if callable(context):

contextMsg = context()

elif hasattr(context, '__abort_context__'):

contextMsg = context.__abort_context__()

else:

contextMsg = str(context)

msg = contextMsg + ":\n" + msg

print(colorize('WARNING: ' + msg, color='magenta', bright=True, stream=sys.stderr), file=sys.stderr)

class Timer():

"""

A simple timing facility.

Example 1:

with Timer('phase'):

phase()

will emit the following as soon as `phase()` returns:

"phase took 2.45 seconds"

Example 2:

times = []

with Timer('phase1', times):

phase1()

with Timer('phase2', times):

phase2()

will not emit anything but will have leave `times` with something like:

[('phase1', 2.45), ('phase2', 1.75)]

See also _show_timestamp.

"""

def __init__(self, name, times=None):

self.name = name

self.times = times

def __enter__(self):

self.start = time.time()

return self

def __exit__(self, t, value, traceback):

elapsed = time.time() - self.start

if self.times is None:

print('{} took {} seconds'.format(self.name, elapsed))

else:

self.times.append((self.name, elapsed))

_last_timestamp = None

def _show_timestamp(label):

"""

Prints the current date and time followed by `label` followed by the

milliseconds elapsed since the last call to this method, if any.

"""

global _last_timestamp

now = datetime.utcnow()

if _last_timestamp is not None:

duration = (now - _last_timestamp).total_seconds() * 1000

print('{}: {} [{:.02f} ms]'.format(now, label, duration))

else:

print('{}: {}'.format(now, label))

_last_timestamp = now

def glob_match_any(patterns, path):

return any((glob_match(pattern, path) for pattern in patterns))

def glob_match(pattern, path):

"""

Matches a path against a pattern using glob's special rules. In particular, the pattern is checked for each part

of the path and files starting with `.` are not matched unless the pattern also starts with a `.`.

:param str pattern: The pattern to match with glob syntax

:param str path: The path to be checked against the pattern

:return: The part of the path that matches or None if the path does not match

"""

pattern_parts = pattern.replace(os.path.sep, '/').split('/')

path_parts = path.replace(os.path.sep, '/').split('/')

if len(path_parts) < len(pattern_parts):

return None

for pattern_part, path_part in zip(pattern_parts, path_parts):

if len(pattern_part) > 0 and pattern_part[0] != '.' and len(path_part) > 0 and path_part[0] == '.':

return None

if not fnmatch.fnmatch(path_part, pattern_part):

return None

return '/'.join(path_parts[:len(pattern_parts)])

### ~~~~~~~~~~~~~ Suite

# Define this machinery early in case other modules want to use them

# Names of commands that don't need a primary suite.

# This cannot be used outside of mx because of implementation restrictions

currently_loading_suite = DynamicVar(None)

_suite_context_free = ['init', 'version', 'urlrewrite']

def _command_function_names(func):

"""

Generates list of guesses for command name based on its function name

"""

if isinstance(func, MxCommand):

func_name = func.command

else:

func_name = func.__name__

command_names = [func_name]

if func_name.endswith('_cli'):

command_names.append(func_name[0:-len('_cli')])

for command_name in command_names:

if '_' in command_name:

command_names.append(command_name.replace("_", "-"))

return command_names

def suite_context_free(func):

"""

Decorator for commands that don't need a primary suite.

"""

_suite_context_free.extend(_command_function_names(func))

return func

# Names of commands that don't need a primary suite but will use one if it can be found.

# This cannot be used outside of mx because of implementation restrictions

_optional_suite_context = ['help', 'paths']

def optional_suite_context(func):

"""

Decorator for commands that don't need a primary suite but will use one if it can be found.

"""

_optional_suite_context.extend(_command_function_names(func))

return func

# Names of commands that need a primary suite but don't need suites to be loaded.

# This cannot be used outside of mx because of implementation restrictions

_no_suite_loading = []

def no_suite_loading(func):

"""

Decorator for commands that need a primary suite but don't need suites to be loaded.

"""

_no_suite_loading.extend(_command_function_names(func))

return func

# Names of commands that need a primary suite but don't need suites to be discovered.

# This cannot be used outside of mx because of implementation restrictions

_no_suite_discovery = []

def no_suite_discovery(func):

"""

Decorator for commands that need a primary suite but don't need suites to be discovered.

"""

_no_suite_discovery.extend(_command_function_names(func))

return func

class SuiteModel:

"""

Defines how to locate a URL/path for a suite, including imported suites.

Conceptually a SuiteModel is defined a primary suite URL/path,

and a map from suite name to URL/path for imported suites.

Subclasses define a specfic implementation.

"""

def __init__(self):

self.primaryDir = None

self.suitenamemap = {}

def find_suite_dir(self, suite_import):

"""locates the URL/path for suite_import or None if not found"""

abort('find_suite_dir not implemented')

def set_primary_dir(self, d):

"""informs that d is the primary suite directory"""

self._primaryDir = d

def importee_dir(self, importer_dir, suite_import, check_alternate=True):

"""

returns the directory path for an import of suite_import.name, given importer_dir.

For a "src" suite model, if check_alternate == True and if suite_import specifies an alternate URL,

check whether path exists and if not, return the alternate.

"""

abort('importee_dir not implemented')

def nestedsuites_dirname(self):

"""Returns the dirname that contains any nested suites if the model supports that"""

return None

def _search_dir(self, searchDir, suite_import):

if suite_import.suite_dir:

sd = _is_suite_dir(suite_import.suite_dir, _mxDirName(suite_import.name))

assert sd

return sd

if not exists(searchDir):

return None

found = []

for dd in os.listdir(searchDir):

if suite_import.in_subdir:

candidate = join(searchDir, dd, suite_import.name)

else:

candidate = join(searchDir, dd)

sd = _is_suite_dir(candidate, _mxDirName(suite_import.name))

if sd is not None:

found.append(sd)

if len(found) == 0:

return None

elif len(found) == 1:

return found[0]

else:

abort("Multiple suites match the import {}:\n{}".format(suite_import.name, "\n".join(found)))

def verify_imports(self, suites, args):

"""Ensure that the imports are consistent."""

def _check_exists(self, suite_import, path, check_alternate=True):

if check_alternate and suite_import.urlinfos is not None and not exists(path):

return suite_import.urlinfos

return path

@staticmethod

def create_suitemodel(opts):

envKey = 'MX__SUITEMODEL'

default = os.environ.get(envKey, 'sibling')

name = getattr(opts, 'suitemodel') or default

# Communicate the suite model to mx subprocesses

os.environ[envKey] = name

if name.startswith('sibling'):

return SiblingSuiteModel(_primary_suite_path, name)

elif name.startswith('nested'):

return NestedImportsSuiteModel(_primary_suite_path, name)

else:

abort('unknown suitemodel type: ' + name)

@staticmethod

def siblings_dir(suite_dir):

if exists(suite_dir):

_, primary_vc_root = VC.get_vc_root(suite_dir, abortOnError=False)

if not primary_vc_root:

suite_parent = dirname(suite_dir)

# Use the heuristic of a 'ci.hocon' or '.mx_vcs_root' file being

# at the root of a repo that contains multiple suites.

hocon = join(suite_parent, 'ci.hocon')

mx_vcs_root = join(suite_parent, '.mx_vcs_root')

if exists(hocon) or exists(mx_vcs_root):

return dirname(suite_parent)

return suite_parent

else:

primary_vc_root = suite_dir

return dirname(primary_vc_root)

@staticmethod

def _checked_to_importee_tuple(checked, suite_import):

""" Converts the result of `_check_exists` to a tuple containing the result of `_check_exists` and

the directory in which the importee can be found.

If the result of checked is the urlinfos list, this path is relative to where the repository would be checked out.

"""

if isinstance(checked, list):

return checked, suite_import.name if suite_import.in_subdir else None

else:

return checked, join(checked, suite_import.name) if suite_import.in_subdir else checked