DeepFuzzSL/configure at master · 50417/DeepFuzzSL

executable file
419 lines (342 loc) · 13.3 KB
#!/usr/bin/env python
"""Configure and build the phd toolchain.
Creates the //config.pbtxt proto.
Unlike the other files in this repository, this is not intended to be executed
under a bazel environment, so must be compatible with whatever system python
interpreter the user has, be it version 2 or 3.
from __future__ import print_function
import argparse
import errno
import hashlib
import logging
import subprocess
# The path to the root of the PhD repository, i.e. the directory which this file
PHD_ROOT = os.path.dirname(os.path.realpath(__file__))
# The path to the config proto file which this script generates.
CONFIG_PROTO_PATH = os.path.join(PHD_ROOT, 'config.pbtxt')
def YesNoPrompt(question, default, noninteractive):
  """Ask a yes/no question and return user's answer.
    question: The question that is presented to the user.
    default: is the presumed answer if the user just hits <Enter>.
      It must be a string "yes" or "no".
    noninteractive: If True, just print the question and return the default
      response, do not prompt the user for it.
    True if the user answered "yes", or False for "no".
    ValueError: If the default is not one of: {"yes", "no"}.
  valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False}
  if default == "yes":
    prompt = " [Y/n] "
  elif default == "no":
    prompt = " [y/N] "
    raise ValueError("Invalid default answer: '{}'".format(default))
  while True:
    sys.stdout.write(question + prompt)
    if noninteractive:
      sys.stdout.write(default + '\n')
      return valid[default]
    # Python3 renamed raw_input() to input().
    # https://docs.python.org/3/whatsnew/3.0.html
    if sys.version_info >= (3, 0):
      choice = input().lower()
      choice = raw_input().lower()
    if choice == '':
      sys.stdout.write(default + '\n')
      return valid[default]
    elif choice in valid:
      return valid[choice]
      sys.stdout.write("Please respond with 'yes' or 'no' "
                       "(or 'y' or 'n').\n")
def Mkdir(path):
  """Create a directory, including any parent directories.
    path: The directory to create.
    os.makedirs(path)
  except OSError as exc:  # Python >2.5
    if exc.errno == errno.EEXIST and os.path.isdir(path):
      raise
def GetConfigId(args):
  """Get the ID of the configuration.
  The configuration ID is a sha256 which uniquely identifies both the set of
  configuration options requested by the user, and the contents of this file.
  If either this file or the options change, the return value of this function
  will change.
  Arguments:
    args: The arguments namespace which is returned by
      ArgumentParser.parse_args().
    A 64 character hexadecimal string.
  sha256 = hashlib.sha256()
  with open(__file__, 'rb') as f:
    sha256.update(f.read())
    for arg in sorted(args.__dict__.keys()):
      # The --print_id argument does not modify the ID.
      if arg not in {'print_id', 'noninteractive', 'cache_result'}:
        sha256.update('{}'.format(args.__dict__[arg]).encode('utf-8'))
  return sha256.hexdigest()
def GetCudaHome():
  """Get the CUDA home directory.
  This is hardcoded to the version of CUDA which the version of TensorFlow used
  in this repo requires.
  return os.environ.get('CUDA_HOME', '/usr/local/cuda-9.0')
def GuessIfCudaIsAvailable():
  """Guess if CUDA is available on the host."""
  nvcc_path = os.path.join(GetCudaHome(), 'bin/nvcc')
  is_available = os.path.isfile(nvcc_path)
  if is_available:
    logging.info("CUDA found at: '%s'.", nvcc_path)
    logging.info("CUDA not found at: '%s'.", nvcc_path)
  return is_available
def IsGitRepo():
  """Determine if it is a git repo."""
  return os.path.isdir(os.path.join(os.path.dirname(__file__), '.git'))
def GetGitRemote():
  """Get the git remote for this repo."""
  remotes = subprocess.check_output(
      ['git', '-C', os.path.dirname(__file__), 'remote', '-v']).decode('utf-8')
  return remotes.split('\n')[0].split()[1]
def RewriteGitSubmodulesToHttps():
  """Rewrite git@ prefixed submodules to https://."""
  with open(os.path.join(os.path.dirname(__file__), '.gitmodules')) as f:
    modules = f.read()
  new_modules = re.sub(r'(url\s*=\s*)git@([^:]+):', r'\1https://\2/', modules)
  assert 'git@' not in new_modules
  with open(os.path.join(os.path.dirname(__file__), '.gitmodules'), 'w') as f:
    f.write(new_modules)
def RunAsHomebrewUser(cmd, subprocess_fn=subprocess.check_call):
  if os.path.isdir('/home/linuxbrew'):
    return subprocess_fn(
        ['sudo', '-H', '-u', 'linuxbrew', 'bash', '-c', ' '.join(cmd)])
    return subprocess_fn(cmd)
def GetConfigurationOptions(argv):
  """Parse command line arguments and ask yes/no questions.
  Arguments:
    argv: A list of command line arguments.
     The arguments namespace.
  yes_no_prompts = []
  parser = argparse.ArgumentParser(
      description='Configure this project for building.',
      epilog=('Please report issues at '
              '<https://github.com/ChrisCummins/phd/issues>.'))
  # --print_id.
  parser.add_argument('--print_id', dest='print_id', action='store_true',
                      help='Print the ID of the configuration and exit.')
  # --noninteractive.
  parser.add_argument(
      '--noninteractive', action='store_true',
      help=('Disable interactive yes/no prompts. The default values for any '
            'yes/no prompts will be used, unless command line arguments are '
            'provided.'))
  # --[no]cache_result.
  parser.add_argument(
      '--cache_result', dest='cache_result', action='store_true',
      help=('Cache the results of configure so that subsequent runs of this '
            'script will be faster.'))
  parser.add_argument(
      '--nocache_result', dest='cache_result', action='store_false',
      help='Disable caching of configure result.')
  parser.set_defaults(cache_result=True)
  # --[no]with_cuda.
  parser.add_argument(
      '--with_cuda', dest='with_cuda', action='store_true',
      help=('Enable CUDA support. This requires that CUDA is installed on the '
            'host machine.'))
  parser.add_argument(
      '--nowith_cuda', dest='with_cuda', action='store_false',
      help='Disable CUDA support.')
  parser.set_defaults(with_cuda=GuessIfCudaIsAvailable())
  yes_no_prompts.append(('with_cuda', 'Enable CUDA support?'))
  # --[no]update_git_submodules.
  parser.add_argument(
      '--update_git_submodules', dest='update_git_submodules',
      action='store_true', help='Checkout and update git submodules.')
  parser.add_argument(
      '--noupdate_git_submodules', dest='update_git_submodules',
      action='store_false', help='Do not checkout and update git submodules.')
  parser.set_defaults(update_git_submodules=True)
  yes_no_prompts.append(('update_git_submodules', 'Update git submodules?'))
  # --[no]symlink_python.
  parser.add_argument(
      '--symlink_python', dest='symlink_python',
      action='store_true', help='Symlink python to ~/.local/bin/python.')
  parser.add_argument(
      '--nosymlink_python', dest='symlink_python',
      action='store_false', help='Do not symlink Python.')
  parser.set_defaults(symlink_python=True)
  yes_no_prompts.append(
      ('symlink_python', 'Symlink python to ~/.local/bin/python?'))
  args = parser.parse_args(argv)
  # Prompt the user for options.
  if not args.print_id:
    for field_name, question in yes_no_prompts:
      default_val = getattr(args, field_name)
      val = YesNoPrompt(question, 'yes' if default_val else 'no',
                        args.noninteractive)
      setattr(args, field_name, val)
  return args
def main(argv):
  logging.basicConfig(level=logging.DEBUG, format='%(message)s')
  args = GetConfigurationOptions(argv)
  os.chdir(PHD_ROOT)
  # Calculate config ID and compare against cached ID (if it exists).
  config_id = GetConfigId(args)
  if args.print_id:
    print(config_id)
  logging.info('Config ID: %s', config_id)
  # Uname in lowercase.
  uname_path = '/usr/bin/uname'
  if os.path.isfile('/bin/uname'):
    uname_path = '/bin/uname'
  uname = subprocess.check_output(
      [uname_path], universal_newlines=True).rstrip().lower()
  if IsGitRepo():
    git_remote = GetGitRemote()
    logging.info('Git remote: %s', git_remote)
    # Rewrite git@ to https:// if required.
    if git_remote.startswith('https://'):
      logging.info('Rewriting .gitmodules to use https://')
      #RewriteGitSubmodulesToHttps()
  # Python2 doesn't have subprocess.DEVNULL defined.
  DEVNULL = open(os.devnull, 'w')
  if args.update_git_submodules:
    subprocess.check_call(
        ['git', '-C', PHD_ROOT, 'submodule', 'update', '--init', '--recursive'],
        stdout=DEVNULL)
  # Find the path of a supported python binary.
  # TODO(cec): Add 'python3.7' to list once this issue is fixed:
  #   https://github.com/tensorflow/tensorflow/issues/20517
  # Homebrew has updated to python3.7, but TensorFlow does not yet support it.
  SUPPORTED_PYTHON_VERSIONS = ['python3.6']
  python_path = None
  for python in SUPPORTED_PYTHON_VERSIONS:
      python_path = subprocess.check_output(
          ['which', python], universal_newlines=True).rstrip()
      break
    except subprocess.CalledProcessError:
  if not python_path:
    print('fatal: no suitable python version found!\nSupported versions: {}'
          .format(SUPPORTED_PYTHON_VERSIONS), file=sys.stderr)
    sys.exit(1)
  assert os.path.isfile(python_path)
  base_requirements_txt_path = os.path.join(
      PHD_ROOT, 'tools', 'requirements.txt')
  assert os.path.isfile(base_requirements_txt_path)
  with open(base_requirements_txt_path) as f:
    base_requirements = f.read()
  tensorflow_build_in_path = os.path.join(
      PHD_ROOT, 'third_party/py/tensorflow/BUILD.in')
  tensorflow_build_path = os.path.join(
      PHD_ROOT, 'third_party/py/tensorflow/BUILD')
  with open(tensorflow_build_in_path) as f:
    tensorflow_build = f.read()
  if args.with_cuda:
    requirements_txt = base_requirements.replace(
        'tensorflow=', 'tensorflow-gpu=')
    tensorflow_build = tensorflow_build.replace(
        'requirement("tensorflow")',
        'requirement("tensorflow-gpu")')
    requirements_txt = base_requirements
  requirements_txt_path = os.path.join(PHD_ROOT, 'requirements.txt')
  with open(requirements_txt_path, 'w') as f:
    f.write('# GENERATED BY ./configure. DO NOT EDIT!\n')
    f.write(requirements_txt)
  with open(tensorflow_build_path, 'w') as f:
    f.write('# GENERATED BY ./configure. DO NOT EDIT!\n')
    f.write(tensorflow_build)
  # Symlink python to ~/.local/bin/python.
  if args.symlink_python:
    Mkdir(os.path.expanduser('~/.local/bin'))
    if not os.path.islink(os.path.expanduser('~/.local/bin/python')):
      print('Created ~/.local/bin/python')
      subprocess.check_call(
          ['ln', '-s', python_path, os.path.expanduser('~/.local/bin/python')])
  # Create //config.pbtxt.
  options_to_exclude = {'print_id', 'noninteractive', 'cache_result'}
  options_to_export = {k: args.__dict__[k] for k in args.__dict__
                       if k not in options_to_exclude}
  options = '\n'.join(
      ['  {k}: {v}'.format(k=k, v='true' if options_to_export[k] else 'false')
       for k in options_to_export])
  with open(os.path.join(PHD_ROOT, 'config.pbtxt'), 'w') as f:
    f.write("""\
# Global configuration options. Generated by ./configure. DO NOT EDIT!
# File: //config/proto/config.proto
# Proto: phd.GlobalConfig
uname: "{uname}"
configure_id: "{config_id}"
with_cuda: {with_cuda}
  repo_root: "{phd_root}"
  python: "{python_path}"
""".format(uname=uname, config_id=config_id, phd_root=PHD_ROOT,
           with_cuda='true' if args.with_cuda else 'false',
           options=options, python_path=python_path))
  with open(os.path.join(PHD_ROOT, '.env'), 'w') as f:
    f.write("""\
#!/bin/bash
# Shell environment for the project. Generated by ./configure. DO NOT EDIT!
export PHD="{phd_root}"
if [[ -d "$HOME/venv/phd/bin" ]]; then
  # If there is a virtualenv, use it. Note that even if it does exist, bazel
  # will go ahead and ignore it, so we still need to rely on the system python
  # being the required version.
  . "$HOME/venv/phd/bin/activate"
  # Export a dummy virtualenv.
  # TODO(cec): Add a better way of signalling that we're in the phd env from
  # the command line.
  export VIRTUAL_ENV=phd
# Export a dummy virtualenv.
# TODO(cec): Add a better way of signalling that we're in the phd env from
# the command line.
export VIRTUAL_ENV=phd
# Ensure that default python is 3.6.
echo $PATH | grep $HOME/.local/bin >/dev/null || export PATH="$HOME/.local/bin:$PATH"
export PYTHON="{python_path}"
export PYTHONPATH={phd_root}:{phd_root}/lib:{phd_root}/bazel-genfiles
# TODO(cec): Remove this workaround once the underlying issue is fixed.
# See: https://github.com/bazelbuild/rules_docker/issues/293
export BAZEL_PYTHON=$PHD/tools/py3_wrapper.sh
# TODO(cec): Remove whatever is setting $CC to clang, rather than unset here.
  bazel build //deeplearning/clgen
  {phd_root}/bazel-phd/bazel-out/*-py3-opt/bin/deeplearning/clgen/clgen $@
""".format(phd_root=PHD_ROOT, python_path=python_path))
if __name__ == '__main__':
    main(sys.argv[1:])
  except KeyboardInterrupt:
    print('\ninterrupt', file=sys.stderr)
Provide feedback

Saved searches

Use saved searches to filter your results more quickly

FilesExpand file tree

configure

Latest commit

History

configure

File metadata and controls
