build: Manage mspdbsrv instances during Win builds

peter-b · peter-b · commit ae5a76c2fd6f · 2017-02-16T09:09:51.000Z
Builds were stalling and timing out on Windows, and this appeared
to be due to `mspdbsrv` instances hanging around after the build
had been completed.

Further investigation revealed a host of issues related to
`mspdbsrv` process management.  This patch updates the Windows
buildbot build script to prevent as many of these issues as
possible.
diff --git a/buildbot.py b/buildbot.py
@@ -31,6 +31,9 @@
 import shutil
 import tarfile
 
+# LiveCode build configuration script
+import config
+
 # The set of platforms for which this branch supports automated builds
 BUILDBOT_PLATFORMS = ('linux-x86', 'linux-x86_64', 'android-armv6', 'mac',
     'ios', 'win-x86', 'emscripten')
@@ -77,6 +80,9 @@ def get_buildtype():
 def get_build_edition():
     return os.environ.get('BUILD_EDITION', 'community')
 
+def get_git_commit():
+    return subprocess.check_output(['git','rev-parse','HEAD']).strip()
+
 ################################################################
 # Defer to buildbot.mk
 ################################################################
@@ -91,7 +97,6 @@ def exec_buildbot_make(target):
 ################################################################
 
 def exec_configure(args):
-    import config
     print('config.py ' + ' '.join(args))
     sys.exit(config.configure(args))
 
@@ -118,15 +123,69 @@ def exec_make(target):
     print(' '.join(args))
     sys.exit(subprocess.call(args))
 
+# mspdbsrv is the service used by Visual Studio to collect debug
+# data during compilation.  One instance is shared by all C++
+# compiler instances and threads.  It poses a unique challenge in
+# several ways:
+#
+# - If not running when the build job starts, the build job will
+#   automatically spawn it as soon as it needs to emit debug symbols.
+#   There's no way to prevent this from happening.
+#
+# - The build job _doesn't_ automatically clean it up when it finishes
+#
+# - By default, mspdbsrv inherits its parent process' file handles,
+#   including (unfortunately) some log handles owned by Buildbot.  This
+#   can prevent Buildbot from detecting that the compile job is finished
+#
+# - If a compile job starts and detects an instance of mspdbsrv already
+#   running, by default it will reuse it.  So, if you have a compile
+#   job A running, and start a second job B, job B will use job A's
+#   instance of mspdbsrv.  If you kill mspdbsrv when job A finishes,
+#   job B will die horribly.  To make matters worse, the version of
+#   mspdbsrv should match the version of Visual Studio being used.
+#
+# This class works around these problems:
+#
+# - It sets the _MSPDBSRV_ENDPOINT_ to a value that's probably unique to
+#   the build, to prevent other builds on the same machine from sharing
+#   the same mspdbsrv endpoint
+#
+# - It launches mspdbsrv with _all_ file handles closed, so that it
+#   can't block the build from being detected as finished.
+#
+# - It explicitly kills mspdbsrv after the build job has finished.
+#
+# - It wraps all of this into a context manager, so mspdbsrv gets killed
+#   even if a Python exception causes a non-local exit.
+class UniqueMspdbsrv(object):
+    def __enter__(self):
+        os.environ['_MSPDBSRV_ENDPOINT_'] = '{}-{}-{}'.format(
+            get_build_edition(), get_git_commit()[0:8], get_buildtype())
+
+        mspdbsrv_exe = os.path.join(config.get_program_files_x86(),
+            'Microsoft Visual Studio 10.0\\Common7\\IDE\\mspdbsrv.exe')
+        args = [mspdbsrv_exe, '-start', '-shutdowntime', '-1']
+        print(' '.join(args))
+        self.proc = subprocess.Popen(args, close_fds=True)
+        return self
+
+    def __exit__(self, type, value, traceback):
+        self.proc.terminate()
+        return False
+
 def exec_msbuild(platform):
     # Run the make.cmd batch script; it's run using Wine if this is
     # not actually a Windows system.
     cwd = 'build-' + platform
 
     if _platform.system() == 'Windows':
-        args = ['cmd', '/C', '..\\make.cmd']
-        print(' '.join(args))
-        sys.exit(subprocess.call(args, cwd=cwd))
+        with UniqueMspdbsrv() as mspdbsrv:
+            args = ['cmd', '/C', '..\\make.cmd']
+            print(' '.join(args))
+            result = subprocess.call(args, cwd=cwd)
+
+        sys.exit(result)
 
     else:
         args = ['wine', 'cmd', '/K', '..\\make.cmd']
diff --git a/make.cmd b/make.cmd
@@ -18,8 +18,8 @@ ECHO %ProgramFilesBase%
 @REM Run this with its CWD outside the build tree so that
 @REM the fact it hangs around does not interfere with
 @REM cleaning up the build tree.
-@pushd %ProgramFilesBase%
-@start /min /b mspdbsrv -start -spawn -shutdowntime -1
+@pushd \
+@REM @start /min mspdbsrv -start -spawn -shutdowntime -1
 @popd
 
 @REM Select the correct build mode.
