[sagemath] 01/01: Re-enable parallel build but using subprocess instead
Ximin Luo
infinity0 at debian.org
Tue Sep 19 21:59:15 UTC 2017
This is an automated email from the git hooks/post-receive script.
infinity0 pushed a commit to branch master
in repository sagemath.
commit d6bd3b7ec076a1cbdd1809797e44a08c83f99d23
Author: Ximin Luo <infinity0 at debian.org>
Date: Tue Sep 19 23:59:08 2017 +0200
Re-enable parallel build but using subprocess instead
---
debian/changelog | 1 +
debian/patches/d1-sage-env.patch | 2 +-
.../patches/df-disable-custom-sphinx-logger.patch | 19 ------
debian/patches/df-subprocess-sphinx.patch | 40 ++++++++++++
debian/patches/dt-disable-parallel-sphinx.patch | 20 ------
debian/patches/series | 4 +-
.../patches/u2-better-sphinx-failure-modes.patch | 75 ++++++++++++++++++++++
7 files changed, 119 insertions(+), 42 deletions(-)
diff --git a/debian/changelog b/debian/changelog
index 3e82770..a8b137a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -3,6 +3,7 @@ sagemath (8.0-8) UNRELEASED; urgency=medium
[ Ximin Luo ]
* Set a higher test timeout for slower architectures.
* Don't ignore timed-out tests on i386, they work fine.
+ * Re-enable parallel build but using subprocess instead.
[ Tobias Hansen ]
* Add upstream patch to build with Sphinx (>= 1.6):
diff --git a/debian/patches/d1-sage-env.patch b/debian/patches/d1-sage-env.patch
index f30c258..6c5d618 100644
--- a/debian/patches/d1-sage-env.patch
+++ b/debian/patches/d1-sage-env.patch
@@ -182,7 +182,7 @@ Author: Tobias Hansen <thansen at debian.org>, Ximin Luo <infinity0 at debian.org>
--- a/sage/src/sage_setup/docbuild/__init__.py
+++ b/sage/src/sage_setup/docbuild/__init__.py
-@@ -396,7 +396,7 @@
+@@ -412,7 +412,7 @@
document_name = document.split('/')[1]
# the sage directory within a subdocument, for example
diff --git a/debian/patches/df-disable-custom-sphinx-logger.patch b/debian/patches/df-disable-custom-sphinx-logger.patch
deleted file mode 100644
index 863d5a0..0000000
--- a/debian/patches/df-disable-custom-sphinx-logger.patch
+++ /dev/null
@@ -1,19 +0,0 @@
-Description: Disable a convoluted "build log buffering" mechanism which was causing problems.
-Author: Ximin Luo <infinity0 at debian.org>
-Forwarded: not-needed
----
-This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
---- a/sage/src/sage_setup/docbuild/sphinxbuild.py
-+++ b/sage/src/sage_setup/docbuild/sphinxbuild.py
-@@ -205,8 +205,9 @@
- saved_stderr = sys.stderr
-
- try:
-- sys.stdout = SageSphinxLogger(sys.stdout, os.path.basename(output_dir))
-- sys.stderr = SageSphinxLogger(sys.stderr, os.path.basename(output_dir))
-+ # Sage is too smart for their own good, this poorly-implemented "file" replacement breaks stuff
-+ #sys.stdout = SageSphinxLogger(sys.stdout, os.path.basename(output_dir))
-+ #sys.stderr = SageSphinxLogger(sys.stderr, os.path.basename(output_dir))
- sphinx.cmdline.main(sys.argv)
- finally:
- sys.stdout = saved_stdout
diff --git a/debian/patches/df-subprocess-sphinx.patch b/debian/patches/df-subprocess-sphinx.patch
new file mode 100644
index 0000000..72bf359
--- /dev/null
+++ b/debian/patches/df-subprocess-sphinx.patch
@@ -0,0 +1,40 @@
+Description: Use subprocess to run parallel sphinx build
+ The upstream sphinx parallel build segfaults because we don't patch Python,
+ see our revert-lazy-import patches for details. (Those reverts are needed to
+ make the serial build not segfault, but the parallel build still segfaults.)
+ .
+ We used to disable parallel build completely, but it was too slow. This
+ alternative method of running the parallel build seems to work OK.
+ .
+ As of SageMath 8.0 the RAM usage seems to be more reasonable (~3-4GB for 4
+ threads), but please complain on the mailing list if this eats your machine.
+Author: Ximin Luo <infinity0 at debian.org>
+Forwarded: TBD
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/sage/src/sage_setup/docbuild/__init__.py
++++ b/sage/src/sage_setup/docbuild/__init__.py
+@@ -98,10 +98,9 @@
+ logger.debug(build_command)
+
+ # Run Sphinx with Sage's special logger
+- sys.argv = ["sphinx-build"] + build_command.split()
+- from .sphinxbuild import runsphinx
++ args = "python -um sage_setup.docbuild.sphinxbuild -N".split() + build_command.split()
+ try:
+- runsphinx()
++ subprocess.check_call(args)
+ except Exception:
+ # https://bugs.python.org/issue13831Ups, multiprocessing loses the stack trace
+ logger.exception("sphinx job failed:")
+--- a/sage/src/sage_setup/docbuild/sphinxbuild.py
++++ b/sage/src/sage_setup/docbuild/sphinxbuild.py
+@@ -216,3 +216,8 @@
+
+ if exception is not None:
+ raise exception
++
++if __name__ == '__main__':
++ import sys
++ sys.argv[0] = "sphinx-build"
++ runsphinx()
diff --git a/debian/patches/dt-disable-parallel-sphinx.patch b/debian/patches/dt-disable-parallel-sphinx.patch
deleted file mode 100644
index 1c6aace..0000000
--- a/debian/patches/dt-disable-parallel-sphinx.patch
+++ /dev/null
@@ -1,20 +0,0 @@
-Description: Temporarily disable parallel sphinx build to make component failures more obvious
- Before we completely remove this patch, we should (1) figure out how to make
- the parallel-build fail-fast in case its components fail in the future; and
- (2) cap the number of threads used - it seems that each thread uses 2-6 GB of
- RAM (unclear what the exact figure is, it varies between different people)
-Author: Ximin Luo <infinity0 at debian.org>
-Forwarded: not-needed
----
-This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
---- a/sage/src/sage_setup/docbuild/__init__.py
-+++ b/sage/src/sage_setup/docbuild/__init__.py
-@@ -234,7 +234,7 @@
- inventory = builder_helper('inventory')
-
-
--if NUM_THREADS > 1:
-+if False:
- def build_many(target, args):
- from multiprocessing import Pool
- pool = Pool(NUM_THREADS, maxtasksperchild=1)
diff --git a/debian/patches/series b/debian/patches/series
index 01f06d8..dba9718 100644
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -26,6 +26,7 @@ u2-fix-less-trivial-test-cases.patch
u2-fix-sympow-cachedir.patch
u2-allow-override-sage-local.patch
u2-workaround-sympow-malloc-perturb.patch
+u2-better-sphinx-failure-modes.patch
# Patch Sage to work with dependency Debian packages
# These won't change even if Debian and Sage use the same version
@@ -61,15 +62,14 @@ d1-disable-post-install-tests.patch
df-revert-lazy-imports-22755.patch
df-revert-lazy-import-dependent-on-python-patch.patch
df-revert-minor-feature-dependent-on-python-patch.patch
-df-disable-custom-sphinx-logger.patch
df-python_security.patch
df-sage-gap-reset-doctest.patch
df-silence_debian_bug_780186.patch
df-silence_ecl_opt_signals.patch
+df-subprocess-sphinx.patch
# Temporary patches for whatever reason
# They will eventually be gotten rid of
-dt-disable-parallel-sphinx.patch
dt-version-glpk-4.60-extra-hacky-fixes.patch
dt-r-no-readline.patch
dt-more-fix-test-cases.patch
diff --git a/debian/patches/u2-better-sphinx-failure-modes.patch b/debian/patches/u2-better-sphinx-failure-modes.patch
new file mode 100644
index 0000000..665cf00
--- /dev/null
+++ b/debian/patches/u2-better-sphinx-failure-modes.patch
@@ -0,0 +1,75 @@
+Description: Better failure modes for the docbuild
+ - Make non-Exception BaseExceptions like SignalError fail the docbuild early.
+ - Don't strip lines in the custom logger, because it messes with stack traces.
+ - Slight refactoring of the build_many stuff to make it easier to play with.
+Author: Ximin Luo <infinity0 at debian.org>
+Forwarded: TBD
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/sage/src/sage_setup/docbuild/__init__.py
++++ b/sage/src/sage_setup/docbuild/__init__.py
+@@ -103,8 +103,19 @@
+ try:
+ runsphinx()
+ except Exception:
++ # https://bugs.python.org/issue13831Ups, multiprocessing loses the stack trace
++ logger.exception("sphinx job failed:")
+ if ABORT_ON_ERROR:
+ raise
++ # this allows us to catch errors like SignalError (raised by cysignals
++ # during a segfault) which inherit from BaseException directly
++ except BaseException as e:
++ # https://bugs.python.org/issue13831Ups, multiprocessing loses the stack trace
++ logger.exception("sphinx job failed extraordinarily:")
++ if ABORT_ON_ERROR:
++ # later on, multiprocessing also ignores non-Exception BaseExceptions so
++ # wrap our non-Exception BaseException in an Exception. hey, dawg.
++ raise Exception(str(e), e)
+
+ if "/latex" in output_dir:
+ logger.warning("LaTeX file written to {}".format(output_dir))
+@@ -234,8 +245,8 @@
+ inventory = builder_helper('inventory')
+
+
+-if NUM_THREADS > 1:
+- def build_many(target, args):
++if True:
++ def build_many_parallel(target, args):
+ from multiprocessing import Pool
+ pool = Pool(NUM_THREADS, maxtasksperchild=1)
+ # map_async handles KeyboardInterrupt correctly. Plain map and
+@@ -250,8 +261,8 @@
+ if ABORT_ON_ERROR:
+ raise
+ return ret
+-else:
+- def build_many(target, args):
++
++ def build_many_serial(target, args):
+ results = []
+
+ for arg in args:
+@@ -263,6 +274,11 @@
+
+ return results
+
++if NUM_THREADS > 1:
++ build_many = build_many_parallel
++else:
++ build_many = build_many_serial
++
+
+ ##########################################
+ # Parallel Building Ref Manual #
+--- a/sage/src/sage_setup/docbuild/sphinxbuild.py
++++ b/sage/src/sage_setup/docbuild/sphinxbuild.py
+@@ -118,7 +118,7 @@
+ return
+ for (old, new) in self.replacements:
+ line = old.sub(new, line)
+- line = self._prefix + ' ' + line.strip() + '\n'
++ line = self._prefix + ' ' + line + '\n'
+ if not self._color:
+ line = self.ansi_color.sub('', line)
+ self._stream.write(line)
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-science/packages/sagemath.git
More information about the debian-science-commits
mailing list