bug#22533: Python bytecode reproducibility

[Gábor Boskovits]

2018-03-04 20:18 GMT+01:00 Ricardo Wurmus <address@hidden>:

I have applied this patch locally:


diff --git a/gnu/packages/python.scm b/gnu/packages/python.scm
index 5f701701a..0d1ecc3c6 100644
--- a/gnu/packages/python.scm
+++ b/gnu/packages/python.scm
@@ -359,8 +359,42 @@ data types.")
                               "Lib/ctypes/test/test_win32.py" ; fails on aarch64
                               "Lib/test/test_fcntl.py")) ; fails on aarch64
                   #t))))
-    (arguments (substitute-keyword-arguments (package-arguments python-2)
-                 ((#:tests? _) #t)))
+    (arguments
+     (substitute-keyword-arguments (package-arguments python-2)
+       ((#:tests? _) #t)
+       ((#:phases phases)
+        `(modify-phases ,phases
+           (add-after 'unpack 'patch-timestamp-for-pyc-files
+             (lambda _
+               ;; We set DETERMINISTIC_BUILD to only override the mtime when
+               ;; building with Guix, lest we break auto-compilation in
+               ;; environments.
+               (setenv "DETERMINISTIC_BUILD" "1")
+               (substitute* "Lib/py_compile.py"
+                 (("source_stats\\['mtime'\\]")
+                  "(1 if 'DETERMINISTIC_BUILD' in os.environ else source_stats['mtime'])"))
+
+               ;; Use deterministic hashes for strings, bytes, and datetime
+               ;; objects.
+               (setenv "PYTHONHASHSEED" "0")
+
+               ;; Reset mtime when validating bytecode header.
+               (substitute* "Lib/importlib/_bootstrap_external.py"
+                 (("source_mtime = int\\(source_stats\\['mtime'\\]\\)")
+                  "source_mtime = 1"))
+               #t))
+           (add-after 'unpack 'disable-timestamp-tests
+             (lambda _
+               (substitute* "Lib/test/test_importlib/source/test_file_loader.py"
+                 (("test_bad_marshal")
+                  "disable_test_bad_marshal")
+                 (("test_no_marshal")
+                  "disable_test_no_marshal")
+                 (("test_non_code_marshal")
+                  "disable_test_non_code_marshal"))
+               #t))
+           (add-before 'check 'allow-non-deterministic-compilation
+             (lambda _ (unsetenv "DETERMINISTIC_BUILD") #t))))))
     (native-search-paths
      (list (search-path-specification
             (variable "PYTHONPATH")


It allows me to build python-six and python-sip reproducibly.  It does
not fix problems with Python 2, and I haven’t yet tested if it causes
any new problems.

It’s a little worrying that I had to disable three more tests that I
think shouldn’t have failed.

Ok, I've checked the test issue again. If we change the _bootstrap_external.py

substitution to:

"source_mtime = 1 if 'DETERMINISTIC_BUILD' in _os.environ else int(source_stats['mtime'])"

the test do not fail any more. WDYT?

 

What do you think?

--
Ricardo

GPG: BCA6 89B6 3655 3801 C3C6  2150 197A 5888 235F ACAC
https://elephly.net