From mboxrd@z Thu Jan 1 00:00:00 1970 From: ludo@gnu.org (Ludovic =?UTF-8?Q?Court=C3=A8s?=) Subject: bug#22533: Non-determinism in python-3 ".pyc" bytecode Date: Wed, 06 Apr 2016 10:29:57 +0200 Message-ID: <87oa9nuol6.fsf@gnu.org> References: <20160202051544.GA11744@jasmine> <87powezvyo.fsf@gnu.org> <20160204231708.GA1297@jasmine> <56FB0C14.9010909@gmail.com> Mime-Version: 1.0 Content-Type: multipart/mixed; boundary="=-=-=" Return-path: Received: from eggs.gnu.org ([2001:4830:134:3::10]:40455) by lists.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1anirh-0002yv-Oj for bug-guix@gnu.org; Wed, 06 Apr 2016 04:31:07 -0400 Received: from Debian-exim by eggs.gnu.org with spam-scanned (Exim 4.71) (envelope-from ) id 1anire-0006A0-FF for bug-guix@gnu.org; Wed, 06 Apr 2016 04:31:05 -0400 Received: from debbugs.gnu.org ([208.118.235.43]:38618) by eggs.gnu.org with esmtp (Exim 4.71) (envelope-from ) id 1anire-00069w-Bk for bug-guix@gnu.org; Wed, 06 Apr 2016 04:31:02 -0400 Received: from Debian-debbugs by debbugs.gnu.org with local (Exim 4.84_2) (envelope-from ) id 1anire-0007kD-5F for bug-guix@gnu.org; Wed, 06 Apr 2016 04:31:02 -0400 Sender: "Debbugs-submit" Resent-Message-ID: In-Reply-To: <56FB0C14.9010909@gmail.com> (Cyril Roelandt's message of "Wed, 30 Mar 2016 01:13:24 +0200") List-Id: Bug reports for GNU Guix List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , Errors-To: bug-guix-bounces+gcggb-bug-guix=m.gmane.org@gnu.org Sender: bug-guix-bounces+gcggb-bug-guix=m.gmane.org@gnu.org To: Cyril Roelandt Cc: 22533@debbugs.gnu.org --=-=-= Content-Type: text/plain Cyril Roelandt skribis: > Here is a version of the patch that works with the upstream Python, but > that I cannot get to work with our Guix recipe. At first sight the patch LGTM. How does it not work for you? :-) I applied this: --=-=-= Content-Type: text/x-patch Content-Disposition: inline diff --git a/gnu/packages/patches/python-3-deterministic-build-info.patch b/gnu/packages/patches/python-3-deterministic-build-info.patch index 22c372a..bdf9f20 100644 --- a/gnu/packages/patches/python-3-deterministic-build-info.patch +++ b/gnu/packages/patches/python-3-deterministic-build-info.patch @@ -15,3 +15,28 @@ We cannot pass it in CPPFLAGS due to whitespace in the DATE string. #ifndef DATE #ifdef __DATE__ #define DATE __DATE__ + +--- Lib/importlib/_bootstrap.py ++++ Lib/importlib/_bootstrap.py +@@ -1443,7 +1443,8 @@ class SourceLoader(_LoaderBasics): + Implementing this method allows the loader to read bytecode files. + Raises IOError when the path cannot be handled. + """ +- return {'mtime': self.path_mtime(path)} ++ return {'mtime': float(_os.environ.get(b'SOURCE_DATE_EPOCH', ++ st.st_mtime))} + + def _cache_bytecode(self, source_path, cache_path, data): + """Optional method which writes data (bytes) to a file path (a str). +@@ -1580,7 +1581,10 @@ class SourceFileLoader(FileLoader, SourceLoader): + def path_stats(self, path): + """Return the metadata for the path.""" + st = _path_stat(path) +- return {'mtime': st.st_mtime, 'size': st.st_size} ++ return { ++ 'mtime': float(_os.environ.get(b'SOURCE_DATE_EPOCH', st.st_mtime)), ++ 'size': st.st_size ++ } + + def _cache_bytecode(self, source_path, bytecode_path, data): + # Adapt between the two APIs --=-=-= Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: quoted-printable =E2=80=A6 and that leads to these test failures: --8<---------------cut here---------------start------------->8--- $ ./pre-inst-env guix build python@3 --rounds=3D2 -K [...] =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D FAIL: test_bad_marshal (test.test_importlib.source.test_file_loader.Source_= SourceLoaderBadBytecodeTestPEP302) ---------------------------------------------------------------------- Traceback (most recent call last): File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/util.py", line 22, in wrapper to_return =3D fxn(*args, **kwargs) File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 452, in test_bad_marshal self._test_bad_marshal() File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 342, in _test_bad_marshal self.import_(file_path, '_temp') AssertionError: EOFError not raised =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D FAIL: test_no_marshal (test.test_importlib.source.test_file_loader.Source_S= ourceLoaderBadBytecodeTestPEP302) ---------------------------------------------------------------------- Traceback (most recent call last): File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/util.py", line 22, in wrapper to_return =3D fxn(*args, **kwargs) File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 441, in test_no_marshal self._test_no_marshal() File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 322, in _test_no_marshal self.import_(file_path, '_temp') AssertionError: EOFError not raised =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D FAIL: test_non_code_marshal (test.test_importlib.source.test_file_loader.So= urce_SourceLoaderBadBytecodeTestPEP302) ---------------------------------------------------------------------- Traceback (most recent call last): File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/util.py", line 22, in wrapper to_return =3D fxn(*args, **kwargs) File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 445, in test_non_code_marshal self._test_non_code_marshal() File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 331, in _test_non_code_marsh= al self.import_(file_path, '_temp') AssertionError: ImportError not raised =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D FAIL: test_old_timestamp (test.test_importlib.source.test_file_loader.Sourc= e_SourceLoaderBadBytecodeTestPEP302) ---------------------------------------------------------------------- Traceback (most recent call last): File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/util.py", line 22, in wrapper to_return =3D fxn(*args, **kwargs) File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 471, in test_old_timestamp self.assertEqual(bytecode_file.read(4), source_timestamp) AssertionError: b'\x01\x00\x00\x00' !=3D b'\x7f\xc7\x04W' =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D FAIL: test_bad_marshal (test.test_importlib.source.test_file_loader.Source_= SourceLoaderBadBytecodeTestPEP451) ---------------------------------------------------------------------- Traceback (most recent call last): File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/util.py", line 22, in wrapper to_return =3D fxn(*args, **kwargs) File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 452, in test_bad_marshal self._test_bad_marshal() File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 342, in _test_bad_marshal self.import_(file_path, '_temp') AssertionError: EOFError not raised =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D FAIL: test_no_marshal (test.test_importlib.source.test_file_loader.Source_S= ourceLoaderBadBytecodeTestPEP451) ---------------------------------------------------------------------- Traceback (most recent call last): File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/util.py", line 22, in wrapper to_return =3D fxn(*args, **kwargs) File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 441, in test_no_marshal self._test_no_marshal() File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 322, in _test_no_marshal self.import_(file_path, '_temp') AssertionError: EOFError not raised =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D FAIL: test_non_code_marshal (test.test_importlib.source.test_file_loader.So= urce_SourceLoaderBadBytecodeTestPEP451) ---------------------------------------------------------------------- Traceback (most recent call last): File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/util.py", line 22, in wrapper to_return =3D fxn(*args, **kwargs) File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 445, in test_non_code_marshal self._test_non_code_marshal() File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 331, in _test_non_code_marsh= al self.import_(file_path, '_temp') AssertionError: ImportError not raised =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D= =3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D=3D FAIL: test_old_timestamp (test.test_importlib.source.test_file_loader.Sourc= e_SourceLoaderBadBytecodeTestPEP451) ---------------------------------------------------------------------- Traceback (most recent call last): File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/util.py", line 22, in wrapper to_return =3D fxn(*args, **kwargs) File "/tmp/guix-build-python-minimal-3.4.3.drv-0/Python-3.4.3/Lib/test/te= st_importlib/source/test_file_loader.py", line 471, in test_old_timestamp self.assertEqual(bytecode_file.read(4), source_timestamp) AssertionError: b'\x01\x00\x00\x00' !=3D b'\x7f\xc7\x04W' ---------------------------------------------------------------------- Ran 951 tests in 1.102s FAILED (failures=3D8, skipped=3D19, expected failures=3D1) Makefile:958: recipe for target 'test' failed --8<---------------cut here---------------end--------------->8--- =E2=80=98test_old_timestamp=E2=80=99 clearly needs to be adjusted to accoun= t for the change. The others have to do with the bytecode loader, so it=E2=80=99s probably a similar story. Could you look into it? Perhaps you tested with SOURCE_DATE_EPOCH unset? Thanks for working on this, it=E2=80=99s an important bug to fix! Ludo=E2=80=99. --=-=-=--