From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from localhost (localhost [127.0.0.1]) by arlo.cworth.org (Postfix) with ESMTP id 897756DE01E6 for ; Sun, 27 Jan 2019 14:04:38 -0800 (PST) X-Virus-Scanned: Debian amavisd-new at cworth.org X-Spam-Flag: NO X-Spam-Score: 0.041 X-Spam-Level: X-Spam-Status: No, score=0.041 tagged_above=-999 required=5 tests=[AWL=0.041, DKIMWL_WL_MED=0.001, DKIM_SIGNED=0.1, DKIM_VALID=-0.1, RCVD_IN_DNSWL_NONE=-0.0001, RCVD_IN_MSPIKE_H2=-0.001] autolearn=disabled Received: from arlo.cworth.org ([127.0.0.1]) by localhost (arlo.cworth.org [127.0.0.1]) (amavisd-new, port 10024) with ESMTP id F5RDqn2Qp8YN for ; Sun, 27 Jan 2019 14:04:37 -0800 (PST) Received: from mail-lj1-f194.google.com (mail-lj1-f194.google.com [209.85.208.194]) by arlo.cworth.org (Postfix) with ESMTPS id E9B276DE01E1 for ; Sun, 27 Jan 2019 14:04:36 -0800 (PST) Received: by mail-lj1-f194.google.com with SMTP id c19-v6so12564208lja.5 for ; Sun, 27 Jan 2019 14:04:36 -0800 (PST) DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=nikula-org.20150623.gappssmtp.com; s=20150623; h=from:to:cc:subject:date:message-id:in-reply-to:references :mime-version:content-transfer-encoding; bh=D6Ghty2eUMXJUk4TloU0s0ZSGTIknh/8mmCVpgrdA1Q=; b=MclkP+xJt4z1gBkGRz9obRB6KRDoFpQsKUO5jTypKLjMtBtDCISMYYhm31Lyzb1a+Z KLKmWLYR4F0i2/5aoislY7KTqlFVNp8lMQyIzLCfBiqCcsnbbDprYeuKT7WTUDLPhl9+ TwnKZC1LIEX+BHtRTpVu0rmRTI2DtJneWUPOq++MynzVlijLEbJNRgT8iwEwWorHDTMu Jv88XUOFYq2jSCGK3FPt4mfxDiN6ufbktJWbTSf6NyA3v6JuZoSqnprqzzltY1MzIdmb pkbD/XdoeEo9nDnpHgTqfcj1u3fqijL+xQ15hV/XA5WNfenwLohHXPIspR69OJ0Tt8A1 XiNA== X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=1e100.net; s=20161025; h=x-gm-message-state:from:to:cc:subject:date:message-id:in-reply-to :references:mime-version:content-transfer-encoding; bh=D6Ghty2eUMXJUk4TloU0s0ZSGTIknh/8mmCVpgrdA1Q=; b=sziNS7M9j66+c7UhSE8RM8qsJ9FpA1nz02FRUMKy0a24HxLCh00CGHqgYtunDdi83n 3QhWKQHsRXRXbvebr7o6MvKR5LpDW5+DT454vRIg5Lt9q8xSjpkuYBwAYOfCJIlXHu+0 zIabD+/rtcn/pQZkIsh8e/YC/abzJra5ZigQbXbPbr/Fx7YQEWOVDvDyfOox1bxEYr5/ eykAGp8qYyKr3hFg27wzzuE+NiuIvaLFXA4RFjBBVu5lnv/hgYVk11OGT89Kr/JoaMp9 FSH9cWZM+i8yEk8ycwQKn1NejNHGPu54FaZ9WoD13JQ6QnlAtcjcJ4Y1CeNddxYqJ5AR Wphw== X-Gm-Message-State: AJcUukehQEtjTJt1IR3XRz0inNBUZCvHxnIdejM3CZy491wgX41whRKX kIBhJDnkwo7OePI23m6W8F9wZRXs53fbIg== X-Google-Smtp-Source: ALg8bN4nJb91ocYSHrXfo2suWr4m6hGJQmPSglAoA9H7xMjnkniOVw5mEnLGgiLUqlBSsijlPSg4xA== X-Received: by 2002:a2e:1b47:: with SMTP id b68-v6mr15157246ljb.104.1548626674725; Sun, 27 Jan 2019 14:04:34 -0800 (PST) Received: from localhost (82-203-142-74.bb.dnainternet.fi. [82.203.142.74]) by smtp.gmail.com with ESMTPSA id w12sm2630428lfe.80.2019.01.27.14.04.33 (version=TLS1_2 cipher=ECDHE-RSA-CHACHA20-POLY1305 bits=256/256); Sun, 27 Jan 2019 14:04:34 -0800 (PST) From: Jani Nikula To: notmuch@notmuchmail.org Subject: [PATCH v2 2/4] test: dynamically generate directive tests Date: Mon, 28 Jan 2019 00:04:27 +0200 Message-Id: <20190127220429.9206-2-jani@nikula.org> X-Mailer: git-send-email 2.20.1 In-Reply-To: <20190127220429.9206-1-jani@nikula.org> References: <20190127220429.9206-1-jani@nikula.org> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-BeenThere: notmuch@notmuchmail.org X-Mailman-Version: 2.1.29 Precedence: list List-Id: "Use and development of the notmuch mail system." List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Sun, 27 Jan 2019 22:04:38 -0000 Similar to the parser test transition to dynamically generate test methods, but with more test separation added. After this, the Sphinx build is done independently for each test case, and separately for the actual directive output and the expected output. This removes any potential C domain interactions between test cases and expected/actual inputs. The change does mean the Sphinx build is run roughly 50x times per full test run, at the current number of test cases. This is somewhat offset by the ability to run individual directive test cases: $ test/test_cautodoc.py DirectiveTest.test_example_10_macro --- test/{sphinx => }/conf.py | 0 test/sphinx/index.rst | 20 ------------- test/test_cautodoc.py | 61 ++++++++++++++------------------------- 3 files changed, 21 insertions(+), 60 deletions(-) rename test/{sphinx => }/conf.py (100%) delete mode 100644 test/sphinx/index.rst diff --git a/test/sphinx/conf.py b/test/conf.py similarity index 100% rename from test/sphinx/conf.py rename to test/conf.py diff --git a/test/sphinx/index.rst b/test/sphinx/index.rst deleted file mode 100644 index 20404ce8ee5e..000000000000 --- a/test/sphinx/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. Hawkmoth Test documentation master file, created by - sphinx-quickstart on Wed Dec 12 13:11:26 2018. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Welcome to Hawkmoth Test's documentation! -========================================= - -.. toctree:: - :maxdepth: 2 - :caption: Contents: - - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/test/test_cautodoc.py b/test/test_cautodoc.py index 3fb98af8629d..848f2105a1da 100755 --- a/test/test_cautodoc.py +++ b/test/test_cautodoc.py @@ -9,54 +9,35 @@ import unittest import testenv from sphinx_testing import with_app -class DirectiveTest(unittest.TestCase): - - def _setup_src(self, srcdir, testcase_in): - testcase_out = testenv.modify_filename(testcase_in, dir=srcdir) - - # use the pre-generated rst as comparison data - shutil.copyfile(testenv.modify_filename(testcase_in, ext='stdout'), - testenv.modify_filename(testcase_out, ext='expected.rst')) - - # set up an rst file to run the extension - shutil.copyfile(testcase_in, testcase_out) - options = testenv.get_testcase_options(testcase_in) +@with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='text') +def _get_output(input_filename, app, status, warning, **options): + shutil.copyfile(input_filename, + testenv.modify_filename(input_filename, dir=app.srcdir)) - with open(testenv.modify_filename(testcase_out, ext='output.rst'), 'w') as file: - fmt = '.. c:autodoc:: {source}\n' - file.write(fmt.format(source=os.path.basename(testcase_out))) - for key in options.keys(): - fmt = ' :{key}: {value}\n' - file.write(fmt.format(key=key, value=options[key])) + with open(os.path.join(app.srcdir, 'index.rst'), 'w') as file: + fmt = '.. c:autodoc:: {source}\n' + file.write(fmt.format(source=os.path.basename(input_filename))) + for key in options.keys(): + fmt = ' :{key}: {value}\n' + file.write(fmt.format(key=key, value=options[key])) - def _check_out(self, outdir, testcase_in): - testcase_out = testenv.modify_filename(testcase_in, dir=outdir) + app.build() - # compare output from the pre-generated rst against the output generated - # by the extension + return testenv.read_file(os.path.join(app.outdir, 'index.txt')) - output = testenv.read_file(testenv.modify_filename(testcase_out, - ext='output.txt')) +@with_app(confdir=testenv.testdir, create_new_srcdir=True, buildername='text') +def _get_expected(input_filename, app, status, warning, **options): + shutil.copyfile(testenv.modify_filename(input_filename, ext='stdout'), + os.path.join(app.srcdir, 'index.rst')) - expected = testenv.read_file(testenv.modify_filename(testcase_out, - ext='expected.txt')) + app.build() - self.assertEqual(expected, output) + return testenv.read_file(os.path.join(app.outdir, 'index.txt')) - # Use copy_srcdir_to_tmpdir=False and outdir='some-dir' for debugging - @with_app(srcdir=os.path.join(testenv.testdir, 'sphinx'), - buildername='text', copy_srcdir_to_tmpdir=True) - def test_directive(self, app, status, warning): - testcases = list(testenv.get_testcases(testenv.testdir)) - - for f in testcases: - self._setup_src(app.srcdir, f) - - app.build() +class DirectiveTest(unittest.TestCase): + pass - for f in testcases: - with self.subTest(source=os.path.basename(f)): - self._check_out(app.outdir, os.path.basename(f)) +testenv.assign_test_methods(DirectiveTest, _get_output, _get_expected) if __name__ == '__main__': unittest.main() -- 2.20.1