Integration tests: Port the ordering plugin to pytest
Ordered integration tests may now be run with pytest. https://fedorahosted.org/freeipa/ticket/4610 Reviewed-By: Tomas Babej <tbabej@redhat.com>
This commit is contained in:
1 parent
0cb12f3cde
commit
3a9a98b285
7 files changed
+146
-108
No files matched your search
@@ -306,6 +306,7 @@ Requires: %{name}-python = %{version}-%{release}
|
||||
Requires: tar
|
||||
Requires: xz
|
||||
Requires: python-nose
|
||||
Requires: pytest >= 2.6
|
||||
Requires: python-paste
|
||||
Requires: python-coverage
|
||||
Requires: python-polib
|
||||
|
||||
@@ -1,106 +0,0 @@
|
||||
# Authors:
|
||||
# Petr Viktorin <pviktori@redhat.com>
|
||||
#
|
||||
# Copyright (C) 2013 Red Hat
|
||||
# see file 'COPYING' for use and warranty information
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""A Nose plugin that allows ordered test cases"""
|
||||
|
||||
import os
|
||||
import unittest
|
||||
import inspect
|
||||
|
||||
from nose.plugins import Plugin
|
||||
import nose.loader
|
||||
import nose.util
|
||||
|
||||
|
||||
def ordered(cls):
|
||||
"""Decorator that marks a test class as ordered
|
||||
|
||||
Methods within the marked class will be executed in definition order
|
||||
(or more strictly, in ordered by the line number where they're defined).
|
||||
|
||||
Subclasses of unittest.TestCase can not be ordered.
|
||||
|
||||
Generator methods will not be ordered by this plugin.
|
||||
"""
|
||||
cls._order_plugin__ordered = True
|
||||
assert not isinstance(cls, unittest.TestCase), (
|
||||
"A unittest.TestCase may not be ordered.")
|
||||
return cls
|
||||
|
||||
|
||||
class OrderTests(Plugin):
|
||||
name = 'ordered-tests'
|
||||
|
||||
def options(self, parser, env=os.environ):
|
||||
super(OrderTests, self).options(parser, env=env)
|
||||
|
||||
def configure(self, options, conf):
|
||||
super(OrderTests, self).configure(options, conf)
|
||||
if not self.enabled:
|
||||
return
|
||||
|
||||
def loadTestsFromTestClass(self, cls):
|
||||
"""Sort methods of ordered test cases by co_firstlineno"""
|
||||
if not getattr(cls, '_order_plugin__ordered', False):
|
||||
return
|
||||
loader = nose.loader.TestLoader()
|
||||
|
||||
def wanted(attr):
|
||||
item = getattr(cls, attr, None)
|
||||
if not inspect.ismethod(item):
|
||||
return False
|
||||
if nose.util.isgenerator(item.im_func):
|
||||
return False
|
||||
return loader.selector.wantMethod(item)
|
||||
|
||||
def sort_with_respect_to_overriding(func):
|
||||
"""
|
||||
Sorts the methods in respect with the parent classes.
|
||||
|
||||
The methods are sorted with respect to the inheritance chain,
|
||||
methods that were defined in the same class are sorted by the line
|
||||
number on which they were defined.
|
||||
"""
|
||||
|
||||
# Check each *ordered* class in MRO for definition of func method
|
||||
for i, parent_class in enumerate(reversed(cls.mro())):
|
||||
if getattr(parent_class, '_order_plugin__ordered', False):
|
||||
method = getattr(parent_class, func.__name__, None)
|
||||
if method:
|
||||
# This sorts methods as tuples (position of the class
|
||||
# in the inheritance chain, position of the method
|
||||
# within that class)
|
||||
return 0, i, method.func_code.co_firstlineno
|
||||
|
||||
# Weird case fallback
|
||||
# Method name not in any of the classes in MRO, run it last
|
||||
return 1, func.func_code.co_firstlineno
|
||||
|
||||
methods = [getattr(cls, case) for case in dir(cls) if wanted(case)]
|
||||
methods.sort(key=sort_with_respect_to_overriding)
|
||||
cases = [loader.makeTest(m, cls) for m in methods]
|
||||
return cases
|
||||
|
||||
def wantMethod(self, method):
|
||||
"""Hide non-TestCase methods from the normal loader"""
|
||||
im_class = getattr(method, 'im_class', None)
|
||||
if im_class and getattr(im_class, '_order_plugin__ordered', False):
|
||||
if nose.util.isgenerator(method.im_func):
|
||||
return True
|
||||
return False
|
||||
@@ -8,6 +8,7 @@
|
||||
python_classes = test_ Test
|
||||
addopts = --doctest-modules
|
||||
-p ipatests.pytest_plugins.declarative
|
||||
-p ipatests.pytest_plugins.ordering
|
||||
# Ignore files for doc tests.
|
||||
# TODO: ideally, these should all use __name__=='__main__' guards
|
||||
--ignore=setup.py
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
# Authors:
|
||||
# Petr Viktorin <pviktori@redhat.com>
|
||||
#
|
||||
# Copyright (C) 2014 Red Hat
|
||||
# see file 'COPYING' for use and warranty information
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Pytest plugin for IPA
|
||||
|
||||
Adds support for the @pytest.mark.source_order decorator which,
|
||||
when applied to a class, runs the test methods in source order.
|
||||
|
||||
See test_ordering for an example.
|
||||
"""
|
||||
|
||||
import unittest
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def ordered(cls):
|
||||
"""Decorator that marks a test class as ordered
|
||||
|
||||
Methods within the marked class will be executed in definition order
|
||||
(or more strictly, in ordered by the line number where they're defined).
|
||||
|
||||
Subclasses of unittest.TestCase can not be ordered.
|
||||
|
||||
Generator methods will not be ordered by this plugin.
|
||||
"""
|
||||
cls._order_plugin__ordered = True
|
||||
assert not isinstance(cls, unittest.TestCase), (
|
||||
"A unittest.TestCase may not be ordered.")
|
||||
cls = pytest.mark.source_order(cls)
|
||||
return cls
|
||||
|
||||
|
||||
def decorate_items(items):
|
||||
node_indexes = {}
|
||||
for index, item in enumerate(items):
|
||||
try:
|
||||
func = item.function
|
||||
except AttributeError:
|
||||
yield (index, ), item
|
||||
continue
|
||||
|
||||
key = (index, )
|
||||
for node in reversed(item.listchain()):
|
||||
# Find the corresponding class
|
||||
if isinstance(node, pytest.Class):
|
||||
cls = node.cls
|
||||
else:
|
||||
continue
|
||||
if getattr(cls, '_order_plugin__ordered', False):
|
||||
node_index = node_indexes.setdefault(node, index)
|
||||
# Find first occurence of the method in class hierarchy
|
||||
for i, parent_class in enumerate(reversed(cls.mro())):
|
||||
if getattr(parent_class, '_order_plugin__ordered', False):
|
||||
method = getattr(parent_class, func.__name__, None)
|
||||
if method:
|
||||
# Sort methods as tuples (position of the class
|
||||
# in the inheritance chain, position of the method
|
||||
# within that class)
|
||||
key = (node_index, 0,
|
||||
i, method.func_code.co_firstlineno, node)
|
||||
break
|
||||
else:
|
||||
# Weird case fallback
|
||||
# Method name not in any of the classes in MRO, run it last
|
||||
key = node_index, 1, func.func_code.co_firstlineno, node
|
||||
break
|
||||
yield key, item
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(session, config, items):
|
||||
items[:] = [item for i, item in sorted(decorate_items(items))]
|
||||
@@ -24,7 +24,7 @@ import nose
|
||||
from ipapython.ipa_log_manager import log_mgr
|
||||
from ipatests.test_integration.config import get_global_config
|
||||
from ipatests.test_integration import tasks
|
||||
from ipatests.order_plugin import ordered
|
||||
from ipatests.pytest_plugins.ordering import ordered
|
||||
|
||||
log = log_mgr.get_logger(__name__)
|
||||
|
||||
|
||||
@@ -31,7 +31,7 @@ from ipaplatform.paths import paths
|
||||
from ipapython.dn import DN
|
||||
from ipatests.test_integration.base import IntegrationTest
|
||||
from ipatests.test_integration import tasks
|
||||
from ipatests.order_plugin import ordered
|
||||
from ipatests.pytest_plugins.ordering import ordered
|
||||
|
||||
_DEFAULT = object()
|
||||
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
# Authors:
|
||||
# Petr Viktorin <pviktori@redhat.com>
|
||||
#
|
||||
# Copyright (C) 2014 Red Hat
|
||||
# see file 'COPYING' for use and warranty information
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
"""Test the ordering of tests
|
||||
|
||||
IPA integration tests, marked with `@ordered`, require tests to be run
|
||||
in a specific order:
|
||||
- Base classes first
|
||||
- Within a class, test methods are ordered according to source line
|
||||
"""
|
||||
|
||||
from ipatests.pytest_plugins.ordering import ordered
|
||||
|
||||
|
||||
@ordered
|
||||
class TestBase(object):
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
cls.value = 'unchanged'
|
||||
|
||||
def test_d_first(self):
|
||||
type(self).value = 'changed once'
|
||||
|
||||
|
||||
class TestChild(TestBase):
|
||||
def test_b_third(self):
|
||||
assert type(self).value == 'changed twice'
|
||||
type(self).value = 'changed thrice'
|
||||
|
||||
def test_a_fourth(self):
|
||||
assert type(self).value == 'changed thrice'
|
||||
|
||||
|
||||
def test_c_second(self):
|
||||
assert type(self).value == 'changed once'
|
||||
type(self).value = 'changed twice'
|
||||
TestBase.test_c_second = test_c_second
|
||||
del test_c_second
|
||||
Reference in new issue
Block a user