Integration tests: Port the ordering plugin to pytest

Ordered integration tests may now be run with pytest.

https://fedorahosted.org/freeipa/ticket/4610

Reviewed-By: Tomas Babej <tbabej@redhat.com>
This commit is contained in:
Petr Viktorin authored and Tomas Babej committed 2014-11-21 12:14:44 +01:00
1 parent 0cb12f3cde
commit 3a9a98b285
7 files changed
+146 -108

No files matched your search

+1
View File
@@ -306,6 +306,7 @@ Requires: %{name}-python = %{version}-%{release}
Requires: tar
Requires: xz
Requires: python-nose
Requires: pytest >= 2.6
Requires: python-paste
Requires: python-coverage
Requires: python-polib
-106
View File
@@ -1,106 +0,0 @@
# Authors:
# Petr Viktorin <pviktori@redhat.com>
#
# Copyright (C) 2013 Red Hat
# see file 'COPYING' for use and warranty information
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""A Nose plugin that allows ordered test cases"""
import os
import unittest
import inspect
from nose.plugins import Plugin
import nose.loader
import nose.util
def ordered(cls):
"""Decorator that marks a test class as ordered
Methods within the marked class will be executed in definition order
(or more strictly, in ordered by the line number where they're defined).
Subclasses of unittest.TestCase can not be ordered.
Generator methods will not be ordered by this plugin.
"""
cls._order_plugin__ordered = True
assert not isinstance(cls, unittest.TestCase), (
"A unittest.TestCase may not be ordered.")
return cls
class OrderTests(Plugin):
name = 'ordered-tests'
def options(self, parser, env=os.environ):
super(OrderTests, self).options(parser, env=env)
def configure(self, options, conf):
super(OrderTests, self).configure(options, conf)
if not self.enabled:
return
def loadTestsFromTestClass(self, cls):
"""Sort methods of ordered test cases by co_firstlineno"""
if not getattr(cls, '_order_plugin__ordered', False):
return
loader = nose.loader.TestLoader()
def wanted(attr):
item = getattr(cls, attr, None)
if not inspect.ismethod(item):
return False
if nose.util.isgenerator(item.im_func):
return False
return loader.selector.wantMethod(item)
def sort_with_respect_to_overriding(func):
"""
Sorts the methods in respect with the parent classes.
The methods are sorted with respect to the inheritance chain,
methods that were defined in the same class are sorted by the line
number on which they were defined.
"""
# Check each *ordered* class in MRO for definition of func method
for i, parent_class in enumerate(reversed(cls.mro())):
if getattr(parent_class, '_order_plugin__ordered', False):
method = getattr(parent_class, func.__name__, None)
if method:
# This sorts methods as tuples (position of the class
# in the inheritance chain, position of the method
# within that class)
return 0, i, method.func_code.co_firstlineno
# Weird case fallback
# Method name not in any of the classes in MRO, run it last
return 1, func.func_code.co_firstlineno
methods = [getattr(cls, case) for case in dir(cls) if wanted(case)]
methods.sort(key=sort_with_respect_to_overriding)
cases = [loader.makeTest(m, cls) for m in methods]
return cases
def wantMethod(self, method):
"""Hide non-TestCase methods from the normal loader"""
im_class = getattr(method, 'im_class', None)
if im_class and getattr(im_class, '_order_plugin__ordered', False):
if nose.util.isgenerator(method.im_func):
return True
return False
+1
View File
@@ -8,6 +8,7 @@
python_classes = test_ Test
addopts = --doctest-modules
-p ipatests.pytest_plugins.declarative
-p ipatests.pytest_plugins.ordering
# Ignore files for doc tests.
# TODO: ideally, these should all use __name__=='__main__' guards
--ignore=setup.py
+88
View File
@@ -0,0 +1,88 @@
# Authors:
# Petr Viktorin <pviktori@redhat.com>
#
# Copyright (C) 2014 Red Hat
# see file 'COPYING' for use and warranty information
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Pytest plugin for IPA
Adds support for the @pytest.mark.source_order decorator which,
when applied to a class, runs the test methods in source order.
See test_ordering for an example.
"""
import unittest
import pytest
def ordered(cls):
"""Decorator that marks a test class as ordered
Methods within the marked class will be executed in definition order
(or more strictly, in ordered by the line number where they're defined).
Subclasses of unittest.TestCase can not be ordered.
Generator methods will not be ordered by this plugin.
"""
cls._order_plugin__ordered = True
assert not isinstance(cls, unittest.TestCase), (
"A unittest.TestCase may not be ordered.")
cls = pytest.mark.source_order(cls)
return cls
def decorate_items(items):
node_indexes = {}
for index, item in enumerate(items):
try:
func = item.function
except AttributeError:
yield (index, ), item
continue
key = (index, )
for node in reversed(item.listchain()):
# Find the corresponding class
if isinstance(node, pytest.Class):
cls = node.cls
else:
continue
if getattr(cls, '_order_plugin__ordered', False):
node_index = node_indexes.setdefault(node, index)
# Find first occurence of the method in class hierarchy
for i, parent_class in enumerate(reversed(cls.mro())):
if getattr(parent_class, '_order_plugin__ordered', False):
method = getattr(parent_class, func.__name__, None)
if method:
# Sort methods as tuples (position of the class
# in the inheritance chain, position of the method
# within that class)
key = (node_index, 0,
i, method.func_code.co_firstlineno, node)
break
else:
# Weird case fallback
# Method name not in any of the classes in MRO, run it last
key = node_index, 1, func.func_code.co_firstlineno, node
break
yield key, item
def pytest_collection_modifyitems(session, config, items):
items[:] = [item for i, item in sorted(decorate_items(items))]
+1 -1
View File
@@ -24,7 +24,7 @@ import nose
from ipapython.ipa_log_manager import log_mgr
from ipatests.test_integration.config import get_global_config
from ipatests.test_integration import tasks
from ipatests.order_plugin import ordered
from ipatests.pytest_plugins.ordering import ordered
log = log_mgr.get_logger(__name__)
+1 -1
View File
@@ -31,7 +31,7 @@ from ipaplatform.paths import paths
from ipapython.dn import DN
from ipatests.test_integration.base import IntegrationTest
from ipatests.test_integration import tasks
from ipatests.order_plugin import ordered
from ipatests.pytest_plugins.ordering import ordered
_DEFAULT = object()
@@ -0,0 +1,54 @@
# Authors:
# Petr Viktorin <pviktori@redhat.com>
#
# Copyright (C) 2014 Red Hat
# see file 'COPYING' for use and warranty information
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""Test the ordering of tests
IPA integration tests, marked with `@ordered`, require tests to be run
in a specific order:
- Base classes first
- Within a class, test methods are ordered according to source line
"""
from ipatests.pytest_plugins.ordering import ordered
@ordered
class TestBase(object):
@classmethod
def setup_class(cls):
cls.value = 'unchanged'
def test_d_first(self):
type(self).value = 'changed once'
class TestChild(TestBase):
def test_b_third(self):
assert type(self).value == 'changed twice'
type(self).value = 'changed thrice'
def test_a_fourth(self):
assert type(self).value == 'changed thrice'
def test_c_second(self):
assert type(self).value == 'changed once'
type(self).value = 'changed twice'
TestBase.test_c_second = test_c_second
del test_c_second