quicklinks

home contents
install changelog
examples customize
issues[bb] contact

Table Of Contents

Previous topic

mysetup pattern: application specific test fixtures

Next topic

Changing standard (Python) test discovery

parametrizing tests

py.test allows to easily implement your own custom parametrization scheme for tests. Here we provide some examples for inspiration and re-use.

generating parameters combinations, depending on command line

Let’s say we want to execute a test with different parameters and the parameter range shall be determined by a command line argument. Let’s first write a simple computation test:

# content of test_compute.py

def test_compute(param1):
    assert param1 < 4

Now we add a test configuration like this:

# content of conftest.py

def pytest_addoption(parser):
    parser.addoption("--all", action="store_true",
        help="run all combinations")

def pytest_generate_tests(metafunc):
    if 'param1' in metafunc.funcargnames:
        if metafunc.config.option.all:
            end = 5
        else:
            end = 2
        for i in range(end):
            metafunc.addcall(funcargs={'param1': i})

This means that we only run 2 tests if we do not pass --all:

$ py.test -q test_compute.py
collecting ... collected 2 items
..
2 passed in 0.01 seconds

We run only two computations, so we see two dots. let’s run the full monty:

$ py.test -q --all
collecting ... collected 5 items
....F
================================= FAILURES =================================
_____________________________ test_compute[4] ______________________________

param1 = 4

    def test_compute(param1):
>       assert param1 < 4
E       assert 4 < 4

test_compute.py:3: AssertionError
1 failed, 4 passed in 0.03 seconds

As expected when running the full range of param1 values we’ll get an error on the last one.

Deferring the setup of parametrizing resources

The parametrization of test functions happens at collection time. It is often a good idea to setup possibly expensive resources only when the actual test is run. Here is a simple example how you can achieve that:

# content of test_backends.py

import pytest
def test_db_initialized(db):
    # a dummy test
    if db.__class__.__name__ == "DB2":
        pytest.fail("deliberately failing for demo purposes")

Now we add a test configuration that takes care to generate two invocations of the test_db_initialized function and furthermore a factory that creates a database object when each test is actually run:

# content of conftest.py

def pytest_generate_tests(metafunc):
    if 'db' in metafunc.funcargnames:
        metafunc.addcall(param="d1")
        metafunc.addcall(param="d2")

class DB1:
    "one database object"
class DB2:
    "alternative database object"

def pytest_funcarg__db(request):
    if request.param == "d1":
        return DB1()
    elif request.param == "d2":
        return DB2()
    else:
        raise ValueError("invalid internal test config")

Let’s first see how it looks like at collection time:

$ py.test test_backends.py --collectonly
=========================== test session starts ============================
platform linux2 -- Python 2.6.6 -- pytest-2.0.3
collecting ... collected 2 items
<Module 'test_backends.py'>
  <Function 'test_db_initialized[0]'>
  <Function 'test_db_initialized[1]'>

=============================  in 0.01 seconds =============================

And then when we run the test:

$ py.test -q test_backends.py
collecting ... collected 2 items
.F
================================= FAILURES =================================
__________________________ test_db_initialized[1] __________________________

db = <conftest.DB2 instance at 0x2bf7bd8>

    def test_db_initialized(db):
        # a dummy test
        if db.__class__.__name__ == "DB2":
>           pytest.fail("deliberately failing for demo purposes")
E           Failed: deliberately failing for demo purposes

test_backends.py:6: Failed
1 failed, 1 passed in 0.03 seconds

Now you see that one invocation of the test passes and another fails, as it to be expected.

Parametrizing test methods through per-class configuration

Here is an example pytest_generate_function function implementing a parametrization scheme similar to Michael Foords unittest parameterizer in a lot less code:

# content of ./test_parametrize.py
import pytest

def pytest_generate_tests(metafunc):
    # called once per each test function
    for funcargs in metafunc.cls.params[metafunc.function.__name__]:
        # schedule a new test function run with applied **funcargs
        metafunc.addcall(funcargs=funcargs)

class TestClass:
    # a map specifying multiple argument sets for a test method
    params = {
        'test_equals': [dict(a=1, b=2), dict(a=3, b=3), ],
        'test_zerodivision': [dict(a=1, b=0), dict(a=3, b=2)],
    }

    def test_equals(self, a, b):
        assert a == b

    def test_zerodivision(self, a, b):
        pytest.raises(ZeroDivisionError, "a/b")

Running it means we are two tests for each test functions, using the respective settings:

$ py.test -q
collecting ... collected 6 items
.FF..F
================================= FAILURES =================================
__________________________ test_db_initialized[1] __________________________

db = <conftest.DB2 instance at 0x19bcb90>

    def test_db_initialized(db):
        # a dummy test
        if db.__class__.__name__ == "DB2":
>           pytest.fail("deliberately failing for demo purposes")
E           Failed: deliberately failing for demo purposes

test_backends.py:6: Failed
_________________________ TestClass.test_equals[0] _________________________

self = <test_parametrize.TestClass instance at 0x19ca8c0>, a = 1, b = 2

    def test_equals(self, a, b):
>       assert a == b
E       assert 1 == 2

test_parametrize.py:17: AssertionError
______________________ TestClass.test_zerodivision[1] ______________________

self = <test_parametrize.TestClass instance at 0x19cd4d0>, a = 3, b = 2

    def test_zerodivision(self, a, b):
>       pytest.raises(ZeroDivisionError, "a/b")
E       Failed: DID NOT RAISE

test_parametrize.py:20: Failed
3 failed, 3 passed in 0.05 seconds

Parametrizing test methods through a decorator

Modifying the previous example we can also allow decorators for parametrizing test methods:

# content of test_parametrize2.py

import pytest

# test support code
def params(funcarglist):
    def wrapper(function):
        function.funcarglist = funcarglist
        return function
    return wrapper

def pytest_generate_tests(metafunc):
    for funcargs in getattr(metafunc.function, 'funcarglist', ()):
        metafunc.addcall(funcargs=funcargs)

# actual test code
class TestClass:
    @params([dict(a=1, b=2), dict(a=3, b=3), ])
    def test_equals(self, a, b):
        assert a == b

    @params([dict(a=1, b=0), dict(a=3, b=2)])
    def test_zerodivision(self, a, b):
        pytest.raises(ZeroDivisionError, "a/b")

Running it gives similar results as before:

$ py.test -q test_parametrize2.py
collecting ... collected 4 items
F..F
================================= FAILURES =================================
_________________________ TestClass.test_equals[0] _________________________

self = <test_parametrize2.TestClass instance at 0x1cf1170>, a = 1, b = 2

    @params([dict(a=1, b=2), dict(a=3, b=3), ])
    def test_equals(self, a, b):
>       assert a == b
E       assert 1 == 2

test_parametrize2.py:19: AssertionError
______________________ TestClass.test_zerodivision[1] ______________________

self = <test_parametrize2.TestClass instance at 0x1d02170>, a = 3, b = 2

    @params([dict(a=1, b=0), dict(a=3, b=2)])
    def test_zerodivision(self, a, b):
>       pytest.raises(ZeroDivisionError, "a/b")
E       Failed: DID NOT RAISE

test_parametrize2.py:23: Failed
2 failed, 2 passed in 0.03 seconds

checking serialization between Python interpreters

Here is a stripped down real-life example of using parametrized testing for testing serialization between different interpreters. We define a test_basic_objects function which is to be run with different sets of arguments for its three arguments:

* ``python1``: first python interpreter
* ``python2``: second python interpreter
* ``obj``: object to be dumped from first interpreter and loaded into second interpreter
"""
module containing a parametrized tests testing cross-python
serialization via the pickle module.
"""
import py

pythonlist = ['python2.4', 'python2.5', 'python2.6', 'python2.7', 'python2.8']

def pytest_generate_tests(metafunc):
    if 'python1' in metafunc.funcargnames:
        assert 'python2' in metafunc.funcargnames
        for obj in metafunc.function.multiarg.kwargs['obj']:
            for py1 in pythonlist:
                for py2 in pythonlist:
                    metafunc.addcall(id="%s-%s-%s" % (py1, py2, obj),
                        param=(py1, py2, obj))

@py.test.mark.multiarg(obj=[42, {}, {1:3},])
def test_basic_objects(python1, python2, obj):
    python1.dumps(obj)
    python2.load_and_is_true("obj == %s" % obj)

def pytest_funcarg__python1(request):
    tmpdir = request.getfuncargvalue("tmpdir")
    picklefile = tmpdir.join("data.pickle")
    return Python(request.param[0], picklefile)

def pytest_funcarg__python2(request):
    python1 = request.getfuncargvalue("python1")
    return Python(request.param[1], python1.picklefile)

def pytest_funcarg__obj(request):
    return request.param[2]

class Python:
    def __init__(self, version, picklefile):
        self.pythonpath = py.path.local.sysfind(version)
        if not self.pythonpath:
            py.test.skip("%r not found" %(version,))
        self.picklefile = picklefile
    def dumps(self, obj):
        dumpfile = self.picklefile.dirpath("dump.py")
        dumpfile.write(py.code.Source("""
            import pickle
            f = open(%r, 'wb')
            s = pickle.dump(%r, f)
            f.close()
        """ % (str(self.picklefile), obj)))
        py.process.cmdexec("%s %s" %(self.pythonpath, dumpfile))

    def load_and_is_true(self, expression):
        loadfile = self.picklefile.dirpath("load.py")
        loadfile.write(py.code.Source("""
            import pickle
            f = open(%r, 'rb')
            obj = pickle.load(f)
            f.close()
            res = eval(%r)
            if not res:
                raise SystemExit(1)
        """ % (str(self.picklefile), expression)))
        print (loadfile)
        py.process.cmdexec("%s %s" %(self.pythonpath, loadfile))

Running it (with Python-2.4 through to Python2.7 installed):

. $ py.test -q multipython.py
collecting ... collected 75 items
....s....s....s....ssssss....s....s....s....ssssss....s....s....s....ssssss
48 passed, 27 skipped in 2.04 seconds