Parsing out data from pytest assert and converting into a string

Question

I want to collect data about test to an external database, where I would collect all assert and parse the expected value versus the actual result.

To describe what I want in code:

test_something.py:

def test_me_1(send_to_test):
    """
    Docstring to pad even more
    """
    x,y = send_to_test(3)
    assert x == 3**2
    assert y == 3**3
    assert y == 3**2 # I want this to fail intentionally


conftest.py:

@pytest.fixture(scope='function')
def send_to_test() -> Callable:
    return lambda x: (x ** 2, x ** 3)

The data I want to collect is (Any other way is fine as long as I get this data):

test_data = [{'func': 'test_me_1', 'variable': 'x', 'expected_value': 9, 'actual_value': 9},
             {'func': 'test_me_1', 'variable': 'y', 'expected_value': 27, 'actual_value': 27},
             {'func': 'test_me_1', 'variable': 'y', 'expected_value': 27, 'actual_value': 9}]

My current idea is a wrapper, that can easily be implemented to existing test and will collect the assert statements and their value.
Collecting the number of assert statements is straightforward if I use AST, but these are good only for static analysis. I've also tried to use the inspect module, but have hard time figuring out how to use it for this purpose

My poor attempt thus far (print statements are temporary since I want to collect the data)

def get_this(func: Callable):
    @functools.wraps(func)
    def wrapper(*arg, **kwargs):
        print(func.__name__)
        func_content = inspect.getsource(func)
        func_root = ast.parse(func_content)
        for node in func_root.body:
            for content in node.body:
                if isinstance(content, ast.Assert):
                    print(content)
        return func(*arg, **kwargs)

    return wrapper

Answer 1

You could try this:

import inspect
import json
from typing import Callable

import pytest


@pytest.fixture(scope="function")
def send_to_test() -> Callable:
    return lambda x: (x ** 2, x ** 3)

# Helper function to run and monitor a test
def run_test(test_func, variable, expected_value, actual_value, data):
    try:
        with open("./data.json", "r") as f:
            data = json.load(f)
    except FileNotFoundError:
        data = []

    data.append(
        {
            "func": test_func,
            "variable": variable,
            "expected_value": expected_value,
            "actual_value": actual_value,
        }
    )
    with open("./data.json", "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=4)

    assert actual_value == expected_value


# Reformatted tests
def test_me_1(send_to_test):
    # setup
    data = []
    x, y = send_to_test(3)

    # tests
    run_test(
        test_func=inspect.stack()[0][3],
        variable="x",
        expected_value=3 ** 2,
        actual_value=x,
        data=data,
    )
    run_test(
        test_func=inspect.stack()[0][3],
        variable="y",
        expected_value=3 ** 3,
        actual_value=y,
        data=data,
    )
    run_test(
        test_func=inspect.stack()[0][3],
        variable="y",
        expected_value=3 ** 2,
        actual_value=y,
        data=data,
    )

And so, when you run pytest, a new data.jsonfile is created with the expected content:

[
    {
        "func": "test_me_1",
        "variable": "x",
        "expected_value": 9,
        "actual_value": 9
    },
    {
        "func": "test_me_1",
        "variable": "y",
        "expected_value": 27,
        "actual_value": 27
    },
    {
        "func": "test_me_1",
        "variable": "y",
        "expected_value": 9,
        "actual_value": 27
    }
]