Skip to content

Pyramid level

pyramid_level

Pyramid-level soft-signal rule — R1+R2+R3 core.

Classifies every tests/**/test_*.py function into unit / integration / e2e based on attr-IO, fixture arguments, taint of tmp_path, and import provenance (public vs internal). Mismatches between the classified level and the folder the test lives in are reported as findings.

R4 (conftest fixture-IO resolution) and R5 (mock neutralisation) are stubbed here as identities — ticket #4b replaces the two placeholders.

Finding

Bases: BaseModel

One classification verdict for a single test function.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/pyramid_level.py
Python
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
class Finding(BaseModel):  # type: ignore[explicit-any]  # pydantic synthesizes __init__(**data: Any)
    """One classification verdict for a single test function."""

    path: str
    function: str
    level: str
    reason: str
    current_level: str
    has_real_io: bool
    has_subprocess: bool
    io_signals: list[str] = Field(default_factory=list)
    imports_public: list[str] = Field(default_factory=list)
    imports_internal: list[str] = Field(default_factory=list)
    suggested_file: str = ""
    severity: Severity = Severity.WARNING

    model_config = {"extra": "forbid"}

PyramidCheckResult

Bases: CheckResult

:class:CheckResult subclass exposing findings and score.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/pyramid_level.py
Python
 97
 98
 99
100
101
102
103
class PyramidCheckResult(CheckResult):  # type: ignore[explicit-any]  # pydantic synthesizes __init__(**data: Any)
    """:class:`CheckResult` subclass exposing ``findings`` and ``score``."""

    findings: list[Finding] = Field(default_factory=list)
    score: int = 100

    model_config = {"extra": "forbid"}

PyramidLevelRule dataclass

Bases: ProjectRule

Report tests whose classified pyramid level mismatches their folder.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/pyramid_level.py
Python
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
@register_rule("test_quality")
@dataclass
class PyramidLevelRule(ProjectRule):
    """Report tests whose classified pyramid level mismatches their folder."""

    strict_mismatches: bool = True

    @property
    def rule_id(self) -> str:
        """Stable identifier for this rule."""
        return "TEST_QUALITY_PYRAMID_LEVEL"

    def check(self, project_path: Path) -> PyramidCheckResult:
        """Classify tests in ``project_path`` against their pyramid folder."""
        tests_dir = project_path / "tests"
        if not tests_dir.exists():
            return PyramidCheckResult(
                rule_id=self.rule_id,
                passed=True,
                message="no tests/ directory",
                severity=Severity.INFO,
                score=100,
            )

        all_findings = scan_package(project_path)
        mismatches = _filter_mismatches(all_findings, tests_dir)
        count = len(mismatches) if self.strict_mismatches else 0
        score = max(0, 100 - count * _SCORE_PENALTY)
        passed = count == 0
        message = (
            "pyramid levels match folder layout"
            if passed
            else f"{count} test(s) mis-located vs. classified pyramid level"
        )
        details: dict[str, object] = {
            "mismatches": [f.model_dump() for f in mismatches],
            "total": len(mismatches),
        }
        text = render_mismatch_text(mismatches, project_path) if mismatches else None
        fix_hint = (
            None
            if passed
            else "Move tests to matching pyramid dir (use /pyramid-relocate)"
        )
        return PyramidCheckResult(
            rule_id=self.rule_id,
            passed=passed,
            message=message,
            severity=Severity.WARNING,
            findings=all_findings,
            score=score,
            details=details,
            text=text,
            fix_hint=fix_hint,
        )
rule_id property

Stable identifier for this rule.

check(project_path)

Classify tests in project_path against their pyramid folder.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/pyramid_level.py
Python
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
def check(self, project_path: Path) -> PyramidCheckResult:
    """Classify tests in ``project_path`` against their pyramid folder."""
    tests_dir = project_path / "tests"
    if not tests_dir.exists():
        return PyramidCheckResult(
            rule_id=self.rule_id,
            passed=True,
            message="no tests/ directory",
            severity=Severity.INFO,
            score=100,
        )

    all_findings = scan_package(project_path)
    mismatches = _filter_mismatches(all_findings, tests_dir)
    count = len(mismatches) if self.strict_mismatches else 0
    score = max(0, 100 - count * _SCORE_PENALTY)
    passed = count == 0
    message = (
        "pyramid levels match folder layout"
        if passed
        else f"{count} test(s) mis-located vs. classified pyramid level"
    )
    details: dict[str, object] = {
        "mismatches": [f.model_dump() for f in mismatches],
        "total": len(mismatches),
    }
    text = render_mismatch_text(mismatches, project_path) if mismatches else None
    fix_hint = (
        None
        if passed
        else "Move tests to matching pyramid dir (use /pyramid-relocate)"
    )
    return PyramidCheckResult(
        rule_id=self.rule_id,
        passed=passed,
        message=message,
        severity=Severity.WARNING,
        findings=all_findings,
        score=score,
        details=details,
        text=text,
        fix_hint=fix_hint,
    )

classify_level(*, has_real_io, has_subprocess, has_in_package_subprocess, imports_public, imports_internal)

Return (level, reason) for the given soft signals.

has_subprocess preserves raw subprocess diagnostics, while has_in_package_subprocess is the narrower signal that promotes a test to e2e. R2 public-only rescue MUST fire before the generic has_public → integration branch; otherwise pure-function unit tests at integration/ would be mis-classified.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/pyramid_level.py
Python
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
def classify_level(
    *,
    has_real_io: bool,
    has_subprocess: bool,
    has_in_package_subprocess: bool,
    imports_public: bool,
    imports_internal: bool,
) -> tuple[str, str]:
    """Return ``(level, reason)`` for the given soft signals.

    ``has_subprocess`` preserves raw subprocess diagnostics, while
    ``has_in_package_subprocess`` is the narrower signal that promotes a test
    to e2e. R2 public-only rescue MUST fire before the generic
    ``has_public \u2192 integration`` branch; otherwise pure-function unit tests at
    integration/ would be mis-classified.
    """
    if has_in_package_subprocess:
        return "e2e", "in-package CLI invocation via subprocess (end-to-end)"
    if not has_real_io and imports_public and not imports_internal:
        return "unit", "public API import, no real I/O (pure function unit test)"
    if has_real_io:
        if imports_public:
            detail = " + public import"
        elif imports_internal:
            detail = " + internal import"
        else:
            detail = " without package import"
        return "integration", f"real I/O{detail} (integration)"
    if imports_internal:
        return "unit", "internal import, no real I/O (unit)"
    return "unit", "no real I/O, no package import (unit)"

has_in_package_subprocess_invocation(*, call, module_ast, project_scripts)

Return true when call invokes a declared package script.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/_shared.py
Python
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
def has_in_package_subprocess_invocation(
    *,
    call: ast.Call,
    module_ast: ast.Module,
    project_scripts: set[str],
) -> bool:
    """Return true when *call* invokes a declared package script."""
    if not project_scripts:
        return False
    argv = _argv_from_call(call, module_ast)
    if argv is None:
        return False
    return _argv_contains_package_entrypoint(argv, project_scripts)

load_project_scripts(pkg_root)

Return scripts declared by [project.scripts] in pyproject.toml.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/_shared.py
Python
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
def load_project_scripts(pkg_root: Path) -> set[str]:
    """Return scripts declared by ``[project.scripts]`` in pyproject.toml."""
    pyproject = pkg_root / "pyproject.toml"
    if not pyproject.exists():
        return set()
    with pyproject.open("rb") as handle:
        data = tomllib.load(handle)
    scripts = data.get("project", {}).get("scripts", {})
    if not isinstance(scripts, dict):
        return set()
    return {name for name in scripts if isinstance(name, str)}

relpath(path, project_path)

Return path relative to project_path if possible, else absolute.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/pyramid_level.py
Python
1060
1061
1062
1063
1064
1065
def relpath(path: str, project_path: Path) -> str:
    """Return *path* relative to *project_path* if possible, else absolute."""
    try:
        return str(Path(path).relative_to(project_path))
    except ValueError:
        return path

render_mismatch_text(mismatches, project_path)

Render top-N mismatched findings as a compact bullet list.

Paths are relativized to project_path to keep the text dense; falls back to the absolute path if the finding lives outside the project root.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/pyramid_level.py
Python
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
def render_mismatch_text(mismatches: list[Finding], project_path: Path) -> str:
    """Render top-N mismatched findings as a compact bullet list.

    Paths are relativized to *project_path* to keep the text dense; falls
    back to the absolute path if the finding lives outside the project root.
    """
    lines = [
        f"• {relpath(f.path, project_path)}:{f.function} "
        f"{f.current_level}→{f.level} ({f.reason})"
        for f in mismatches[:_MAX_TEXT_ITEMS]
    ]
    if len(mismatches) > _MAX_TEXT_ITEMS:
        lines.append(f"(+{len(mismatches) - _MAX_TEXT_ITEMS} more)")
    return "\n".join(lines)

scan_package(pkg_root)

Scan every test file under <pkg_root>/tests and return findings.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/pyramid_level.py
Python
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
def scan_package(pkg_root: Path) -> list[Finding]:
    """Scan every test file under ``<pkg_root>/tests`` and return findings."""
    tests_dir = pkg_root / "tests"
    if not tests_dir.exists():
        return []
    pkg_prefixes = get_pkg_prefixes(pkg_root)
    init_all = get_init_all(pkg_root)
    findings: list[Finding] = []
    for test_file, tree in iter_test_files(pkg_root):
        if tree is None:
            continue
        findings.extend(
            scan_test_file(test_file, tree, pkg_root, pkg_prefixes, init_all, tests_dir)
        )
    return findings

scan_test_file(test_file, tree, pkg_root, pkg_prefixes, init_all, tests_dir)

Classify every test_* function in tree and return findings.

Emits one :class:Finding per function whose classified level differs from its folder-derived current level.

Source code in packages/axm-audit/src/axm_audit/core/rules/test_quality/pyramid_level.py
Python
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
def scan_test_file(  # noqa: PLR0913
    test_file: Path,
    tree: ast.Module,
    pkg_root: Path,
    pkg_prefixes: set[str],
    init_all: set[str] | None,
    tests_dir: Path,
) -> list[Finding]:
    """Classify every ``test_*`` function in *tree* and return findings.

    Emits one :class:`Finding` per function whose classified level differs
    from its folder-derived current level.
    """
    ctx = _build_scan_context(
        test_file, tree, pkg_root, pkg_prefixes, init_all, tests_dir
    )
    return [
        _classify_test_function(ctx, node)
        for node in ast.walk(tree)
        if isinstance(node, ast.FunctionDef) and node.name.startswith("test_")
    ]