Pyramid route-finding script

find_pyramid_routes.py

# Analysis of Hypothesis Python app routes as part of
# https://github.com/hypothesis/playbook/issues/1797#issuecomment-2397242000.

import os
import ast
import sys


def find_py_files(root_dir):
    excluded_dirs = ["node_modules", "build", "test", "tests", "third_party"]

    py_files = []
    for dirpath, dirnames, filenames in os.walk(root_dir):
        # Ignore hidden and excluded directories
        dirnames[:] = [
            d for d in dirnames if not d.startswith(".") and d not in excluded_dirs
        ]
        for filename in filenames:
            if filename.endswith(".py"):
                py_files.append(os.path.join(dirpath, filename))
    return py_files


class RouteCallVisitor(ast.NodeVisitor):
    """
    Scan for Pyramid `add_route` calls and invoke a callback for each one found.
    """

    def __init__(self, on_route):
        super().__init__()
        self.on_route = on_route

    def visit_Call(self, node):
        # Check if the call is to a method named 'add_route'
        if isinstance(node.func, ast.Attribute) and node.func.attr == "add_route":
            if len(node.args) >= 2:
                route = node.args[0]
                handler = node.args[1]
                # Try to extract the values if possible (this may fail for complex expressions)
                if isinstance(route, (ast.Str, ast.Constant)) and isinstance(
                    handler, (ast.Str, ast.Constant)
                ):
                    route_name = route.s if isinstance(route, ast.Str) else route.value
                    handler_name = (
                        handler.s if isinstance(route, ast.Str) else handler.value
                    )
                    self.on_route(route_name, handler_name)
                else:
                    print("Unhandled call:", ast.dump(node), file=sys.stderr)
            else:
                print("Not enough args:", ast.dump(node), file=sys.stderr)
                pass

        # Continue traversing the AST
        self.generic_visit(node)


def process_file(filepath, on_route):
    try:
        with open(filepath, "r") as f:
            file_content = f.read()
        tree = ast.parse(file_content)
        visitor = RouteCallVisitor(on_route)
        visitor.visit(tree)
    except (SyntaxError, IOError) as e:
        print(f"Error processing {filepath}: {e}", file=sys.stderr)


def main(root_dirs: list[str]):
    # Find `.py` files
    py_files = []
    for root_dir in root_dirs:
        py_files += find_py_files(root_dir)

    # Scan Python files and collect (route_name, pattern) tuples for each
    # `add_route` call.
    all_routes = []

    def on_route(route, handler):
        all_routes.append((route, handler))

    for py_file in py_files:
        process_file(py_file, on_route)

    # Extract and print a sorted list of the unique first path segments,
    # eg. ("/api/foo/bar" => "api").
    def extract_first_path_segment(path):
        segments = path.split("/")
        if len(segments) < 2:
            return ""
        return segments[1]

    first_parts = sorted(
        list(
            set(
                [
                    extract_first_path_segment(route_handler[1])
                    for route_handler in all_routes
                ]
            )
        )
    )
    for first_part in first_parts:
        if not first_part:
            # Skip empty path segment from index route
            continue
        print(first_part)


if __name__ == "__main__":
    # Assume we're in the parent directory of Hypothesis project checkouts.
    # root_dirs = ["h", "lms", "via", "viahtml", "bouncer", "checkmate"]
    root_dirs = ["h", "lms"]
    main(root_dirs)