parse_data_structs.py

#!/usr/bin/env python3
# SPDX-License-Identifier: GPL-2.0
# Copyright (c) 2016-2025 by Mauro Carvalho Chehab <[email protected]>.
# pylint: disable=R0912,R0915

"""
Parse a source file or header, creating ReStructured Text cross references.

It accepts an optional file to change the default symbol reference or to
suppress symbols from the output.

It is capable of identifying defines, functions, structs, typedefs,
enums and enum symbols and create cross-references for all of them.
It is also capable of distinguish #define used for specifying a Linux
ioctl.

The optional rules file contains a set of rules like:

    ignore ioctl VIDIOC_ENUM_FMT
    replace ioctl VIDIOC_DQBUF vidioc_qbuf
    replace define V4L2_EVENT_MD_FL_HAVE_FRAME_SEQ :c:type:`v4l2_event_motion_det`
"""

import os
import re
import sys


class ParseDataStructs:
    """
    Creates an enriched version of a Kernel header file with cross-links
    to each C data structure type.

    It is meant to allow having a more comprehensive documentation, where
    uAPI headers will create cross-reference links to the code.

    It is capable of identifying defines, functions, structs, typedefs,
    enums and enum symbols and create cross-references for all of them.
    It is also capable of distinguish #define used for specifying a Linux
    ioctl.

    By default, it create rules for all symbols and defines, but it also
    allows parsing an exception file. Such file contains a set of rules
    using the syntax below:

    1. Ignore rules:

        ignore <type> <symbol>`

    Removes the symbol from reference generation.

    2. Replace rules:

        replace <type> <old_symbol> <new_reference>

       Replaces how old_symbol with a new reference. The new_reference can be:

        - A simple symbol name;
        - A full Sphinx reference.

    3. Namespace rules

        namespace <namespace>

       Sets C namespace to be used during cross-reference generation. Can
       be overridden by replace rules.

    On ignore and replace rules, <type> can be:
        - ioctl: for defines that end with _IO*, e.g. ioctl definitions
        - define: for other defines
        - symbol: for symbols defined within enums;
        - typedef: for typedefs;
        - enum: for the name of a non-anonymous enum;
        - struct: for structs.

    Examples:

        ignore define __LINUX_MEDIA_H
        ignore ioctl VIDIOC_ENUM_FMT
        replace ioctl VIDIOC_DQBUF vidioc_qbuf
        replace define V4L2_EVENT_MD_FL_HAVE_FRAME_SEQ :c:type:`v4l2_event_motion_det`

        namespace MC
    """

    # Parser regexes with multiple ways to capture enums and structs
    RE_ENUMS = [
        re.compile(r"^\s*enum\s+([\w_]+)\s*\{"),
        re.compile(r"^\s*enum\s+([\w_]+)\s*$"),
        re.compile(r"^\s*typedef\s*enum\s+([\w_]+)\s*\{"),
        re.compile(r"^\s*typedef\s*enum\s+([\w_]+)\s*$"),
    ]
    RE_STRUCTS = [
        re.compile(r"^\s*struct\s+([_\w][\w\d_]+)\s*\{"),
        re.compile(r"^\s*struct\s+([_\w][\w\d_]+)$"),
        re.compile(r"^\s*typedef\s*struct\s+([_\w][\w\d_]+)\s*\{"),
        re.compile(r"^\s*typedef\s*struct\s+([_\w][\w\d_]+)$"),
    ]

    # FIXME: the original code was written a long time before Sphinx C
    # domain to have multiple namespaces. To avoid to much turn at the
    # existing hyperlinks, the code kept using "c:type" instead of the
    # right types. To change that, we need to change the types not only
    # here, but also at the uAPI media documentation.
    DEF_SYMBOL_TYPES = {
        "ioctl": {
            "prefix": "\\ ",
            "suffix": "\\ ",
            "ref_type": ":ref",
            "description": "IOCTL Commands",
        },
        "define": {
            "prefix": "\\ ",
            "suffix": "\\ ",
            "ref_type": ":ref",
            "description": "Macros and Definitions",
        },
        # We're calling each definition inside an enum as "symbol"
        "symbol": {
            "prefix": "\\ ",
            "suffix": "\\ ",
            "ref_type": ":ref",
            "description": "Enumeration values",
        },
        "typedef": {
            "prefix": "\\ ",
            "suffix": "\\ ",
            "ref_type": ":c:type",
            "description": "Type Definitions",
        },
        # This is the description of the enum itself
        "enum": {
            "prefix": "\\ ",
            "suffix": "\\ ",
            "ref_type": ":c:type",
            "description": "Enumerations",
        },
        "struct": {
            "prefix": "\\ ",
            "suffix": "\\ ",
            "ref_type": ":c:type",
            "description": "Structures",
        },
    }

    def __init__(self, debug: bool = False):
        """Initialize internal vars"""
        self.debug = debug
        self.data = ""

        self.symbols = {}

        self.namespace = None
        self.ignore = []
        self.replace = []

        for symbol_type in self.DEF_SYMBOL_TYPES:
            self.symbols[symbol_type] = {}

    def read_exceptions(self, fname: str):
        if not fname:
            return

        name = os.path.basename(fname)

        with open(fname, "r", encoding="utf-8", errors="backslashreplace") as f:
            for ln, line in enumerate(f):
                ln += 1
                line = line.strip()
                if not line or line.startswith("#"):
                    continue

                # ignore rules
                match = re.match(r"^ignore\s+(\w+)\s+(\S+)", line)

                if match:
                    self.ignore.append((ln, match.group(1), match.group(2)))
                    continue

                # replace rules
                match = re.match(r"^replace\s+(\S+)\s+(\S+)\s+(\S+)", line)
                if match:
                    self.replace.append((ln, match.group(1), match.group(2),
                                         match.group(3)))
                    continue

                match = re.match(r"^namespace\s+(\S+)", line)
                if match:
                    self.namespace = match.group(1)
                    continue

                sys.exit(f"{name}:{ln}: invalid line: {line}")

    def apply_exceptions(self):
        """
        Process exceptions file with rules to ignore or replace references.
        """

        # Handle ignore rules
        for ln, c_type, symbol in self.ignore:
            if c_type not in self.DEF_SYMBOL_TYPES:
                sys.exit(f"{name}:{ln}: {c_type} is invalid")

            d = self.symbols[c_type]
            if symbol in d:
                del d[symbol]

        # Handle replace rules
        for ln, c_type, old, new in self.replace:
            if c_type not in self.DEF_SYMBOL_TYPES:
                sys.exit(f"{name}:{ln}: {c_type} is invalid")

            reftype = None

            # Parse reference type when the type is specified

            match = re.match(r"^\:c\:(\w+)\:\`(.+)\`", new)
            if match:
                reftype = f":c:{match.group(1)}"
                new = match.group(2)
            else:
                match = re.search(r"(\:ref)\:\`(.+)\`", new)
                if match:
                    reftype = match.group(1)
                    new = match.group(2)

            # If the replacement rule doesn't have a type, get default
            if not reftype:
                reftype = self.DEF_SYMBOL_TYPES[c_type].get("ref_type")
                if not reftype:
                    reftype = self.DEF_SYMBOL_TYPES[c_type].get("real_type")

            new_ref = f"{reftype}:`{old} <{new}>`"

            # Change self.symbols to use the replacement rule
            if old in self.symbols[c_type]:
                (_, ln) = self.symbols[c_type][old]
                self.symbols[c_type][old] = (new_ref, ln)
            else:
                print(f"{name}:{ln}: Warning: can't find {old} {c_type}")

    def store_type(self, ln, symbol_type: str, symbol: str,
                   ref_name: str = None, replace_underscores: bool = True):
        """
        Stores a new symbol at self.symbols under symbol_type.

        By default, underscores are replaced by "-"
        """
        defs = self.DEF_SYMBOL_TYPES[symbol_type]

        prefix = defs.get("prefix", "")
        suffix = defs.get("suffix", "")
        ref_type = defs.get("ref_type")

        # Determine ref_link based on symbol type
        if ref_type or self.namespace:
            if not ref_name:
                ref_name = symbol.lower()

            # c-type references don't support hash
            if ref_type == ":ref" and replace_underscores:
                ref_name = ref_name.replace("_", "-")

            # C domain references may have namespaces
            if ref_type.startswith(":c:"):
                if self.namespace:
                    ref_name = f"{self.namespace}.{ref_name}"

            if ref_type:
                ref_link = f"{ref_type}:`{symbol} <{ref_name}>`"
            else:
                ref_link = f"`{symbol} <{ref_name}>`"
        else:
            ref_link = symbol

        self.symbols[symbol_type][symbol] = (f"{prefix}{ref_link}{suffix}", ln)

    def store_line(self, line):
        """Stores a line at self.data, properly indented"""
        line = "    " + line.expandtabs()
        self.data += line.rstrip(" ")

    def parse_file(self, file_in: str, exceptions: str = None):
        """Reads a C source file and get identifiers"""
        self.data = ""
        is_enum = False
        is_comment = False
        multiline = ""

        self.read_exceptions(exceptions)

        with open(file_in, "r",
                  encoding="utf-8", errors="backslashreplace") as f:
            for line_no, line in enumerate(f):
                self.store_line(line)
                line = line.strip("\n")

                # Handle continuation lines
                if line.endswith(r"\\"):
                    multiline += line[-1]
                    continue

                if multiline:
                    line = multiline + line
                    multiline = ""

                # Handle comments. They can be multilined
                if not is_comment:
                    if re.search(r"/\*.*", line):
                        is_comment = True
                    else:
                        # Strip C99-style comments
                        line = re.sub(r"(//.*)", "", line)

                if is_comment:
                    if re.search(r".*\*/", line):
                        is_comment = False
                    else:
                        multiline = line
                        continue

                # At this point, line variable may be a multilined statement,
                # if lines end with \ or if they have multi-line comments
                # With that, it can safely remove the entire comments,
                # and there's no need to use re.DOTALL for the logic below

                line = re.sub(r"(/\*.*\*/)", "", line)
                if not line.strip():
                    continue

                # It can be useful for debug purposes to print the file after
                # having comments stripped and multi-lines grouped.
                if self.debug > 1:
                    print(f"line {line_no + 1}: {line}")

                # Now the fun begins: parse each type and store it.

                # We opted for a two parsing logic here due to:
                # 1. it makes easier to debug issues not-parsed symbols;
                # 2. we want symbol replacement at the entire content, not
                #    just when the symbol is detected.

                if is_enum:
                    match = re.match(r"^\s*([_\w][\w\d_]+)\s*[\,=]?", line)
                    if match:
                        self.store_type(line_no, "symbol", match.group(1))
                    if "}" in line:
                        is_enum = False
                    continue

                match = re.match(r"^\s*#\s*define\s+([\w_]+)\s+_IO", line)
                if match:
                    self.store_type(line_no, "ioctl", match.group(1),
                                    replace_underscores=False)
                    continue

                match = re.match(r"^\s*#\s*define\s+([\w_]+)(\s+|$)", line)
                if match:
                    self.store_type(line_no, "define", match.group(1))
                    continue

                match = re.match(r"^\s*typedef\s+([_\w][\w\d_]+)\s+(.*)\s+([_\w][\w\d_]+);",
                                 line)
                if match:
                    name = match.group(2).strip()
                    symbol = match.group(3)
                    self.store_type(line_no, "typedef", symbol, ref_name=name)
                    continue

                for re_enum in self.RE_ENUMS:
                    match = re_enum.match(line)
                    if match:
                        self.store_type(line_no, "enum", match.group(1))
                        is_enum = True
                        break

                for re_struct in self.RE_STRUCTS:
                    match = re_struct.match(line)
                    if match:
                        self.store_type(line_no, "struct", match.group(1))
                        break

        self.apply_exceptions()

    def debug_print(self):
        """
        Print debug information containing the replacement rules per symbol.
        To make easier to check, group them per type.
        """
        if not self.debug:
            return

        for c_type, refs in self.symbols.items():
            if not refs:  # Skip empty dictionaries
                continue

            print(f"{c_type}:")

            for symbol, (ref, ln) in sorted(refs.items()):
                print(f"  #{ln:<5d} {symbol} -> {ref}")

            print()

    def gen_output(self):
        """Write the formatted output to a file."""

        # Avoid extra blank lines
        text = re.sub(r"\s+$", "", self.data) + "\n"
        text = re.sub(r"\n\s+\n", "\n\n", text)

        # Escape Sphinx special characters
        text = re.sub(r"([\_\`\*\<\>\&\\\\:\/\|\%\$\#\{\}\~\^])", r"\\\1", text)

        # Source uAPI files may have special notes. Use bold font for them
        text = re.sub(r"DEPRECATED", "**DEPRECATED**", text)

        # Delimiters to catch the entire symbol after escaped
        start_delim = r"([ \n\t\(=\*\@])"
        end_delim = r"(\s|,|\\=|\\:|\;|\)|\}|\{)"

        # Process all reference types
        for ref_dict in self.symbols.values():
            for symbol, (replacement, _) in ref_dict.items():
                symbol = re.escape(re.sub(r"([\_\`\*\<\>\&\\\\:\/])", r"\\\1", symbol))
                text = re.sub(fr'{start_delim}{symbol}{end_delim}',
                              fr'\1{replacement}\2', text)

        # Remove "\ " where not needed: before spaces and at the end of lines
        text = re.sub(r"\\ ([\n ])", r"\1", text)
        text = re.sub(r" \\ ", " ", text)

        return text

    def gen_toc(self):
        """
        Create a list of symbols to be part of a TOC contents table
        """
        text = []

        # Sort symbol types per description
        symbol_descriptions = []
        for k, v in self.DEF_SYMBOL_TYPES.items():
            symbol_descriptions.append((v['description'], k))

        symbol_descriptions.sort()

        # Process each category
        for description, c_type in symbol_descriptions:

            refs = self.symbols[c_type]
            if not refs:  # Skip empty categories
                continue

            text.append(f"{description}")
            text.append("-" * len(description))
            text.append("")

            # Sort symbols alphabetically
            for symbol, (ref, ln) in sorted(refs.items()):
                text.append(f"- LINENO_{ln}: {ref}")

            text.append("")  # Add empty line between categories

        return "\n".join(text)

    def write_output(self, file_in: str, file_out: str, toc: bool):
        title = os.path.basename(file_in)

        if toc:
            text = self.gen_toc()
        else:
            text = self.gen_output()

        with open(file_out, "w", encoding="utf-8", errors="backslashreplace") as f:
            f.write(".. -*- coding: utf-8; mode: rst -*-\n\n")
            f.write(f"{title}\n")
            f.write("=" * len(title) + "\n\n")

            if not toc:
                f.write(".. parsed-literal::\n\n")

            f.write(text)