annotate_params.py

#!/usr/bin/env python3

"""
This script fetches online ArduPilot parameter documentation (if not cached) and adds it to the specified file
 or to all *.param and *.parm files in the specified directory.

1. Checks if a local cache of the XML file exists in the target directory or on the directory of the target file:
 - If it does, the script loads the file content.
 - If it doesn't, the script sends a GET request to the URL to fetch the XML data for the requested vehicle type.
2. Parses the XML data and creates a dictionary of parameter documentation.
3. DELETES all comments that start at the beginning of a line
4. Adds the parameter documentation to the target file or to all *.param,*.parm files in the target directory.

Supports AP_Periph, AntennaTracker, ArduCopter, ArduPlane, ArduSub, Blimp, Heli, Rover and SITL vehicle types
Supports both Mission Planner and MAVProxy file formats
Supports sorting the parameters
Has unit tests with 88% coverage

AP_FLAKE8_CLEAN

Author: Amilcar do Carmo Lucas, IAV GmbH
"""

import os
import glob
import re
from typing import Any, Dict, List, Tuple
import xml.etree.ElementTree as ET
import argparse
import logging

# URL of the XML file
BASE_URL = "https://autotest.ardupilot.org/Parameters/"

PARAM_DEFINITION_XML_FILE = "apm.pdef.xml"

# ArduPilot parameter names start with a capital letter and can have capital letters, numbers and _
PARAM_NAME_REGEX = r'^[A-Z][A-Z_0-9]*'
PARAM_NAME_MAX_LEN = 16
VERSION = '1.0'


def arg_parser():
    parser = argparse.ArgumentParser(description='Fetches on-line ArduPilot parameter documentation and adds it to the '
                                     'specified file or to all *.param and *.parm files in the specified directory.')
    parser.add_argument('target',
                        help='The target file or directory.',
                        )
    parser.add_argument('-s', '--sort',
                        choices=['none', 'missionplanner', 'mavproxy'],
                        default='none',
                        help='Sort the parameters in the file. Defaults to not sorting.',
                        )
    parser.add_argument('-t', '--vehicle-type',
                        choices=['AP_Periph', 'AntennaTracker', 'ArduCopter', 'ArduPlane',
                                 'ArduSub', 'Blimp', 'Heli', 'Rover', 'SITL'],
                        default='ArduCopter',
                        help='The type of the vehicle. Defaults to ArduCopter',
                        )
    parser.add_argument('--verbose', action='store_true',
                        help='Increase output verbosity, print ReadOnly parameter list. Defaults to false',
                        )
    parser.add_argument('-v', '--version', action='version', version=f'%(prog)s {VERSION}',
                        help='Display version information and exit.',
                        )
    args = parser.parse_args()

    if args.verbose:
        logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
    else:
        logging.basicConfig(level=logging.WARNING, format='%(levelname)s: %(message)s')

    return args


def get_xml_data(base_url: str, directory: str, filename: str) -> ET.Element:
    """
    Fetches XML data from a local file or a URL.

    Args:
        base_url (str): The base URL for fetching the XML file.
        directory (str): The directory where the XML file is expected.
        filename (str): The name of the XML file.

    Returns:
        ET.Element: The root element of the parsed XML data.
    """
    file_path = os.path.join(directory, filename)
    # Check if the locally cached file exists
    if os.path.isfile(file_path):
        # Load the file content relative to the script location
        with open(file_path, "r", encoding="utf-8") as file:
            xml_data = file.read()
    else:
        # No locally cached file exists, get it from the internet
        try:
            import requests  # pylint: disable=C0415
        except ImportError:
            logging.error("The requests package was not found")
            logging.error("Please install it by running 'pip install requests' in your terminal.")
            raise SystemExit("requests package is not installed")  # pylint: disable=W0707
        try:
            # Send a GET request to the URL
            response = requests.get(base_url + filename, timeout=5)
        except requests.exceptions.RequestException as e:
            logging.error("Unable to fetch XML data: %s", e)
            raise SystemExit("unable to fetch online XML documentation")  # pylint: disable=W0707
        # Get the text content of the response
        xml_data = response.text
        # Write the content to a file
        with open(os.path.join(directory, filename), "w", encoding="utf-8") as file:
            file.write(xml_data)

    # Parse the XML data
    root = ET.fromstring(xml_data)
    return root


def remove_prefix(text: str, prefix: str) -> str:
    """
    Removes a prefix from a string.

    Args:
        text (str): The original string.
        prefix (str): The prefix to remove.

    Returns:
        str: The string without the prefix.
    """
    if text.startswith(prefix):
        return text[len(prefix):]
    return text


def split_into_lines(string_to_split: str, maximum_line_length: int) -> List[str]:
    """
    Splits a string into lines of a maximum length.

    Args:
        string_to_split (str): The string to split.
        maximum_line_length (int): The maximum length of a line.

    Returns:
        List[str]: The list of lines.
    """
    doc_lines = re.findall(
        r".{1," + str(maximum_line_length) + r"}(?:\s|$)", string_to_split
    )
    # Remove trailing whitespace from each line
    return [line.rstrip() for line in doc_lines]


def create_doc_dict(root: ET.Element, vehicle_type: str) -> Dict[str, Any]:
    """
    Create a dictionary of parameter documentation from the root element of the parsed XML data.

    Args:
        root (ET.Element): The root element of the parsed XML data.

    Returns:
        Dict[str, Any]: A dictionary of parameter documentation.
    """
    # Dictionary to store the parameter documentation
    doc = {}

    # Use the findall method with an XPath expression to find all "param" elements
    for param in root.findall(".//param"):
        name = param.get("name")
        # Remove the <vehicle_type>: prefix from the name if it exists
        name = remove_prefix(name, vehicle_type + ":")

        human_name = param.get("humanName")
        documentation = split_into_lines(param.get("documentation"), 100)
        # the keys are the "name" attribute of the "field" sub-elements
        # the values are the text content of the "field" sub-elements
        fields = {field.get("name"): field.text for field in param.findall("field")}
        # if Units and UnitText exist, combine them into a single element
        delete_unit_text = False
        for key, value in fields.items():
            if key == "Units" and "UnitText" in fields:
                fields[key] = f"{value} ({fields['UnitText']})"
                delete_unit_text = True
        if delete_unit_text:
            del fields['UnitText']
        # the keys are the "code" attribute of the "values/value" sub-elements
        # the values are the text content of the "values/value" sub-elements
        values = {value.get("code"): value.text for value in param.findall("values/value")}

        # Dictionary with "Parameter names" as keys and the values is a
        # dictionary with "humanName", "documentation" attributes and
        # "fields", "values" sub-elements.
        doc[name] = {
            "humanName": human_name,
            "documentation": documentation,
            "fields": fields,
            "values": values,
        }

    return doc


def format_columns(values: Dict[str, Any], max_width: int = 105, max_columns: int = 4) -> List[str]:
    """
    Formats a dictionary of values into column-major horizontally aligned columns.
    It uses at most max_columns columns

    Args:
        values (Dict[str, Any]): The dictionary of values to format.
        max_width (int, optional): The maximum number of characters on all columns. Defaults to 105.

    Returns:
        List[str]: The list of formatted strings.
    """
    # Convert the dictionary into a list of strings
    strings = [f"{k}: {v}" for k, v in values.items()]

    if (not strings) or (len(strings) == 0):
        return []

    # Calculate the maximum length of the strings
    max_len = max(len(s) for s in strings)

    # Determine the number of columns
    # Column distribution will only happen if it results in more than 5 rows
    # The strings will be distributed evenly across up-to max_columns columns.
    for num_cols in range(max_columns, 0, -1):
        if len(strings) // num_cols > 5 and (max_len + 2) * num_cols < max_width:
            break

    # Calculate the column width
    col_width = max_width // num_cols

    num_rows = (len(strings) + num_cols - 1) // num_cols

    formatted_strings = []
    for j in range(num_rows):
        row = []
        for i in range(num_cols):
            if i*num_rows + j < len(strings):
                if i < num_cols - 1 and ((i+1)*num_rows + j < len(strings)):
                    row.append(strings[i*num_rows + j].ljust(col_width))
                else:
                    row.append(strings[i*num_rows + j])
        formatted_strings.append(" ".join(row))

    return formatted_strings


def extract_parameter_name(item: str) -> str:
    """
    Extract the parameter name from a line. Very simple to be used in sorting
    """
    item = item.strip()
    match = re.match(PARAM_NAME_REGEX, item)
    return match.group(0) if match else item


def missionplanner_sort(item: str) -> Tuple[str, ...]:
    """
    MissionPlanner parameter sorting function
    """
    # Split the parameter name by underscore
    parts = extract_parameter_name(item).split("_")
    # Compare the parts separately
    return tuple(parts)


def extract_parameter_name_and_validate(line: str, filename: str, line_nr: int) -> str:
    """
    Extracts the parameter name from a line and validates it.
    Args:
        line (str): The line to extract the parameter name from.
    Returns:
        str: The extracted parameter name.
    Raises:
        SystemExit: If the line is invalid or the parameter name is too long or invalid.
    """
    # Extract the parameter name
    match = re.match(PARAM_NAME_REGEX, line)
    if match:
        param_name = match.group(0)
    else:
        logging.error("Invalid line %d in file %s: %s", line_nr, filename, line)
        raise SystemExit("Invalid line in input file")
    param_len = len(param_name)
    param_sep = line[param_len] # the character following the parameter name must be a separator
    if param_sep not in {',', ' ', '\t'}:
        logging.error("Invalid parameter name %s on line %d in file %s", param_name, line_nr,
                      filename)
        raise SystemExit("Invalid parameter name")
    if param_len > PARAM_NAME_MAX_LEN:
        logging.error("Too long parameter name on line %d in file %s", line_nr, filename)
        raise SystemExit("Too long parameter name")
    return param_name


def update_parameter_documentation(doc: Dict[str, Any], target: str = '.', sort_type: str = 'none') -> None:
    """
    Updates the parameter documentation in the target file or in all *.param,*.parm files of the target directory.

    This function iterates over all the ArduPilot parameter files in the target directory or file.
    For each file, it DELETES all comments that start at the beginning of a line, optionally sorts the
    parameter names and checks if the parameter name is in the dictionary of parameter documentation.
    If it is, it prefixes the line with comment derived from the dictionary element.
    If it's not, it copies the parameter line 1-to-1.
    After processing all the parameters in a file, it writes the new lines back to the file.

    Args:
        doc (Dict[str, Any]): A dictionary of parameter documentation.
        target (str, optional): The target directory or file. Defaults to '.'.
        sort_type (str, optional): The type of sorting to apply to the parameters.
                                   Can be 'none', 'missionplanner', or 'mavproxy'. Defaults to 'none'.
    """
    # Check if the target is a file or a directory
    if os.path.isfile(target):
        # If it's a file, process only that file
        param_files = [target]
    elif os.path.isdir(target):
        # If it's a directory, process all .param and .parm files in that directory
        param_files = glob.glob(os.path.join(target, "*.param")) \
                    + glob.glob(os.path.join(target, "*.parm"))
    else:
        raise ValueError(f"Target '{target}' is neither a file nor a directory.")

    # Iterate over all the target ArduPilot parameter files
    for param_file in param_files:

        # Read the entire file contents
        with open(param_file, "r", encoding="utf-8") as file:
            lines = file.readlines()

        new_lines = []
        total_params = 0
        documented_params = 0
        undocumented_params = []
        is_first_param_in_file = True  # pylint: disable=C0103
        if sort_type == "missionplanner":
            lines.sort(key=missionplanner_sort)
        if sort_type == "mavproxy":
            lines.sort(key=extract_parameter_name)
        for n, line in enumerate(lines, start=1):
            line = line.strip()
            if not line.startswith("#") and line:
                param_name = extract_parameter_name_and_validate(line, param_file, n)

                if param_name in doc:
                    # If the parameter name is in the dictionary,
                    #  prefix the line with comment derived from the dictionary element
                    data = doc[param_name]
                    prefix_parts = [
                        f"{data['humanName']}",
                    ]
                    prefix_parts += data["documentation"]
                    for key, value in data["fields"].items():
                        prefix_parts.append(f"{key}: {value}")
                    prefix_parts += format_columns(data["values"])
                    doc_text = "\n# ".join(prefix_parts)  # pylint: disable=C0103
                    if not is_first_param_in_file:
                        new_lines.append("\n")
                    new_lines.append(f"# {doc_text}\n{line}\n")
                    documented_params += 1
                else:
                    # If the parameter name is in not the dictionary, copy the parameter line 1-to-1
                    new_lines.append(f"{line}\n")
                    undocumented_params.append(param_name)
                total_params += 1
                is_first_param_in_file = False

        if total_params == documented_params:
            logging.info("Read file %s with %d parameters, all got documented",
                         param_file, total_params)
        else:
            logging.warning("Read file %s with %d parameters, but only %s of which got documented",
                            param_file, total_params, documented_params)
            logging.warning("No documentation found for: %s", ", ".join(undocumented_params))

        # Write the new file contents to the file
        with open(param_file, "w", encoding="utf-8") as file:
            file.writelines(new_lines)


def print_read_only_params(doc):
    """
    Print the names of read-only parameters.

    Args:
        doc (dict): A dictionary of parameter documentation.
    """
    logging.info("ReadOnly parameters:")
    for param_name, param_value in doc.items():
        if 'ReadOnly' in param_value['fields'] and param_value['fields']['ReadOnly']:
            logging.info(param_name)


def main():
    args = arg_parser()
    try:
        xml_dir = args.target if os.path.isdir(args.target) else os.path.dirname(os.path.realpath(args.target))
        xml_root = get_xml_data(BASE_URL + args.vehicle_type + "/", xml_dir, PARAM_DEFINITION_XML_FILE)
        doc_dict = create_doc_dict(xml_root, args.vehicle_type)
        update_parameter_documentation(doc_dict, args.target, args.sort)
        if args.verbose:
            print_read_only_params(doc_dict)
    except Exception as exp:  # pylint: disable=W0718
        logging.fatal(exp)
        exit(1)  # pylint: disable=R1722


if __name__ == "__main__":
    main()