uORB docs parser (#24977)

Tools/msg/generate_msg_docs.py

@@ -8,6 +8,803 @@ Also generates docs/en/middleware/dds_topics.md from dds_topics.yaml
 import os
 import argparse
 import sys
+import re
+
+VALID_FIELDS = { #Note, also have to add the message types as those can be fields
+    'uint64',
+    'uint16',
+    'uint8',
+    'uint32'
+}
+
+ALLOWED_UNITS = set(["m", "m/s", "m/s^2", "(m/s)^2", "deg", "deg/s", "rad", "rad/s", "rad^2", "rpm" ,"V", "A", "mA", "mAh", "W", "dBm", "h", "s", "ms", "us", "Ohm", "MB", "Kb/s", "degC","Pa","%","-"])
+invalid_units = set()
+ALLOWED_FRAMES = set(["NED","Body"])
+ALLOWED_INVALID_VALUES = set(["NaN", "0"])
+ALLOWED_CONSTANTS_NOT_IN_ENUM = set(["ORB_QUEUE_LENGTH","MESSAGE_VERSION"])
+
+class Error:
+    def __init__(self, type, message, linenumber=None, issueString = None, field = None):
+        self.type = type
+        self.message = message
+        self.linenumber = linenumber
+        self.issueString = issueString
+        self.field = field
+
+    def display_error(self):
+        #print(f"Debug: Error: display_error")
+
+
+        if 'trailing_whitespace' == self.type:
+            if self.issueString.strip():    
+                print(f"NOTE: Line has trailing whitespace ({self.message}: {self.linenumber}): {self.issueString}")
+            else:
+                print(f"NOTE: Line has trailing whitespace ({self.message}: {self.linenumber})")
+        elif 'leading_whitespace_field_or_constant' == self.type:
+            print(f"NOTE: Whitespace before field or constant ({self.message}: {self.linenumber}): {self.issueString}")             
+        elif 'field_or_constant_has_multiple_whitepsace' == self.type:
+            print(f"NOTE: Field/constant has more than one sequential whitespace character ({self.message}: {self.linenumber}): {self.issueString}") 
+        elif 'empty_start_line' == self.type:
+            print(f"NOTE: Empty line at start of file ({self.message}: {self.linenumber})")
+        elif 'internal_comment' == self.type:
+            print(f"NOTE: Internal Comment ({self.message}: {self.linenumber})\n {self.issueString}")
+        elif 'internal_comment_empty' == self.type:
+            print(f"NOTE: Empty Internal Comment ({self.message}: {self.linenumber})")
+        elif 'summary_missing' == self.type:
+            print(f"WARNING: No message description ({self.message})")
+        elif 'topic_error' == self.type:
+            print(f"NOTE: TOPIC ISSUE: {self.issueString}")
+        elif 'unknown_unit' == self.type:
+            print(f"WARNING: Unknown Unit: [{self.issueString}] on `{self.field}` ({self.message}: {self.linenumber})")
+        elif 'constant_not_in_assigned_enum' == self.type:
+            print(f"WARNING: `{self.issueString}` constant: Prefix not in `@enum` field metadata ({self.message}: {self.linenumber})")
+        elif 'unknown_invalid_value' == self.type:
+            print(f"WARNING: Unknown @invalid value: [{self.issueString}] on `{self.field}` ({self.message}: {self.linenumber})")
+        elif 'unknown_frame' == self.type:
+            print(f"WARNING: Unknown @frame: [{self.issueString}] on `{self.field}` ({self.message}: {self.linenumber})")
+        elif 'command_no_params_pipes' == self.type:
+            print(f"WARNING: `{self.field}` command has no parameters (pipes): [{self.issueString}] ({self.message}: {self.linenumber})")
+        elif 'command_missing_params' == self.type:
+            print(f"WARNING: `{self.field}` command missing params - should be 7 params surrounded by 8 pipes: [{self.issueString}] ({self.message}: {self.linenumber})")
+        elif 'command_too_many_params' == self.type:
+            print(f"WARNING: `{self.field}` command too many params (should be 7). Extras: [{self.issueString}] ({self.message}: {self.linenumber})")
+
+
+        else:
+            self.display_info()
+
+    def display_info(self):
+        """
+        Display info about an error.
+        Used as a fallback if error does not have specific printout in display_error()
+        """
+        #print(f"Debug: Error: display_info")
+        print(f" type: {self.type}, message: {self.message}, linenumber: {self.linenumber}, issueString: {self.issueString}, field: {self.field}")
+
+class Enum:
+    def __init__(self, name, parentMessage):
+        self.name = name
+        self.parent = parentMessage
+        self.enumValues = dict()
+
+    def display_info(self):
+        """
+        Display info about an enum
+        """
+        print(f"Debug: Enum: display_info")
+        print(f" name: {self.name}")
+        for key, value in self.enumValues.items():
+            value.display_info()
+
+class ConstantValue:
+    def __init__(self, name, type, value, comment, line_number):
+        self.name = name.strip()
+        self.type = type.strip()
+        self.value = value.strip()
+        self.comment = comment
+        self.line_number = line_number
+
+        if not self.value:
+            print(f"Debug WARNING: NO VALUE in ConstantValue: {self.name}")  ## TODO make into ERROR
+            exit()
+
+        # TODO if value or name are empty, error
+
+    def display_info(self):
+        print(f"Debug: ConstantValue: display_info")
+        print(f" name: {self.name}, type: {self.type}, value: {self.value}, comment: {self.comment}, line: {self.line_number}")
+
+
+class CommandParam:
+    """
+    Represents an individual param in a command constant
+    Encapsulates parsing of the param to extract units etc.
+    """
+
+    def __init__(self, num, paramText, line_number, parentCommand):
+        self.paramNum = num
+        self.paramText = paramText.strip()
+        self.enum = None
+        self.range = None
+        #self.type = type
+        self.units = []
+        self.enums = []
+        self.minValue = None
+        self.maxValue = None
+        self.invalidValue = None
+        self.frameValue = None
+        self.lineNumber = line_number
+        self.parent = parentCommand
+        self.parentMessage = self.parent.parent
+
+        match = None
+        if self.paramText:
+            match = re.match(r'^((?:\[[^\]]*\]\s*)+)(.*)$', paramText)
+        self.description = paramText
+        bracketed_part = None
+        if match:
+            bracketed_part = match.group(1).strip() # .strip() removes trailing whitespace from the bracketed part
+            self.description = match.group(2).strip()
+        if bracketed_part:
+          # get units
+            bracket_content_matches = re.findall(r'\[(.*?)\]', bracketed_part)
+            #print(f"DEBUG: bracket_content_matches: {bracket_content_matches}")
+            for item in bracket_content_matches:
+                item = item.strip()
+                if item.startswith('@'): # Not a unit:
+                    if item.startswith('@enum'):
+                        item = item.split(" ")
+                        enum = item[1].strip()
+                        if enum and enum not in self.enums:
+                            self.enums.append(enum)
+
+                        # Create parent enum objects for any enums created in this step
+                        for enumName in self.enums:
+                            if not enumName in self.parentMessage.enums:
+                                self.parentMessage.enums[enumName]=Enum(enumName,self.parentMessage)
+
+                    elif item.startswith('@range'):
+                        item = item[6:].strip().split(",")
+                        self.range = item
+                        self.minValue = item[0].strip()
+                        self.maxValue = item[1].strip()
+                    elif item.startswith('@invalid'):
+                        self.invalidValue = item[8:].strip()
+                        #TODO: Do we require a description? (not currently)
+                        if self.invalidValue.split(" ")[0] not in ALLOWED_INVALID_VALUES:
+                            print(f"TODO: Command param do not support @invalid: {self.invalidValue}")
+                            """
+                            error = Error("unknown_invalid_value", self.parent.filename, self.lineNumber, self.invalidValue, self.name)
+                            #error.display_error()
+                            if not "unknown_invalid_value" in self.parent.errors:
+                                self.parent.errors["unknown_invalid_value"] = []
+                            self.parent.errors["unknown_invalid_value"].append(error)
+                            """
+
+                    elif item.startswith('@frame'):
+                        self.frameValue = item[6:].strip()
+                        print(f"TODO: Command param do not support @frame: {self.frameValue}")
+                        """
+                        if self.frameValue not in ALLOWED_FRAMES:
+                            error = Error("unknown_frame", self.parent.filename, self.lineNumber, self.frameValue, self.name)
+                            #error.display_error()
+                            if not "unknown_frame" in self.parent.errors:
+                                self.parent.errors["unknown_frame"] = []
+                            self.parent.errors["unknown_frame"].append(error)
+                        """    
+                    else:
+                        print(f"WARNING: Unhandled metadata in message comment: {item}")
+                        # TODO - report errors for different kinds of metadata
+                        exit()
+
+                else: # bracket is a unit
+                    unit = item.strip()
+
+                    if item == "-":
+                        unit = ""
+                    
+                    if unit and unit not in self.units:
+                        self.units.append(unit) 
+
+                    if unit not in ALLOWED_UNITS:
+                        invalid_units.add(unit)
+                        error = Error("unknown_unit", self.parentMessage.filename, self.lineNumber, unit, self.parent.name)
+                        #error.display_error()
+                        if not "unknown_unit" in self.parentMessage.errors:
+                            self.parentMessage.errors["unknown_unit"] = []
+                        self.parentMessage.errors["unknown_unit"].append(error)
+
+
+    def display_info(self):
+        print(f"Debug: CommandParam: display_info")
+        print(f" id: {self.paramNum}")
+        print(f"   paramText: {self.paramText}\n  unit:  {self.units}\n  enums: {self.enums}\n  lineNumber: {self.lineNumber}\n  range: {self.range}\n  minValue: {self.minValue}\n  maxValue: {self.maxValue}\n  invalidValue: {self.invalidValue}\n  frameValue: {self.frameValue}\n  parent: {self.parent}\n  ")
+
+
+        
+class CommandConstant:
+    """
+    Represents a constant that is a command definition.
+    Encapsulates parsing of the command format.
+    The individual params are further parsed in CommandParam
+    """
+    def __init__(self, name, type, value, comment, line_number, parentMessage):
+        self.name = name.strip()
+        self.type = type.strip()
+        self.value = value.strip()
+        self.comment = comment
+        self.line_number = line_number
+        self.parent = parentMessage
+
+        self.description = self.comment
+        self.param1 = None
+        self.param2 = None
+        self.param3 = None
+        self.param4 = None
+        self.param5 = None
+        self.param6 = None
+        self.param7 = None
+
+        if not self.value:
+            print(f"Debug WARNING: NO VALUE in CommandConstant: {self.name}")  ## TODO make into ERROR
+            exit()
+
+        if not self.comment: # This is an bug for a command
+            #print(f"Debug WARNING: NO COMMENT in CommandConstant: {self.name}")  ## TODO make into ERROR
+            return
+            
+        # Parse command comment to get the description and parameters.
+        # print(f"Debug CommandConstant: {self.comment}") 
+        if not "|" in self.comment:
+            # This is an error for a command constant
+            error = Error("command_no_params_pipes", self.parent.filename, self.line_number, self.comment, self.name)
+            #error.display_error()
+            if not "command_no_params_pipes" in self.parent.errors:
+                self.parent.errors["command_no_params_pipes"] = []
+            self.parent.errors["command_no_params_pipes"].append(error)
+            return
+        
+        # Split on pipes
+        commandSplit = self.comment.split("|")
+        if len(commandSplit) < 9:
+            # Should 7 pipes, so each command is fully surrounded
+            error = Error("command_missing_params", self.parent.filename, self.line_number, self.comment, self.name)
+            #error.display_error()
+            if not "command_missing_params" in self.parent.errors:
+                self.parent.errors["command_missing_params"] = []
+            self.parent.errors["command_missing_params"].append(error)
+
+        self.description = commandSplit[0].strip()
+        self.description = self.description if self.description else None
+
+        params_to_update = commandSplit[1:8]
+
+        for i, value in enumerate(params_to_update, start=1):
+            if value.strip():
+                # parse the param
+                param = CommandParam(i, value, self.line_number, self)
+                #param.display_info() # DEBUG CODE XXX
+                setattr(self, f"param{i}", param)
+                # parse the param
+
+        if len(commandSplit) > 8:
+            extras = commandSplit[8:]
+            error = Error("command_too_many_params", self.parent.filename, self.line_number, extras, self.name)
+            if not "command_too_many_params" in self.parent.errors:
+                self.parent.errors["command_too_many_params"] = []
+            self.parent.errors["command_too_many_params"].append(error)
+
+
+        # TODO if value or name are empty, error
+
+    def markdown_out(self):
+        #print("DEBUG: CommandConstant.markdown_out")
+        output = f"""### {self.name} ({self.value})
+
+{self.description}
+
+Param | Units | Range/Enum | Description
+--- | --- | --- | ---
+"""
+        for i in range(1, 8):
+            attr_name = f"param{i}"
+            # getattr returns None if the attribute doesn't exist
+            val = getattr(self, attr_name, None)
+
+            if val is not None:
+                rangeVal = ""
+                if val.minValue or val.maxValue:
+                    rangeVal = f"[{val.minValue if val.minValue else '-'} : {val.maxValue if val.maxValue else '-' }]"
+
+                output+=f"{i} | {", ".join(val.units)}|{', '.join(f"[{e}](#{e})" for e in val.enums)}{rangeVal} | {val.description}\n"
+            else:
+                output+=f"{i} | | | ?\n"                
+
+        output+=f"\n"
+        return output
+
+
+    def display_info(self):
+        print(f"Debug: CommandConstant: display_info")
+        print(f" name: {self.name}, type: {self.type}, value: {self.value}, comment: {self.comment}, line: {self.line_number}")
+        print(f"   description: {self.description}\n  param1: {self.param1}\n  param2: {self.param2}\n  param3: {self.param3}\n  param4: {self.param4}\n  param5: {self.param5}\n  param6: {self.param6}\n  param7: {self.param7}")
+
+class MessageField:
+    """
+    Represents a field.
+    Encapsulates parsing of the field information.
+    """
+    def __init__(self, name, type, comment, line_number, parentMessage):
+        self.name = name
+        self.type = type
+        self.comment = comment
+        self.unit = None
+        self.enums = None
+        self.minValue = None
+        self.maxValue = None
+        self.invalidValue = None
+        self.frameValue = None
+        self.lineNumber = line_number
+        self.parent = parentMessage
+
+        #print(f"MessageComment: {comment}")
+        match = None
+        if self.comment:
+            match = re.match(r'^((?:\[[^\]]*\]\s*)+)(.*)$', comment)
+        self.description = comment
+        bracketed_part = None
+        if match:
+            bracketed_part = match.group(1).strip() # .strip() removes trailing whitespace from the bracketed part
+            self.description = match.group(2).strip()
+        if bracketed_part:
+          # get units
+            bracket_content_matches = re.findall(r'\[(.*?)\]', bracketed_part)
+            #print(f"bracket_content_matches: {bracket_content_matches}")
+            for item in bracket_content_matches:
+                item = item.strip()
+                if item.startswith('@'): # Not a unit:
+                    if item.startswith('@enum'):
+                        item = item.split(" ")
+                        self.enums = item[1:]
+                        # Create parent enum objects
+                        for enumName in self.enums:
+                            if not enumName in parentMessage.enums:
+                                parentMessage.enums[enumName]=Enum(enumName,parentMessage)
+                    elif item.startswith('@range'):
+                        item = item[6:].strip().split(",")
+                        self.minValue = item[0].strip()
+                        self.maxValue = item[1].strip()
+                    elif item.startswith('@invalid'):
+                        self.invalidValue = item[8:].strip()
+                        #TODO: Do we require a description? (not currently)
+                        if self.invalidValue.split(" ")[0] not in ALLOWED_INVALID_VALUES:
+                            error = Error("unknown_invalid_value", self.parent.filename, self.lineNumber, self.invalidValue, self.name)
+                            #error.display_error()
+                            if not "unknown_invalid_value" in self.parent.errors:
+                                self.parent.errors["unknown_invalid_value"] = []
+                            self.parent.errors["unknown_invalid_value"].append(error)
+                    elif item.startswith('@frame'):
+                        self.frameValue = item[6:].strip()
+                        if self.frameValue not in ALLOWED_FRAMES:
+                            error = Error("unknown_frame", self.parent.filename, self.lineNumber, self.frameValue, self.name)
+                            #error.display_error()
+                            if not "unknown_frame" in self.parent.errors:
+                                self.parent.errors["unknown_frame"] = []
+                            self.parent.errors["unknown_frame"].append(error)
+                    else:
+                        print(f"WARNING: Unhandled metadata in message comment: {item}")
+                        # TODO - report errors for different kinds of metadata
+                        exit()
+
+                else: # bracket is a unit
+                    self.unit = item
+
+                    if self.unit not in ALLOWED_UNITS:
+                        invalid_units.add(self.unit)
+                        error = Error("unknown_unit", self.parent.filename, self.lineNumber, self.unit, self.name)
+                        #error.display_error()
+                        if not "unknown_unit" in self.parent.errors:
+                            self.parent.errors["unknown_unit"] = []
+                        self.parent.errors["unknown_unit"].append(error)
+
+                    if item == "-":
+                        self.unit = ""
+
+
+    def display_info(self):
+        print(f"Debug: MessageField: display_info")
+        print(f" name: {self.name}, type: {self.type}, description: {self.description}, enums: {self.enums}, minValue: {self.minValue}, maxValue: {self.maxValue}, invalidValue: {self.invalidValue}, frameValue: {self.frameValue}")
+
+
+class UORBMessage:
+    """
+    Represents a whole message, including fields, enums, commands, constants.
+    The parser function delegates the parsing of each part of the message to 
+    more appropriate classes, once the specific type of line has been identified.
+    """
+
+    def __init__(self, filename):
+
+        self.filename = filename
+        msg_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../../msg")
+        self.msg_filename = os.path.join(msg_path, self.filename)
+        self.name = os.path.splitext(os.path.basename(msg_file))[0]
+        self.shortDescription = ""
+        self.longDescription = ""
+        self.fields = []
+        self.constantFields = dict()
+        self.commandConstants = dict()
+        self.enums = dict()
+        self.output_file = os.path.join(output_dir, f"{self.name}.md")
+        self.topics = []
+        self.errors = dict()
+
+        self.parseFile()
+
+        if args.errors:
+            #print(f"DEBUG: args.errors: {args.errors}")
+            if args.error_messages:
+                messages = args.error_messages.split(" ")
+                #print(f"DEBUG: args.errors: {messages},self.name: {self.name}")
+                if self.name in messages:
+                    self.reportErrors()
+                    #print(f"Debug: {self.name} in {messages}")
+            else:
+              self.reportErrors()
+
+    def reportErrors(self):
+        #print(f"Debug: UORBMessage: reportErrors()")
+        for errorType, errors in self.errors.items():
+            for error in errors:
+               error.display_error()
+
+    def markdown_out(self):
+        #print(f"Debug: UORBMessage: markdown_out()")
+
+        # Add page header (forces wide pages)
+        markdown = f"""---
+pageClass: is-wide-page
+---
+
+# {self.name} (UORB message)
+
+"""
+        ## Append description info if present
+        markdown += f"{self.shortDescription}\n\n" if self.shortDescription else ""
+        markdown += f"{self.longDescription}\n\n" if self.longDescription else ""
+
+        topicList = " ".join(self.topics)
+        markdown += f"**TOPICS:** {topicList}\n\n"
+
+        # Generate field docs
+        markdown += f"## Fields\n\n"
+        markdown += "Name | Type | Unit [Frame] | Range/Enum | Description\n"
+        markdown += "--- | --- | --- | --- | ---\n"
+        for field in self.fields:
+            unit = f"{field.unit}" if field.unit else ""
+            frame = f"[{field.frameValue}]" if field.frameValue else ""
+            unit = f"{unit} {frame}"
+            unit.strip()
+            unit = f" {unit}"
+
+            value = " "
+            if field.enums:
+                value = ""
+                for enum in field.enums:
+                    value += f"[{enum}](#{enum})"
+                value = value.strip()
+                value = f"{value}"
+            elif field.minValue or field.maxValue:
+                value = f"[{field.minValue if field.minValue else '-'} : {field.maxValue if field.maxValue else '-' }]"
+
+            description = f" {field.description}" if field.description else ""
+            invalid = f" (Invalid: {field.invalidValue}) " if field.invalidValue else ""
+            markdown += f"{field.name} | `{field.type}` |{unit}|{value}|{description}{invalid}\n"
+
+        # Generate table for command docs
+        if len(self.commandConstants) > 0:
+            #print("DEBUGCOMMAND")
+            markdown += f"\n## Commands\n\n"
+
+            """
+            markdown += "Name | Type | Value | Description\n"
+            markdown += "--- | --- | --- |---\n"
+            for name, command in self.commandConstants.items():
+                description = f" {command.comment} " if enum.comment else " "
+                markdown += f'<a href="#{name}"></a> {name} | `{command.type}` | {command.value} |{description}\n'            
+            """
+            for commandConstant in self.commandConstants.values():
+                #print(commandConstant)
+                markdown += commandConstant.markdown_out()     
+
+        # Generate enum docs
+        if len(self.enums) > 0:
+            markdown += f"\n## Enums\n"
+
+            for name, enum in self.enums.items():
+                markdown += f"\n### {name} {{#{name}}}\n\n"
+
+                markdown += "Name | Type | Value | Description\n"
+                markdown += "--- | --- | --- | ---\n"
+
+                for enumValueName, enumValue in enum.enumValues.items():
+                    description = f" {enumValue.comment} " if enumValue.comment else " "
+                    markdown += f'<a href="#{enumValueName}"></a> {enumValueName} | `{enumValue.type}` | {enumValue.value} |{description}\n'
+
+        # Generate table for constants docs
+        if len(self.constantFields) > 0:
+            markdown += f"\n## Constants\n\n"
+            markdown += "Name | Type | Value | Description\n"
+            markdown += "--- | --- | --- |---\n"
+            for name, enum in self.constantFields.items():
+                description = f" {enum.comment} " if enum.comment else " "
+                markdown += f'<a href="#{name}"></a> {name} | `{enum.type}` | {enum.value} |{description}\n'
+
+
+
+        # Append msg contents to the end
+        with open(self.msg_filename, 'r') as source_file:
+            msg_contents = source_file.read()
+            msg_contents = msg_contents.strip()
+
+        #Format markdown using msg name, comment, url, contents.
+        markdown += f"""
+
+## Source Message
+
+[Source file (GitHub)](https://github.com/PX4/PX4-Autopilot/blob/main/msg/{self.filename})
+
+::: details Click here to see original file
+
+```c
+{msg_contents}
+```
+
+:::
+"""
+
+        with open(self.output_file, 'w') as content_file:
+            content_file.write(markdown)
+
+        #exit()
+
+
+    def display_info(self):
+        print(f"UORBMessage: display_info")
+        print(f" name: {self.name}")
+        print(f" filename: {self.filename}, ")
+        print(f" msg_filename: {self.msg_filename}, ")
+        print(f"self.shortDescription: {self.shortDescription}")
+        print(f"self.longDescription: {self.longDescription}")
+        print(f"self.enums: {self.enums}")
+
+        for enum, enumObject in self.enums.items():
+            enumObject.display_info()
+
+        # Output our data so far
+        for field in self.fields:
+            field.display_info()
+
+        for enumvalue in self.constantFields:
+            print(enumvalue)
+            self.constantFields[enumvalue].display_info()
+
+    def handleField(self, line, line_number, parentMessage):
+        #print(f"debug: handleField: (line): \n {line}")
+        # Note, here we know we don't have a comment or a topic.
+        # We expect it to be a field.
+
+        # Check field doesn't have leading whitespace (trailing spaces already checked)
+        if line[:1].isspace(): # Returns True for ' ', '\t', '\n', '\r', etc.
+            #print("First character is whitespace")
+            error = Error("leading_whitespace_field_or_constant", self.filename, line_number, line)
+            if not "leading_whitespace_field_or_constant" in self.errors:
+                self.errors["leading_whitespace_field_or_constant"] = []
+                self.errors["leading_whitespace_field_or_constant"].append(error)
+
+        # Now we can parse the stripped line
+        fieldOrConstant = line.strip()
+
+        # Check that the field or constant has only single whitespace separators
+        stripped_fieldOrConstant = re.sub(r'\s+', ' ', fieldOrConstant) # Collapse all spaces to a single space (LHS already stripped).
+        if stripped_fieldOrConstant != fieldOrConstant:
+            #print("Field/Constant has multiple whitespace characters") # Since the collapsed version shows them.
+            error = Error("field_or_constant_has_multiple_whitepsace", self.filename, line_number, line)
+            if not "field_or_constant_has_multiple_whitepsace" in self.errors:
+                self.errors["field_or_constant_has_multiple_whitepsace"] = []
+                self.errors["field_or_constant_has_multiple_whitepsace"].append(error)
+
+        fieldOrConstant = stripped_fieldOrConstant
+
+
+
+        comment = None
+        if "#" in line:
+            commentExtract = line.split("#", 1) # Split once on left-most '#'
+            fieldOrConstant = commentExtract[0].strip()
+            comment = commentExtract[-1].strip()
+
+        if "=" not in fieldOrConstant:
+            # Is a field:
+            field = fieldOrConstant.split(" ")
+            type = field[0].strip()
+            name = field[1].strip()
+            field = MessageField(name, type, comment, line_number, parentMessage)
+            self.fields.append(field)
+        else:
+            temp = fieldOrConstant.split("=")
+            value = temp[-1]
+            typeAndName = temp[0].split(" ")
+            type = typeAndName[0]
+            name = typeAndName[1]
+            if name.startswith("VEHICLE_CMD_") and parentMessage.name == 'VehicleCommand': #it's a command.
+                #print(f"DEBUG: startswith VEHICLE_CMD_ {name}")
+                commandConstant = CommandConstant(name, type, value, comment, line_number, parentMessage)
+                #commandConstant.display_info()
+                self.commandConstants[name]=commandConstant
+            else: #it's a constant (or part of an enum)
+                constantField = ConstantValue(name, type, value, comment, line_number)
+                self.constantFields[name]=constantField
+
+
+    def parseFile(self):
+        initial_block_lines = []
+        #stopping_token = None
+        found_first_relevant_content = False
+        gettingInitialComments = False
+        gettingFields = False
+
+        with open(self.msg_filename, 'r', encoding='utf-8') as uorbfile:
+            lines = uorbfile.read().splitlines()
+            for line_number, line in enumerate(lines, 1):
+
+                if line != line.rstrip():
+                    #print(f"[{self.filename}] Trailing whitespace on line {line_number}: XX{line}YY")
+                    error = Error("trailing_whitespace", self.filename, line_number, line)
+                    if not "trailing_whitespace" in self.errors:
+                        self.errors["trailing_whitespace"] = []
+                    self.errors["trailing_whitespace"].append(error)
+
+                #print(f"line: {line}")
+                stripped_line = re.sub(r'\s+', ' ', line).strip() # Collapse all spaces to a single space and strip stuff off end.
+                #print(f"stripped_line: {stripped_line}")
+                # TODO? Perhaps report whitespace if the size of those two is different and it is empty
+                # Or perhaps we just fix it on request
+
+                isEmptyLine = False if line.strip() else True
+                if not found_first_relevant_content and isEmptyLine: #Empty line
+                    #print(f"{self.filename}: Empty line at start of file: [{line_number}]\n {line}")
+                    error = Error("empty_start_line", self.filename, line_number, line)
+                    if not "empty_start_line" in self.errors:
+                        self.errors["empty_start_line"] = []
+                    self.errors["empty_start_line"].append(error)
+                    #error.display_error()
+                    continue
+                if not found_first_relevant_content and not isEmptyLine:
+                    found_first_relevant_content = True
+
+                    if stripped_line.startswith("#"):
+                        gettingInitialComments = True
+                    else:
+                        gettingInitialComments = False
+                        gettingFields = True
+
+                if gettingInitialComments and stripped_line.startswith("#"):
+                    stripped_line=stripped_line[1:].strip()
+                    #print(f"DEBUG: gettingInitialComments: comment line: {stripped_line}")
+                    initial_block_lines.append(stripped_line)
+                else:
+                    gettingInitialComments = False
+                    gettingFields = True #Getting fields and constants
+                if gettingFields:
+                    if isEmptyLine:
+                        continue # empty line
+                    if stripped_line.startswith("# TOPICS "):
+                        stripped_line = stripped_line[9:]
+                        stripped_line = stripped_line.split(" ")
+                        self.topics+= stripped_line
+                        # Note, default topic and topic errors handled after all lines parsed
+                        continue
+                    if stripped_line.startswith("#"):
+                        # Its an internal comment
+                        stripped_line=stripped_line[1:].strip()
+                        
+                        if stripped_line:
+                            #print(f"{self.filename}: Internal comment: [{line_number}]\n {line}")
+                            error = Error("internal_comment", self.filename, line_number, line)
+                            if not "internal_comment" in self.errors:
+                                self.errors["internal_comment"] = []
+                            self.errors["internal_comment"].append(error)
+                        else:
+                            #print(f"{self.filename}: Empty internal comment: [{line_number}]\n {line}")
+                            error = Error("internal_comment_empty", self.filename, line_number, line)
+                            if not "internal_comment_empty" in self.errors:
+                                self.errors["internal_comment_empty"] = []
+                            self.errors["internal_comment_empty"].append(error)
+                            #pass # Empty comment
+                        continue
+                    
+                    # Must be a field or a comment.
+                    self.handleField(line, line_number, parentMessage=self)
+
+            # Fix up topics if the topic is empty
+            def camel_to_snake(name):
+                # Match upper case not at start of string
+                s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
+                # Handle cases with multiple capital first letter
+                return re.sub('([A-Z]+)([A-Z][a-z]*)', r'\1_\2', s1).lower()
+
+            defaultTopic = camel_to_snake(self.name)
+            if len(self.topics) == 0:
+                # We have no topic declared, so set the default topic
+                self.topics.append(defaultTopic)
+            elif len(self.topics) == 1:
+                # We have 1 topic declared - either it is default or there is some issue.
+                if defaultTopic in self.topics:
+                    # Declared topic is default topic
+                    error = Error("topic_error", self.filename, "", f"WARNING: TOPIC {defaultTopic} unnecessarily declared for {self.name}")
+                else:
+                    # Declared topic is not default topic
+                    error = Error("topic_error", self.filename, "", f"NOTE: TOPIC {self.topics[1]}: Only Declared topic is not default topic {defaultTopic} for {self.name}")
+                if not "topic_error" in self.errors:
+                    self.errors["topic_error"] = []
+                    self.errors["topic_error"].append(error)
+            elif len(self.topics) > 1:
+                if defaultTopic not in self.topics:
+                    error = Error("topic_error", self.filename, "", f"NOTE: TOPIC - Default topic {defaultTopic} for {self.name} not in {self.topics}")
+
+            # Parse our short and long description
+            #print(f"DEBUG: initial_block_lines: {initial_block_lines}")
+            doingLongDescription = False
+            for summaryline in initial_block_lines:
+                if not self.shortDescription and summaryline.strip() == '':
+                    continue
+                if not doingLongDescription and not summaryline.strip() == '':
+                   self.shortDescription += f" {summaryline}"
+                   self.shortDescription = self.shortDescription.strip()
+                   if not self.shortDescription[-1:] == ".": # Add terminating fullstop if not present.
+                       self.shortDescription += "."
+                if not doingLongDescription and summaryline.strip() == '':
+                   doingLongDescription = True
+                   continue
+                if doingLongDescription:
+                    self.longDescription += f"{summaryline}\n"
+
+            if self.longDescription:
+                self.longDescription.strip()
+
+            if not self.shortDescription:
+                # Summary has not been defined
+                error = Error("summary_missing", self.filename)
+                if not "summary_missing" in self.errors:
+                    self.errors["summary_missing"] = []
+                self.errors["summary_missing"].append(error)
+
+
+            # TODO Parse our constantValues into enums, leaving only constants
+            constantValuesToRemove = []
+            #print(f"DEBUG: Self.enums: {self.enums}")
+            for enumName, enumObject in self.enums.items():
+                for enumValueName, enumValueObject in self.constantFields.items():
+                    if enumValueName.startswith(enumName):
+                        # Copy this value into the object (cant be duplicate because parent is dict)
+                        enumObject.enumValues[enumValueName]=enumValueObject
+                        constantValuesToRemove.append(enumValueName)
+            # Now delete the original enumvalues
+            for enumValName in constantValuesToRemove:
+                del self.constantFields[enumValName]
+            constantsNotAssignedToEnums = len(self.constantFields)
+            if constantsNotAssignedToEnums > 0:
+                #print(f"Debug: WARNING constantsNotAssignedToEnums: {constantsNotAssignedToEnums}")
+                for enumValueName, enumValue in self.constantFields.items():
+                    if enumValueName in ALLOWED_CONSTANTS_NOT_IN_ENUM: # Ignore constants
+                        pass
+                    else:
+                        error = Error("constant_not_in_assigned_enum", self.filename, enumValue.line_number, enumValueName)
+                        if not "constant_not_in_assigned_enum" in self.errors:
+                            self.errors["constant_not_in_assigned_enum"] = []
+                        self.errors["constant_not_in_assigned_enum"].append(error)
+                # TODO Maybe present as list of possible enums.
 
 
 import yaml
@@ -127,83 +924,50 @@ if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description='Generate docs from .msg files')
     parser.add_argument('-d', dest='dir', help='output directory', required=True)
+    parser.add_argument('-e', dest='errors', action='store_true', help='Report errors')
+    parser.add_argument('-m', dest='error_messages', help='Message to report errors against (by default all)')
     args = parser.parse_args()
 
     output_dir = args.dir
     if not os.path.isdir(output_dir):
+        print(f"making output_dir {output_dir}")
         os.mkdir(output_dir)
 
     msg_path = os.path.join(os.path.dirname(os.path.realpath(__file__)),"../../msg")
     msg_files = get_msgs_list(msg_path)
+
     msg_files.sort()
 
     versioned_msgs_list = ''
     unversioned_msgs_list = ''
+    msgTypes = set()
 
     for msg_file in msg_files:
+        # Add messages to set of allowed types (compound types)
+        #msg_type = msg_file.rsplit('/')[-1]
+        #msg_type = msg_type.rsplit('\\')[-1]
+        #msg_type = msg_type.rsplit('.')[0]
         msg_name = os.path.splitext(os.path.basename(msg_file))[0]
-        output_file = os.path.join(output_dir, msg_name+'.md')
-        msg_filename = os.path.join(msg_path, msg_file)
-        print("{:} -> {:}".format(msg_filename, output_file))
+        msgTypes.add(msg_name)
 
-        #Format msg url
-        msg_url="[source file](https://github.com/PX4/PX4-Autopilot/blob/main/msg/%s)" % msg_file
-
-        msg_description = ""
-        summary_description = ""
-
-        #Get msg description (first non-empty comment line from top of msg)
-        with open(msg_filename, 'r') as lineparser:
-            line = lineparser.readline()
-            while line.startswith('#') or (line.strip() == ''):
-                print('DEBUG: line: %s' % line)
-                line=line[1:].strip()+'\n'
-                stripped_line=line.strip()
-                if msg_description and not summary_description and stripped_line=='':
-                    summary_description = msg_description.strip()
-
-                msg_description+=line
-                line = lineparser.readline()
-            msg_description=msg_description.strip()
-            if not summary_description and msg_description:
-                summary_description = msg_description
-            print('msg_description: Z%sZ' % msg_description)
-            print('summary_description: Z%sZ' % summary_description)
-            summary_description
-        msg_contents = ""
-        #Get msg contents (read the file)
-        with open(msg_filename, 'r') as source_file:
-            msg_contents = source_file.read()
-
-        #Format markdown using msg name, comment, url, contents.
-        markdown_output="""# %s (UORB message)
-
-%s
-
-%s
-
-```c
-%s
-```
-""" % (msg_name, msg_description, msg_url, msg_contents)
-
-        with open(output_file, 'w') as content_file:
-            content_file.write(markdown_output)
+    for msg_file in msg_files:
+        message = UORBMessage(msg_file)
+        # Any additional tests that can't be in UORBMessage parser go here.
+        message.markdown_out()
 
         # Categorize as versioned or unversioned
         if "versioned" in msg_file:
-            versioned_msgs_list += '- [%s](%s.md)' % (msg_name, msg_name)
-            if summary_description:
-                versioned_msgs_list += " — %s" % summary_description
+            versioned_msgs_list += f"- [{message.name}]({message.name}.md)"
+            if message.shortDescription:
+                versioned_msgs_list += f" — {message.shortDescription}"
             versioned_msgs_list += "\n"
         else:
-            unversioned_msgs_list += '- [%s](%s.md)' % (msg_name, msg_name)
-            if summary_description:
-                unversioned_msgs_list += " — %s" % summary_description
+            unversioned_msgs_list += f"- [{message.name}]({message.name}.md)"
+            if message.shortDescription:
+                unversioned_msgs_list += f" — {message.shortDescription}"
             unversioned_msgs_list += "\n"
-
     # Write out the index.md file
-    index_text="""# uORB Message Reference
+    index_text=f"""# uORB Message Reference
 
 ::: info
 This list is [auto-generated](https://github.com/PX4/PX4-Autopilot/blob/main/Tools/msg/generate_msg_docs.py) from the source code.
@@ -218,14 +982,14 @@ Graphs showing how these are used [can be found here](../middleware/uorb_graph.m
 
 ## Versioned Messages
 
-%s
+{versioned_msgs_list}
 
 ## Unversioned Messages
 
-%s
-    """ % (versioned_msgs_list, unversioned_msgs_list)
+{unversioned_msgs_list}
+    """
     index_file = os.path.join(output_dir, 'index.md')
-    with open(index_file, 'w') as content_file:
+    with open(index_file, 'w', encoding='utf-8') as content_file:
             content_file.write(index_text)
 
     generate_dds_yaml_doc(msg_files)

docs/.vitepress/theme/style.css

@@ -160,3 +160,44 @@
  .vp-doc img {
   display: inline; /* block by default set by vitepress */
 }
+
+
+/**
+ * Custom styles for wide pages
+ * -------------------------------------------------------------------------- */
+
+.is-wide-page .content-container {
+  max-width: 100% !important;
+}
+@media (min-width: 1280px) {
+  .is-wide-page .content {
+    min-width: 940px !important;
+  }
+}
+
+/* Make page width larger  */
+@media (min-width: 1440px) {
+  .is-wide-page .VPSidebar {
+    padding-left: 32px !important;
+    width: var(--vp-sidebar-width) !important;
+  }
+  .is-wide-page .VPContent.has-sidebar {
+    padding-left: var(--vp-sidebar-width) !important;
+    padding-right: 0 !important;
+  }
+
+  .is-wide-page .VPNavBar.has-sidebar .title {
+    padding-left: 32px !important;
+  }
+
+  .is-wide-page .VPNavBar.has-sidebar .content {
+    padding-left: var(--vp-sidebar-width) !important;
+    padding-right: 32px !important;
+  }
+
+  /* Very hacky */
+  .is-wide-page .VPNavBar.has-sidebar #local-search {
+    z-index: 10;
+  }
+
+}