cmake remove circular linking and reorganize

- px4_add_module now requires MAIN
 - px4_add_library doesn't automatically link
This commit is contained in:
Daniel Agar
2018-04-29 21:48:54 -04:00
parent a8bc3d187f
commit ea3acb7121
576 changed files with 1406 additions and 2014 deletions
+1
View File
@@ -0,0 +1 @@
This folder contains a python library used by px_process_params.py
+1
View File
@@ -0,0 +1 @@
__all__ = ["srcscanner", "srcparser", "xmlout", "dokuwikiout", "dokuwikirpc", "scope"]
@@ -0,0 +1,44 @@
from xml.sax.saxutils import escape
import codecs
class DokuWikiTablesOutput():
def __init__(self, groups):
result = ("====== Parameter Reference ======\n"
"<note>**This list is auto-generated from the source code** and contains the most recent parameter documentation.</note>\n"
"\n")
for group in groups:
result += "==== %s ====\n\n" % group.GetName()
result += "|< 100% 25% 45% 10% 10% 10% >|\n"
result += "^ Name ^ Description ^ Min ^ Max ^ Default ^\n"
result += "^ ::: ^ Comment ^^^^\n"
for param in group.GetParams():
code = param.GetName()
def_val = param.GetDefault()
name = param.GetFieldValue("short_desc")
min_val = param.GetFieldValue("min")
max_val = param.GetFieldValue("max")
long_desc = param.GetFieldValue("long_desc")
if name == code:
name = ""
else:
name = name.replace("\n", " ")
name = name.replace("|", "%%|%%")
name = name.replace("^", "%%^%%")
result += "| **%s** |" % code
result += " %s |" % name
result += " %s |" % (min_val or "")
result += " %s |" % (max_val or "")
result += " %s |" % (def_val or "")
result += "\n"
if long_desc is not None:
result += "| ::: | <div>%s</div> ||||\n" % long_desc
result += "\n"
self.output = result;
def Save(self, filename):
with codecs.open(filename, 'w', 'utf-8') as f:
f.write(self.output)
@@ -0,0 +1,16 @@
try:
import xmlrpclib
except ImportError:
import xmlrpc.client as xmlrpclib
# See https://www.dokuwiki.org/devel:xmlrpc for a list of available functions!
# Usage example:
# xmlrpc = dokuwikirpc.get_xmlrpc(url, username, password)
# print(xmlrpc.dokuwiki.getVersion())
def get_xmlrpc(url, username, password):
#proto, url = url.split("://")
#url = proto + "://" + username + ":" + password + "@" + url + "/lib/exe/xmlrpc.php"
url += "/lib/exe/xmlrpc.php?u=" + username + "&p=" + password
return xmlrpclib.ServerProxy(url)
+105
View File
@@ -0,0 +1,105 @@
from xml.sax.saxutils import escape
import codecs
class MarkdownTablesOutput():
def __init__(self, groups):
result = ("# Parameter Reference\n"
"> **Note** **This list is auto-generated from the source code** and contains the most recent parameter documentation.\n"
"\n")
for group in groups:
result += '## %s\n\n' % group.GetName()
#Check if scope (module where parameter is defined) is the same for all parameters in the group.
# If so then display just once about the table.
scope_set = set()
for param in group.GetParams():
scope_set.add(param.GetFieldValue("scope"))
if len(scope_set)==1:
result+='\nThe module where these parameters are defined is: *%s*.\n\n' % list(scope_set)[0]
result += '<table style="width: 100%; table-layout:fixed; font-size:1.5rem; overflow: auto; display:block;">\n'
result += ' <colgroup><col style="width: 23%"><col style="width: 46%"><col style="width: 11%"><col style="width: 11%"><col style="width: 9%"></colgroup>\n'
result += ' <thead>\n'
result += ' <tr><th>Name</th><th>Description</th><th>Min > Max (Incr.)</th><th>Default</th><th>Units</th></tr>\n'
result += ' </thead>\n'
result += '<tbody>\n'
for param in group.GetParams():
code = param.GetName()
name = param.GetFieldValue("short_desc") or ''
long_desc = param.GetFieldValue("long_desc") or ''
min_val = param.GetFieldValue("min") or ''
max_val = param.GetFieldValue("max") or ''
increment = param.GetFieldValue("increment") or ''
def_val = param.GetDefault() or ''
unit = param.GetFieldValue("unit") or ''
type = param.GetType()
reboot_required = param.GetFieldValue("reboot_required") or ''
#board = param.GetFieldValue("board") or '' ## Disabled as no board values are defined in any parameters!
#decimal = param.GetFieldValue("decimal") or '' #Disabled as is intended for GCS not people
#field_codes = param.GetFieldCodes() ## Disabled as not needed for display.
#boolean = param.GetFieldValue("boolean") # or '' # Disabled - does not appear useful.
# Format values for display.
# Display min/max/increment value based on what values are defined.
max_min_combined = ''
if min_val or max_val:
if not min_val:
min_val='?'
if not max_val:
max_val='?'
max_min_combined+='%s > %s ' % (min_val, max_val)
if increment:
max_min_combined+='(%s)' % increment
if long_desc is not '':
long_desc = '<p><strong>Comment:</strong> %s</p>' % long_desc
if name == code:
name = ""
code='<strong id="%s">%s</strong>' % (code, code)
if reboot_required:
reboot_required='<p><b>Reboot required:</b> %s</p>\n' % reboot_required
scope=''
if not len(scope_set)==1 or len(scope_set)==0:
scope = param.GetFieldValue("scope") or ''
if scope:
scope='<p><b>Module:</b> %s</p>\n' % scope
enum_codes=param.GetEnumCodes() or '' # Gets numerical values for parameter.
enum_output=''
# Format codes and their descriptions for display.
if enum_codes:
enum_output+='<strong>Values:</strong><ul>'
enum_codes=sorted(enum_codes,key=float)
for item in enum_codes:
enum_output+='\n<li><strong>%s:</strong> %s</li> \n' % (item, param.GetEnumValue(item))
enum_output+='</ul>\n'
bitmask_list=param.GetBitmaskList() #Gets bitmask values for parameter
bitmask_output=''
#Format bitmask values
if bitmask_list:
bitmask_output+='<strong>Bitmask:</strong><ul>'
for bit in bitmask_list:
bit_text = param.GetBitmaskBit(bit)
bitmask_output+=' <li><strong>%s:</strong> %s</li> \n' % (bit, bit_text)
bitmask_output+='</ul>\n'
result += '<tr>\n <td style="vertical-align: top;">%s (%s)</td>\n <td style="vertical-align: top;"><p>%s</p>%s %s %s %s %s</td>\n <td style="vertical-align: top;">%s</td>\n <td style="vertical-align: top;">%s </td>\n <td style="vertical-align: top;">%s</td>\n</tr>\n' % (code,type,name, long_desc, enum_output, bitmask_output, reboot_required, scope, max_min_combined,def_val,unit)
#Close the table.
result += '</tbody></table>\n\n'
self.output = result
def Save(self, filename):
with codecs.open(filename, 'w', 'utf-8') as f:
f.write(self.output)
+32
View File
@@ -0,0 +1,32 @@
import os
import re
class Scope(object):
"""
Single parameter group
"""
re_deep_lines = re.compile(r'.*\/.*\/')
def __init__(self,):
self.scope = set()
def __str__(self):
return self.scope.__str__()
def Add(self, scope):
"""
Add Scope to set
"""
self.scope.add(scope)
def Has(self, scope):
"""
Check for existance
"""
if len(self.scope) == 0:
return True
# Anything in the form xxxxx/yyyyy/zzzzz....
# is treated as xxxxx/yyyyy
while (self.re_deep_lines.match(scope)):
scope = os.path.dirname(scope)
return scope in self.scope
+404
View File
@@ -0,0 +1,404 @@
import sys
import re
import math
global default_var
default_var = {}
class ParameterGroup(object):
"""
Single parameter group
"""
def __init__(self, name):
self.name = name
self.params = []
def AddParameter(self, param):
"""
Add parameter to the group
"""
self.params.append(param)
def GetName(self):
"""
Get parameter group name
"""
return self.name
def GetParams(self):
"""
Returns the parsed list of parameters. Every parameter is a Parameter
object. Note that returned object is not a copy. Modifications affect
state of the parser.
"""
return sorted(self.params, key=lambda param: param.name)
class Parameter(object):
"""
Single parameter
"""
# Define sorting order of the fields
priority = {
"board": 9,
"short_desc": 8,
"long_desc": 7,
"min": 5,
"max": 4,
"unit": 3,
"decimal": 2,
# all others == 0 (sorted alphabetically)
}
def __init__(self, name, type, default = ""):
self.fields = {}
self.values = {}
self.bitmask = {}
self.name = name
self.type = type
self.default = default
self.volatile = "false"
self.category = ""
def GetName(self):
return self.name
def GetType(self):
return self.type
def GetDefault(self):
return self.default
def GetCategory(self):
return self.category.title()
def GetVolatile(self):
return self.volatile
def SetField(self, code, value):
"""
Set named field value
"""
self.fields[code] = value
def SetEnumValue(self, code, value):
"""
Set named enum value
"""
self.values[code] = value
def SetBitmaskBit(self, index, bit):
"""
Set named enum value
"""
self.bitmask[index] = bit
def SetVolatile(self):
"""
Set volatile flag
"""
self.volatile = "true"
def SetCategory(self, category):
"""
Set param category
"""
self.category = category
def GetFieldCodes(self):
"""
Return list of existing field codes in convenient order
"""
keys = self.fields.keys()
keys = sorted(keys)
keys = sorted(keys, key=lambda x: self.priority.get(x, 0), reverse=True)
return keys
def GetFieldValue(self, code):
"""
Return value of the given field code or None if not found.
"""
fv = self.fields.get(code)
if not fv:
# required because python 3 sorted does not accept None
return ""
return fv
def GetEnumCodes(self):
"""
Return list of existing value codes in convenient order
"""
return sorted(self.values.keys(), key=float)
def GetEnumValue(self, code):
"""
Return value of the given enum code or None if not found.
"""
fv = self.values.get(code)
if not fv:
# required because python 3 sorted does not accept None
return ""
return fv
def GetBitmaskList(self):
"""
Return list of existing bitmask codes in convenient order
"""
keys = self.bitmask.keys()
return sorted(keys, key=float)
def GetBitmaskBit(self, index):
"""
Return value of the given bitmask code or None if not found.
"""
fv = self.bitmask.get(index)
if not fv:
# required because python 3 sorted does not accept None
return ""
return fv
class SourceParser(object):
"""
Parses provided data and stores all found parameters internally.
"""
re_split_lines = re.compile(r'[\r\n]+')
re_comment_start = re.compile(r'^\/\*\*')
re_comment_content = re.compile(r'^\*\s*(.*)')
re_comment_tag = re.compile(r'@([a-zA-Z][a-zA-Z0-9_]*)\s*(.*)')
re_comment_end = re.compile(r'(.*?)\s*\*\/')
re_parameter_definition = re.compile(r'PARAM_DEFINE_([A-Z_][A-Z0-9_]*)\s*\(([A-Z_][A-Z0-9_]*)\s*,\s*([^ ,\)]+)\s*\)\s*;')
re_px4_parameter_definition = re.compile(r'PX4_PARAM_DEFINE_([A-Z_][A-Z0-9_]*)\s*\(([A-Z_][A-Z0-9_]*)\s*\)\s*;')
re_px4_param_default_definition = re.compile(r'#define\s*PARAM_([A-Z_][A-Z0-9_]*)\s*([^ ,\)]+)\s*')
re_cut_type_specifier = re.compile(r'[a-z]+$')
re_is_a_number = re.compile(r'^-?[0-9\.]')
re_remove_dots = re.compile(r'\.+$')
re_remove_carriage_return = re.compile('\n+')
valid_tags = set(["group", "board", "min", "max", "unit", "decimal", "increment", "reboot_required", "value", "boolean", "bit", "category", "volatile"])
# Order of parameter groups
priority = {
# All other groups = 0 (sort alphabetically)
"Miscellaneous": -10
}
def __init__(self):
self.param_groups = {}
def Parse(self, scope, contents):
"""
Incrementally parse program contents and append all found parameters
to the list.
"""
# This code is essentially a comment-parsing grammar. "state"
# represents parser state. It contains human-readable state
# names.
state = None
for line in self.re_split_lines.split(contents):
line = line.strip()
# Ignore empty lines
if line == "":
continue
if self.re_comment_start.match(line):
state = "wait-short"
short_desc = None
long_desc = None
tags = {}
def_values = {}
def_bitmask = {}
elif state is not None and state != "comment-processed":
m = self.re_comment_end.search(line)
if m:
line = m.group(1)
last_comment_line = True
else:
last_comment_line = False
m = self.re_comment_content.match(line)
if m:
comment_content = m.group(1)
if comment_content == "":
# When short comment ends with empty comment line,
# start waiting for the next part - long comment.
if state == "wait-short-end":
state = "wait-long"
else:
m = self.re_comment_tag.match(comment_content)
if m:
tag, desc = m.group(1, 2)
if (tag == "value"):
# Take the meta info string and split the code and description
metainfo = desc.split(" ", 1)
def_values[metainfo[0]] = metainfo[1]
elif (tag == "bit"):
# Take the meta info string and split the code and description
metainfo = desc.split(" ", 1)
def_bitmask[metainfo[0]] = metainfo[1]
else:
tags[tag] = desc
current_tag = tag
state = "wait-tag-end"
elif state == "wait-short":
# Store first line of the short description
short_desc = comment_content
state = "wait-short-end"
elif state == "wait-short-end":
# Append comment line to the short description
short_desc += "\n" + comment_content
elif state == "wait-long":
# Store first line of the long description
long_desc = comment_content
state = "wait-long-end"
elif state == "wait-long-end":
# Append comment line to the long description
long_desc += "\n" + comment_content
elif state == "wait-tag-end":
# Append comment line to the tag text
tags[current_tag] += "\n" + comment_content
else:
raise AssertionError(
"Invalid parser state: %s" % state)
elif not last_comment_line:
# Invalid comment line (inside comment, but not starting with
# "*" or "*/". Reset parsed content.
state = None
if last_comment_line:
state = "comment-processed"
else:
tp = None
name = None
defval = ""
# Non-empty line outside the comment
m = self.re_px4_param_default_definition.match(line)
# Default value handling
if m:
name_m, defval_m = m.group(1,2)
default_var[name_m] = defval_m
m = self.re_parameter_definition.match(line)
if m:
tp, name, defval = m.group(1, 2, 3)
else:
m = self.re_px4_parameter_definition.match(line)
if m:
tp, name = m.group(1, 2)
if (name+'_DEFAULT') in default_var:
defval = default_var[name+'_DEFAULT']
if tp is not None:
# Remove trailing type specifier from numbers: 0.1f => 0.1
if defval != "" and self.re_is_a_number.match(defval):
defval = self.re_cut_type_specifier.sub('', defval)
param = Parameter(name, tp, defval)
param.SetField("scope", scope)
param.SetField("short_desc", name)
# If comment was found before the parameter declaration,
# inject its data into the newly created parameter.
group = "Miscellaneous"
if state == "comment-processed":
if short_desc is not None:
param.SetField("short_desc", self.re_remove_dots.sub('', short_desc))
if long_desc is not None:
long_desc = self.re_remove_carriage_return.sub(' ', long_desc)
param.SetField("long_desc", long_desc)
for tag in tags:
if tag == "group":
group = tags[tag]
elif tag == "volatile":
param.SetVolatile()
elif tag == "category":
param.SetCategory(tags[tag])
elif tag not in self.valid_tags:
sys.stderr.write("Skipping invalid documentation tag: '%s'\n" % tag)
return False
else:
param.SetField(tag, tags[tag])
for def_value in def_values:
param.SetEnumValue(def_value, def_values[def_value])
for def_bit in def_bitmask:
param.SetBitmaskBit(def_bit, def_bitmask[def_bit])
# Store the parameter
if group not in self.param_groups:
self.param_groups[group] = ParameterGroup(group)
self.param_groups[group].AddParameter(param)
state = None
return True
def IsNumber(self, numberString):
try:
float(numberString)
return True
except ValueError:
return False
def Validate(self):
"""
Validates the parameter meta data.
"""
seenParamNames = []
for group in self.GetParamGroups():
for param in group.GetParams():
name = param.GetName()
if len(name) > 16:
sys.stderr.write("Parameter Name {0} is too long (Limit is 16)\n".format(name))
return False
board = param.GetFieldValue("board")
# Check for duplicates
name_plus_board = name + "+" + board
for seenParamName in seenParamNames:
if seenParamName == name_plus_board:
sys.stderr.write("Duplicate parameter definition: {0}\n".format(name_plus_board))
return False
seenParamNames.append(name_plus_board)
# Validate values
default = param.GetDefault()
min = param.GetFieldValue("min")
max = param.GetFieldValue("max")
#sys.stderr.write("{0} default:{1} min:{2} max:{3}\n".format(name, default, min, max))
if default != "" and not self.IsNumber(default):
sys.stderr.write("Default value not number: {0} {1}\n".format(name, default))
return False
# if default != "" and "." not in default:
# sys.stderr.write("Default value does not contain dot (e.g. 10 needs to be written as 10.0): {0} {1}\n".format(name, default))
# return False
if min != "":
if not self.IsNumber(min):
sys.stderr.write("Min value not number: {0} {1}\n".format(name, min))
return False
if default != "" and float(default) < float(min):
sys.stderr.write("Default value is smaller than min: {0} default:{1} min:{2}\n".format(name, default, min))
return False
if max != "":
if not self.IsNumber(max):
sys.stderr.write("Max value not number: {0} {1}\n".format(name, max))
return False
if default != "" and float(default) > float(max):
sys.stderr.write("Default value is larger than max: {0} default:{1} max:{2}\n".format(name, default, max))
return False
for code in param.GetEnumCodes():
if not self.IsNumber(code):
sys.stderr.write("Min value not number: {0} {1}\n".format(name, code))
return False
if param.GetEnumValue(code) == "":
sys.stderr.write("Description for enum value is empty: {0} {1}\n".format(name, code))
return False
for index in param.GetBitmaskList():
if not self.IsNumber(index):
sys.stderr.write("bit value not number: {0} {1}\n".format(name, index))
return False
if not int(min) <= math.pow(2, int(index)) <= int(max):
sys.stderr.write("Bitmask bit must be between {0} and {1}: {2} {3}\n".format(min, max, name, math.pow(2, int(index))))
return False
if param.GetBitmaskBit(index) == "":
sys.stderr.write("Description for bitmask bit is empty: {0} {1}\n".format(name, index))
return False
return True
def GetParamGroups(self):
"""
Returns the parsed list of parameters. Every parameter is a Parameter
object. Note that returned object is not a copy. Modifications affect
state of the parser.
"""
groups = self.param_groups.values()
groups = sorted(groups, key=lambda x: x.GetName())
groups = sorted(groups, key=lambda x: self.priority.get(x.GetName(), 0), reverse=True)
return groups
@@ -0,0 +1,50 @@
import os
import re
import codecs
import sys
class SourceScanner(object):
"""
Traverses directory tree, reads all source files, and passes their contents
to the Parser.
"""
def ScanDir(self, srcdirs, parser):
"""
Scans provided path and passes all found contents to the parser using
parser.Parse method.
"""
extensions1 = tuple([".h"])
extensions2 = tuple([".c"])
for srcdir in srcdirs:
for filename in os.listdir(srcdir):
if filename.endswith(extensions1):
path = os.path.join(srcdir, filename)
if not self.ScanFile(path, parser):
return False
for filename in os.listdir(srcdir):
if filename.endswith(extensions2):
path = os.path.join(srcdir, filename)
if not self.ScanFile(path, parser):
return False
return True
def ScanFile(self, path, parser):
"""
Scans provided file and passes its contents to the parser using
parser.Parse method.
"""
# Extract the scope: it is the directory within the repo. Either it
# starts directly with 'src/module/abc', or it has the form 'x/y/z/src/module/abc'.
# The output is 'module/abc' in both cases.
prefix = "^(|.*" + os.path.sep + ")src" + os.path.sep
scope = re.sub(prefix.replace("\\", "/"), "", os.path.dirname(os.path.relpath(path)).replace("\\", "/"))
with codecs.open(path, 'r', 'utf-8') as f:
try:
contents = f.read()
except:
contents = ''
print('Failed reading file: %s, skipping content.' % path)
pass
return parser.Parse(scope, contents)
+82
View File
@@ -0,0 +1,82 @@
import xml.etree.ElementTree as ET
import codecs
def indent(elem, level=0):
i = "\n" + level*" "
if len(elem):
if not elem.text or not elem.text.strip():
elem.text = i + " "
if not elem.tail or not elem.tail.strip():
elem.tail = i
for elem in elem:
indent(elem, level+1)
if not elem.tail or not elem.tail.strip():
elem.tail = i
else:
if level and (not elem.tail or not elem.tail.strip()):
elem.tail = i
class XMLOutput():
def __init__(self, groups, board, inject_xml_file_name):
xml_parameters = ET.Element("parameters")
xml_version = ET.SubElement(xml_parameters, "version")
xml_version.text = "3"
xml_version = ET.SubElement(xml_parameters, "parameter_version_major")
xml_version.text = "1"
xml_version = ET.SubElement(xml_parameters, "parameter_version_minor")
xml_version.text = "15"
importtree = ET.parse(inject_xml_file_name)
injectgroups = importtree.getroot().findall("group")
for igroup in injectgroups:
xml_parameters.append(igroup)
last_param_name = ""
board_specific_param_set = False
for group in groups:
xml_group = ET.SubElement(xml_parameters, "group")
xml_group.attrib["name"] = group.GetName()
for param in group.GetParams():
if (last_param_name == param.GetName() and not board_specific_param_set) or last_param_name != param.GetName():
xml_param = ET.SubElement(xml_group, "parameter")
xml_param.attrib["name"] = param.GetName()
xml_param.attrib["default"] = param.GetDefault()
xml_param.attrib["type"] = param.GetType()
if (param.GetVolatile() == "true"):
xml_param.attrib["volatile"] = param.GetVolatile()
if (param.GetCategory()):
xml_param.attrib["category"] = param.GetCategory()
last_param_name = param.GetName()
for code in param.GetFieldCodes():
value = param.GetFieldValue(code)
if code == "board":
if value == board:
board_specific_param_set = True
xml_field = ET.SubElement(xml_param, code)
xml_field.text = value
else:
xml_group.remove(xml_param)
else:
xml_field = ET.SubElement(xml_param, code)
xml_field.text = value
if last_param_name != param.GetName():
board_specific_param_set = False
if len(param.GetEnumCodes()) > 0:
xml_values = ET.SubElement(xml_param, "values")
for code in param.GetEnumCodes():
xml_value = ET.SubElement(xml_values, "value")
xml_value.attrib["code"] = code;
xml_value.text = param.GetEnumValue(code)
if len(param.GetBitmaskList()) > 0:
xml_values = ET.SubElement(xml_param, "bitmask")
for index in param.GetBitmaskList():
xml_value = ET.SubElement(xml_values, "bit")
xml_value.attrib["index"] = index;
xml_value.text = param.GetBitmaskBit(index)
indent(xml_parameters)
self.xml_document = ET.ElementTree(xml_parameters)
def Save(self, filename):
self.xml_document.write(filename, encoding="UTF-8")