scripts: Enhance glibcpp to do basic macro processing

Reviewed-by: Siddhesh Poyarekar <siddhesh@sourceware.org>

1 files changed, 317 insertions, 0 deletions

diff --git a/scripts/glibcpp.py b/scripts/glibcpp.py
index b44c6a4392..455459a609 100644
--- a/scripts/glibcpp.py
+++ b/scripts/glibcpp.py
@@ -33,7 +33,9 @@ Accepts non-ASCII characters only within comments and strings.
 """
 
 import collections
+import operator
 import re
+import sys
 
 # Caution: The order of the outermost alternation matters.
 # STRING must be before BAD_STRING, CHARCONST before BAD_CHARCONST,
@@ -210,3 +212,318 @@ def tokenize_c(file_contents, reporter):
 
         yield tok
         pos = mo.end()
+
+class MacroDefinition(collections.namedtuple('MacroDefinition',
+                                             'name_token args body error')):
+    """A preprocessor macro definition.
+
+    name_token is the Token_ for the name.
+
+    args is None for a macro that is not function-like.  Otherwise, it
+    is a tuple that contains the macro argument name tokens.
+
+    body is a tuple that contains the tokens that constitue the body
+    of the macro definition (excluding whitespace).
+
+    error is None if no error was detected, or otherwise a problem
+    description associated with this macro definition.
+
+    """
+
+    @property
+    def function(self):
+        """Return true if the macro is function-like."""
+        return self.args is not None
+
+    @property
+    def name(self):
+        """Return the name of the macro being defined."""
+        return self.name_token.text
+
+    @property
+    def line(self):
+        """Return the line number of the macro defintion."""
+        return self.name_token.line
+
+    @property
+    def args_lowered(self):
+        """Return the macro argument list as a list of strings"""
+        if self.function:
+            return [token.text for token in self.args]
+        else:
+            return None
+
+    @property
+    def body_lowered(self):
+        """Return the macro body as a list of strings."""
+        return [token.text for token in self.body]
+
+def macro_definitions(tokens):
+    """A generator for C macro definitions among tokens.
+
+    The generator yields MacroDefinition objects.
+
+    tokens must be iterable, yielding Token_ objects.
+
+    """
+
+    macro_name = None
+    macro_start = False # Set to false after macro name and one otken.
+    macro_args = None # Set to a list during the macro argument sequence.
+    in_macro_args = False # True while processing macro identifier-list.
+    error = None
+    body = []
+
+    for token in tokens:
+        if token.context == 'define' and macro_name is None \
+           and token.kind == 'IDENT':
+            # Starting up macro processing.
+            if macro_start:
+                # First identifier is the macro name.
+                macro_name = token
+            else:
+                # Next token is the name.
+                macro_start = True
+            continue
+
+        if macro_name is None:
+            # Drop tokens not in macro definitions.
+            continue
+
+        if token.context != 'define':
+            # End of the macro definition.
+            if in_macro_args and error is None:
+                error = 'macro definition ends in macro argument list'
+            yield MacroDefinition(macro_name, macro_args, tuple(body), error)
+            # No longer in a macro definition.
+            macro_name = None
+            macro_start = False
+            macro_args = None
+            in_macro_args = False
+            error = None
+            body.clear()
+            continue
+
+        if macro_start:
+            # First token after the macro name.
+            macro_start = False
+            if token.kind == 'PUNCTUATOR' and token.text == '(':
+                macro_args = []
+                in_macro_args = True
+            continue
+
+        if in_macro_args:
+            if token.kind == 'IDENT' \
+               or (token.kind == 'PUNCTUATOR' and token.text == '...'):
+                # Macro argument or ... placeholder.
+                macro_args.append(token)
+            if token.kind == 'PUNCTUATOR':
+                if token.text == ')':
+                    macro_args = tuple(macro_args)
+                    in_macro_args = False
+                elif token.text == ',':
+                    pass # Skip.  Not a full syntax check.
+                elif error is None:
+                    error = 'invalid punctuator in macro argument list: ' \
+                        + repr(token.text)
+            elif error is None:
+                error = 'invalid {} token in macro argument list'.format(
+                    token.kind)
+            continue
+
+        if token.kind not in ('WHITESPACE', 'BLOCK_COMMENT'):
+            body.append(token)
+
+    # Emit the macro in case the last line does not end with a newline.
+    if macro_name is not None:
+        if in_macro_args and error is None:
+            error = 'macro definition ends in macro argument list'
+        yield MacroDefinition(macro_name, macro_args, tuple(body), error)
+
+# Used to split UL etc. suffixes from numbers such as 123UL.
+RE_SPLIT_INTEGER_SUFFIX = re.compile(r'([^ullULL]+)([ullULL]*)')
+
+BINARY_OPERATORS = {
+    '+': operator.add,
+    '<<': operator.lshift,
+}
+
+# Use the general-purpose dict type if it is order-preserving.
+if (sys.version_info[0], sys.version_info[1]) <= (3, 6):
+    OrderedDict = collections.OrderedDict
+else:
+    OrderedDict = dict
+
+def macro_eval(macro_defs, reporter):
+    """Compute macro values
+
+    macro_defs is the output from macro_definitions.  reporter is an
+    object that accepts reporter.error(line_number, message) and
+    reporter.note(line_number, message) calls to report errors
+    and error context invocations.
+
+    The returned dict contains the values of macros which are not
+    function-like, pairing their names with their computed values.
+
+    The current implementation is incomplete.  It is deliberately not
+    entirely faithful to C, even in the implemented parts.  It checks
+    that macro replacements follow certain syntactic rules even if
+    they are never evaluated.
+
+    """
+
+    # Unevaluated macro definitions by name.
+    definitions = OrderedDict()
+    for md in macro_defs:
+        if md.name in definitions:
+            reporter.error(md.line, 'macro {} redefined'.format(md.name))
+            reporter.note(definitions[md.name].line,
+                          'location of previous definition')
+        else:
+            definitions[md.name] = md
+
+    # String to value mappings for fully evaluated macros.
+    evaluated = OrderedDict()
+
+    # String to macro definitions during evaluation.  Nice error
+    # reporting relies on determinstic iteration order.
+    stack = OrderedDict()
+
+    def eval_token(current, token):
+        """Evaluate one macro token.
+
+        Integers and strings are returned as such (the latter still
+        quoted).  Identifiers are expanded.
+
+        None indicates an empty expansion or an error.
+
+        """
+
+        if token.kind == 'PP_NUMBER':
+            value = None
+            m = RE_SPLIT_INTEGER_SUFFIX.match(token.text)
+            if m:
+                try:
+                    value = int(m.group(1), 0)
+                except ValueError:
+                    pass
+            if value is None:
+                reporter.error(token.line,
+                    'invalid number {!r} in definition of {}'.format(
+                        token.text, current.name))
+            return value
+
+        if token.kind == 'STRING':
+            return token.text
+
+        if token.kind == 'CHARCONST' and len(token.text) == 3:
+            return ord(token.text[1])
+
+        if token.kind == 'IDENT':
+            name = token.text
+            result = eval1(current, name)
+            if name not in evaluated:
+                evaluated[name] = result
+            return result
+
+        reporter.error(token.line,
+            'unrecognized {!r} in definition of {}'.format(
+                token.text, current.name))
+        return None
+
+
+    def eval1(current, name):
+        """Evaluate one name.
+
+        The name is looked up and the macro definition evaluated
+        recursively if necessary.  The current argument is the macro
+        definition being evaluated.
+
+        None as a return value indicates an error.
+
+        """
+
+        # Fast path if the value has already been evaluated.
+        if name in evaluated:
+            return evaluated[name]
+
+        try:
+            md = definitions[name]
+        except KeyError:
+            reporter.error(current.line,
+                'reference to undefined identifier {} in definition of {}'
+                           .format(name, current.name))
+            return None
+
+        if md.name in stack:
+            # Recursive macro definition.
+            md = stack[name]
+            reporter.error(md.line,
+                'macro definition {} refers to itself'.format(md.name))
+            for md1 in reversed(list(stack.values())):
+                if md1 is md:
+                    break
+                reporter.note(md1.line,
+                              'evaluated from {}'.format(md1.name))
+            return None
+
+        stack[md.name] = md
+        if md.function:
+            reporter.error(current.line,
+                'attempt to evaluate function-like macro {}'.format(name))
+            reporter.note(md.line, 'definition of {}'.format(md.name))
+            return None
+
+        try:
+            body = md.body
+            if len(body) == 0:
+                # Empty expansion.
+                return None
+
+            # Remove surrounding ().
+            if body[0].text == '(' and body[-1].text == ')':
+                body = body[1:-1]
+                had_parens = True
+            else:
+                had_parens = False
+
+            if len(body) == 1:
+                return eval_token(md, body[0])
+
+            # Minimal expression evaluator for binary operators.
+            op = body[1].text
+            if len(body) == 3 and op in BINARY_OPERATORS:
+                if not had_parens:
+                    reporter.error(body[1].line,
+                        'missing parentheses around {} expression'.format(op))
+                    reporter.note(md.line,
+                                  'in definition of macro {}'.format(md.name))
+
+                left = eval_token(md, body[0])
+                right = eval_token(md, body[2])
+
+                if type(left) != type(1):
+                    reporter.error(left.line,
+                        'left operand of {} is not an integer'.format(op))
+                    reporter.note(md.line,
+                                  'in definition of macro {}'.format(md.name))
+                if type(right) != type(1):
+                    reporter.error(left.line,
+                        'right operand of {} is not an integer'.format(op))
+                    reporter.note(md.line,
+                                  'in definition of macro {}'.format(md.name))
+                return BINARY_OPERATORS[op](left, right)
+
+            reporter.error(md.line,
+                'uninterpretable macro token sequence: {}'.format(
+                    ' '.join(md.body_lowered)))
+            return None
+        finally:
+            del stack[md.name]
+
+    # Start of main body of macro_eval.
+    for md in definitions.values():
+        name = md.name
+        if name not in evaluated and not md.function:
+            evaluated[name] = eval1(md, name)
+    return evaluated