| #!/usr/bin/env python |
| # |
| # Copyright 2007 Neal Norwitz |
| # Portions Copyright 2007 Google Inc. |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """Generate an Abstract Syntax Tree (AST) for C++.""" |
| |
| __author__ = 'nnorwitz@google.com (Neal Norwitz)' |
| |
| |
| # TODO: |
| # * Tokens should never be exported, need to convert to Nodes |
| # (return types, parameters, etc.) |
| # * Handle static class data for templatized classes |
| # * Handle casts (both C++ and C-style) |
| # * Handle conditions and loops (if/else, switch, for, while/do) |
| # |
| # TODO much, much later: |
| # * Handle #define |
| # * exceptions |
| |
| |
| try: |
| # Python 3.x |
| import builtins |
| except ImportError: |
| # Python 2.x |
| import __builtin__ as builtins |
| |
| import sys |
| import traceback |
| |
| from cpp import keywords |
| from cpp import tokenize |
| from cpp import utils |
| |
| |
| if not hasattr(builtins, 'reversed'): |
| # Support Python 2.3 and earlier. |
| def reversed(seq): |
| for i in range(len(seq)-1, -1, -1): |
| yield seq[i] |
| |
| if not hasattr(builtins, 'next'): |
| # Support Python 2.5 and earlier. |
| def next(obj): |
| return obj.next() |
| |
| |
| VISIBILITY_PUBLIC, VISIBILITY_PROTECTED, VISIBILITY_PRIVATE = range(3) |
| |
| FUNCTION_NONE = 0x00 |
| FUNCTION_CONST = 0x01 |
| FUNCTION_VIRTUAL = 0x02 |
| FUNCTION_PURE_VIRTUAL = 0x04 |
| FUNCTION_CTOR = 0x08 |
| FUNCTION_DTOR = 0x10 |
| FUNCTION_ATTRIBUTE = 0x20 |
| FUNCTION_UNKNOWN_ANNOTATION = 0x40 |
| FUNCTION_THROW = 0x80 |
| FUNCTION_OVERRIDE = 0x100 |
| |
| """ |
| These are currently unused. Should really handle these properly at some point. |
| |
| TYPE_MODIFIER_INLINE = 0x010000 |
| TYPE_MODIFIER_EXTERN = 0x020000 |
| TYPE_MODIFIER_STATIC = 0x040000 |
| TYPE_MODIFIER_CONST = 0x080000 |
| TYPE_MODIFIER_REGISTER = 0x100000 |
| TYPE_MODIFIER_VOLATILE = 0x200000 |
| TYPE_MODIFIER_MUTABLE = 0x400000 |
| |
| TYPE_MODIFIER_MAP = { |
| 'inline': TYPE_MODIFIER_INLINE, |
| 'extern': TYPE_MODIFIER_EXTERN, |
| 'static': TYPE_MODIFIER_STATIC, |
| 'const': TYPE_MODIFIER_CONST, |
| 'register': TYPE_MODIFIER_REGISTER, |
| 'volatile': TYPE_MODIFIER_VOLATILE, |
| 'mutable': TYPE_MODIFIER_MUTABLE, |
| } |
| """ |
| |
| _INTERNAL_TOKEN = 'internal' |
| _NAMESPACE_POP = 'ns-pop' |
| |
| |
| # TODO(nnorwitz): use this as a singleton for templated_types, etc |
| # where we don't want to create a new empty dict each time. It is also const. |
| class _NullDict(object): |
| __contains__ = lambda self: False |
| keys = values = items = iterkeys = itervalues = iteritems = lambda self: () |
| |
| |
| # TODO(nnorwitz): move AST nodes into a separate module. |
| class Node(object): |
| """Base AST node.""" |
| |
| def __init__(self, start, end): |
| self.start = start |
| self.end = end |
| |
| def IsDeclaration(self): |
| """Returns bool if this node is a declaration.""" |
| return False |
| |
| def IsDefinition(self): |
| """Returns bool if this node is a definition.""" |
| return False |
| |
| def IsExportable(self): |
| """Returns bool if this node exportable from a header file.""" |
| return False |
| |
| def Requires(self, node): |
| """Does this AST node require the definition of the node passed in?""" |
| return False |
| |
| def XXX__str__(self): |
| return self._StringHelper(self.__class__.__name__, '') |
| |
| def _StringHelper(self, name, suffix): |
| if not utils.DEBUG: |
| return '%s(%s)' % (name, suffix) |
| return '%s(%d, %d, %s)' % (name, self.start, self.end, suffix) |
| |
| def __repr__(self): |
| return str(self) |
| |
| |
| class Define(Node): |
| def __init__(self, start, end, name, definition): |
| Node.__init__(self, start, end) |
| self.name = name |
| self.definition = definition |
| |
| def __str__(self): |
| value = '%s %s' % (self.name, self.definition) |
| return self._StringHelper(self.__class__.__name__, value) |
| |
| |
| class Include(Node): |
| def __init__(self, start, end, filename, system): |
| Node.__init__(self, start, end) |
| self.filename = filename |
| self.system = system |
| |
| def __str__(self): |
| fmt = '"%s"' |
| if self.system: |
| fmt = '<%s>' |
| return self._StringHelper(self.__class__.__name__, fmt % self.filename) |
| |
| |
| class Goto(Node): |
| def __init__(self, start, end, label): |
| Node.__init__(self, start, end) |
| self.label = label |
| |
| def __str__(self): |
| return self._StringHelper(self.__class__.__name__, str(self.label)) |
| |
| |
| class Expr(Node): |
| def __init__(self, start, end, expr): |
| Node.__init__(self, start, end) |
| self.expr = expr |
| |
| def Requires(self, node): |
| # TODO(nnorwitz): impl. |
| return False |
| |
| def __str__(self): |
| return self._StringHelper(self.__class__.__name__, str(self.expr)) |
| |
| |
| class Return(Expr): |
| pass |
| |
| |
| class Delete(Expr): |
| pass |
| |
| |
| class Friend(Expr): |
| def __init__(self, start, end, expr, namespace): |
| Expr.__init__(self, start, end, expr) |
| self.namespace = namespace[:] |
| |
| |
| class Using(Node): |
| def __init__(self, start, end, names): |
| Node.__init__(self, start, end) |
| self.names = names |
| |
| def __str__(self): |
| return self._StringHelper(self.__class__.__name__, str(self.names)) |
| |
| |
| class Parameter(Node): |
| def __init__(self, start, end, name, parameter_type, default): |
| Node.__init__(self, start, end) |
| self.name = name |
| self.type = parameter_type |
| self.default = default |
| |
| def Requires(self, node): |
| # TODO(nnorwitz): handle namespaces, etc. |
| return self.type.name == node.name |
| |
| def __str__(self): |
| name = str(self.type) |
| suffix = '%s %s' % (name, self.name) |
| if self.default: |
| suffix += ' = ' + ''.join([d.name for d in self.default]) |
| return self._StringHelper(self.__class__.__name__, suffix) |
| |
| |
| class _GenericDeclaration(Node): |
| def __init__(self, start, end, name, namespace): |
| Node.__init__(self, start, end) |
| self.name = name |
| self.namespace = namespace[:] |
| |
| def FullName(self): |
| prefix = '' |
| if self.namespace and self.namespace[-1]: |
| prefix = '::'.join(self.namespace) + '::' |
| return prefix + self.name |
| |
| def _TypeStringHelper(self, suffix): |
| if self.namespace: |
| names = [n or '<anonymous>' for n in self.namespace] |
| suffix += ' in ' + '::'.join(names) |
| return self._StringHelper(self.__class__.__name__, suffix) |
| |
| |
| # TODO(nnorwitz): merge with Parameter in some way? |
| class VariableDeclaration(_GenericDeclaration): |
| def __init__(self, start, end, name, var_type, initial_value, namespace): |
| _GenericDeclaration.__init__(self, start, end, name, namespace) |
| self.type = var_type |
| self.initial_value = initial_value |
| |
| def Requires(self, node): |
| # TODO(nnorwitz): handle namespaces, etc. |
| return self.type.name == node.name |
| |
| def ToString(self): |
| """Return a string that tries to reconstitute the variable decl.""" |
| suffix = '%s %s' % (self.type, self.name) |
| if self.initial_value: |
| suffix += ' = ' + self.initial_value |
| return suffix |
| |
| def __str__(self): |
| return self._StringHelper(self.__class__.__name__, self.ToString()) |
| |
| |
| class Typedef(_GenericDeclaration): |
| def __init__(self, start, end, name, alias, namespace): |
| _GenericDeclaration.__init__(self, start, end, name, namespace) |
| self.alias = alias |
| |
| def IsDefinition(self): |
| return True |
| |
| def IsExportable(self): |
| return True |
| |
| def Requires(self, node): |
| # TODO(nnorwitz): handle namespaces, etc. |
| name = node.name |
| for token in self.alias: |
| if token is not None and name == token.name: |
| return True |
| return False |
| |
| def __str__(self): |
| suffix = '%s, %s' % (self.name, self.alias) |
| return self._TypeStringHelper(suffix) |
| |
| |
| class _NestedType(_GenericDeclaration): |
| def __init__(self, start, end, name, fields, namespace): |
| _GenericDeclaration.__init__(self, start, end, name, namespace) |
| self.fields = fields |
| |
| def IsDefinition(self): |
| return True |
| |
| def IsExportable(self): |
| return True |
| |
| def __str__(self): |
| suffix = '%s, {%s}' % (self.name, self.fields) |
| return self._TypeStringHelper(suffix) |
| |
| |
| class Union(_NestedType): |
| pass |
| |
| |
| class Enum(_NestedType): |
| pass |
| |
| |
| class Class(_GenericDeclaration): |
| def __init__(self, start, end, name, bases, templated_types, body, namespace): |
| _GenericDeclaration.__init__(self, start, end, name, namespace) |
| self.bases = bases |
| self.body = body |
| self.templated_types = templated_types |
| |
| def IsDeclaration(self): |
| return self.bases is None and self.body is None |
| |
| def IsDefinition(self): |
| return not self.IsDeclaration() |
| |
| def IsExportable(self): |
| return not self.IsDeclaration() |
| |
| def Requires(self, node): |
| # TODO(nnorwitz): handle namespaces, etc. |
| if self.bases: |
| for token_list in self.bases: |
| # TODO(nnorwitz): bases are tokens, do name comparison. |
| for token in token_list: |
| if token.name == node.name: |
| return True |
| # TODO(nnorwitz): search in body too. |
| return False |
| |
| def __str__(self): |
| name = self.name |
| if self.templated_types: |
| name += '<%s>' % self.templated_types |
| suffix = '%s, %s, %s' % (name, self.bases, self.body) |
| return self._TypeStringHelper(suffix) |
| |
| |
| class Struct(Class): |
| pass |
| |
| |
| class Function(_GenericDeclaration): |
| def __init__(self, start, end, name, return_type, parameters, |
| modifiers, templated_types, body, namespace): |
| _GenericDeclaration.__init__(self, start, end, name, namespace) |
| converter = TypeConverter(namespace) |
| self.return_type = converter.CreateReturnType(return_type) |
| self.parameters = converter.ToParameters(parameters) |
| self.modifiers = modifiers |
| self.body = body |
| self.templated_types = templated_types |
| |
| def IsDeclaration(self): |
| return self.body is None |
| |
| def IsDefinition(self): |
| return self.body is not None |
| |
| def IsExportable(self): |
| if self.return_type and 'static' in self.return_type.modifiers: |
| return False |
| return None not in self.namespace |
| |
| def Requires(self, node): |
| if self.parameters: |
| # TODO(nnorwitz): parameters are tokens, do name comparison. |
| for p in self.parameters: |
| if p.name == node.name: |
| return True |
| # TODO(nnorwitz): search in body too. |
| return False |
| |
| def __str__(self): |
| # TODO(nnorwitz): add templated_types. |
| suffix = ('%s %s(%s), 0x%02x, %s' % |
| (self.return_type, self.name, self.parameters, |
| self.modifiers, self.body)) |
| return self._TypeStringHelper(suffix) |
| |
| |
| class Method(Function): |
| def __init__(self, start, end, name, in_class, return_type, parameters, |
| modifiers, templated_types, body, namespace): |
| Function.__init__(self, start, end, name, return_type, parameters, |
| modifiers, templated_types, body, namespace) |
| # TODO(nnorwitz): in_class could also be a namespace which can |
| # mess up finding functions properly. |
| self.in_class = in_class |
| |
| |
| class Type(_GenericDeclaration): |
| """Type used for any variable (eg class, primitive, struct, etc).""" |
| |
| def __init__(self, start, end, name, templated_types, modifiers, |
| reference, pointer, array): |
| """ |
| Args: |
| name: str name of main type |
| templated_types: [Class (Type?)] template type info between <> |
| modifiers: [str] type modifiers (keywords) eg, const, mutable, etc. |
| reference, pointer, array: bools |
| """ |
| _GenericDeclaration.__init__(self, start, end, name, []) |
| self.templated_types = templated_types |
| if not name and modifiers: |
| self.name = modifiers.pop() |
| self.modifiers = modifiers |
| self.reference = reference |
| self.pointer = pointer |
| self.array = array |
| |
| def __str__(self): |
| prefix = '' |
| if self.modifiers: |
| prefix = ' '.join(self.modifiers) + ' ' |
| name = str(self.name) |
| if self.templated_types: |
| name += '<%s>' % self.templated_types |
| suffix = prefix + name |
| if self.reference: |
| suffix += '&' |
| if self.pointer: |
| suffix += '*' |
| if self.array: |
| suffix += '[]' |
| return self._TypeStringHelper(suffix) |
| |
| # By definition, Is* are always False. A Type can only exist in |
| # some sort of variable declaration, parameter, or return value. |
| def IsDeclaration(self): |
| return False |
| |
| def IsDefinition(self): |
| return False |
| |
| def IsExportable(self): |
| return False |
| |
| |
| class TypeConverter(object): |
| |
| def __init__(self, namespace_stack): |
| self.namespace_stack = namespace_stack |
| |
| def _GetTemplateEnd(self, tokens, start): |
| count = 1 |
| end = start |
| while 1: |
| token = tokens[end] |
| end += 1 |
| if token.name == '<': |
| count += 1 |
| elif token.name == '>': |
| count -= 1 |
| if count == 0: |
| break |
| return tokens[start:end-1], end |
| |
| def ToType(self, tokens): |
| """Convert [Token,...] to [Class(...), ] useful for base classes. |
| For example, code like class Foo : public Bar<x, y> { ... }; |
| the "Bar<x, y>" portion gets converted to an AST. |
| |
| Returns: |
| [Class(...), ...] |
| """ |
| result = [] |
| name_tokens = [] |
| reference = pointer = array = False |
| |
| def AddType(templated_types): |
| # Partition tokens into name and modifier tokens. |
| names = [] |
| modifiers = [] |
| for t in name_tokens: |
| if keywords.IsKeyword(t.name): |
| modifiers.append(t.name) |
| else: |
| names.append(t.name) |
| name = ''.join(names) |
| if name_tokens: |
| result.append(Type(name_tokens[0].start, name_tokens[-1].end, |
| name, templated_types, modifiers, |
| reference, pointer, array)) |
| del name_tokens[:] |
| |
| i = 0 |
| end = len(tokens) |
| while i < end: |
| token = tokens[i] |
| if token.name == '<': |
| new_tokens, new_end = self._GetTemplateEnd(tokens, i+1) |
| AddType(self.ToType(new_tokens)) |
| # If there is a comma after the template, we need to consume |
| # that here otherwise it becomes part of the name. |
| i = new_end |
| reference = pointer = array = False |
| elif token.name == ',': |
| AddType([]) |
| reference = pointer = array = False |
| elif token.name == '*': |
| pointer = True |
| elif token.name == '&': |
| reference = True |
| elif token.name == '[': |
| pointer = True |
| elif token.name == ']': |
| pass |
| else: |
| name_tokens.append(token) |
| i += 1 |
| |
| if name_tokens: |
| # No '<' in the tokens, just a simple name and no template. |
| AddType([]) |
| return result |
| |
| def DeclarationToParts(self, parts, needs_name_removed): |
| name = None |
| default = [] |
| if needs_name_removed: |
| # Handle default (initial) values properly. |
| for i, t in enumerate(parts): |
| if t.name == '=': |
| default = parts[i+1:] |
| name = parts[i-1].name |
| if name == ']' and parts[i-2].name == '[': |
| name = parts[i-3].name |
| i -= 1 |
| parts = parts[:i-1] |
| break |
| else: |
| if parts[-1].token_type == tokenize.NAME: |
| name = parts.pop().name |
| else: |
| # TODO(nnorwitz): this is a hack that happens for code like |
| # Register(Foo<T>); where it thinks this is a function call |
| # but it's actually a declaration. |
| name = '???' |
| modifiers = [] |
| type_name = [] |
| other_tokens = [] |
| templated_types = [] |
| i = 0 |
| end = len(parts) |
| while i < end: |
| p = parts[i] |
| if keywords.IsKeyword(p.name): |
| modifiers.append(p.name) |
| elif p.name == '<': |
| templated_tokens, new_end = self._GetTemplateEnd(parts, i+1) |
| templated_types = self.ToType(templated_tokens) |
| i = new_end - 1 |
| # Don't add a spurious :: to data members being initialized. |
| next_index = i + 1 |
| if next_index < end and parts[next_index].name == '::': |
| i += 1 |
| elif p.name in ('[', ']', '='): |
| # These are handled elsewhere. |
| other_tokens.append(p) |
| elif p.name not in ('*', '&', '>'): |
| # Ensure that names have a space between them. |
| if (type_name and type_name[-1].token_type == tokenize.NAME and |
| p.token_type == tokenize.NAME): |
| type_name.append(tokenize.Token(tokenize.SYNTAX, ' ', 0, 0)) |
| type_name.append(p) |
| else: |
| other_tokens.append(p) |
| i += 1 |
| type_name = ''.join([t.name for t in type_name]) |
| return name, type_name, templated_types, modifiers, default, other_tokens |
| |
| def ToParameters(self, tokens): |
| if not tokens: |
| return [] |
| |
| result = [] |
| name = type_name = '' |
| type_modifiers = [] |
| pointer = reference = array = False |
| first_token = None |
| default = [] |
| |
| def AddParameter(end): |
| if default: |
| del default[0] # Remove flag. |
| parts = self.DeclarationToParts(type_modifiers, True) |
| (name, type_name, templated_types, modifiers, |
| unused_default, unused_other_tokens) = parts |
| parameter_type = Type(first_token.start, first_token.end, |
| type_name, templated_types, modifiers, |
| reference, pointer, array) |
| p = Parameter(first_token.start, end, name, |
| parameter_type, default) |
| result.append(p) |
| |
| template_count = 0 |
| for s in tokens: |
| if not first_token: |
| first_token = s |
| if s.name == '<': |
| template_count += 1 |
| elif s.name == '>': |
| template_count -= 1 |
| if template_count > 0: |
| type_modifiers.append(s) |
| continue |
| |
| if s.name == ',': |
| AddParameter(s.start) |
| name = type_name = '' |
| type_modifiers = [] |
| pointer = reference = array = False |
| first_token = None |
| default = [] |
| elif s.name == '*': |
| pointer = True |
| elif s.name == '&': |
| reference = True |
| elif s.name == '[': |
| array = True |
| elif s.name == ']': |
| pass # Just don't add to type_modifiers. |
| elif s.name == '=': |
| # Got a default value. Add any value (None) as a flag. |
| default.append(None) |
| elif default: |
| default.append(s) |
| else: |
| type_modifiers.append(s) |
| AddParameter(tokens[-1].end) |
| return result |
| |
| def CreateReturnType(self, return_type_seq): |
| if not return_type_seq: |
| return None |
| start = return_type_seq[0].start |
| end = return_type_seq[-1].end |
| _, name, templated_types, modifiers, default, other_tokens = \ |
| self.DeclarationToParts(return_type_seq, False) |
| names = [n.name for n in other_tokens] |
| reference = '&' in names |
| pointer = '*' in names |
| array = '[' in names |
| return Type(start, end, name, templated_types, modifiers, |
| reference, pointer, array) |
| |
| def GetTemplateIndices(self, names): |
| # names is a list of strings. |
| start = names.index('<') |
| end = len(names) - 1 |
| while end > 0: |
| if names[end] == '>': |
| break |
| end -= 1 |
| return start, end+1 |
| |
| class AstBuilder(object): |
| def __init__(self, token_stream, filename, in_class='', visibility=None, |
| namespace_stack=[]): |
| self.tokens = token_stream |
| self.filename = filename |
| # TODO(nnorwitz): use a better data structure (deque) for the queue. |
| # Switching directions of the "queue" improved perf by about 25%. |
| # Using a deque should be even better since we access from both sides. |
| self.token_queue = [] |
| self.namespace_stack = namespace_stack[:] |
| self.in_class = in_class |
| if in_class is None: |
| self.in_class_name_only = None |
| else: |
| self.in_class_name_only = in_class.split('::')[-1] |
| self.visibility = visibility |
| self.in_function = False |
| self.current_token = None |
| # Keep the state whether we are currently handling a typedef or not. |
| self._handling_typedef = False |
| |
| self.converter = TypeConverter(self.namespace_stack) |
| |
| def HandleError(self, msg, token): |
| printable_queue = list(reversed(self.token_queue[-20:])) |
| sys.stderr.write('Got %s in %s @ %s %s\n' % |
| (msg, self.filename, token, printable_queue)) |
| |
| def Generate(self): |
| while 1: |
| token = self._GetNextToken() |
| if not token: |
| break |
| |
| # Get the next token. |
| self.current_token = token |
| |
| # Dispatch on the next token type. |
| if token.token_type == _INTERNAL_TOKEN: |
| if token.name == _NAMESPACE_POP: |
| self.namespace_stack.pop() |
| continue |
| |
| try: |
| result = self._GenerateOne(token) |
| if result is not None: |
| yield result |
| except: |
| self.HandleError('exception', token) |
| raise |
| |
| def _CreateVariable(self, pos_token, name, type_name, type_modifiers, |
| ref_pointer_name_seq, templated_types, value=None): |
| reference = '&' in ref_pointer_name_seq |
| pointer = '*' in ref_pointer_name_seq |
| array = '[' in ref_pointer_name_seq |
| var_type = Type(pos_token.start, pos_token.end, type_name, |
| templated_types, type_modifiers, |
| reference, pointer, array) |
| return VariableDeclaration(pos_token.start, pos_token.end, |
| name, var_type, value, self.namespace_stack) |
| |
| def _GenerateOne(self, token): |
| if token.token_type == tokenize.NAME: |
| if (keywords.IsKeyword(token.name) and |
| not keywords.IsBuiltinType(token.name)): |
| method = getattr(self, 'handle_' + token.name) |
| return method() |
| elif token.name == self.in_class_name_only: |
| # The token name is the same as the class, must be a ctor if |
| # there is a paren. Otherwise, it's the return type. |
| # Peek ahead to get the next token to figure out which. |
| next = self._GetNextToken() |
| self._AddBackToken(next) |
| if next.token_type == tokenize.SYNTAX and next.name == '(': |
| return self._GetMethod([token], FUNCTION_CTOR, None, True) |
| # Fall through--handle like any other method. |
| |
| # Handle data or function declaration/definition. |
| syntax = tokenize.SYNTAX |
| temp_tokens, last_token = \ |
| self._GetVarTokensUpTo(syntax, '(', ';', '{', '[') |
| temp_tokens.insert(0, token) |
| if last_token.name == '(': |
| # If there is an assignment before the paren, |
| # this is an expression, not a method. |
| expr = bool([e for e in temp_tokens if e.name == '=']) |
| if expr: |
| new_temp = self._GetTokensUpTo(tokenize.SYNTAX, ';') |
| temp_tokens.append(last_token) |
| temp_tokens.extend(new_temp) |
| last_token = tokenize.Token(tokenize.SYNTAX, ';', 0, 0) |
| |
| if last_token.name == '[': |
| # Handle array, this isn't a method, unless it's an operator. |
| # TODO(nnorwitz): keep the size somewhere. |
| # unused_size = self._GetTokensUpTo(tokenize.SYNTAX, ']') |
| temp_tokens.append(last_token) |
| if temp_tokens[-2].name == 'operator': |
| temp_tokens.append(self._GetNextToken()) |
| else: |
| temp_tokens2, last_token = \ |
| self._GetVarTokensUpTo(tokenize.SYNTAX, ';') |
| temp_tokens.extend(temp_tokens2) |
| |
| if last_token.name == ';': |
| # Handle data, this isn't a method. |
| parts = self.converter.DeclarationToParts(temp_tokens, True) |
| (name, type_name, templated_types, modifiers, default, |
| unused_other_tokens) = parts |
| |
| t0 = temp_tokens[0] |
| names = [t.name for t in temp_tokens] |
| if templated_types: |
| start, end = self.converter.GetTemplateIndices(names) |
| names = names[:start] + names[end:] |
| default = ''.join([t.name for t in default]) |
| return self._CreateVariable(t0, name, type_name, modifiers, |
| names, templated_types, default) |
| if last_token.name == '{': |
| self._AddBackTokens(temp_tokens[1:]) |
| self._AddBackToken(last_token) |
| method_name = temp_tokens[0].name |
| method = getattr(self, 'handle_' + method_name, None) |
| if not method: |
| # Must be declaring a variable. |
| # TODO(nnorwitz): handle the declaration. |
| return None |
| return method() |
| return self._GetMethod(temp_tokens, 0, None, False) |
| elif token.token_type == tokenize.SYNTAX: |
| if token.name == '~' and self.in_class: |
| # Must be a dtor (probably not in method body). |
| token = self._GetNextToken() |
| # self.in_class can contain A::Name, but the dtor will only |
| # be Name. Make sure to compare against the right value. |
| if (token.token_type == tokenize.NAME and |
| token.name == self.in_class_name_only): |
| return self._GetMethod([token], FUNCTION_DTOR, None, True) |
| # TODO(nnorwitz): handle a lot more syntax. |
| elif token.token_type == tokenize.PREPROCESSOR: |
| # TODO(nnorwitz): handle more preprocessor directives. |
| # token starts with a #, so remove it and strip whitespace. |
| name = token.name[1:].lstrip() |
| if name.startswith('include'): |
| # Remove "include". |
| name = name[7:].strip() |
| assert name |
| # Handle #include \<newline> "header-on-second-line.h". |
| if name.startswith('\\'): |
| name = name[1:].strip() |
| assert name[0] in '<"', token |
| assert name[-1] in '>"', token |
| system = name[0] == '<' |
| filename = name[1:-1] |
| return Include(token.start, token.end, filename, system) |
| if name.startswith('define'): |
| # Remove "define". |
| name = name[6:].strip() |
| assert name |
| value = '' |
| for i, c in enumerate(name): |
| if c.isspace(): |
| value = name[i:].lstrip() |
| name = name[:i] |
| break |
| return Define(token.start, token.end, name, value) |
| if name.startswith('if') and name[2:3].isspace(): |
| condition = name[3:].strip() |
| if condition.startswith('0') or condition.startswith('(0)'): |
| self._SkipIf0Blocks() |
| return None |
| |
| def _GetTokensUpTo(self, expected_token_type, expected_token): |
| return self._GetVarTokensUpTo(expected_token_type, expected_token)[0] |
| |
| def _GetVarTokensUpTo(self, expected_token_type, *expected_tokens): |
| last_token = self._GetNextToken() |
| tokens = [] |
| while (last_token.token_type != expected_token_type or |
| last_token.name not in expected_tokens): |
| tokens.append(last_token) |
| last_token = self._GetNextToken() |
| return tokens, last_token |
| |
| # TODO(nnorwitz): remove _IgnoreUpTo() it shouldn't be necessary. |
| def _IgnoreUpTo(self, token_type, token): |
| unused_tokens = self._GetTokensUpTo(token_type, token) |
| |
| def _SkipIf0Blocks(self): |
| count = 1 |
| while 1: |
| token = self._GetNextToken() |
| if token.token_type != tokenize.PREPROCESSOR: |
| continue |
| |
| name = token.name[1:].lstrip() |
| if name.startswith('endif'): |
| count -= 1 |
| if count == 0: |
| break |
| elif name.startswith('if'): |
| count += 1 |
| |
| def _GetMatchingChar(self, open_paren, close_paren, GetNextToken=None): |
| if GetNextToken is None: |
| GetNextToken = self._GetNextToken |
| # Assumes the current token is open_paren and we will consume |
| # and return up to the close_paren. |
| count = 1 |
| token = GetNextToken() |
| while 1: |
| if token.token_type == tokenize.SYNTAX: |
| if token.name == open_paren: |
| count += 1 |
| elif token.name == close_paren: |
| count -= 1 |
| if count == 0: |
| break |
| yield token |
| token = GetNextToken() |
| yield token |
| |
| def _GetParameters(self): |
| return self._GetMatchingChar('(', ')') |
| |
| def GetScope(self): |
| return self._GetMatchingChar('{', '}') |
| |
| def _GetNextToken(self): |
| if self.token_queue: |
| return self.token_queue.pop() |
| return next(self.tokens) |
| |
| def _AddBackToken(self, token): |
| if token.whence == tokenize.WHENCE_STREAM: |
| token.whence = tokenize.WHENCE_QUEUE |
| self.token_queue.insert(0, token) |
| else: |
| assert token.whence == tokenize.WHENCE_QUEUE, token |
| self.token_queue.append(token) |
| |
| def _AddBackTokens(self, tokens): |
| if tokens: |
| if tokens[-1].whence == tokenize.WHENCE_STREAM: |
| for token in tokens: |
| token.whence = tokenize.WHENCE_QUEUE |
| self.token_queue[:0] = reversed(tokens) |
| else: |
| assert tokens[-1].whence == tokenize.WHENCE_QUEUE, tokens |
| self.token_queue.extend(reversed(tokens)) |
| |
| def GetName(self, seq=None): |
| """Returns ([tokens], next_token_info).""" |
| GetNextToken = self._GetNextToken |
| if seq is not None: |
| it = iter(seq) |
| GetNextToken = lambda: next(it) |
| next_token = GetNextToken() |
| tokens = [] |
| last_token_was_name = False |
| while (next_token.token_type == tokenize.NAME or |
| (next_token.token_type == tokenize.SYNTAX and |
| next_token.name in ('::', '<'))): |
| # Two NAMEs in a row means the identifier should terminate. |
| # It's probably some sort of variable declaration. |
| if last_token_was_name and next_token.token_type == tokenize.NAME: |
| break |
| last_token_was_name = next_token.token_type == tokenize.NAME |
| tokens.append(next_token) |
| # Handle templated names. |
| if next_token.name == '<': |
| tokens.extend(self._GetMatchingChar('<', '>', GetNextToken)) |
| last_token_was_name = True |
| next_token = GetNextToken() |
| return tokens, next_token |
| |
| def GetMethod(self, modifiers, templated_types): |
| return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') |
| assert len(return_type_and_name) >= 1 |
| return self._GetMethod(return_type_and_name, modifiers, templated_types, |
| False) |
| |
| def _GetMethod(self, return_type_and_name, modifiers, templated_types, |
| get_paren): |
| template_portion = None |
| if get_paren: |
| token = self._GetNextToken() |
| assert token.token_type == tokenize.SYNTAX, token |
| if token.name == '<': |
| # Handle templatized dtors. |
| template_portion = [token] |
| template_portion.extend(self._GetMatchingChar('<', '>')) |
| token = self._GetNextToken() |
| assert token.token_type == tokenize.SYNTAX, token |
| assert token.name == '(', token |
| |
| name = return_type_and_name.pop() |
| # Handle templatized ctors. |
| if name.name == '>': |
| index = 1 |
| while return_type_and_name[index].name != '<': |
| index += 1 |
| template_portion = return_type_and_name[index:] + [name] |
| del return_type_and_name[index:] |
| name = return_type_and_name.pop() |
| elif name.name == ']': |
| rt = return_type_and_name |
| assert rt[-1].name == '[', return_type_and_name |
| assert rt[-2].name == 'operator', return_type_and_name |
| name_seq = return_type_and_name[-2:] |
| del return_type_and_name[-2:] |
| name = tokenize.Token(tokenize.NAME, 'operator[]', |
| name_seq[0].start, name.end) |
| # Get the open paren so _GetParameters() below works. |
| unused_open_paren = self._GetNextToken() |
| |
| # TODO(nnorwitz): store template_portion. |
| return_type = return_type_and_name |
| indices = name |
| if return_type: |
| indices = return_type[0] |
| |
| # Force ctor for templatized ctors. |
| if name.name == self.in_class and not modifiers: |
| modifiers |= FUNCTION_CTOR |
| parameters = list(self._GetParameters()) |
| del parameters[-1] # Remove trailing ')'. |
| |
| # Handling operator() is especially weird. |
| if name.name == 'operator' and not parameters: |
| token = self._GetNextToken() |
| assert token.name == '(', token |
| parameters = list(self._GetParameters()) |
| del parameters[-1] # Remove trailing ')'. |
| |
| token = self._GetNextToken() |
| while token.token_type == tokenize.NAME: |
| modifier_token = token |
| token = self._GetNextToken() |
| if modifier_token.name == 'const': |
| modifiers |= FUNCTION_CONST |
| elif modifier_token.name == '__attribute__': |
| # TODO(nnorwitz): handle more __attribute__ details. |
| modifiers |= FUNCTION_ATTRIBUTE |
| assert token.name == '(', token |
| # Consume everything between the (parens). |
| unused_tokens = list(self._GetMatchingChar('(', ')')) |
| token = self._GetNextToken() |
| elif modifier_token.name == 'throw': |
| modifiers |= FUNCTION_THROW |
| assert token.name == '(', token |
| # Consume everything between the (parens). |
| unused_tokens = list(self._GetMatchingChar('(', ')')) |
| token = self._GetNextToken() |
| elif modifier_token.name == 'override': |
| modifiers |= FUNCTION_OVERRIDE |
| elif modifier_token.name == modifier_token.name.upper(): |
| # HACK(nnorwitz): assume that all upper-case names |
| # are some macro we aren't expanding. |
| modifiers |= FUNCTION_UNKNOWN_ANNOTATION |
| else: |
| self.HandleError('unexpected token', modifier_token) |
| |
| assert token.token_type == tokenize.SYNTAX, token |
| # Handle ctor initializers. |
| if token.name == ':': |
| # TODO(nnorwitz): anything else to handle for initializer list? |
| while token.name != ';' and token.name != '{': |
| token = self._GetNextToken() |
| |
| # Handle pointer to functions that are really data but look |
| # like method declarations. |
| if token.name == '(': |
| if parameters[0].name == '*': |
| # name contains the return type. |
| name = parameters.pop() |
| # parameters contains the name of the data. |
| modifiers = [p.name for p in parameters] |
| # Already at the ( to open the parameter list. |
| function_parameters = list(self._GetMatchingChar('(', ')')) |
| del function_parameters[-1] # Remove trailing ')'. |
| # TODO(nnorwitz): store the function_parameters. |
| token = self._GetNextToken() |
| assert token.token_type == tokenize.SYNTAX, token |
| assert token.name == ';', token |
| return self._CreateVariable(indices, name.name, indices.name, |
| modifiers, '', None) |
| # At this point, we got something like: |
| # return_type (type::*name_)(params); |
| # This is a data member called name_ that is a function pointer. |
| # With this code: void (sq_type::*field_)(string&); |
| # We get: name=void return_type=[] parameters=sq_type ... field_ |
| # TODO(nnorwitz): is return_type always empty? |
| # TODO(nnorwitz): this isn't even close to being correct. |
| # Just put in something so we don't crash and can move on. |
| real_name = parameters[-1] |
| modifiers = [p.name for p in self._GetParameters()] |
| del modifiers[-1] # Remove trailing ')'. |
| return self._CreateVariable(indices, real_name.name, indices.name, |
| modifiers, '', None) |
| |
| if token.name == '{': |
| body = list(self.GetScope()) |
| del body[-1] # Remove trailing '}'. |
| else: |
| body = None |
| if token.name == '=': |
| token = self._GetNextToken() |
| |
| if token.name == 'default' or token.name == 'delete': |
| # Ignore explicitly defaulted and deleted special members |
| # in C++11. |
| token = self._GetNextToken() |
| else: |
| # Handle pure-virtual declarations. |
| assert token.token_type == tokenize.CONSTANT, token |
| assert token.name == '0', token |
| modifiers |= FUNCTION_PURE_VIRTUAL |
| token = self._GetNextToken() |
| |
| if token.name == '[': |
| # TODO(nnorwitz): store tokens and improve parsing. |
| # template <typename T, size_t N> char (&ASH(T (&seq)[N]))[N]; |
| tokens = list(self._GetMatchingChar('[', ']')) |
| token = self._GetNextToken() |
| |
| assert token.name == ';', (token, return_type_and_name, parameters) |
| |
| # Looks like we got a method, not a function. |
| if len(return_type) > 2 and return_type[-1].name == '::': |
| return_type, in_class = \ |
| self._GetReturnTypeAndClassName(return_type) |
| return Method(indices.start, indices.end, name.name, in_class, |
| return_type, parameters, modifiers, templated_types, |
| body, self.namespace_stack) |
| return Function(indices.start, indices.end, name.name, return_type, |
| parameters, modifiers, templated_types, body, |
| self.namespace_stack) |
| |
| def _GetReturnTypeAndClassName(self, token_seq): |
| # Splitting the return type from the class name in a method |
| # can be tricky. For example, Return::Type::Is::Hard::To::Find(). |
| # Where is the return type and where is the class name? |
| # The heuristic used is to pull the last name as the class name. |
| # This includes all the templated type info. |
| # TODO(nnorwitz): if there is only One name like in the |
| # example above, punt and assume the last bit is the class name. |
| |
| # Ignore a :: prefix, if exists so we can find the first real name. |
| i = 0 |
| if token_seq[0].name == '::': |
| i = 1 |
| # Ignore a :: suffix, if exists. |
| end = len(token_seq) - 1 |
| if token_seq[end-1].name == '::': |
| end -= 1 |
| |
| # Make a copy of the sequence so we can append a sentinel |
| # value. This is required for GetName will has to have some |
| # terminating condition beyond the last name. |
| seq_copy = token_seq[i:end] |
| seq_copy.append(tokenize.Token(tokenize.SYNTAX, '', 0, 0)) |
| names = [] |
| while i < end: |
| # Iterate through the sequence parsing out each name. |
| new_name, next = self.GetName(seq_copy[i:]) |
| assert new_name, 'Got empty new_name, next=%s' % next |
| # We got a pointer or ref. Add it to the name. |
| if next and next.token_type == tokenize.SYNTAX: |
| new_name.append(next) |
| names.append(new_name) |
| i += len(new_name) |
| |
| # Now that we have the names, it's time to undo what we did. |
| |
| # Remove the sentinel value. |
| names[-1].pop() |
| # Flatten the token sequence for the return type. |
| return_type = [e for seq in names[:-1] for e in seq] |
| # The class name is the last name. |
| class_name = names[-1] |
| return return_type, class_name |
| |
| def handle_bool(self): |
| pass |
| |
| def handle_char(self): |
| pass |
| |
| def handle_int(self): |
| pass |
| |
| def handle_long(self): |
| pass |
| |
| def handle_short(self): |
| pass |
| |
| def handle_double(self): |
| pass |
| |
| def handle_float(self): |
| pass |
| |
| def handle_void(self): |
| pass |
| |
| def handle_wchar_t(self): |
| pass |
| |
| def handle_unsigned(self): |
| pass |
| |
| def handle_signed(self): |
| pass |
| |
| def _GetNestedType(self, ctor): |
| name = None |
| name_tokens, token = self.GetName() |
| if name_tokens: |
| name = ''.join([t.name for t in name_tokens]) |
| |
| # Handle forward declarations. |
| if token.token_type == tokenize.SYNTAX and token.name == ';': |
| return ctor(token.start, token.end, name, None, |
| self.namespace_stack) |
| |
| if token.token_type == tokenize.NAME and self._handling_typedef: |
| self._AddBackToken(token) |
| return ctor(token.start, token.end, name, None, |
| self.namespace_stack) |
| |
| # Must be the type declaration. |
| fields = list(self._GetMatchingChar('{', '}')) |
| del fields[-1] # Remove trailing '}'. |
| if token.token_type == tokenize.SYNTAX and token.name == '{': |
| next = self._GetNextToken() |
| new_type = ctor(token.start, token.end, name, fields, |
| self.namespace_stack) |
| # A name means this is an anonymous type and the name |
| # is the variable declaration. |
| if next.token_type != tokenize.NAME: |
| return new_type |
| name = new_type |
| token = next |
| |
| # Must be variable declaration using the type prefixed with keyword. |
| assert token.token_type == tokenize.NAME, token |
| return self._CreateVariable(token, token.name, name, [], '', None) |
| |
| def handle_struct(self): |
| # Special case the handling typedef/aliasing of structs here. |
| # It would be a pain to handle in the class code. |
| name_tokens, var_token = self.GetName() |
| if name_tokens: |
| next_token = self._GetNextToken() |
| is_syntax = (var_token.token_type == tokenize.SYNTAX and |
| var_token.name[0] in '*&') |
| is_variable = (var_token.token_type == tokenize.NAME and |
| next_token.name == ';') |
| variable = var_token |
| if is_syntax and not is_variable: |
| variable = next_token |
| temp = self._GetNextToken() |
| if temp.token_type == tokenize.SYNTAX and temp.name == '(': |
| # Handle methods declared to return a struct. |
| t0 = name_tokens[0] |
| struct = tokenize.Token(tokenize.NAME, 'struct', |
| t0.start-7, t0.start-2) |
| type_and_name = [struct] |
| type_and_name.extend(name_tokens) |
| type_and_name.extend((var_token, next_token)) |
| return self._GetMethod(type_and_name, 0, None, False) |
| assert temp.name == ';', (temp, name_tokens, var_token) |
| if is_syntax or (is_variable and not self._handling_typedef): |
| modifiers = ['struct'] |
| type_name = ''.join([t.name for t in name_tokens]) |
| position = name_tokens[0] |
| return self._CreateVariable(position, variable.name, type_name, |
| modifiers, var_token.name, None) |
| name_tokens.extend((var_token, next_token)) |
| self._AddBackTokens(name_tokens) |
| else: |
| self._AddBackToken(var_token) |
| return self._GetClass(Struct, VISIBILITY_PUBLIC, None) |
| |
| def handle_union(self): |
| return self._GetNestedType(Union) |
| |
| def handle_enum(self): |
| return self._GetNestedType(Enum) |
| |
| def handle_auto(self): |
| # TODO(nnorwitz): warn about using auto? Probably not since it |
| # will be reclaimed and useful for C++0x. |
| pass |
| |
| def handle_register(self): |
| pass |
| |
| def handle_const(self): |
| pass |
| |
| def handle_inline(self): |
| pass |
| |
| def handle_extern(self): |
| pass |
| |
| def handle_static(self): |
| pass |
| |
| def handle_virtual(self): |
| # What follows must be a method. |
| token = token2 = self._GetNextToken() |
| if token.name == 'inline': |
| # HACK(nnorwitz): handle inline dtors by ignoring 'inline'. |
| token2 = self._GetNextToken() |
| if token2.token_type == tokenize.SYNTAX and token2.name == '~': |
| return self.GetMethod(FUNCTION_VIRTUAL + FUNCTION_DTOR, None) |
| assert token.token_type == tokenize.NAME or token.name == '::', token |
| return_type_and_name = self._GetTokensUpTo(tokenize.SYNTAX, '(') # ) |
| return_type_and_name.insert(0, token) |
| if token2 is not token: |
| return_type_and_name.insert(1, token2) |
| return self._GetMethod(return_type_and_name, FUNCTION_VIRTUAL, |
| None, False) |
| |
| def handle_volatile(self): |
| pass |
| |
| def handle_mutable(self): |
| pass |
| |
| def handle_public(self): |
| assert self.in_class |
| self.visibility = VISIBILITY_PUBLIC |
| |
| def handle_protected(self): |
| assert self.in_class |
| self.visibility = VISIBILITY_PROTECTED |
| |
| def handle_private(self): |
| assert self.in_class |
| self.visibility = VISIBILITY_PRIVATE |
| |
| def handle_friend(self): |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') |
| assert tokens |
| t0 = tokens[0] |
| return Friend(t0.start, t0.end, tokens, self.namespace_stack) |
| |
| def handle_static_cast(self): |
| pass |
| |
| def handle_const_cast(self): |
| pass |
| |
| def handle_dynamic_cast(self): |
| pass |
| |
| def handle_reinterpret_cast(self): |
| pass |
| |
| def handle_new(self): |
| pass |
| |
| def handle_delete(self): |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') |
| assert tokens |
| return Delete(tokens[0].start, tokens[0].end, tokens) |
| |
| def handle_typedef(self): |
| token = self._GetNextToken() |
| if (token.token_type == tokenize.NAME and |
| keywords.IsKeyword(token.name)): |
| # Token must be struct/enum/union/class. |
| method = getattr(self, 'handle_' + token.name) |
| self._handling_typedef = True |
| tokens = [method()] |
| self._handling_typedef = False |
| else: |
| tokens = [token] |
| |
| # Get the remainder of the typedef up to the semi-colon. |
| tokens.extend(self._GetTokensUpTo(tokenize.SYNTAX, ';')) |
| |
| # TODO(nnorwitz): clean all this up. |
| assert tokens |
| name = tokens.pop() |
| indices = name |
| if tokens: |
| indices = tokens[0] |
| if not indices: |
| indices = token |
| if name.name == ')': |
| # HACK(nnorwitz): Handle pointers to functions "properly". |
| if (len(tokens) >= 4 and |
| tokens[1].name == '(' and tokens[2].name == '*'): |
| tokens.append(name) |
| name = tokens[3] |
| elif name.name == ']': |
| # HACK(nnorwitz): Handle arrays properly. |
| if len(tokens) >= 2: |
| tokens.append(name) |
| name = tokens[1] |
| new_type = tokens |
| if tokens and isinstance(tokens[0], tokenize.Token): |
| new_type = self.converter.ToType(tokens)[0] |
| return Typedef(indices.start, indices.end, name.name, |
| new_type, self.namespace_stack) |
| |
| def handle_typeid(self): |
| pass # Not needed yet. |
| |
| def handle_typename(self): |
| pass # Not needed yet. |
| |
| def _GetTemplatedTypes(self): |
| result = {} |
| tokens = list(self._GetMatchingChar('<', '>')) |
| len_tokens = len(tokens) - 1 # Ignore trailing '>'. |
| i = 0 |
| while i < len_tokens: |
| key = tokens[i].name |
| i += 1 |
| if keywords.IsKeyword(key) or key == ',': |
| continue |
| type_name = default = None |
| if i < len_tokens: |
| i += 1 |
| if tokens[i-1].name == '=': |
| assert i < len_tokens, '%s %s' % (i, tokens) |
| default, unused_next_token = self.GetName(tokens[i:]) |
| i += len(default) |
| else: |
| if tokens[i-1].name != ',': |
| # We got something like: Type variable. |
| # Re-adjust the key (variable) and type_name (Type). |
| key = tokens[i-1].name |
| type_name = tokens[i-2] |
| |
| result[key] = (type_name, default) |
| return result |
| |
| def handle_template(self): |
| token = self._GetNextToken() |
| assert token.token_type == tokenize.SYNTAX, token |
| assert token.name == '<', token |
| templated_types = self._GetTemplatedTypes() |
| # TODO(nnorwitz): for now, just ignore the template params. |
| token = self._GetNextToken() |
| if token.token_type == tokenize.NAME: |
| if token.name == 'class': |
| return self._GetClass(Class, VISIBILITY_PRIVATE, templated_types) |
| elif token.name == 'struct': |
| return self._GetClass(Struct, VISIBILITY_PUBLIC, templated_types) |
| elif token.name == 'friend': |
| return self.handle_friend() |
| self._AddBackToken(token) |
| tokens, last = self._GetVarTokensUpTo(tokenize.SYNTAX, '(', ';') |
| tokens.append(last) |
| self._AddBackTokens(tokens) |
| if last.name == '(': |
| return self.GetMethod(FUNCTION_NONE, templated_types) |
| # Must be a variable definition. |
| return None |
| |
| def handle_true(self): |
| pass # Nothing to do. |
| |
| def handle_false(self): |
| pass # Nothing to do. |
| |
| def handle_asm(self): |
| pass # Not needed yet. |
| |
| def handle_class(self): |
| return self._GetClass(Class, VISIBILITY_PRIVATE, None) |
| |
| def _GetBases(self): |
| # Get base classes. |
| bases = [] |
| while 1: |
| token = self._GetNextToken() |
| assert token.token_type == tokenize.NAME, token |
| # TODO(nnorwitz): store kind of inheritance...maybe. |
| if token.name not in ('public', 'protected', 'private'): |
| # If inheritance type is not specified, it is private. |
| # Just put the token back so we can form a name. |
| # TODO(nnorwitz): it would be good to warn about this. |
| self._AddBackToken(token) |
| else: |
| # Check for virtual inheritance. |
| token = self._GetNextToken() |
| if token.name != 'virtual': |
| self._AddBackToken(token) |
| else: |
| # TODO(nnorwitz): store that we got virtual for this base. |
| pass |
| base, next_token = self.GetName() |
| bases_ast = self.converter.ToType(base) |
| assert len(bases_ast) == 1, bases_ast |
| bases.append(bases_ast[0]) |
| assert next_token.token_type == tokenize.SYNTAX, next_token |
| if next_token.name == '{': |
| token = next_token |
| break |
| # Support multiple inheritance. |
| assert next_token.name == ',', next_token |
| return bases, token |
| |
| def _GetClass(self, class_type, visibility, templated_types): |
| class_name = None |
| class_token = self._GetNextToken() |
| if class_token.token_type != tokenize.NAME: |
| assert class_token.token_type == tokenize.SYNTAX, class_token |
| token = class_token |
| else: |
| # Skip any macro (e.g. storage class specifiers) after the |
| # 'class' keyword. |
| next_token = self._GetNextToken() |
| if next_token.token_type == tokenize.NAME: |
| self._AddBackToken(next_token) |
| else: |
| self._AddBackTokens([class_token, next_token]) |
| name_tokens, token = self.GetName() |
| class_name = ''.join([t.name for t in name_tokens]) |
| bases = None |
| if token.token_type == tokenize.SYNTAX: |
| if token.name == ';': |
| # Forward declaration. |
| return class_type(class_token.start, class_token.end, |
| class_name, None, templated_types, None, |
| self.namespace_stack) |
| if token.name in '*&': |
| # Inline forward declaration. Could be method or data. |
| name_token = self._GetNextToken() |
| next_token = self._GetNextToken() |
| if next_token.name == ';': |
| # Handle data |
| modifiers = ['class'] |
| return self._CreateVariable(class_token, name_token.name, |
| class_name, |
| modifiers, token.name, None) |
| else: |
| # Assume this is a method. |
| tokens = (class_token, token, name_token, next_token) |
| self._AddBackTokens(tokens) |
| return self.GetMethod(FUNCTION_NONE, None) |
| if token.name == ':': |
| bases, token = self._GetBases() |
| |
| body = None |
| if token.token_type == tokenize.SYNTAX and token.name == '{': |
| assert token.token_type == tokenize.SYNTAX, token |
| assert token.name == '{', token |
| |
| ast = AstBuilder(self.GetScope(), self.filename, class_name, |
| visibility, self.namespace_stack) |
| body = list(ast.Generate()) |
| |
| if not self._handling_typedef: |
| token = self._GetNextToken() |
| if token.token_type != tokenize.NAME: |
| assert token.token_type == tokenize.SYNTAX, token |
| assert token.name == ';', token |
| else: |
| new_class = class_type(class_token.start, class_token.end, |
| class_name, bases, None, |
| body, self.namespace_stack) |
| |
| modifiers = [] |
| return self._CreateVariable(class_token, |
| token.name, new_class, |
| modifiers, token.name, None) |
| else: |
| if not self._handling_typedef: |
| self.HandleError('non-typedef token', token) |
| self._AddBackToken(token) |
| |
| return class_type(class_token.start, class_token.end, class_name, |
| bases, templated_types, body, self.namespace_stack) |
| |
| def handle_namespace(self): |
| token = self._GetNextToken() |
| # Support anonymous namespaces. |
| name = None |
| if token.token_type == tokenize.NAME: |
| name = token.name |
| token = self._GetNextToken() |
| self.namespace_stack.append(name) |
| assert token.token_type == tokenize.SYNTAX, token |
| # Create an internal token that denotes when the namespace is complete. |
| internal_token = tokenize.Token(_INTERNAL_TOKEN, _NAMESPACE_POP, |
| None, None) |
| internal_token.whence = token.whence |
| if token.name == '=': |
| # TODO(nnorwitz): handle aliasing namespaces. |
| name, next_token = self.GetName() |
| assert next_token.name == ';', next_token |
| self._AddBackToken(internal_token) |
| else: |
| assert token.name == '{', token |
| tokens = list(self.GetScope()) |
| # Replace the trailing } with the internal namespace pop token. |
| tokens[-1] = internal_token |
| # Handle namespace with nothing in it. |
| self._AddBackTokens(tokens) |
| return None |
| |
| def handle_using(self): |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') |
| assert tokens |
| return Using(tokens[0].start, tokens[0].end, tokens) |
| |
| def handle_explicit(self): |
| assert self.in_class |
| # Nothing much to do. |
| # TODO(nnorwitz): maybe verify the method name == class name. |
| # This must be a ctor. |
| return self.GetMethod(FUNCTION_CTOR, None) |
| |
| def handle_this(self): |
| pass # Nothing to do. |
| |
| def handle_operator(self): |
| # Pull off the next token(s?) and make that part of the method name. |
| pass |
| |
| def handle_sizeof(self): |
| pass |
| |
| def handle_case(self): |
| pass |
| |
| def handle_switch(self): |
| pass |
| |
| def handle_default(self): |
| token = self._GetNextToken() |
| assert token.token_type == tokenize.SYNTAX |
| assert token.name == ':' |
| |
| def handle_if(self): |
| pass |
| |
| def handle_else(self): |
| pass |
| |
| def handle_return(self): |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') |
| if not tokens: |
| return Return(self.current_token.start, self.current_token.end, None) |
| return Return(tokens[0].start, tokens[0].end, tokens) |
| |
| def handle_goto(self): |
| tokens = self._GetTokensUpTo(tokenize.SYNTAX, ';') |
| assert len(tokens) == 1, str(tokens) |
| return Goto(tokens[0].start, tokens[0].end, tokens[0].name) |
| |
| def handle_try(self): |
| pass # Not needed yet. |
| |
| def handle_catch(self): |
| pass # Not needed yet. |
| |
| def handle_throw(self): |
| pass # Not needed yet. |
| |
| def handle_while(self): |
| pass |
| |
| def handle_do(self): |
| pass |
| |
| def handle_for(self): |
| pass |
| |
| def handle_break(self): |
| self._IgnoreUpTo(tokenize.SYNTAX, ';') |
| |
| def handle_continue(self): |
| self._IgnoreUpTo(tokenize.SYNTAX, ';') |
| |
| |
| def BuilderFromSource(source, filename): |
| """Utility method that returns an AstBuilder from source code. |
| |
| Args: |
| source: 'C++ source code' |
| filename: 'file1' |
| |
| Returns: |
| AstBuilder |
| """ |
| return AstBuilder(tokenize.GetTokens(source), filename) |
| |
| |
| def PrintIndentifiers(filename, should_print): |
| """Prints all identifiers for a C++ source file. |
| |
| Args: |
| filename: 'file1' |
| should_print: predicate with signature: bool Function(token) |
| """ |
| source = utils.ReadFile(filename, False) |
| if source is None: |
| sys.stderr.write('Unable to find: %s\n' % filename) |
| return |
| |
| #print('Processing %s' % actual_filename) |
| builder = BuilderFromSource(source, filename) |
| try: |
| for node in builder.Generate(): |
| if should_print(node): |
| print(node.name) |
| except KeyboardInterrupt: |
| return |
| except: |
| pass |
| |
| |
| def PrintAllIndentifiers(filenames, should_print): |
| """Prints all identifiers for each C++ source file in filenames. |
| |
| Args: |
| filenames: ['file1', 'file2', ...] |
| should_print: predicate with signature: bool Function(token) |
| """ |
| for path in filenames: |
| PrintIndentifiers(path, should_print) |
| |
| |
| def main(argv): |
| for filename in argv[1:]: |
| source = utils.ReadFile(filename) |
| if source is None: |
| continue |
| |
| print('Processing %s' % filename) |
| builder = BuilderFromSource(source, filename) |
| try: |
| entire_ast = filter(None, builder.Generate()) |
| except KeyboardInterrupt: |
| return |
| except: |
| # Already printed a warning, print the traceback and continue. |
| traceback.print_exc() |
| else: |
| if utils.DEBUG: |
| for ast in entire_ast: |
| print(ast) |
| |
| |
| if __name__ == '__main__': |
| main(sys.argv) |
