gn: Reduce stack memory use while parsing.
BUG=641454
Review-Url: https://codereview.chromium.org/2290713002
Cr-Original-Commit-Position: refs/heads/master@{#415181}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 9e92b9d9ef75b26947a46ed3bc2735fddc888b3f
diff --git a/tools/gn/parser.cc b/tools/gn/parser.cc
index e256d2d..76d4580 100644
--- a/tools/gn/parser.cc
+++ b/tools/gn/parser.cc
@@ -281,7 +281,9 @@
};
Parser::Parser(const std::vector<Token>& tokens, Err* err)
- : err_(err), cur_(0) {
+ : invalid_token_(Location(), Token::INVALID, base::StringPiece()),
+ err_(err),
+ cur_(0) {
for (const auto& token : tokens) {
switch (token.type()) {
case Token::LINE_COMMENT:
@@ -377,20 +379,20 @@
return true;
}
-Token Parser::Consume(Token::Type type, const char* error_message) {
+const Token& Parser::Consume(Token::Type type, const char* error_message) {
Token::Type types[1] = { type };
return Consume(types, 1, error_message);
}
-Token Parser::Consume(Token::Type* types,
- size_t num_types,
- const char* error_message) {
+const Token& Parser::Consume(Token::Type* types,
+ size_t num_types,
+ const char* error_message) {
if (has_error()) {
// Don't overwrite current error, but make progress through tokens so that
// a loop that's expecting a particular token will still terminate.
if (!at_end())
cur_++;
- return Token(Location(), Token::INVALID, base::StringPiece());
+ return invalid_token_;
}
if (at_end()) {
const char kEOFMsg[] = "I hit EOF instead.";
@@ -398,7 +400,7 @@
*err_ = Err(Location(), error_message, kEOFMsg);
else
*err_ = Err(tokens_[tokens_.size() - 1], error_message, kEOFMsg);
- return Token(Location(), Token::INVALID, base::StringPiece());
+ return invalid_token_;
}
for (size_t i = 0; i < num_types; ++i) {
@@ -406,10 +408,10 @@
return Consume();
}
*err_ = Err(cur_token(), error_message);
- return Token(Location(), Token::INVALID, base::StringPiece());
+ return invalid_token_;
}
-Token Parser::Consume() {
+const Token& Parser::Consume() {
return tokens_[cur_++];
}
@@ -421,7 +423,7 @@
if (at_end())
return std::unique_ptr<ParseNode>();
- Token token = Consume();
+ const Token& token = Consume();
PrefixFunc prefix = expressions_[token.type()].prefix;
if (prefix == nullptr) {
@@ -437,15 +439,15 @@
while (!at_end() && !IsStatementBreak(cur_token().type()) &&
precedence <= expressions_[cur_token().type()].precedence) {
- token = Consume();
- InfixFunc infix = expressions_[token.type()].infix;
+ const Token& next_token = Consume();
+ InfixFunc infix = expressions_[next_token.type()].infix;
if (infix == nullptr) {
- *err_ = Err(token,
- std::string("Unexpected token '") +
- token.value().as_string() + std::string("'"));
+ *err_ = Err(next_token, std::string("Unexpected token '") +
+ next_token.value().as_string() +
+ std::string("'"));
return std::unique_ptr<ParseNode>();
}
- left = (this->*infix)(std::move(left), token);
+ left = (this->*infix)(std::move(left), next_token);
if (has_error())
return std::unique_ptr<ParseNode>();
}
@@ -453,27 +455,27 @@
return left;
}
-std::unique_ptr<ParseNode> Parser::Block(Token token) {
+std::unique_ptr<ParseNode> Parser::Block(const Token& token) {
// This entrypoint into ParseBlock means it's part of an expression and we
// always want the result.
return ParseBlock(token, BlockNode::RETURNS_SCOPE);
}
-std::unique_ptr<ParseNode> Parser::Literal(Token token) {
+std::unique_ptr<ParseNode> Parser::Literal(const Token& token) {
return base::WrapUnique(new LiteralNode(token));
}
-std::unique_ptr<ParseNode> Parser::Name(Token token) {
+std::unique_ptr<ParseNode> Parser::Name(const Token& token) {
return IdentifierOrCall(std::unique_ptr<ParseNode>(), token);
}
-std::unique_ptr<ParseNode> Parser::BlockComment(Token token) {
+std::unique_ptr<ParseNode> Parser::BlockComment(const Token& token) {
std::unique_ptr<BlockCommentNode> comment(new BlockCommentNode());
comment->set_comment(token);
return std::move(comment);
}
-std::unique_ptr<ParseNode> Parser::Group(Token token) {
+std::unique_ptr<ParseNode> Parser::Group(const Token& token) {
std::unique_ptr<ParseNode> expr = ParseExpression();
if (has_error())
return std::unique_ptr<ParseNode>();
@@ -481,7 +483,7 @@
return expr;
}
-std::unique_ptr<ParseNode> Parser::Not(Token token) {
+std::unique_ptr<ParseNode> Parser::Not(const Token& token) {
std::unique_ptr<ParseNode> expr = ParseExpression(PRECEDENCE_PREFIX + 1);
if (has_error())
return std::unique_ptr<ParseNode>();
@@ -496,7 +498,7 @@
return std::move(unary_op);
}
-std::unique_ptr<ParseNode> Parser::List(Token node) {
+std::unique_ptr<ParseNode> Parser::List(const Token& node) {
std::unique_ptr<ParseNode> list(ParseList(node, Token::RIGHT_BRACKET, true));
if (!has_error() && !at_end())
Consume(Token::RIGHT_BRACKET, "Expected ']'");
@@ -505,7 +507,7 @@
std::unique_ptr<ParseNode> Parser::BinaryOperator(
std::unique_ptr<ParseNode> left,
- Token token) {
+ const Token& token) {
std::unique_ptr<ParseNode> right =
ParseExpression(expressions_[token.type()].precedence + 1);
if (!right) {
@@ -524,14 +526,14 @@
std::unique_ptr<ParseNode> Parser::IdentifierOrCall(
std::unique_ptr<ParseNode> left,
- Token token) {
+ const Token& token) {
std::unique_ptr<ListNode> list(new ListNode);
list->set_begin_token(token);
list->set_end(base::WrapUnique(new EndNode(token)));
std::unique_ptr<BlockNode> block;
bool has_arg = false;
if (LookAhead(Token::LEFT_PAREN)) {
- Token start_token = Consume();
+ const Token& start_token = Consume();
// Parsing a function call.
has_arg = true;
if (Match(Token::RIGHT_PAREN)) {
@@ -563,7 +565,7 @@
}
std::unique_ptr<ParseNode> Parser::Assignment(std::unique_ptr<ParseNode> left,
- Token token) {
+ const Token& token) {
if (left->AsIdentifier() == nullptr && left->AsAccessor() == nullptr) {
*err_ = Err(left.get(),
"The left-hand side of an assignment must be an identifier, "
@@ -584,7 +586,7 @@
}
std::unique_ptr<ParseNode> Parser::Subscript(std::unique_ptr<ParseNode> left,
- Token token) {
+ const Token& token) {
// TODO: Maybe support more complex expressions like a[0][0]. This would
// require work on the evaluator too.
if (left->AsIdentifier() == nullptr) {
@@ -603,7 +605,7 @@
}
std::unique_ptr<ParseNode> Parser::DotOperator(std::unique_ptr<ParseNode> left,
- Token token) {
+ const Token& token) {
if (left->AsIdentifier() == nullptr) {
*err_ = Err(left.get(), "May only use \".\" for identifiers.",
"The thing on the left hand side of the dot must be an identifier\n"
@@ -627,7 +629,7 @@
}
// Does not Consume the start or end token.
-std::unique_ptr<ListNode> Parser::ParseList(Token start_token,
+std::unique_ptr<ListNode> Parser::ParseList(const Token& start_token,
Token::Type stop_before,
bool allow_trailing_comma) {
std::unique_ptr<ListNode> list(new ListNode);
@@ -709,7 +711,7 @@
return stmt;
}
if (!has_error()) {
- Token token = cur_or_last_token();
+ const Token& token = cur_or_last_token();
*err_ = Err(token, "Expecting assignment or function call.");
}
return std::unique_ptr<ParseNode>();
@@ -717,7 +719,7 @@
}
std::unique_ptr<BlockNode> Parser::ParseBlock(
- Token begin_brace,
+ const Token& begin_brace,
BlockNode::ResultMode result_mode) {
if (has_error())
return std::unique_ptr<BlockNode>();
diff --git a/tools/gn/parser.h b/tools/gn/parser.h
index 22acb47..69dc8cf 100644
--- a/tools/gn/parser.h
+++ b/tools/gn/parser.h
@@ -17,9 +17,9 @@
#include "tools/gn/parse_tree.h"
class Parser;
-typedef std::unique_ptr<ParseNode> (Parser::*PrefixFunc)(Token token);
+typedef std::unique_ptr<ParseNode> (Parser::*PrefixFunc)(const Token& token);
typedef std::unique_ptr<ParseNode> (
- Parser::*InfixFunc)(std::unique_ptr<ParseNode> left, Token token);
+ Parser::*InfixFunc)(std::unique_ptr<ParseNode> left, const Token& token);
extern const char kGrammar_Help[];
@@ -58,29 +58,29 @@
std::unique_ptr<ParseNode> ParseExpression(int precedence);
// |PrefixFunc|s used in parsing expressions.
- std::unique_ptr<ParseNode> Block(Token token);
- std::unique_ptr<ParseNode> Literal(Token token);
- std::unique_ptr<ParseNode> Name(Token token);
- std::unique_ptr<ParseNode> Group(Token token);
- std::unique_ptr<ParseNode> Not(Token token);
- std::unique_ptr<ParseNode> List(Token token);
- std::unique_ptr<ParseNode> BlockComment(Token token);
+ std::unique_ptr<ParseNode> Block(const Token& token);
+ std::unique_ptr<ParseNode> Literal(const Token& token);
+ std::unique_ptr<ParseNode> Name(const Token& token);
+ std::unique_ptr<ParseNode> Group(const Token& token);
+ std::unique_ptr<ParseNode> Not(const Token& token);
+ std::unique_ptr<ParseNode> List(const Token& token);
+ std::unique_ptr<ParseNode> BlockComment(const Token& token);
// |InfixFunc|s used in parsing expressions.
std::unique_ptr<ParseNode> BinaryOperator(std::unique_ptr<ParseNode> left,
- Token token);
+ const Token& token);
std::unique_ptr<ParseNode> IdentifierOrCall(std::unique_ptr<ParseNode> left,
- Token token);
+ const Token& token);
std::unique_ptr<ParseNode> Assignment(std::unique_ptr<ParseNode> left,
- Token token);
+ const Token& token);
std::unique_ptr<ParseNode> Subscript(std::unique_ptr<ParseNode> left,
- Token token);
+ const Token& token);
std::unique_ptr<ParseNode> DotOperator(std::unique_ptr<ParseNode> left,
- Token token);
+ const Token& token);
// Helper to parse a comma separated list, optionally allowing trailing
// commas (allowed in [] lists, not in function calls).
- std::unique_ptr<ListNode> ParseList(Token start_token,
+ std::unique_ptr<ListNode> ParseList(const Token& start_token,
Token::Type stop_before,
bool allow_trailing_comma);
@@ -88,7 +88,7 @@
std::unique_ptr<ParseNode> ParseStatement();
// Expects to be passed the token corresponding to the '{' and that the
// current token is the one following the '{'.
- std::unique_ptr<BlockNode> ParseBlock(Token being_brace,
+ std::unique_ptr<BlockNode> ParseBlock(const Token& begin_brace,
BlockNode::ResultMode result_mode);
std::unique_ptr<ParseNode> ParseCondition();
@@ -105,11 +105,11 @@
bool LookAhead(Token::Type type);
bool Match(Token::Type type);
- Token Consume(Token::Type type, const char* error_message);
- Token Consume(Token::Type* types,
- size_t num_types,
- const char* error_message);
- Token Consume();
+ const Token& Consume(Token::Type type, const char* error_message);
+ const Token& Consume(Token::Type* types,
+ size_t num_types,
+ const char* error_message);
+ const Token& Consume();
// Call this only if !at_end().
const Token& cur_token() const { return tokens_[cur_]; }
@@ -128,6 +128,7 @@
static ParserHelper expressions_[Token::NUM_TYPES];
+ Token invalid_token_;
Err* err_;
// Current index into the tokens.
