Support `format --read-tree=json`

Allows roundtripping to json and back with `gn format --dump-tree=json
a.gn` and then `gn format --read-tree=json a.gn`. The preexisting
.json format is not the most pleasant to work with, but this does at
least allow for programmatic editing of the gn files with knowledge of
structure, going beyond naive string replacement.

For example, a python script to change `deps` inside `source_sets` from
`=` to `+=` to support a BUILDCONFIG.gn change:
https://gist.github.com/sgraham/bd9ffee312f307d5f417019a9c0f0777

The .json format was extended slightly, mostly to include location
information which is necessary to faithfully roundtrip including
whitespace. If the input .gn is formatted by `gn format`, then it can
roundtrip to json without change. This was confirmed by running
--dump-tree=json => --read-tree=json on all .gn and .gni files in both
the Fuchsia and Chromium trees
(https://gist.github.com/sgraham/bc561f19cd354746dd2469103e1f97d4), as
well as by adding a roundtrip step to the formatter tests in-repo.

Change-Id: Ie10bbbcb32b8184400ca9bb1f6ab63e5ca9c89e7
Reviewed-on: https://gn-review.googlesource.com/c/gn/+/10700
Reviewed-by: Brett Wilson <brettw@chromium.org>
Commit-Queue: Scott Graham <scottmg@chromium.org>
diff --git a/src/gn/command_format.cc b/src/gn/command_format.cc
index 20b3ccb..e94fb4f 100644
--- a/src/gn/command_format.cc
+++ b/src/gn/command_format.cc
@@ -10,6 +10,7 @@
 
 #include "base/command_line.h"
 #include "base/files/file_util.h"
+#include "base/json/json_reader.h"
 #include "base/json/json_writer.h"
 #include "base/macros.h"
 #include "base/strings/string_split.h"
@@ -35,9 +36,10 @@
 
 const char kSwitchDryRun[] = "dry-run";
 const char kSwitchDumpTree[] = "dump-tree";
-const char kSwitchDumpTreeText[] = "text";
-const char kSwitchDumpTreeJSON[] = "json";
+const char kSwitchReadTree[] = "read-tree";
 const char kSwitchStdin[] = "stdin";
+const char kSwitchTreeTypeJSON[] = "json";
+const char kSwitchTreeTypeText[] = "text";
 
 const char kFormat[] = "format";
 const char kFormat_HelpShort[] = "format: Format .gn files.";
@@ -73,11 +75,18 @@
       Read input from stdin and write to stdout rather than update a file
       in-place.
 
+  --read-tree=json
+      Reads an AST from stdin in the format output by --dump-tree=json and
+      uses that as the parse tree. (The only read-tree format currently
+      supported is json.) The given .gn file will be overwritten. This can be
+      used to programmatically transform .gn files.
+
 Examples
   gn format //some/BUILD.gn //some/other/BUILD.gn //and/another/BUILD.gn
   gn format some\\BUILD.gn
   gn format /abspath/some/BUILD.gn
   gn format --stdin
+  gn format --read-tree=json //rewritten/BUILD.gn
 )";
 
 namespace {
@@ -324,7 +333,7 @@
         if (have_empty_line && continuation) {
           Print("# ");
         }
-        Print(split_on_spaces[j]) ;
+        Print(split_on_spaces[j]);
         Print(" ");
         if (split_on_spaces[j] != "#") {
           have_empty_line = false;
@@ -403,7 +412,7 @@
         lhs == "public")
       const_cast<ListNode*>(list)->SortAsStringsList();
     else if (base::EndsWith(lhs, "deps", base::CompareCase::SENSITIVE) ||
-        lhs == "visibility")
+             lhs == "visibility")
       const_cast<ListNode*>(list)->SortAsTargetsList();
   }
 }
@@ -519,7 +528,7 @@
     RETURN_IF_SET(SuffixCommentTreeWalk(binop->right()));
   } else if (const BlockNode* block = node->AsBlock()) {
     RETURN_IF_SET(SuffixCommentTreeWalk(block->End()));
-  } else if (const ConditionNode* condition = node->AsConditionNode()) {
+  } else if (const ConditionNode* condition = node->AsCondition()) {
     RETURN_IF_SET(SuffixCommentTreeWalk(condition->if_false()));
     RETURN_IF_SET(SuffixCommentTreeWalk(condition->if_true()));
     RETURN_IF_SET(SuffixCommentTreeWalk(condition->condition()));
@@ -869,7 +878,7 @@
   } else if (const BlockNode* block = root->AsBlock()) {
     Sequence(kSequenceStyleBracedBlock, block->statements(), block->End(),
              false);
-  } else if (const ConditionNode* condition = root->AsConditionNode()) {
+  } else if (const ConditionNode* condition = root->AsCondition()) {
     Print("if (");
     CHECK(at_end.empty());
     Expr(condition->condition(), kPrecedenceLowest, ") {");
@@ -1206,21 +1215,17 @@
 
 void DoFormat(const ParseNode* root,
               TreeDumpMode dump_tree,
-              std::string* output) {
-#if defined(OS_WIN)
-    // Set stdout to binary mode to prevent converting newlines to \r\n.
-    _setmode(_fileno(stdout), _O_BINARY);
-#endif
-
+              std::string* output,
+              std::string* dump_output) {
   if (dump_tree == TreeDumpMode::kPlainText) {
     std::ostringstream os;
     RenderToText(root->GetJSONNode(), 0, os);
-    fprintf(stdout, "%s", os.str().c_str());
+    *dump_output = os.str();
   } else if (dump_tree == TreeDumpMode::kJSON) {
     std::string os;
     base::JSONWriter::WriteWithOptions(
         root->GetJSONNode(), base::JSONWriter::OPTIONS_PRETTY_PRINT, &os);
-    fprintf(stdout, "%s", os.c_str());
+    *dump_output = os;
   }
 
   Printer pr;
@@ -1230,9 +1235,18 @@
 
 }  // namespace
 
+bool FormatJsonToString(const std::string& json, std::string* output) {
+  base::JSONReader reader;
+  std::unique_ptr<base::Value> json_root = reader.Read(json);
+  std::unique_ptr<ParseNode> root = ParseNode::BuildFromJSON(*json_root);
+  DoFormat(root.get(), TreeDumpMode::kInactive, output, nullptr);
+  return true;
+}
+
 bool FormatStringToString(const std::string& input,
                           TreeDumpMode dump_tree,
-                          std::string* output) {
+                          std::string* output,
+                          std::string* dump_output) {
   SourceFile source_file;
   InputFile file(source_file);
   file.SetContents(input);
@@ -1252,11 +1266,17 @@
     return false;
   }
 
-  DoFormat(parse_node.get(), dump_tree, output);
+  DoFormat(parse_node.get(), dump_tree, output, dump_output);
   return true;
 }
 
 int RunFormat(const std::vector<std::string>& args) {
+#if defined(OS_WIN)
+  // Set to binary mode to prevent converting newlines to \r\n.
+  _setmode(_fileno(stdout), _O_BINARY);
+  _setmode(_fileno(stderr), _O_BINARY);
+#endif
+
   bool dry_run =
       base::CommandLine::ForCurrentProcess()->HasSwitch(kSwitchDryRun);
   TreeDumpMode dump_tree = TreeDumpMode::kInactive;
@@ -1264,16 +1284,16 @@
     std::string tree_type =
         base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
             kSwitchDumpTree);
-    if (tree_type == kSwitchDumpTreeJSON) {
+    if (tree_type == kSwitchTreeTypeJSON) {
       dump_tree = TreeDumpMode::kJSON;
-    } else if (tree_type.empty() || tree_type == kSwitchDumpTreeText) {
+    } else if (tree_type.empty() || tree_type == kSwitchTreeTypeText) {
       dump_tree = TreeDumpMode::kPlainText;
     } else {
       Err(Location(), tree_type +
                           " is an invalid value for --dump-tree. Specify "
                           "\"" +
-                          kSwitchDumpTreeText + "\" or \"" +
-                          kSwitchDumpTreeJSON + "\".\n")
+                          kSwitchTreeTypeText + "\" or \"" +
+                          kSwitchTreeTypeJSON + "\".\n")
           .PrintToStdout();
       return 1;
     }
@@ -1297,12 +1317,10 @@
     }
     std::string input = ReadStdin();
     std::string output;
-    if (!FormatStringToString(input, dump_tree, &output))
+    std::string dump_output;
+    if (!FormatStringToString(input, dump_tree, &output, &dump_output))
       return 1;
-#if defined(OS_WIN)
-    // Set stdout to binary mode to prevent converting newlines to \r\n.
-    _setmode(_fileno(stdout), _O_BINARY);
-#endif
+    printf("%s", dump_output.c_str());
     printf("%s", output.c_str());
     return 0;
   }
@@ -1317,6 +1335,45 @@
   SourceDir source_dir =
       SourceDirForCurrentDirectory(setup.build_settings().root_path());
 
+  if (base::CommandLine::ForCurrentProcess()->HasSwitch(kSwitchReadTree)) {
+    std::string tree_type =
+        base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
+            kSwitchReadTree);
+    if (tree_type != kSwitchTreeTypeJSON) {
+      Err(Location(), "Only json supported for read-tree.\n").PrintToStdout();
+      return 1;
+    }
+
+    if (args.size() != 1) {
+      Err(Location(),
+          "Expect exactly one .gn when reading tree from json on stdin.\n")
+          .PrintToStdout();
+      return 1;
+    }
+    Err err;
+    SourceFile file =
+        source_dir.ResolveRelativeFile(Value(nullptr, args[0]), &err);
+    if (err.has_error()) {
+      err.PrintToStdout();
+      return 1;
+    }
+    base::FilePath to_format = setup.build_settings().GetFullPath(file);
+    std::string output;
+    FormatJsonToString(ReadStdin(), &output);
+    if (base::WriteFile(to_format, output.data(),
+                        static_cast<int>(output.size())) == -1) {
+      Err(Location(), std::string("Failed to write output to \"") +
+                          FilePathToUTF8(to_format) + std::string("\"."))
+          .PrintToStdout();
+      return 1;
+    }
+    if (!quiet) {
+      printf("Wrote rebuilt from json to '%s'.\n",
+             FilePathToUTF8(to_format).c_str());
+    }
+    return 0;
+  }
+
   // TODO(scottmg): Eventually, this list of files should be processed in
   // parallel.
   int exit_code = 0;
@@ -1340,10 +1397,13 @@
     }
 
     std::string output_string;
-    if (!FormatStringToString(original_contents, dump_tree, &output_string)) {
+    std::string dump_output_string;
+    if (!FormatStringToString(original_contents, dump_tree, &output_string,
+                              &dump_output_string)) {
       exit_code = 1;
       continue;
     }
+    printf("%s", dump_output_string.c_str());
     if (dump_tree == TreeDumpMode::kInactive) {
       if (dry_run) {
         if (original_contents != output_string) {
@@ -1365,7 +1425,7 @@
         }
         if (!quiet) {
           printf("Wrote formatted to '%s'.\n",
-                FilePathToUTF8(to_format).c_str());
+                 FilePathToUTF8(to_format).c_str());
         }
       }
     }
diff --git a/src/gn/command_format.h b/src/gn/command_format.h
index 785b070..0eb667f 100644
--- a/src/gn/command_format.h
+++ b/src/gn/command_format.h
@@ -24,14 +24,12 @@
   kJSON
 };
 
-bool FormatFileToString(Setup* setup,
-                        const SourceFile& file,
-                        TreeDumpMode dump_tree,
-                        std::string* output);
+bool FormatJsonToString(const std::string& input, std::string* output);
 
 bool FormatStringToString(const std::string& input,
                           TreeDumpMode dump_tree,
-                          std::string* output);
+                          std::string* output,
+                          std::string* dump_output);
 
 }  // namespace commands
 
diff --git a/src/gn/command_format_unittest.cc b/src/gn/command_format_unittest.cc
index 2141f6f..9112dd3 100644
--- a/src/gn/command_format_unittest.cc
+++ b/src/gn/command_format_unittest.cc
@@ -33,13 +33,21 @@
                                FILE_PATH_LITERAL(".golden")),               \
         &expected));                                                        \
     EXPECT_TRUE(commands::FormatStringToString(                             \
-        input, commands::TreeDumpMode::kInactive, &out));                   \
+        input, commands::TreeDumpMode::kInactive, &out, nullptr));          \
     EXPECT_EQ(expected, out);                                               \
     /* Make sure formatting the output doesn't cause further changes. */    \
     std::string out_again;                                                  \
     EXPECT_TRUE(commands::FormatStringToString(                             \
-        out, commands::TreeDumpMode::kInactive, &out_again));               \
+        out, commands::TreeDumpMode::kInactive, &out_again, nullptr));      \
     ASSERT_EQ(out, out_again);                                              \
+    /* Make sure we can roundtrip to json without any changes. */           \
+    std::string as_json;                                                    \
+    std::string unused;                                                     \
+    EXPECT_TRUE(commands::FormatStringToString(                             \
+        out_again, commands::TreeDumpMode::kJSON, &unused, &as_json));      \
+    std::string rewritten;                                                  \
+    EXPECT_TRUE(commands::FormatJsonToString(as_json, &rewritten));         \
+    ASSERT_EQ(out, rewritten);                                              \
   }
 
 // These are expanded out this way rather than a runtime loop so that
diff --git a/src/gn/parse_tree.cc b/src/gn/parse_tree.cc
index e62b87e..74bc2af 100644
--- a/src/gn/parse_tree.cc
+++ b/src/gn/parse_tree.cc
@@ -26,6 +26,15 @@
 const char kJsonBeforeComment[] = "before_comment";
 const char kJsonSuffixComment[] = "suffix_comment";
 const char kJsonAfterComment[] = "after_comment";
+const char kJsonLocation[] = "location";
+const char kJsonLocationBeginLine[] = "begin_line";
+const char kJsonLocationBeginColumn[] = "begin_column";
+const char kJsonLocationEndLine[] = "end_line";
+const char kJsonLocationEndColumn[] = "end_column";
+
+// Used by Block and List.
+const char kJsonBeginToken[] = "begin_token";
+const char kJsonEnd[] = "end";
 
 namespace {
 
@@ -85,6 +94,68 @@
   return std::string_view();
 }
 
+void AddLocationJSONNodes(base::Value* dict, LocationRange location) {
+  base::Value loc(base::Value::Type::DICTIONARY);
+  loc.SetKey(kJsonLocationBeginLine,
+             base::Value(location.begin().line_number()));
+  loc.SetKey(kJsonLocationBeginColumn,
+             base::Value(location.begin().column_number()));
+  loc.SetKey(kJsonLocationEndLine, base::Value(location.end().line_number()));
+  loc.SetKey(kJsonLocationEndColumn,
+             base::Value(location.end().column_number()));
+  dict->SetKey(kJsonLocation, std::move(loc));
+}
+
+Location GetBeginLocationFromJSON(const base::Value& value) {
+  int line =
+      value.FindKey(kJsonLocation)->FindKey(kJsonLocationBeginLine)->GetInt();
+  int column =
+      value.FindKey(kJsonLocation)->FindKey(kJsonLocationBeginColumn)->GetInt();
+  return Location(nullptr, line, column, 0);
+}
+
+void GetCommentsFromJSON(ParseNode* node, const base::Value& value) {
+  Comments* comments = node->comments_mutable();
+
+  Location loc = GetBeginLocationFromJSON(value);
+
+  auto loc_for = [&loc](int line) {
+    return Location(nullptr, loc.line_number() + line, loc.column_number(), 0);
+  };
+
+  if (value.FindKey(kJsonBeforeComment)) {
+    int line = 0;
+    for (const auto& c : value.FindKey(kJsonBeforeComment)->GetList()) {
+      comments->append_before(
+          Token::ClassifyAndMake(loc_for(line), c.GetString()));
+      ++line;
+    }
+  }
+
+  if (value.FindKey(kJsonSuffixComment)) {
+    int line = 0;
+    for (const auto& c : value.FindKey(kJsonSuffixComment)->GetList()) {
+      comments->append_suffix(
+          Token::ClassifyAndMake(loc_for(line), c.GetString()));
+      ++line;
+    }
+  }
+
+  if (value.FindKey(kJsonAfterComment)) {
+    int line = 0;
+    for (const auto& c : value.FindKey(kJsonAfterComment)->GetList()) {
+      comments->append_after(
+          Token::ClassifyAndMake(loc_for(line), c.GetString()));
+      ++line;
+    }
+  }
+}
+
+Token TokenFromValue(const base::Value& value) {
+  return Token::ClassifyAndMake(GetBeginLocationFromJSON(value),
+                                value.FindKey(kJsonNodeValue)->GetString());
+}
+
 }  // namespace
 
 Comments::Comments() = default;
@@ -112,7 +183,7 @@
 const BlockNode* ParseNode::AsBlock() const {
   return nullptr;
 }
-const ConditionNode* ParseNode::AsConditionNode() const {
+const ConditionNode* ParseNode::AsCondition() const {
   return nullptr;
 }
 const EndNode* ParseNode::AsEnd() const {
@@ -140,18 +211,22 @@
   return comments_.get();
 }
 
-base::Value ParseNode::CreateJSONNode(const char* type) const {
+base::Value ParseNode::CreateJSONNode(const char* type,
+                                      LocationRange location) const {
   base::Value dict(base::Value::Type::DICTIONARY);
   dict.SetKey(kJsonNodeType, base::Value(type));
+  AddLocationJSONNodes(&dict, location);
   AddCommentsJSONNodes(&dict);
   return dict;
 }
 
 base::Value ParseNode::CreateJSONNode(const char* type,
-                                      const std::string_view& value) const {
+                                      const std::string_view& value,
+                                      LocationRange location) const {
   base::Value dict(base::Value::Type::DICTIONARY);
   dict.SetKey(kJsonNodeType, base::Value(type));
   dict.SetKey(kJsonNodeValue, base::Value(value));
+  AddLocationJSONNodes(&dict, location);
   AddCommentsJSONNodes(&dict);
   return dict;
 }
@@ -179,6 +254,33 @@
   }
 }
 
+// static
+std::unique_ptr<ParseNode> ParseNode::BuildFromJSON(const base::Value& value) {
+  const std::string& str_type = value.FindKey(kJsonNodeType)->GetString();
+
+#define RETURN_IF_MATCHES_NAME(t)     \
+  if (str_type == t::kDumpNodeName) { \
+    return t::NewFromJSON(value);     \
+  }
+
+  RETURN_IF_MATCHES_NAME(AccessorNode);
+  RETURN_IF_MATCHES_NAME(BinaryOpNode);
+  RETURN_IF_MATCHES_NAME(BlockCommentNode);
+  RETURN_IF_MATCHES_NAME(BlockNode);
+  RETURN_IF_MATCHES_NAME(ConditionNode);
+  RETURN_IF_MATCHES_NAME(EndNode);
+  RETURN_IF_MATCHES_NAME(FunctionCallNode);
+  RETURN_IF_MATCHES_NAME(IdentifierNode);
+  RETURN_IF_MATCHES_NAME(ListNode);
+  RETURN_IF_MATCHES_NAME(LiteralNode);
+  RETURN_IF_MATCHES_NAME(UnaryOpNode);
+
+#undef RETURN_IF_MATCHES_NAME
+
+  NOTREACHED() << str_type;
+  return std::unique_ptr<ParseNode>();
+}
+
 // AccessorNode ---------------------------------------------------------------
 
 AccessorNode::AccessorNode() = default;
@@ -213,16 +315,42 @@
 }
 
 base::Value AccessorNode::GetJSONNode() const {
-  base::Value dict(CreateJSONNode("ACCESSOR", base_.value()));
+  base::Value dict(CreateJSONNode(kDumpNodeName, base_.value(), GetRange()));
   base::Value child(base::Value::Type::LIST);
-  if (subscript_)
+  if (subscript_) {
     child.GetList().push_back(subscript_->GetJSONNode());
-  else if (member_)
+    dict.SetKey(kDumpAccessorKind, base::Value(kDumpAccessorKindSubscript));
+  } else if (member_) {
     child.GetList().push_back(member_->GetJSONNode());
+    dict.SetKey(kDumpAccessorKind, base::Value(kDumpAccessorKindMember));
+  }
   dict.SetKey(kJsonNodeChild, std::move(child));
   return dict;
 }
 
+#define DECLARE_CHILD_AS_LIST_OR_FAIL()                     \
+  const base::Value* child = value.FindKey(kJsonNodeChild); \
+  if (!child || !child->is_list()) {                        \
+    return nullptr;                                         \
+  }
+
+// static
+std::unique_ptr<AccessorNode> AccessorNode::NewFromJSON(
+    const base::Value& value) {
+  auto ret = std::make_unique<AccessorNode>();
+  DECLARE_CHILD_AS_LIST_OR_FAIL();
+  ret->base_ = TokenFromValue(value);
+  const base::Value::ListStorage& children = child->GetList();
+  const std::string& kind = value.FindKey(kDumpAccessorKind)->GetString();
+  if (kind == kDumpAccessorKindSubscript) {
+    ret->subscript_ = ParseNode::BuildFromJSON(children[0]);
+  } else if (kind == kDumpAccessorKindMember) {
+    ret->member_ = IdentifierNode::NewFromJSON(children[0]);
+  }
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
+}
+
 Value AccessorNode::ExecuteSubscriptAccess(Scope* scope, Err* err) const {
   const Value* base_value = scope->GetValue(base_.value(), true);
   if (!base_value) {
@@ -377,7 +505,7 @@
 }
 
 base::Value BinaryOpNode::GetJSONNode() const {
-  base::Value dict(CreateJSONNode("BINARY", op_.value()));
+  base::Value dict(CreateJSONNode(kDumpNodeName, op_.value(), GetRange()));
   base::Value child(base::Value::Type::LIST);
   child.GetList().push_back(left_->GetJSONNode());
   child.GetList().push_back(right_->GetJSONNode());
@@ -385,6 +513,19 @@
   return dict;
 }
 
+// static
+std::unique_ptr<BinaryOpNode> BinaryOpNode::NewFromJSON(
+    const base::Value& value) {
+  auto ret = std::make_unique<BinaryOpNode>();
+  DECLARE_CHILD_AS_LIST_OR_FAIL();
+  const base::Value::ListStorage& children = child->GetList();
+  ret->left_ = ParseNode::BuildFromJSON(children[0]);
+  ret->right_ = ParseNode::BuildFromJSON(children[1]);
+  ret->op_ = TokenFromValue(value);
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
+}
+
 // BlockNode ------------------------------------------------------------------
 
 BlockNode::BlockNode(ResultMode result_mode) : result_mode_(result_mode) {}
@@ -456,24 +597,64 @@
 }
 
 base::Value BlockNode::GetJSONNode() const {
-  base::Value dict(CreateJSONNode("BLOCK"));
+  base::Value dict(CreateJSONNode(kDumpNodeName, GetRange()));
   base::Value statements(base::Value::Type::LIST);
   for (const auto& statement : statements_)
     statements.GetList().push_back(statement->GetJSONNode());
-  if (end_ && end_->comments())
-    statements.GetList().push_back(end_->GetJSONNode());
+  if (end_)
+    dict.SetKey(kJsonEnd, end_->GetJSONNode());
 
-  dict.SetKey("child", std::move(statements));
+  dict.SetKey(kJsonNodeChild, std::move(statements));
+
+  if (result_mode_ == BlockNode::RETURNS_SCOPE) {
+    dict.SetKey(kDumpResultMode, base::Value(kDumpResultModeReturnsScope));
+  } else if (result_mode_ == BlockNode::DISCARDS_RESULT) {
+    dict.SetKey(kDumpResultMode, base::Value(kDumpResultModeDiscardsResult));
+  } else {
+    NOTREACHED();
+  }
+
+  dict.SetKey(kJsonBeginToken, base::Value(begin_token_.value()));
+
   return dict;
 }
 
+// static
+std::unique_ptr<BlockNode> BlockNode::NewFromJSON(const base::Value& value) {
+  const std::string& result_mode = value.FindKey(kDumpResultMode)->GetString();
+  std::unique_ptr<BlockNode> ret;
+
+  if (result_mode == kDumpResultModeReturnsScope) {
+    ret.reset(new BlockNode(BlockNode::RETURNS_SCOPE));
+  } else if (result_mode == kDumpResultModeDiscardsResult) {
+    ret.reset(new BlockNode(BlockNode::DISCARDS_RESULT));
+  } else {
+    NOTREACHED();
+  }
+
+  DECLARE_CHILD_AS_LIST_OR_FAIL();
+  for (const auto& elem : child->GetList()) {
+    ret->statements_.push_back(ParseNode::BuildFromJSON(elem));
+  }
+
+  ret->begin_token_ =
+      Token::ClassifyAndMake(GetBeginLocationFromJSON(value),
+                             value.FindKey(kJsonBeginToken)->GetString());
+  if (value.FindKey(kJsonEnd)) {
+    ret->end_ = EndNode::NewFromJSON(*value.FindKey(kJsonEnd));
+  }
+
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
+}
+
 // ConditionNode --------------------------------------------------------------
 
 ConditionNode::ConditionNode() = default;
 
 ConditionNode::~ConditionNode() = default;
 
-const ConditionNode* ConditionNode::AsConditionNode() const {
+const ConditionNode* ConditionNode::AsCondition() const {
   return this;
 }
 
@@ -512,7 +693,7 @@
 }
 
 base::Value ConditionNode::GetJSONNode() const {
-  base::Value dict = CreateJSONNode("CONDITION");
+  base::Value dict = CreateJSONNode(kDumpNodeName, GetRange());
   base::Value child(base::Value::Type::LIST);
   child.GetList().push_back(condition_->GetJSONNode());
   child.GetList().push_back(if_true_->GetJSONNode());
@@ -523,6 +704,25 @@
   return dict;
 }
 
+// static
+std::unique_ptr<ConditionNode> ConditionNode::NewFromJSON(
+    const base::Value& value) {
+  auto ret = std::make_unique<ConditionNode>();
+
+  DECLARE_CHILD_AS_LIST_OR_FAIL();
+  const base::Value::ListStorage& children = child->GetList();
+
+  ret->if_token_ =
+      Token::ClassifyAndMake(GetBeginLocationFromJSON(value), "if");
+  ret->condition_ = ParseNode::BuildFromJSON(children[0]);
+  ret->if_true_ = BlockNode::NewFromJSON(children[1]);
+  if (children.size() > 2) {
+    ret->if_false_ = ParseNode::BuildFromJSON(children[2]);
+  }
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
+}
+
 // FunctionCallNode -----------------------------------------------------------
 
 FunctionCallNode::FunctionCallNode() = default;
@@ -551,7 +751,8 @@
 }
 
 base::Value FunctionCallNode::GetJSONNode() const {
-  base::Value dict = CreateJSONNode("FUNCTION", function_.value());
+  base::Value dict =
+      CreateJSONNode(kDumpNodeName, function_.value(), GetRange());
   base::Value child(base::Value::Type::LIST);
   child.GetList().push_back(args_->GetJSONNode());
   if (block_) {
@@ -561,6 +762,22 @@
   return dict;
 }
 
+// static
+std::unique_ptr<FunctionCallNode> FunctionCallNode::NewFromJSON(
+    const base::Value& value) {
+  auto ret = std::make_unique<FunctionCallNode>();
+
+  DECLARE_CHILD_AS_LIST_OR_FAIL();
+  const base::Value::ListStorage& children = child->GetList();
+  ret->function_ = TokenFromValue(value);
+  ret->args_ = ListNode::NewFromJSON(children[0]);
+  if (children.size() > 1)
+    ret->block_ = BlockNode::NewFromJSON(children[1]);
+
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
+}
+
 void FunctionCallNode::SetNewLocation(int line_number) {
   Location func_old_loc = function_.location();
   Location func_new_loc =
@@ -616,7 +833,16 @@
 }
 
 base::Value IdentifierNode::GetJSONNode() const {
-  return CreateJSONNode("IDENTIFIER", value_.value());
+  return CreateJSONNode(kDumpNodeName, value_.value(), GetRange());
+}
+
+// static
+std::unique_ptr<IdentifierNode> IdentifierNode::NewFromJSON(
+    const base::Value& value) {
+  auto ret = std::make_unique<IdentifierNode>();
+  ret->set_value(TokenFromValue(value));
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
 }
 
 void IdentifierNode::SetNewLocation(int line_number) {
@@ -665,18 +891,37 @@
 }
 
 base::Value ListNode::GetJSONNode() const {
-  base::Value dict(CreateJSONNode("LIST"));
+  base::Value dict(CreateJSONNode(kDumpNodeName, GetRange()));
   base::Value child(base::Value::Type::LIST);
   for (const auto& cur : contents_) {
     child.GetList().push_back(cur->GetJSONNode());
   }
-  if (end_ && end_->comments()) {
-    child.GetList().push_back(end_->GetJSONNode());
-  }
+  if (end_)
+    dict.SetKey(kJsonEnd, end_->GetJSONNode());
   dict.SetKey(kJsonNodeChild, std::move(child));
+  dict.SetKey(kJsonBeginToken, base::Value(begin_token_.value()));
   return dict;
 }
 
+// static
+std::unique_ptr<ListNode> ListNode::NewFromJSON(const base::Value& value) {
+  auto ret = std::make_unique<ListNode>();
+
+  DECLARE_CHILD_AS_LIST_OR_FAIL();
+  for (const auto& elem : child->GetList()) {
+    ret->contents_.push_back(ParseNode::BuildFromJSON(elem));
+  }
+  ret->begin_token_ =
+      Token::ClassifyAndMake(GetBeginLocationFromJSON(value),
+                             value.FindKey(kJsonBeginToken)->GetString());
+  if (value.FindKey(kJsonEnd)) {
+    ret->end_ = EndNode::NewFromJSON(*value.FindKey(kJsonEnd));
+  }
+
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
+}
+
 template <typename Comparator>
 void ListNode::SortList(Comparator comparator) {
   // Partitions first on BlockCommentNodes and sorts each partition separately.
@@ -875,7 +1120,16 @@
 }
 
 base::Value LiteralNode::GetJSONNode() const {
-  return CreateJSONNode("LITERAL", value_.value());
+  return CreateJSONNode(kDumpNodeName, value_.value(), GetRange());
+}
+
+// static
+std::unique_ptr<LiteralNode> LiteralNode::NewFromJSON(
+    const base::Value& value) {
+  auto ret = std::make_unique<LiteralNode>();
+  ret->value_ = TokenFromValue(value);
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
 }
 
 void LiteralNode::SetNewLocation(int line_number) {
@@ -911,13 +1165,24 @@
 }
 
 base::Value UnaryOpNode::GetJSONNode() const {
-  base::Value dict = CreateJSONNode("UNARY", op_.value());
+  base::Value dict = CreateJSONNode(kDumpNodeName, op_.value(), GetRange());
   base::Value child(base::Value::Type::LIST);
   child.GetList().push_back(operand_->GetJSONNode());
   dict.SetKey(kJsonNodeChild, std::move(child));
   return dict;
 }
 
+// static
+std::unique_ptr<UnaryOpNode> UnaryOpNode::NewFromJSON(
+    const base::Value& value) {
+  auto ret = std::make_unique<UnaryOpNode>();
+  ret->op_ = TokenFromValue(value);
+  DECLARE_CHILD_AS_LIST_OR_FAIL();
+  ret->operand_ = ParseNode::BuildFromJSON(child->GetList()[0]);
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
+}
+
 // BlockCommentNode ------------------------------------------------------------
 
 BlockCommentNode::BlockCommentNode() = default;
@@ -943,8 +1208,17 @@
 
 base::Value BlockCommentNode::GetJSONNode() const {
   std::string escaped;
-  base::EscapeJSONString(std::string(comment_.value()), false, &escaped);
-  return CreateJSONNode("BLOCK_COMMENT", escaped);
+  return CreateJSONNode(kDumpNodeName, comment_.value(), GetRange());
+}
+
+// static
+std::unique_ptr<BlockCommentNode> BlockCommentNode::NewFromJSON(
+    const base::Value& value) {
+  auto ret = std::make_unique<BlockCommentNode>();
+  ret->comment_ = Token(GetBeginLocationFromJSON(value), Token::BLOCK_COMMENT,
+                        value.FindKey(kJsonNodeValue)->GetString());
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
 }
 
 // EndNode ---------------------------------------------------------------------
@@ -971,5 +1245,12 @@
 }
 
 base::Value EndNode::GetJSONNode() const {
-  return CreateJSONNode("END", value_.value());
+  return CreateJSONNode(kDumpNodeName, value_.value(), GetRange());
+}
+
+// static
+std::unique_ptr<EndNode> EndNode::NewFromJSON(const base::Value& value) {
+  auto ret = std::make_unique<EndNode>(TokenFromValue(value));
+  GetCommentsFromJSON(ret.get(), value);
+  return ret;
 }
diff --git a/src/gn/parse_tree.h b/src/gn/parse_tree.h
index cb64661..9ffacc2 100644
--- a/src/gn/parse_tree.h
+++ b/src/gn/parse_tree.h
@@ -83,7 +83,7 @@
   virtual const BinaryOpNode* AsBinaryOp() const;
   virtual const BlockCommentNode* AsBlockComment() const;
   virtual const BlockNode* AsBlock() const;
-  virtual const ConditionNode* AsConditionNode() const;
+  virtual const ConditionNode* AsCondition() const;
   virtual const EndNode* AsEnd() const;
   virtual const FunctionCallNode* AsFunctionCall() const;
   virtual const IdentifierNode* AsIdentifier() const;
@@ -108,12 +108,15 @@
   const Comments* comments() const { return comments_.get(); }
   Comments* comments_mutable();
 
+  static std::unique_ptr<ParseNode> BuildFromJSON(const base::Value& value);
+
  protected:
   // Helper functions for GetJSONNode. Creates and fills a Value object with
   // given type (and value).
-  base::Value CreateJSONNode(const char* type) const;
+  base::Value CreateJSONNode(const char* type, LocationRange location) const;
   base::Value CreateJSONNode(const char* type,
-                             const std::string_view& value) const;
+                             const std::string_view& value,
+                             LocationRange location) const;
 
  private:
   // Helper function for CreateJSONNode.
@@ -161,6 +164,7 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<AccessorNode> NewFromJSON(const base::Value& value);
 
   // Base is the thing on the left of the [] or dot, currently always required
   // to be an identifier token.
@@ -189,6 +193,8 @@
                                    size_t* computed_index,
                                    Err* err) const;
 
+  static constexpr const char* kDumpNodeName = "ACCESSOR";
+
  private:
   Value ExecuteSubscriptAccess(Scope* scope, Err* err) const;
   Value ExecuteArrayAccess(Scope* scope,
@@ -199,6 +205,10 @@
                                     Err* err) const;
   Value ExecuteScopeAccess(Scope* scope, Err* err) const;
 
+  static constexpr const char* kDumpAccessorKind = "accessor_kind";
+  static constexpr const char* kDumpAccessorKindSubscript = "subscript";
+  static constexpr const char* kDumpAccessorKindMember = "member";
+
   Token base_;
 
   // Either index or member will be set according to what type of access this
@@ -223,6 +233,7 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<BinaryOpNode> NewFromJSON(const base::Value& value);
 
   const Token& op() const { return op_; }
   void set_op(const Token& t) { op_ = t; }
@@ -235,6 +246,8 @@
     right_ = std::move(right);
   }
 
+  static constexpr const char* kDumpNodeName = "BINARY";
+
  private:
   std::unique_ptr<ParseNode> left_;
   Token op_;
@@ -268,6 +281,7 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<BlockNode> NewFromJSON(const base::Value& value);
 
   void set_begin_token(const Token& t) { begin_token_ = t; }
   void set_end(std::unique_ptr<EndNode> e) { end_ = std::move(e); }
@@ -282,7 +296,14 @@
     statements_.push_back(std::move(s));
   }
 
+  static constexpr const char* kDumpNodeName = "BLOCK";
+
  private:
+  static constexpr const char* kDumpResultMode = "result_mode";
+  static constexpr const char* kDumpResultModeReturnsScope = "returns_scope";
+  static constexpr const char* kDumpResultModeDiscardsResult =
+      "discards_result";
+
   const ResultMode result_mode_;
 
   // Tokens corresponding to { and }, if any (may be NULL). The end is stored
@@ -302,13 +323,14 @@
   ConditionNode();
   ~ConditionNode() override;
 
-  const ConditionNode* AsConditionNode() const override;
+  const ConditionNode* AsCondition() const override;
   Value Execute(Scope* scope, Err* err) const override;
   LocationRange GetRange() const override;
   Err MakeErrorDescribing(
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<ConditionNode> NewFromJSON(const base::Value& value);
 
   void set_if_token(const Token& token) { if_token_ = token; }
 
@@ -325,6 +347,8 @@
   const ParseNode* if_false() const { return if_false_.get(); }
   void set_if_false(std::unique_ptr<ParseNode> f) { if_false_ = std::move(f); }
 
+  static constexpr const char* kDumpNodeName = "CONDITION";
+
  private:
   // Token corresponding to the "if" string.
   Token if_token_;
@@ -350,6 +374,8 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<FunctionCallNode> NewFromJSON(
+      const base::Value& value);
 
   const Token& function() const { return function_; }
   void set_function(Token t) { function_ = t; }
@@ -362,6 +388,8 @@
 
   void SetNewLocation(int line_number);
 
+  static constexpr const char* kDumpNodeName = "FUNCTION";
+
  private:
   Token function_;
   std::unique_ptr<ListNode> args_;
@@ -385,12 +413,15 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<IdentifierNode> NewFromJSON(const base::Value& value);
 
   const Token& value() const { return value_; }
   void set_value(const Token& t) { value_ = t; }
 
   void SetNewLocation(int line_number);
 
+  static constexpr const char* kDumpNodeName = "IDENTIFIER";
+
  private:
   Token value_;
 
@@ -411,6 +442,7 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<ListNode> NewFromJSON(const base::Value& value);
 
   void set_begin_token(const Token& t) { begin_token_ = t; }
   const Token& Begin() const { return begin_token_; }
@@ -435,6 +467,8 @@
   // Only public for testing.
   std::vector<SortRange> GetSortRanges() const;
 
+  static constexpr const char* kDumpNodeName = "LIST";
+
  private:
   template <typename Comparator>
   void SortList(Comparator comparator);
@@ -464,12 +498,15 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<LiteralNode> NewFromJSON(const base::Value& value);
 
   const Token& value() const { return value_; }
   void set_value(const Token& t) { value_ = t; }
 
   void SetNewLocation(int line_number);
 
+  static constexpr const char* kDumpNodeName = "LITERAL";
+
  private:
   Token value_;
 
@@ -490,6 +527,7 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<UnaryOpNode> NewFromJSON(const base::Value& value);
 
   const Token& op() const { return op_; }
   void set_op(const Token& t) { op_ = t; }
@@ -499,6 +537,8 @@
     operand_ = std::move(operand);
   }
 
+  static constexpr const char* kDumpNodeName = "UNARY";
+
  private:
   Token op_;
   std::unique_ptr<ParseNode> operand_;
@@ -525,10 +565,14 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<BlockCommentNode> NewFromJSON(
+      const base::Value& value);
 
   const Token& comment() const { return comment_; }
   void set_comment(const Token& t) { comment_ = t; }
 
+  static constexpr const char* kDumpNodeName = "BLOCK_COMMENT";
+
  private:
   Token comment_;
 
@@ -553,10 +597,13 @@
       const std::string& msg,
       const std::string& help = std::string()) const override;
   base::Value GetJSONNode() const override;
+  static std::unique_ptr<EndNode> NewFromJSON(const base::Value& value);
 
   const Token& value() const { return value_; }
   void set_value(const Token& t) { value_ = t; }
 
+  static constexpr const char* kDumpNodeName = "END";
+
  private:
   Token value_;
 
diff --git a/src/gn/parser.cc b/src/gn/parser.cc
index 0cb7314..07e9ba3 100644
--- a/src/gn/parser.cc
+++ b/src/gn/parser.cc
@@ -788,7 +788,7 @@
       for (const auto& statement : block->statements())
         TraverseOrder(statement.get(), pre, post);
       TraverseOrder(block->End(), pre, post);
-    } else if (const ConditionNode* condition = root->AsConditionNode()) {
+    } else if (const ConditionNode* condition = root->AsCondition()) {
       TraverseOrder(condition->condition(), pre, post);
       TraverseOrder(condition->if_true(), pre, post);
       TraverseOrder(condition->if_false(), pre, post);
@@ -932,4 +932,10 @@
       RenderToText(n, indent_level + 1, os);
     }
   }
+  const base::Value* end = node.FindKey("end");
+  if (end &&
+      (end->FindKey(kJsonBeforeComment) || end->FindKey(kJsonSuffixComment) ||
+       end->FindKey(kJsonAfterComment))) {
+    RenderToText(*end, indent_level + 1, os);
+  }
 }
diff --git a/src/gn/setup.cc b/src/gn/setup.cc
index 8c402f1..0e3e4a2 100644
--- a/src/gn/setup.cc
+++ b/src/gn/setup.cc
@@ -588,7 +588,7 @@
 
   std::string contents = args_input_file_->contents();
   commands::FormatStringToString(contents, commands::TreeDumpMode::kInactive,
-                                 &contents);
+                                 &contents, nullptr);
 #if defined(OS_WIN)
   // Use Windows lineendings for this file since it will often open in
   // Notepad which can't handle Unix ones.
diff --git a/src/gn/token.cc b/src/gn/token.cc
index 0466ce7..3af9c48 100644
--- a/src/gn/token.cc
+++ b/src/gn/token.cc
@@ -5,12 +5,21 @@
 #include "gn/token.h"
 
 #include "base/logging.h"
+#include "gn/tokenizer.h"
 
 Token::Token() : type_(INVALID), value_() {}
 
 Token::Token(const Location& location, Type t, const std::string_view& v)
     : type_(t), value_(v), location_(location) {}
 
+// static
+Token Token::ClassifyAndMake(const Location& location,
+                             const std::string_view& v) {
+  char first = v.size() > 0 ? v[0] : '\0';
+  char second = v.size() > 1 ? v[1] : '\0';
+  return Token(location, Tokenizer::ClassifyToken(first, second), v);
+}
+
 bool Token::IsIdentifierEqualTo(const char* v) const {
   return type_ == IDENTIFIER && value_ == v;
 }
diff --git a/src/gn/token.h b/src/gn/token.h
index c074620..a11807a 100644
--- a/src/gn/token.h
+++ b/src/gn/token.h
@@ -60,6 +60,9 @@
   Token();
   Token(const Location& location, Type t, const std::string_view& v);
 
+  static Token ClassifyAndMake(const Location& location,
+                               const std::string_view& v);
+
   Type type() const { return type_; }
   const std::string_view& value() const { return value_; }
   const Location& location() const { return location_; }
diff --git a/src/gn/tokenizer.cc b/src/gn/tokenizer.cc
index eec2349..93a6425 100644
--- a/src/gn/tokenizer.cc
+++ b/src/gn/tokenizer.cc
@@ -198,9 +198,8 @@
     Advance();
 }
 
-Token::Type Tokenizer::ClassifyCurrent() const {
-  DCHECK(!at_end());
-  char next_char = cur_char();
+// static
+Token::Type Tokenizer::ClassifyToken(char next_char, char following_char) {
   if (base::IsAsciiDigit(next_char))
     return Token::INTEGER;
   if (next_char == '"')
@@ -237,10 +236,9 @@
   // For the case of '-' differentiate between a negative number and anything
   // else.
   if (next_char == '-') {
-    if (!CanIncrement())
+    if (following_char == '\0')
       return Token::UNCLASSIFIED_OPERATOR;  // Just the minus before end of
                                             // file.
-    char following_char = input_[cur_ + 1];
     if (base::IsAsciiDigit(following_char))
       return Token::INTEGER;
     return Token::UNCLASSIFIED_OPERATOR;
@@ -249,6 +247,13 @@
   return Token::INVALID;
 }
 
+Token::Type Tokenizer::ClassifyCurrent() const {
+  DCHECK(!at_end());
+  char next_char = cur_char();
+  char following_char = CanIncrement() ? input_[cur_ + 1] : '\0';
+  return ClassifyToken(next_char, following_char);
+}
+
 void Tokenizer::AdvanceToEndOfToken(const Location& location,
                                     Token::Type type) {
   switch (type) {
diff --git a/src/gn/tokenizer.h b/src/gn/tokenizer.h
index fb74d2a..d4b347b 100644
--- a/src/gn/tokenizer.h
+++ b/src/gn/tokenizer.h
@@ -50,6 +50,8 @@
 
   static bool IsIdentifierContinuingChar(char c);
 
+  static Token::Type ClassifyToken(char next_char, char following_char);
+
  private:
   // InputFile must outlive the tokenizer and all generated tokens.
   Tokenizer(const InputFile* input_file,