Merge #518

518: Token strings r=philberty a=tromey

This series implements a couple of small changes to improve the handling
of strings attached to Token.

Co-authored-by: Tom Tromey <tom@tromey.com>

2 files changed, 37 insertions, 34 deletions

diff --git a/gcc/rust/lex/rust-lex.cc b/gcc/rust/lex/rust-lex.cc
index dcb57c8..b320401 100644
--- a/gcc/rust/lex/rust-lex.cc
+++ b/gcc/rust/lex/rust-lex.cc
@@ -1,4 +1,4 @@
-// Copyright (C) 2020 Free Software Foundation, Inc.
+// Copyright (C) 2020, 2021 Free Software Foundation, Inc.
 
 // This file is part of GCC.
 
@@ -1436,7 +1436,7 @@ Lexer::parse_byte_string (Location loc)
 
   str.shrink_to_fit ();
 
-  return Token::make_byte_string (loc, str);
+  return Token::make_byte_string (loc, std::move (str));
 }
 
 // Parses a raw byte string.
@@ -1509,7 +1509,7 @@ Lexer::parse_raw_byte_string (Location loc)
 
   str.shrink_to_fit ();
 
-  return Token::make_byte_string (loc, str);
+  return Token::make_byte_string (loc, std::move (str));
 }
 
 // Parses a raw identifier.
@@ -1559,7 +1559,7 @@ Lexer::parse_raw_identifier (Location loc)
     {
       str.shrink_to_fit ();
 
-      return Token::make_identifier (loc, str);
+      return Token::make_identifier (loc, std::move (str));
     }
 }
 
@@ -1623,7 +1623,7 @@ Lexer::parse_string (Location loc)
     }
 
   str.shrink_to_fit ();
-  return Token::make_string (loc, str);
+  return Token::make_string (loc, std::move (str));
 }
 
 // Parses an identifier or keyword.
@@ -1659,7 +1659,7 @@ Lexer::parse_identifier_or_keyword (Location loc)
 
   TokenId keyword = classify_keyword (str);
   if (keyword == IDENTIFIER)
-    return Token::make_identifier (loc, str);
+    return Token::make_identifier (loc, std::move (str));
   else
     return Token::make (keyword, loc);
 }
@@ -1736,7 +1736,7 @@ Lexer::parse_raw_string (Location loc, int initial_hash_count)
 
   str.shrink_to_fit ();
 
-  return Token::make_string (loc, str);
+  return Token::make_string (loc, std::move (str));
 }
 
 template <typename IsDigitFunc>
@@ -1797,7 +1797,7 @@ Lexer::parse_non_decimal_int_literal (Location loc, IsDigitFunc is_digit_func,
 						 : "<insert unknown base>")));
       return nullptr;
     }
-  return Token::make_int (loc, existent_str, type_hint);
+  return Token::make_int (loc, std::move (existent_str), type_hint);
 }
 
 // Parses a hex, binary or octal int literal.
@@ -1889,7 +1889,7 @@ Lexer::parse_decimal_int_or_float (Location loc)
       current_column += length;
 
       str.shrink_to_fit ();
-      return Token::make_float (loc, str, type_hint);
+      return Token::make_float (loc, std::move (str), type_hint);
     }
   else if (current_char == '.' && check_valid_float_dot_end (peek_input (1)))
     {
@@ -1909,7 +1909,7 @@ Lexer::parse_decimal_int_or_float (Location loc)
       current_column += length;
 
       str.shrink_to_fit ();
-      return Token::make_float (loc, str, CORETYPE_UNKNOWN);
+      return Token::make_float (loc, std::move (str), CORETYPE_UNKNOWN);
     }
   else if (current_char == 'E' || current_char == 'e')
     {
@@ -1938,7 +1938,7 @@ Lexer::parse_decimal_int_or_float (Location loc)
       current_column += length;
 
       str.shrink_to_fit ();
-      return Token::make_float (loc, str, type_hint);
+      return Token::make_float (loc, std::move (str), type_hint);
     }
   else
     {
@@ -1952,7 +1952,7 @@ Lexer::parse_decimal_int_or_float (Location loc)
       current_column += length;
 
       str.shrink_to_fit ();
-      return Token::make_int (loc, str, type_hint);
+      return Token::make_int (loc, std::move (str), type_hint);
     }
 }
 
@@ -2026,7 +2026,7 @@ Lexer::parse_char_or_lifetime (Location loc)
 	  current_column += length;
 
 	  str.shrink_to_fit ();
-	  return Token::make_lifetime (loc, str);
+	  return Token::make_lifetime (loc, std::move (str));
 	}
       else
 	{
diff --git a/gcc/rust/lex/rust-token.h b/gcc/rust/lex/rust-token.h
index f76899c..e8d5519 100644
--- a/gcc/rust/lex/rust-token.h
+++ b/gcc/rust/lex/rust-token.h
@@ -1,4 +1,4 @@
-// Copyright (C) 2020 Free Software Foundation, Inc.
+// Copyright (C) 2020, 2021 Free Software Foundation, Inc.
 
 // This file is part of GCC.
 
@@ -249,7 +249,7 @@ private:
   // Token location.
   Location locus;
   // Associated text (if any) of token.
-  std::string *str;
+  std::unique_ptr<std::string> str;
   // TODO: maybe remove issues and just store std::string as value?
   /* Type hint for token based on lexer data (e.g. type suffix). Does not exist
    * for most tokens. */
@@ -262,9 +262,9 @@ private:
   {}
 
   // Token constructor from token id, location, and a string.
-  Token (TokenId token_id, Location location, const std::string &paramStr)
-    : token_id (token_id), locus (location), str (new std::string (paramStr)),
-      type_hint (CORETYPE_UNKNOWN)
+  Token (TokenId token_id, Location location, std::string &&paramStr)
+    : token_id (token_id), locus (location),
+      str (new std::string (std::move (paramStr))), type_hint (CORETYPE_UNKNOWN)
   {}
 
   // Token constructor from token id, location, and a char.
@@ -281,10 +281,10 @@ private:
   {}
 
   // Token constructor from token id, location, a string, and type hint.
-  Token (TokenId token_id, Location location, const std::string &paramStr,
+  Token (TokenId token_id, Location location, std::string &&paramStr,
 	 PrimitiveCoreType parType)
-    : token_id (token_id), locus (location), str (new std::string (paramStr)),
-      type_hint (parType)
+    : token_id (token_id), locus (location),
+      str (new std::string (std::move (paramStr))), type_hint (parType)
   {}
 
 public:
@@ -298,7 +298,7 @@ public:
   Token (Token &&other) = default;
   Token &operator= (Token &&other) = default;
 
-  ~Token () { delete str; }
+  ~Token () = default;
 
   /* TODO: make_shared (which saves a heap allocation) does not work with the
    * private constructor */
@@ -311,34 +311,37 @@ public:
   }
 
   // Makes and returns a new TokenPtr of type IDENTIFIER.
-  static TokenPtr make_identifier (Location locus, const std::string &str)
+  static TokenPtr make_identifier (Location locus, std::string &&str)
   {
     // return std::make_shared<Token> (IDENTIFIER, locus, str);
-    return TokenPtr (new Token (IDENTIFIER, locus, str));
+    return TokenPtr (new Token (IDENTIFIER, locus, std::move (str)));
   }
 
   // Makes and returns a new TokenPtr of type INT_LITERAL.
-  static TokenPtr make_int (Location locus, const std::string &str,
+  static TokenPtr make_int (Location locus, std::string &&str,
 			    PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
   {
     // return std::make_shared<Token> (INT_LITERAL, locus, str, type_hint);
-    return TokenPtr (new Token (INT_LITERAL, locus, str, type_hint));
+    return TokenPtr (
+      new Token (INT_LITERAL, locus, std::move (str), type_hint));
   }
 
   // Makes and returns a new TokenPtr of type FLOAT_LITERAL.
-  static TokenPtr make_float (Location locus, const std::string &str,
+  static TokenPtr make_float (Location locus, std::string &&str,
 			      PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
   {
     // return std::make_shared<Token> (FLOAT_LITERAL, locus, str, type_hint);
-    return TokenPtr (new Token (FLOAT_LITERAL, locus, str, type_hint));
+    return TokenPtr (
+      new Token (FLOAT_LITERAL, locus, std::move (str), type_hint));
   }
 
   // Makes and returns a new TokenPtr of type STRING_LITERAL.
-  static TokenPtr make_string (Location locus, const std::string &str)
+  static TokenPtr make_string (Location locus, std::string &&str)
   {
     // return std::make_shared<Token> (STRING_LITERAL, locus, str,
     // CORETYPE_STR);
-    return TokenPtr (new Token (STRING_LITERAL, locus, str, CORETYPE_STR));
+    return TokenPtr (
+      new Token (STRING_LITERAL, locus, std::move (str), CORETYPE_STR));
   }
 
   // Makes and returns a new TokenPtr of type CHAR_LITERAL.
@@ -356,17 +359,17 @@ public:
   }
 
   // Makes and returns a new TokenPtr of type BYTE_STRING_LITERAL (fix).
-  static TokenPtr make_byte_string (Location locus, const std::string &str)
+  static TokenPtr make_byte_string (Location locus, std::string &&str)
   {
     // return std::make_shared<Token> (BYTE_STRING_LITERAL, locus, str);
-    return TokenPtr (new Token (BYTE_STRING_LITERAL, locus, str));
+    return TokenPtr (new Token (BYTE_STRING_LITERAL, locus, std::move (str)));
   }
 
   // Makes and returns a new TokenPtr of type LIFETIME.
-  static TokenPtr make_lifetime (Location locus, const std::string &str)
+  static TokenPtr make_lifetime (Location locus, std::string &&str)
   {
     // return std::make_shared<Token> (LIFETIME, locus, str);
-    return TokenPtr (new Token (LIFETIME, locus, str));
+    return TokenPtr (new Token (LIFETIME, locus, std::move (str)));
   }
 
   // Gets id of the token.