diff options
Diffstat (limited to 'gcc/rust/lex/rust-token.cc')
| -rw-r--r-- | gcc/rust/lex/rust-token.cc | 18 |
1 files changed, 18 insertions, 0 deletions
diff --git a/gcc/rust/lex/rust-token.cc b/gcc/rust/lex/rust-token.cc index 77ec6cf..9a1132f 100644 --- a/gcc/rust/lex/rust-token.cc +++ b/gcc/rust/lex/rust-token.cc @@ -19,6 +19,7 @@ #include "rust-system.h" #include "rust-token.h" #include "rust-diagnostics.h" +#include "rust-unicode.h" namespace Rust { // Hackily defined way to get token description for enum value using x-macros @@ -150,6 +151,23 @@ Token::get_type_hint_str () const return get_type_hint_string (type_hint); } +std::string +nfc_normalize_token_string (location_t loc, TokenId id, const std::string &str) +{ + if (id == IDENTIFIER || id == LIFETIME) + { + tl::optional<Utf8String> ustring = Utf8String::make_utf8_string (str); + if (ustring.has_value ()) + return ustring.value ().nfc_normalize ().as_string (); + else + rust_internal_error_at (loc, + "identifier '%s' is not a valid UTF-8 string", + str.c_str ()); + } + else + return str; +} + const std::string & Token::get_str () const { |