From 4c34885a5afbda4bad5873668cdc6fddd2d1c29b Mon Sep 17 00:00:00 2001 From: Pierre-Emmanuel Patry Date: Wed, 10 May 2023 15:17:46 +0200 Subject: libproc_macro: Change cpp literal representation Change the literal representation on cpp side to match the new one in rust. This means FFIString had to be implemented on cpp side. A few helper functions has also been introduced. ChangeLog: * libgrust/libproc_macro/Makefile.am: Add ffistring unit to compiled objects list. * libgrust/libproc_macro/Makefile.in: Regenerate. * libgrust/libproc_macro/literal.cc (Literal::drop): Change with a call to ffistring drop function. (Literal::make_literal): Add new helper constructor (Literal__drop): Remove this function. (Literal__string): Likewise. (Literal__byte_string): Likewise. (Literal__from_string): Moved this function. (Literal::make_unsigned): Changed the constructor to match the new layout. (Literal::make_signed): Likewise. (Literal::clone): Reimplement th eclone function. (Literal::make_u8): Changed the constructor, make suffixed by default. (Literal::make_u16): Likewise. (Literal::make_u32): Likewise. (Literal::make_u64): Likewise. (Literal::make_i8): Likewise. (Literal::make_i16): Likewise. (Literal::make_i32): Likewise. (Literal::make_i64): Likewise. (Literal::make_string): Likewise. (Literal::make_byte_string): Likewise. (Literal::make_f32): Likewise. (Literal::make_f64): Likewise. (Literal::make_char): Likewise. (Literal::make_usize): Likewise. (Literal::make_isize): Likewise. (LitKind::make_byte): Add new helper constructor to avoid having to set the payload value. (LitKind::make_char): Likewise. (LitKind::make_integer): Likewise. (LitKind::make_float): Likewise. (LitKind::make_str): Likewise. (LitKind::make_str_raw): Add a new helper constructor which takes the payload value as an argument. (LitKind::make_byte_str): Add new helper constructor to avoid mistakes with payload value. (LitKind::make_byte_str_raw): Add a new helper constructor which takes the payload value as an argument. * libgrust/libproc_macro/literal.h: Add new functions prototype. (enum UnsignedTag): Removed because it is now unused. (struct Payload128): Likewise. (union UnsignedPayload): Likewise. (struct Unsigned): Likewise. (enum SignedTag): Likewise. (union SignedPayload): Likewise. (struct Signed): Likewise. (enum LiteralTag): Likewise. (enum LitKindTag): Likewise. (struct StringPayload): Likewise. (struct ByteStringPayload): Likewise. (union LitKindPayload): Likewise. (struct UnsignedSuffixPayload): Likewise. (struct LitKind): Add new literal kind struct representation to match the enum on rust side. (struct SignedSuffixPayload): Removed because now unused. (struct UsizePayload): Likewise. (struct IsizePayload): Likewise. (struct Float32Payload): Likewise. (struct Float64Payload): Likewise. (union LiteralPayload): Likewise. (struct Literal): Changed the internals of the structure. (Literal__drop): Removed the drop function fom the c interface. (Literal__string): Removed unused function. (Literal__byte_string): Removed unused function. * libgrust/libproc_macro/ffistring.cc: New file. * libgrust/libproc_macro/ffistring.h: New file. gcc/rust/ChangeLog: * lex/rust-token.h: Implement hash for token id enumeration. * util/rust-token-converter.cc (dispatch_float_literals): Update to new internals. (dispatch_integer_literals): Likewise. (convert): Likewise. (string_literal): Remove function. (byte_string_literal): Likewise. (unsigned_literal): Likewise. (signed_literal): Likewise. (from_literal): Update with new internals. Signed-off-by: Pierre-Emmanuel Patry --- gcc/rust/lex/rust-token.h | 12 ++ gcc/rust/util/rust-token-converter.cc | 268 +++++++++------------------------- 2 files changed, 77 insertions(+), 203 deletions(-) (limited to 'gcc') diff --git a/gcc/rust/lex/rust-token.h b/gcc/rust/lex/rust-token.h index 68b41f7..8956d7b 100644 --- a/gcc/rust/lex/rust-token.h +++ b/gcc/rust/lex/rust-token.h @@ -456,4 +456,16 @@ return *str; }; } // namespace Rust +namespace std { +template <> struct hash +{ + size_t operator() (const Rust::PrimitiveCoreType &coretype) const noexcept + { + return hash::type> () ( + static_cast::type> ( + coretype)); + } +}; +} // namespace std + #endif diff --git a/gcc/rust/util/rust-token-converter.cc b/gcc/rust/util/rust-token-converter.cc index 2047ea7..d56493f 100644 --- a/gcc/rust/util/rust-token-converter.cc +++ b/gcc/rust/util/rust-token-converter.cc @@ -17,9 +17,28 @@ #include "rust-lex.h" #include "rust-token-converter.h" #include "libproc_macro/proc_macro.h" +#include "bi-map.h" + +#include namespace Rust { +static const BiMap suffixes + = {{{CORETYPE_F32, "f32"}, + {CORETYPE_F64, "f64"}, + {CORETYPE_U8, "u8"}, + {CORETYPE_U16, "u16"}, + {CORETYPE_U32, "u32"}, + {CORETYPE_U64, "u64"}, + {CORETYPE_U128, "u128"}, + {CORETYPE_I8, "i8"}, + {CORETYPE_I16, "i16"}, + {CORETYPE_I32, "i32"}, + {CORETYPE_I64, "i64"}, + {CORETYPE_I128, "i128"}, + {CORETYPE_ISIZE, "isize"}, + {CORETYPE_USIZE, "usize"}}}; + static void pop_group (std::vector &streams, ProcMacro::Delimiter delim) @@ -35,93 +54,24 @@ static void dispatch_float_literals (ProcMacro::TokenStream &ts, const const_TokenPtr &token) { - std::string::size_type sz; auto str = token->as_string (); - switch (token->get_type_hint ()) - { - case CORETYPE_F32: { - auto value = std::stof (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_f32 (value, sz != str.length ()))); - } - break; - case CORETYPE_F64: { - auto value = std::stod (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_f64 (value, sz != str.length ()))); - } - break; - default: - gcc_unreachable (); - } + auto kind = ProcMacro::LitKind::make_float (); + auto lookup = suffixes.lookup (token->get_type_hint ()); + auto suffix = suffixes.is_iter_ok (lookup) ? lookup->second : ""; + ts.push (ProcMacro::TokenTree::make_tokentree ( + ProcMacro::Literal::make_literal (kind, str, suffix))); } static void dispatch_integer_literals (ProcMacro::TokenStream &ts, const const_TokenPtr &token) { - std::string::size_type sz; auto str = token->as_string (); - unsigned long long uvalue; - long long svalue; - - switch (token->get_type_hint ()) - { - case CORETYPE_U8: - uvalue = std::stoull (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_u8 (uvalue, sz != str.length ()))); - break; - case CORETYPE_U16: - uvalue = std::stoull (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_u16 (uvalue, sz != str.length ()))); - break; - case CORETYPE_U32: - uvalue = std::stoull (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_u32 (uvalue, sz != str.length ()))); - break; - case CORETYPE_U64: - uvalue = std::stoull (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_u32 (uvalue, sz != str.length ()))); - break; - case CORETYPE_I8: - svalue = std::stoll (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_i8 (svalue, sz != str.length ()))); - break; - case CORETYPE_I16: - svalue = std::stoll (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_i16 (svalue, sz != str.length ()))); - break; - case CORETYPE_I32: - svalue = std::stoll (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_i32 (svalue, sz != str.length ()))); - break; - case CORETYPE_I64: - svalue = std::stoll (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_i32 (svalue, sz != str.length ()))); - break; - case CORETYPE_INT: - svalue = std::stoll (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_isize (svalue, sz != str.length ()))); - break; - case CORETYPE_UINT: - uvalue = std::stoull (str, &sz); - ts.push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_usize (uvalue, sz != str.length ()))); - break; - case CORETYPE_UNKNOWN: - default: - gcc_unreachable (); - break; - } + auto kind = ProcMacro::LitKind::make_integer (); + auto lookup = suffixes.lookup (token->get_type_hint ()); + auto suffix = suffixes.is_iter_ok (lookup) ? lookup->second : ""; + ts.push (ProcMacro::TokenTree::make_tokentree ( + ProcMacro::Literal::make_literal (kind, str, suffix))); } ProcMacro::TokenStream @@ -140,21 +90,25 @@ convert (const std::vector &tokens) case INT_LITERAL: dispatch_integer_literals (trees.back (), token); break; - // FIXME: Why does BYTE_CHAR_LITERAL is not handled by rustc ? - case CHAR_LITERAL: // TODO: UTF-8 handling + case CHAR_LITERAL: trees.back ().push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_char (token->as_string ()[0]))); + ProcMacro::Literal::make_literal (ProcMacro::LitKind::make_char (), + token->as_string ()))); break; case STRING_LITERAL: trees.back ().push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_string (token->as_string ()))); + ProcMacro::Literal::make_literal (ProcMacro::LitKind::make_str (), + token->as_string ()))); break; - case BYTE_STRING_LITERAL: { - auto str = token->as_string (); - std::vector data (str.begin (), str.end ()); - trees.back ().push (ProcMacro::TokenTree::make_tokentree ( - ProcMacro::Literal::make_byte_string (data))); - } + case BYTE_CHAR_LITERAL: + trees.back ().push (ProcMacro::TokenTree::make_tokentree ( + ProcMacro::Literal::make_literal (ProcMacro::LitKind::make_byte (), + token->as_string ()))); + break; + case BYTE_STRING_LITERAL: + trees.back ().push (ProcMacro::TokenTree::make_tokentree ( + ProcMacro::Literal::make_literal ( + ProcMacro::LitKind::make_byte_str (), token->as_string ()))); break; // Ident case IDENTIFIER: @@ -321,91 +275,6 @@ from_ident (const ProcMacro::Ident &ident, std::vector &result) result.push_back (lexer.peek_token ()); } -static void -string_literal (const ProcMacro::StringPayload &payload, - std::vector &result) -{ - // TODO: UTF-8 string - result.push_back (Token::make_string ( - Location (), - std::string (reinterpret_cast (payload.data), payload.len))); -} - -static void -byte_string_literal (const ProcMacro::ByteStringPayload &payload, - std::vector &result) -{ - result.push_back (Token::make_byte_string ( - Location (), - std::string (reinterpret_cast (payload.data), payload.size))); -} - -static void -unsigned_literal (const ProcMacro::Unsigned &lit, - std::vector &result) -{ - switch (lit.tag) - { - case ProcMacro::UNSIGNED_8: - result.push_back (Token::make_int (Location (), - std::to_string (lit.payload.unsigned8), - CORETYPE_U8)); - break; - case ProcMacro::UNSIGNED_16: - result.push_back ( - Token::make_int (Location (), std::to_string (lit.payload.unsigned16), - CORETYPE_U16)); - break; - case ProcMacro::UNSIGNED_32: - result.push_back ( - Token::make_int (Location (), std::to_string (lit.payload.unsigned32), - CORETYPE_U32)); - break; - case ProcMacro::UNSIGNED_64: - result.push_back ( - Token::make_int (Location (), std::to_string (lit.payload.unsigned64), - CORETYPE_U64)); - break; - case ProcMacro::UNSIGNED_128: - // TODO: Handle 128 bits - default: - gcc_unreachable (); - } -} - -static void -signed_literal (const ProcMacro::Signed &lit, - std::vector &result) -{ - switch (lit.tag) - { - case ProcMacro::SIGNED_8: - result.push_back (Token::make_int (Location (), - std::to_string (lit.payload.signed8), - CORETYPE_I8)); - break; - case ProcMacro::SIGNED_16: - result.push_back (Token::make_int (Location (), - std::to_string (lit.payload.signed16), - CORETYPE_I16)); - break; - case ProcMacro::SIGNED_32: - result.push_back (Token::make_int (Location (), - std::to_string (lit.payload.signed32), - CORETYPE_I32)); - break; - case ProcMacro::SIGNED_64: - result.push_back (Token::make_int (Location (), - std::to_string (lit.payload.signed64), - CORETYPE_I64)); - break; - case ProcMacro::SIGNED_128: - // TODO: Handle 128 bits - default: - gcc_unreachable (); - } -} - /** * Append the token corresponding to a given Literal to a vector. * @@ -416,46 +285,39 @@ static void from_literal (const ProcMacro::Literal &literal, std::vector &result) { - switch (literal.tag) + auto lookup = suffixes.lookup (literal.suffix.to_string ()); + auto suffix + = suffixes.is_iter_ok (lookup) ? lookup->second : CORETYPE_UNKNOWN; + // FIXME: Add spans instead of empty locations + switch (literal.kind.tag) { - case ProcMacro::STRING: - string_literal (literal.payload.string_payload, result); - break; - case ProcMacro::BYTE_STRING: - byte_string_literal (literal.payload.byte_string_payload, result); + case ProcMacro::BYTE: + result.push_back ( + Token::make_byte_char (Location (), literal.text.to_string ()[0])); break; case ProcMacro::CHAR: result.push_back ( - Token::make_char (Location (), literal.payload.char_payload)); + Token::make_char (Location (), literal.text.to_string ()[0])); break; - case ProcMacro::UNSIGNED: - unsigned_literal (literal.payload.unsigned_payload.value, result); - break; - case ProcMacro::SIGNED: - signed_literal (literal.payload.signed_payload.value, result); - break; - case ProcMacro::USIZE: + case ProcMacro::INTEGER: result.push_back ( - Token::make_int (Location (), - std::to_string (literal.payload.usize_payload.value), - CORETYPE_USIZE)); + Token::make_int (Location (), literal.text.to_string (), suffix)); break; - case ProcMacro::ISIZE: + case ProcMacro::FLOAT: result.push_back ( - Token::make_int (Location (), - std::to_string (literal.payload.isize_payload.value), - CORETYPE_ISIZE)); + Token::make_float (Location (), literal.text.to_string (), suffix)); break; - case ProcMacro::FLOAT32: - result.push_back (Token::make_float ( - Location (), std::to_string (literal.payload.float32_payload.value), - CORETYPE_F32)); + case ProcMacro::STR: + result.push_back ( + Token::make_string (Location (), literal.text.to_string ())); break; - case ProcMacro::FLOAT64: - result.push_back (Token::make_float ( - Location (), std::to_string (literal.payload.float64_payload.value), - CORETYPE_F64)); + case ProcMacro::BYTE_STR: + result.push_back ( + Token::make_byte_string (Location (), literal.text.to_string ())); break; + // FIXME: Handle raw string + case ProcMacro::STR_RAW: + case ProcMacro::BYTE_STR_RAW: default: gcc_unreachable (); } -- cgit v1.1