// Copyright (C) 2020-2025 Free Software Foundation, Inc.
// This file is part of GCC.
// GCC is free software; you can redistribute it and/or modify it under
// the terms of the GNU General Public License as published by the Free
// Software Foundation; either version 3, or (at your option) any later
// version.
// GCC is distributed in the hope that it will be useful, but WITHOUT ANY
// WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
// for more details.
// You should have received a copy of the GNU General Public License
// along with GCC; see the file COPYING3. If not see
// <http://www.gnu.org/licenses/>.
#ifndef RUST_TOKEN_H
#define RUST_TOKEN_H
#include "rust-system.h"
#include "rust-linemap.h"
#include "rust-make-unique.h"
#include "rust-unicode.h"
namespace Rust {
// "Primitive core types" in Rust - the different int and float types, as well
// as some others
enum PrimitiveCoreType
{
CORETYPE_UNKNOWN,
// named primitives
CORETYPE_BOOL,
CORETYPE_CHAR,
CORETYPE_STR,
// okay technically int and uint are arch-dependent (pointer size)
CORETYPE_INT,
CORETYPE_UINT,
// numbered number primitives
CORETYPE_F32,
CORETYPE_F64,
CORETYPE_I8,
CORETYPE_I16,
CORETYPE_I32,
CORETYPE_I64,
CORETYPE_I128,
CORETYPE_U8,
CORETYPE_U16,
CORETYPE_U32,
CORETYPE_U64,
CORETYPE_U128,
// Pure decimals are used for tuple index.
// Also means there is no type hint.
CORETYPE_PURE_DECIMAL,
// arch-dependent pointer sizes
CORETYPE_ISIZE = CORETYPE_INT,
CORETYPE_USIZE = CORETYPE_UINT
};
// RS_TOKEN(name, description)
// RS_TOKEN_KEYWORD_{2015,2018}(name, identifier)
// Keep RS_TOKEN_KEYWORD sorted
/* note that abstract, async, become, box, do, final, macro, override, priv,
* try, typeof, unsized, virtual, and yield are unused */
#define RS_TOKEN_LIST \
RS_TOKEN (FIRST_TOKEN, "<first-token-marker>") \
RS_TOKEN (END_OF_FILE, "end of file") \
RS_TOKEN (EXCLAM, "!") \
RS_TOKEN (NOT_EQUAL, "!=") \
RS_TOKEN (PERCENT, "%") \
RS_TOKEN (PERCENT_EQ, "%=") \
RS_TOKEN (AMP, "&") \
RS_TOKEN (AMP_EQ, "&=") \
RS_TOKEN (LOGICAL_AND, "&&") \
RS_TOKEN (ASTERISK, "*") \
RS_TOKEN (ASTERISK_EQ, "*=") \
RS_TOKEN (PLUS, "+") \
RS_TOKEN (PLUS_EQ, "+=") \
RS_TOKEN (COMMA, ",") \
RS_TOKEN (MINUS, "-") \
RS_TOKEN (MINUS_EQ, "-=") \
RS_TOKEN (RETURN_TYPE, "->") \
RS_TOKEN (DOT, ".") \
RS_TOKEN (DOT_DOT, "..") \
RS_TOKEN (DOT_DOT_EQ, "..=") \
RS_TOKEN (ELLIPSIS, "...") \
RS_TOKEN (DIV, "/") \
RS_TOKEN (DIV_EQ, "/=") \
RS_TOKEN (COLON, ":") \
RS_TOKEN (SEMICOLON, ";") \
RS_TOKEN (LEFT_SHIFT, "<<") \
RS_TOKEN (LEFT_SHIFT_EQ, "<<=") \
RS_TOKEN (LEFT_ANGLE, "<") \
RS_TOKEN (LESS_OR_EQUAL, "<=") \
RS_TOKEN (EQUAL, "=") \
RS_TOKEN (EQUAL_EQUAL, "==") \
RS_TOKEN (MATCH_ARROW, "=>") \
RS_TOKEN (RIGHT_ANGLE, ">") \
RS_TOKEN (GREATER_OR_EQUAL, ">=") \
RS_TOKEN (RIGHT_SHIFT, ">>") \
RS_TOKEN (RIGHT_SHIFT_EQ, ">>=") \
RS_TOKEN (PATTERN_BIND, "@") \
RS_TOKEN (TILDE, "~") \
RS_TOKEN (BACKSLASH, "\\") \
RS_TOKEN (BACKTICK, "`") \
RS_TOKEN (CARET, "^") \
RS_TOKEN (CARET_EQ, "^=") \
RS_TOKEN (PIPE, "|") \
RS_TOKEN (PIPE_EQ, "|=") \
RS_TOKEN (OR, "||") \
RS_TOKEN (QUESTION_MARK, "?") \
RS_TOKEN (HASH, "#") \
/* from here on, dodgy and may not be correct. not operators and may be \
* symbols */ \
/* RS_TOKEN(SPACE, " ") probably too dodgy */ \
/* RS_TOKEN(NEWLINE, "\n")*/ \
RS_TOKEN (SCOPE_RESOLUTION, "::") /* dodgy */ \
RS_TOKEN (SINGLE_QUOTE, "'") /* should i differentiate from lifetime? */ \
RS_TOKEN (DOUBLE_QUOTE, "\"") \
RS_TOKEN (IDENTIFIER, "identifier") \
RS_TOKEN (INT_LITERAL, \
"integer literal") /* do different int and float types need \
different literal types? */ \
RS_TOKEN (FLOAT_LITERAL, "float literal") \
RS_TOKEN (STRING_LITERAL, "string literal") \
RS_TOKEN (CHAR_LITERAL, "character literal") \
RS_TOKEN (BYTE_STRING_LITERAL, "byte string literal") \
RS_TOKEN (RAW_STRING_LITERAL, "raw string literal") \
RS_TOKEN (BYTE_CHAR_LITERAL, "byte character literal") \
RS_TOKEN (LIFETIME, "lifetime") /* TODO: improve token type */ \
/* Have "interpolated" tokens (whatever that means)? identifer, path, type, \
* pattern, */ \
/* expression, statement, block, meta, item in mrustc (but not directly in \
* lexer). */ \
RS_TOKEN (LEFT_PAREN, "(") \
RS_TOKEN (RIGHT_PAREN, ")") \
RS_TOKEN (LEFT_CURLY, "{") \
RS_TOKEN (RIGHT_CURLY, "}") \
RS_TOKEN (LEFT_SQUARE, "[") \
RS_TOKEN (RIGHT_SQUARE, "]") \
/* Macros */ \
RS_TOKEN (DOLLAR_SIGN, "$") \
/* Doc Comments */ \
RS_TOKEN (INNER_DOC_COMMENT, "#![doc]") \
RS_TOKEN (OUTER_DOC_COMMENT, "#[doc]") \
RS_TOKEN_KEYWORD_2015 (ABSTRACT, "abstract") /* unused */ \
RS_TOKEN_KEYWORD_2015 (AS, "as") \
RS_TOKEN_KEYWORD_2018 (ASYNC, "async") /* unused */ \
RS_TOKEN_KEYWORD_2015 (AUTO, "auto") \
RS_TOKEN_KEYWORD_2018 (AWAIT, "await") \
RS_TOKEN_KEYWORD_2015 (BECOME, "become") /* unused */ \
RS_TOKEN_KEYWORD_2015 (BOX, "box") /* unused */ \
RS_TOKEN_KEYWORD_2015 (BREAK, "break") \
RS_TOKEN_KEYWORD_2015 (CONST, "const") \
RS_TOKEN_KEYWORD_2015 (CONTINUE, "continue") \
RS_TOKEN_KEYWORD_2015 (CRATE, "crate") \
RS_TOKEN_KEYWORD_2015 (DO, "do") /* unused */ \
RS_TOKEN_KEYWORD_2018 (DYN, "dyn") \
RS_TOKEN_KEYWORD_2015 (ELSE, "else") \
RS_TOKEN_KEYWORD_2015 (ENUM_KW, "enum") \
RS_TOKEN_KEYWORD_2015 (EXTERN_KW, "extern") \
RS_TOKEN_KEYWORD_2015 (FALSE_LITERAL, "false") \
RS_TOKEN_KEYWORD_2015 (FINAL_KW, "final") /* unused */ \
RS_TOKEN_KEYWORD_2015 (FN_KW, "fn") \
RS_TOKEN_KEYWORD_2015 (FOR, "for") \
RS_TOKEN_KEYWORD_2015 (IF, "if") \
RS_TOKEN_KEYWORD_2015 (IMPL, "impl") \
RS_TOKEN_KEYWORD_2015 (IN, "in") \
RS_TOKEN_KEYWORD_2015 (LET, "let") \
RS_TOKEN_KEYWORD_2015 (LOOP, "loop") \
RS_TOKEN_KEYWORD_2015 (MACRO, "macro") \
RS_TOKEN_KEYWORD_2015 (MATCH_KW, "match") \
RS_TOKEN_KEYWORD_2015 (MOD, "mod") \
RS_TOKEN_KEYWORD_2015 (MOVE, "move") \
RS_TOKEN_KEYWORD_2015 (MUT, "mut") \
RS_TOKEN_KEYWORD_2015 (OVERRIDE_KW, "override") /* unused */ \
RS_TOKEN_KEYWORD_2015 (PRIV, "priv") /* unused */ \
RS_TOKEN_KEYWORD_2015 (PUB, "pub") \
RS_TOKEN_KEYWORD_2015 (REF, "ref") \
RS_TOKEN_KEYWORD_2015 (RETURN_KW, "return") \
RS_TOKEN_KEYWORD_2015 ( \
SELF_ALIAS, "Self") /* mrustc does not treat this as a reserved word*/ \
RS_TOKEN_KEYWORD_2015 (SELF, "self") \
RS_TOKEN_KEYWORD_2015 (STATIC_KW, "static") \
RS_TOKEN_KEYWORD_2015 (STRUCT_KW, "struct") \
RS_TOKEN_KEYWORD_2015 (SUPER, "super") \
RS_TOKEN_KEYWORD_2015 (TRAIT, "trait") \
RS_TOKEN_KEYWORD_2015 (TRUE_LITERAL, "true") \
RS_TOKEN_KEYWORD_2015 (TRY, "try") /* unused */ \
RS_TOKEN_KEYWORD_2015 (TYPE, "type") \
RS_TOKEN_KEYWORD_2015 (TYPEOF, "typeof") /* unused */ \
RS_TOKEN_KEYWORD_2015 (UNDERSCORE, "_") \
RS_TOKEN_KEYWORD_2015 (UNSAFE, "unsafe") \
RS_TOKEN_KEYWORD_2015 (UNSIZED, "unsized") /* unused */ \
RS_TOKEN_KEYWORD_2015 (USE, "use") \
RS_TOKEN_KEYWORD_2015 (VIRTUAL, "virtual") /* unused */ \
RS_TOKEN_KEYWORD_2015 (WHERE, "where") \
RS_TOKEN_KEYWORD_2015 (WHILE, "while") \
RS_TOKEN_KEYWORD_2015 (YIELD, "yield") /* unused */ \
RS_TOKEN (LAST_TOKEN, "<last-token-marker>")
// Contains all token types. Crappy implementation via x-macros.
enum TokenId
{
#define RS_TOKEN(name, _) name,
#define RS_TOKEN_KEYWORD_2015(x, y) RS_TOKEN (x, y)
#define RS_TOKEN_KEYWORD_2018 RS_TOKEN_KEYWORD_2015
RS_TOKEN_LIST
#undef RS_TOKEN_KEYWORD_2015
#undef RS_TOKEN_KEYWORD_2018
#undef RS_TOKEN
};
// dodgy "TokenPtr" declaration with Token forward declaration
class Token;
// A smart pointer (shared_ptr) to Token.
typedef std::shared_ptr<Token> TokenPtr;
// A smart pointer (shared_ptr) to a constant Token.
typedef std::shared_ptr<const Token> const_TokenPtr;
// Hackily defined way to get token description for enum value using x-macros
const char *
get_token_description (TokenId id);
/* Hackily defined way to get token description as a string for enum value using
* x-macros */
const char *
token_id_to_str (TokenId id);
/* checks if a token is a keyword */
bool
token_id_is_keyword (TokenId id);
/* gets the string associated with a keyword */
const std::string &
token_id_keyword_string (TokenId id);
// Get type hint description as a string.
const char *
get_type_hint_string (PrimitiveCoreType type);
/* Normalize string if a token is a identifier */
std::string
nfc_normalize_token_string (location_t loc, TokenId id, const std::string &str);
// Represents a single token. Create using factory static methods.
class Token
{
private:
// Token kind.
TokenId token_id;
// Token location.
location_t locus;
// Associated text (if any) of token.
std::unique_ptr<std::string> str;
// TODO: maybe remove issues and just store std::string as value?
/* Type hint for token based on lexer data (e.g. type suffix). Does not exist
* for most tokens. */
PrimitiveCoreType type_hint;
// Token constructor from token id and location. Has a null string.
Token (TokenId token_id, location_t location)
: token_id (token_id), locus (location), str (nullptr),
type_hint (CORETYPE_UNKNOWN)
{}
// Token constructor from token id, location, and a string.
Token (TokenId token_id, location_t location, std::string &¶mStr)
: token_id (token_id), locus (location), type_hint (CORETYPE_UNKNOWN)
{
// Normalize identifier tokens
str = Rust::make_unique<std::string> (
nfc_normalize_token_string (location, token_id, paramStr));
}
// Token constructor from token id, location, and a char.
Token (TokenId token_id, location_t location, char paramChar)
: token_id (token_id), locus (location),
str (new std::string (1, paramChar)), type_hint (CORETYPE_UNKNOWN)
{
// Do not need to normalize 1byte char
}
// Token constructor from token id, location, and a "codepoint".
Token (TokenId token_id, location_t location, Codepoint paramCodepoint)
: token_id (token_id), locus (location), type_hint (CORETYPE_UNKNOWN)
{
// Normalize identifier tokens
str = Rust::make_unique<std::string> (
nfc_normalize_token_string (location, token_id,
paramCodepoint.as_string ()));
}
// Token constructor from token id, location, a string, and type hint.
Token (TokenId token_id, location_t location, std::string &¶mStr,
PrimitiveCoreType parType)
: token_id (token_id), locus (location), type_hint (parType)
{
// Normalize identifier tokens
str = Rust::make_unique<std::string> (
nfc_normalize_token_string (location, token_id, paramStr));
}
public:
// No default constructor.
Token () = delete;
// Do not copy/assign tokens.
Token (const Token &) = delete;
Token &operator= (const Token &) = delete;
// Allow moving tokens.
Token (Token &&other) = default;
Token &operator= (Token &&other) = default;
~Token () = default;
/* TODO: make_shared (which saves a heap allocation) does not work with the
* private constructor */
// Makes and returns a new TokenPtr (with null string).
static TokenPtr make (TokenId token_id, location_t locus)
{
// return std::make_shared<Token> (token_id, locus);
return TokenPtr (new Token (token_id, locus));
}
// Makes and returns a new TokenPtr of type IDENTIFIER.
static TokenPtr make_identifier (location_t locus, std::string &&str)
{
// return std::make_shared<Token> (IDENTIFIER, locus, str);
return TokenPtr (new Token (IDENTIFIER, locus, std::move (str)));
}
// Makes and returns a new TokenPtr of type INT_LITERAL.
static TokenPtr make_int (location_t locus, std::string &&str,
PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
{
// return std::make_shared<Token> (INT_LITERAL, locus, str, type_hint);
return TokenPtr (
new Token (INT_LITERAL, locus, std::move (str), type_hint));
}
// Makes and returns a new TokenPtr of type FLOAT_LITERAL.
static TokenPtr make_float (location_t locus, std::string &&str,
PrimitiveCoreType type_hint = CORETYPE_UNKNOWN)
{
// return std::make_shared<Token> (FLOAT_LITERAL, locus, str, type_hint);
return TokenPtr (
new Token (FLOAT_LITERAL, locus, std::move (str), type_hint));
}
// Makes and returns a new TokenPtr of type STRING_LITERAL.
static TokenPtr make_string (location_t locus, std::string &&str)
{
// return std::make_shared<Token> (STRING_LITERAL, locus, str,
// CORETYPE_STR);
return TokenPtr (
new Token (STRING_LITERAL, locus, std::move (str), CORETYPE_STR));
}
// Makes and returns a new TokenPtr of type CHAR_LITERAL.
static TokenPtr make_char (location_t locus, Codepoint char_lit)
{
// return std::make_shared<Token> (CHAR_LITERAL, locus, char_lit);
return TokenPtr (new Token (CHAR_LITERAL, locus, char_lit));
}
// Makes and returns a new TokenPtr of type BYTE_CHAR_LITERAL.
static TokenPtr make_byte_char (location_t locus, char byte_char)
{
// return std::make_shared<Token> (BYTE_CHAR_LITERAL, locus, byte_char);
return TokenPtr (new Token (BYTE_CHAR_LITERAL, locus, byte_char));
}
// Makes and returns a new TokenPtr of type BYTE_STRING_LITERAL (fix).
static TokenPtr make_byte_string (location_t locus, std::string &&str)
{
// return std::make_shared<Token> (BYTE_STRING_LITERAL, locus, str);
return TokenPtr (new Token (BYTE_STRING_LITERAL, locus, std::move (str)));
}
// Makes and returns a new TokenPtr of type RAW_STRING_LITERAL.
static TokenPtr make_raw_string (location_t locus, std::string &&str)
{
return TokenPtr (new Token (RAW_STRING_LITERAL, locus, std::move (str)));
}
// Makes and returns a new TokenPtr of type INNER_DOC_COMMENT.
static TokenPtr make_inner_doc_comment (location_t locus, std::string &&str)
{
return TokenPtr (new Token (INNER_DOC_COMMENT, locus, std::move (str)));
}
// Makes and returns a new TokenPtr of type OUTER_DOC_COMMENT.
static TokenPtr make_outer_doc_comment (location_t locus, std::string &&str)
{
return TokenPtr (new Token (OUTER_DOC_COMMENT, locus, std::move (str)));
}
// Makes and returns a new TokenPtr of type LIFETIME.
static TokenPtr make_lifetime (location_t locus, std::string &&str)
{
// return std::make_shared<Token> (LIFETIME, locus, str);
return TokenPtr (new Token (LIFETIME, locus, std::move (str)));
}
// Gets id of the token.
TokenId get_id () const { return token_id; }
// Gets location of the token.
location_t get_locus () const { return locus; }
// Set location of the token.
void set_locus (location_t locus) { this->locus = locus; }
// Gets string description of the token.
const std::string &
get_str () const; /*{
// FIXME: put in header again when fix null problem
//gcc_assert(str != nullptr);
if (str == nullptr) {
error_at(get_locus(), "attempted to get string for '%s', which has no string.
returning empty string instead.", get_token_description()); return "";
}
return *str;
}*/
// Gets token's type hint info.
PrimitiveCoreType get_type_hint () const
{
return type_hint == CORETYPE_PURE_DECIMAL ? CORETYPE_UNKNOWN : type_hint;
}
// diagnostics (error reporting)
const char *get_token_description () const
{
return Rust::get_token_description (token_id);
}
// debugging
const char *token_id_to_str () const
{
return Rust::token_id_to_str (token_id);
}
// debugging
const char *get_type_hint_str () const;
/* Returns whether the token is a literal of any type (int, float, char,
* string, byte char, byte string). */
bool is_literal () const
{
switch (token_id)
{
case INT_LITERAL:
case FLOAT_LITERAL:
case CHAR_LITERAL:
case STRING_LITERAL:
case BYTE_CHAR_LITERAL:
case BYTE_STRING_LITERAL:
case RAW_STRING_LITERAL:
return true;
default:
return false;
}
}
/* Returns whether the token actually has a string (regardless of whether it
* should or not). */
bool has_str () const { return str != nullptr; }
// Returns whether the token should have a string.
bool should_have_str () const
{
return is_literal () || token_id == IDENTIFIER || token_id == LIFETIME;
}
// Returns whether the token is a pure decimal int literal
bool is_pure_decimal () const { return type_hint == CORETYPE_PURE_DECIMAL; }
// Return the token representation as someone would find it in the original
// source code file.
std::string as_string () const;
};
} // namespace Rust
namespace std {
template <> struct hash<Rust::PrimitiveCoreType>
{
size_t operator() (const Rust::PrimitiveCoreType &coretype) const noexcept
{
return hash<std::underlying_type<Rust::PrimitiveCoreType>::type> () (
static_cast<std::underlying_type<Rust::PrimitiveCoreType>::type> (
coretype));
}
};
} // namespace std
#endif