mirrors/ladybird
0
0
Fork 0
mirror of https://github.com/LadybirdBrowser/ladybird.git synced 2025-09-06 09:36:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

LibCpp: Store the text of a token as a StringView member

Browse source
This commit is contained in:
Itamar 2021-03-12 13:11:41 +02:00 • committed by Andreas Kling
commit 26d9485562
Notes: sideshowbarker 2024-07-18 21:25:53 +09:00
3 changed files with 25 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

30
Userland/Libraries/LibCpp/Lexer.cpp
View file

@ -231,8 +231,8 @@ Vector<Token> Lexer::lex()
size_t token_start_index = 0;
Position token_start_position;
auto emit_token = [&](auto type) {
tokens.empend(type, m_position, m_position);
auto emit_single_char_token = [&](auto type) {
tokens.empend(type, m_position, m_position, m_input.substring_view(m_index, 1));
consume();
};
@ -241,7 +241,7 @@ Vector<Token> Lexer::lex()
token_start_position = m_position;
};
auto commit_token = [&](auto type) {
tokens.empend(type, token_start_position, m_previous_position);
tokens.empend(type, token_start_position, m_previous_position, m_input.substring_view(token_start_index, m_index - token_start_index));
};
auto emit_token_equals = [&](auto type, auto equals_type) {
@ -252,7 +252,7 @@ Vector<Token> Lexer::lex()
commit_token(equals_type);
return;
}
emit_token(type);
emit_single_char_token(type);
};
auto match_escape_sequence = [&]() -> size_t {
@ -335,27 +335,27 @@ Vector<Token> Lexer::lex()
continue;
}
if (ch == '(') {
emit_token(Token::Type::LeftParen);
emit_single_char_token(Token::Type::LeftParen);
continue;
}
if (ch == ')') {
emit_token(Token::Type::RightParen);
emit_single_char_token(Token::Type::RightParen);
continue;
}
if (ch == '{') {
emit_token(Token::Type::LeftCurly);
emit_single_char_token(Token::Type::LeftCurly);
continue;
}
if (ch == '}') {
emit_token(Token::Type::RightCurly);
emit_single_char_token(Token::Type::RightCurly);
continue;
}
if (ch == '[') {
emit_token(Token::Type::LeftBracket);
emit_single_char_token(Token::Type::LeftBracket);
continue;
}
if (ch == ']') {
emit_token(Token::Type::RightBracket);
emit_single_char_token(Token::Type::RightBracket);
continue;
}
if (ch == '<') {
@ -406,7 +406,7 @@ Vector<Token> Lexer::lex()
continue;
}
if (ch == ',') {
emit_token(Token::Type::Comma);
emit_single_char_token(Token::Type::Comma);
continue;
}
if (ch == '+') {
@ -504,11 +504,11 @@ Vector<Token> Lexer::lex()
continue;
}
if (ch == '~') {
emit_token(Token::Type::Tilde);
emit_single_char_token(Token::Type::Tilde);
continue;
}
if (ch == '?') {
emit_token(Token::Type::QuestionMark);
emit_single_char_token(Token::Type::QuestionMark);
continue;
}
if (ch == ':') {
@ -528,7 +528,7 @@ Vector<Token> Lexer::lex()
continue;
}
if (ch == ';') {
emit_token(Token::Type::Semicolon);
emit_single_char_token(Token::Type::Semicolon);
continue;
}
if (ch == '.') {
@ -778,7 +778,7 @@ Vector<Token> Lexer::lex()
continue;
}
dbgln("Unimplemented token character: {}", ch);
emit_token(Token::Type::Unknown);
emit_single_char_token(Token::Type::Unknown);
}
return tokens;
}