Skip to content

Commit f26b7aa

Browse files
ngeorngeor
committed
refactor(parser): keyword parser should not handle EOF
keyword parser should not handle EOF explicitly, that was a functionality specific to detecting exit keywords in statements. Also removed the unused keyword from the relevant enum type.
1 parent c5171a2 commit f26b7aa

3 files changed

Lines changed: 78 additions & 49 deletions

File tree

‎rusty_parser/src/core/statements.rs‎

Lines changed: 35 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -6,14 +6,15 @@ use crate::core::statement::statement_p;
66
use crate::core::statement_separator::{comment_separator, common_separator};
77
use crate::input::StringView;
88
use crate::pc_specific::*;
9+
use crate::tokens::{TokenMatcher, peek_token};
910
use crate::*;
1011

1112
macro_rules! zero_or_more_statements {
1213
($exit:expr, ParserError::$err:ident) => {
13-
$crate::core::statements::zero_or_more_statements_p([$exit], Some(ParserError::$err))
14+
$crate::core::statements::zero_or_more_statements_p(&[$exit], Some(ParserError::$err))
1415
};
1516
($($exit:expr),+) => {
16-
$crate::core::statements::zero_or_more_statements_p( [$($exit),+], None)
17+
$crate::core::statements::zero_or_more_statements_p( &[$($exit),+], None)
1718
};
1819
}
1920

@@ -27,7 +28,7 @@ pub(crate) use zero_or_more_statements;
2728
///
2829
/// The custom error can be used to specify a custom error when the exit keyword is not found.
2930
pub fn zero_or_more_statements_p(
30-
exit_keywords: impl IntoIterator<Item = Keyword> + 'static,
31+
exit_keywords: &[Keyword],
3132
custom_err: Option<ParserError>,
3233
) -> impl Parser<StringView, Output = Statements, Error = ParserError> {
3334
one_statement_p(exit_keywords, custom_err)
@@ -39,15 +40,15 @@ pub fn zero_or_more_statements_p(
3940

4041
/// Either parses one statement or detects the exit keyword and stops parsing.
4142
fn one_statement_p(
42-
exit_keywords: impl IntoIterator<Item = Keyword> + 'static,
43+
exit_keywords: &[Keyword],
4344
custom_err: Option<ParserError>,
4445
) -> impl Parser<StringView, bool, Output = StatementPos, Error = ParserError> + SetContext<bool> {
4546
one_statement_or_exit_keyword_p(exit_keywords, custom_err).and_then(
4647
|statement_or_exit_keyword| match statement_or_exit_keyword {
4748
// we parsed a statement, return it
4849
StatementOrExitKeyword::Statement(s) => Ok(s),
4950
// we detected an exit keyword, stop parsing
50-
StatementOrExitKeyword::ExitKeyword(_keyword) => default_parse_error(),
51+
StatementOrExitKeyword::ExitKeyword => default_parse_error(),
5152
},
5253
)
5354
}
@@ -57,7 +58,7 @@ fn one_statement_p(
5758
/// which depending on the context (previously parsed statement)
5859
/// is either the comment separator (EOL) or the regular separator (EOL or colon).
5960
fn one_statement_or_exit_keyword_p(
60-
exit_keywords: impl IntoIterator<Item = Keyword> + 'static,
61+
exit_keywords: &[Keyword],
6162
custom_err: Option<ParserError>,
6263
) -> impl Parser<StringView, bool, Output = StatementOrExitKeyword, Error = ParserError> + SetContext<bool>
6364
{
@@ -101,29 +102,43 @@ fn ctx_demand_separator_p()
101102
}
102103

103104
fn find_exit_keyword_or_demand_statement_p(
104-
exit_keywords: impl IntoIterator<Item = Keyword> + 'static,
105+
exit_keywords: &[Keyword],
105106
custom_err: Option<ParserError>,
106107
) -> impl Parser<StringView, Output = StatementOrExitKeyword, Error = ParserError> {
107108
find_exit_keyword_p(exit_keywords, custom_err).or(demand_statement_p())
108109
}
109110

110111
fn find_exit_keyword_p(
111-
exit_keywords: impl IntoIterator<Item = Keyword> + 'static,
112+
exit_keywords: &[Keyword],
112113
custom_err: Option<ParserError>,
113114
) -> impl Parser<StringView, Output = StatementOrExitKeyword, Error = ParserError> {
114-
// the first parser will return:
115+
// the parser will return:
115116
// Ok if it finds the keyword (peeking)
116-
// Err(false) if it finds something else
117-
// Err(true) if it finds EOF
118-
let p = keyword_p(exit_keywords, true)
119-
.peek()
120-
// Ok(None) if it finds the keyword
121-
.map(StatementOrExitKeyword::ExitKeyword);
117+
// Soft error if it finds something else
118+
// Fatal error if it finds EOF
119+
peek_token().to_option().and_then(move |opt_token| {
120+
match opt_token {
121+
Some(token) => {
122+
for exit_keyword in exit_keywords {
123+
if exit_keyword.matches_token(&token) {
124+
return Ok(StatementOrExitKeyword::ExitKeyword);
125+
}
126+
}
122127

123-
match custom_err {
124-
Some(err) => p.with_fatal_err(err).boxed(),
125-
None => p.boxed(),
126-
}
128+
Err(ParserError::expected(&to_syntax_err(exit_keywords.iter())))
129+
}
130+
None => {
131+
// eof is fatal
132+
match &custom_err {
133+
Some(err) => Err(err.clone()),
134+
None => {
135+
let s = to_syntax_err(exit_keywords.iter());
136+
Err(ParserError::expected(&s).to_fatal())
137+
}
138+
}
139+
}
140+
}
141+
})
127142
}
128143

129144
fn demand_statement_p()
@@ -137,5 +152,5 @@ fn demand_statement_p()
137152

138153
enum StatementOrExitKeyword {
139154
Statement(StatementPos),
140-
ExitKeyword(Keyword),
155+
ExitKeyword,
141156
}

‎rusty_parser/src/pc_specific/keyword.rs‎

Lines changed: 42 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ macro_rules! keyword_of {
2525
$($keyword:expr),+
2626
$(,)?
2727
) => {
28-
$crate::pc_specific::keyword_p([ $($keyword),+ ], false)
28+
$crate::pc_specific::keyword_p([ $($keyword),+ ])
2929
};
3030
}
3131

@@ -34,9 +34,8 @@ pub(crate) use keyword_of;
3434
/// Parses on of the given keywords, possibly treating EOF as a fatal error.
3535
pub fn keyword_p(
3636
keywords: impl IntoIterator<Item = Keyword>,
37-
eof_is_fatal: bool,
3837
) -> impl Parser<StringView, Output = Keyword, Error = ParserError> {
39-
KeywordParser::new(any_token(), keywords, eof_is_fatal)
38+
KeywordParser::new(any_token(), keywords)
4039
}
4140

4241
pub fn keyword_ignoring(k: Keyword) -> impl Parser<StringView, Output = (), Error = ParserError> {
@@ -68,21 +67,17 @@ pub struct KeywordParser<P> {
6867
parser: P,
6968
// using a BTreeSet so that the generated error message is predictable (keywords sorted)
7069
keywords: BTreeSet<Keyword>,
71-
72-
/// If true, EOF will be reported as a fatal error
73-
eof_is_fatal: bool,
7470
}
7571

7672
impl<P> KeywordParser<P> {
77-
pub fn new(parser: P, keywords: impl IntoIterator<Item = Keyword>, eof_is_fatal: bool) -> Self {
73+
pub fn new(parser: P, keywords: impl IntoIterator<Item = Keyword>) -> Self {
7874
let mut keyword_set: BTreeSet<Keyword> = BTreeSet::new();
7975
for keyword in keywords {
8076
keyword_set.insert(keyword);
8177
}
8278
Self {
8379
parser,
8480
keywords: keyword_set,
85-
eof_is_fatal,
8681
}
8782
}
8883
}
@@ -102,12 +97,12 @@ where
10297
Some(keyword) => Ok(keyword),
10398
None => {
10499
input.set_position(original_input);
105-
self.to_syntax_err(false)
100+
self.to_syntax_err()
106101
}
107102
},
108103
Err(err) if err.is_soft() => {
109104
input.set_position(original_input);
110-
self.to_syntax_err(self.eof_is_fatal)
105+
self.to_syntax_err()
111106
}
112107
Err(err) => Err(err),
113108
}
@@ -137,26 +132,45 @@ impl<P> KeywordParser<P> {
137132
}
138133
}
139134

140-
fn to_syntax_err<O>(&self, fatal: bool) -> Result<O, ParserError> {
141-
let mut msg = String::from("Expected: ");
142-
let mut is_first = true;
143-
for k in &self.keywords {
144-
if is_first {
145-
is_first = false;
146-
} else {
147-
msg.push_str(" or ");
148-
}
135+
fn to_syntax_err<O>(&self) -> Result<O, ParserError> {
136+
let s = to_syntax_err(self.keywords.iter());
137+
Err(ParserError::expected(&s))
138+
}
139+
}
149140

150-
// doing &.to_string() to get it in uppercase
151-
msg.push_str(&k.to_string());
152-
}
141+
/// Concatenates the given keywords into a syntax error message.
142+
pub fn to_syntax_err<'a>(keywords: impl Iterator<Item = &'a Keyword>) -> String {
143+
keywords
144+
.map(|k| k.to_string())
145+
.reduce(|a, b| a + " or " + &b)
146+
.unwrap_or_default()
147+
}
153148

154-
let err = if fatal {
155-
ParserError::SyntaxError(msg)
156-
} else {
157-
ParserError::Expected(msg)
158-
};
149+
#[cfg(test)]
150+
mod tests {
151+
use super::*;
152+
153+
#[test]
154+
fn test_to_syntax_err_zero_items() {
155+
let keywords: Vec<Keyword> = vec![];
156+
assert_eq!(to_syntax_err(keywords.iter()), "");
157+
}
158+
159+
#[test]
160+
fn test_to_syntax_err_one_item() {
161+
let keywords = vec![Keyword::If];
162+
assert_eq!(to_syntax_err(keywords.iter()), "IF");
163+
}
164+
165+
#[test]
166+
fn test_to_syntax_err_two_items() {
167+
let keywords = vec![Keyword::If, Keyword::Then];
168+
assert_eq!(to_syntax_err(keywords.iter()), "IF or THEN");
169+
}
159170

160-
Err(err)
171+
#[test]
172+
fn test_to_syntax_err_three_items() {
173+
let keywords = vec![Keyword::If, Keyword::Then, Keyword::Else];
174+
assert_eq!(to_syntax_err(keywords.iter()), "IF or THEN or ELSE");
161175
}
162176
}

‎rusty_parser/src/pc_specific/keyword_map.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,6 @@ where
1414
T: Clone,
1515
{
1616
let keyword_to_value: HashMap<Keyword, T> = mappings.iter().cloned().collect();
17-
keyword_p(mappings.iter().map(|(k, _)| *k), false)
17+
keyword_p(mappings.iter().map(|(k, _)| *k))
1818
.map(move |keyword| keyword_to_value.get(&keyword).unwrap().clone())
1919
}

0 commit comments

Comments
 (0)