Skip to content

Commit 507ad97

Browse files
ngeorngeor
authored
refactor: Add dedicated whitespace module (#120)
* Removed obsolete TODOs * Moved whitespace related parsers to a new module `whitespace` under `pc_specific`. Removed the shortcut in `any_token` that allowed to parse `whitespace` without parsing a token first. * Removed the PaddedByWs trait, replaced it with a function * Removed the unused whitespace function * Added lead_ws and lead_opt_ws functions * use lead_ws in a few cases * use lead_ws in a few cases * use lead_ws and lead_opt_ws a bit more * Adding demand_ws and demand_lead_ws functions * Added function demand_lead_ws_ctx to deal with SetContext being a different trait * Removed opt_and_keep_right * Removed opt_and * Created dedicated parser IgnoringParser for skipping whitespace * Revert "Created dedicated parser IgnoringParser for skipping whitespace" This reverts commit ac14b62. * fix build after rebase
1 parent 12c2050 commit 507ad97

31 files changed

Lines changed: 182 additions & 241 deletions

‎rusty_parser/src/built_ins/line_input.rs‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,15 @@ use rusty_pc::*;
33
use crate::built_ins::common::{encode_opt_file_handle_arg, opt_file_handle_comma_p};
44
use crate::input::StringView;
55
use crate::pc_specific::*;
6-
use crate::tokens::whitespace_ignoring;
76
use crate::{BuiltInSub, ParserError, *};
87
// LINE INPUT variable$
98
// LINE INPUT #file-number%, variable$
109
pub fn parse() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
11-
seq4(
10+
seq3(
1211
keyword_pair(Keyword::Line, Keyword::Input),
13-
whitespace_ignoring(),
14-
opt_file_handle_comma_p(),
12+
demand_lead_ws(opt_file_handle_comma_p()),
1513
expression_pos_p().or_expected("#file-number or variable"),
16-
|_, _, opt_file_number_pos, variable| {
14+
|_, opt_file_number_pos, variable| {
1715
let mut args: Expressions = encode_opt_file_handle_arg(opt_file_number_pos);
1816
// add the LINE INPUT variable
1917
args.push(variable);

‎rusty_parser/src/built_ins/open.rs‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
use rusty_common::*;
2-
use rusty_pc::and::IgnoringBothCombiner;
32
use rusty_pc::*;
43

54
use crate::input::StringView;
65
use crate::pc_specific::*;
7-
use crate::tokens::{equal_sign_ws, whitespace_ignoring};
6+
use crate::tokens::equal_sign_ws;
87
use crate::{BuiltInSub, ParserError, *};
98
pub fn parse() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
109
seq6(
@@ -65,7 +64,7 @@ fn parse_file_number_p() -> impl Parser<StringView, Output = ExpressionPos, Erro
6564

6665
fn parse_len_p() -> impl Parser<StringView, Output = ExpressionPos, Error = ParserError> {
6766
seq3(
68-
whitespace_ignoring().and(keyword_ignoring(Keyword::Len), IgnoringBothCombiner),
67+
lead_ws(keyword_ignoring(Keyword::Len)),
6968
equal_sign_ws(),
7069
expression_pos_p().or_expected("expression after LEN ="),
7170
|_, _, e| e,

‎rusty_parser/src/core/declaration.rs‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,9 @@
1-
use rusty_pc::and::opt_and_keep_right;
21
use rusty_pc::*;
32

43
use crate::core::name::{bare_name_p, name_p};
54
use crate::core::param_name::parameter_pos_p;
65
use crate::input::StringView;
76
use crate::pc_specific::*;
8-
use crate::tokens::whitespace_ignoring;
97
use crate::{ParserError, *};
108

119
// Declaration ::= DECLARE<ws+>(FunctionDeclaration|SubDeclaration)
@@ -53,12 +51,7 @@ pub fn sub_declaration_p()
5351

5452
// result ::= "" | "(" ")" | "(" parameter (,parameter)* ")"
5553
fn declaration_parameters_p() -> impl Parser<StringView, Output = Parameters, Error = ParserError> {
56-
// TODO remove the need for the double .or_default()
57-
opt_and_keep_right(
58-
whitespace_ignoring(),
59-
in_parenthesis(csv(parameter_pos_p()).or_default()),
60-
)
61-
.or_default()
54+
lead_opt_ws(in_parenthesis(csv(parameter_pos_p()).or_default())).or_default()
6255
}
6356

6457
#[cfg(test)]

‎rusty_parser/src/core/def_type.rs‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ use rusty_pc::*;
33
use crate::error::ParserError;
44
use crate::input::StringView;
55
use crate::pc_specific::*;
6-
use crate::tokens::{TokenType, any_token_of, minus_sign, whitespace_ignoring};
6+
use crate::tokens::{TokenType, any_token_of, minus_sign};
77
use crate::{Keyword, LetterRange, TypeQualifier};
88

99
/// Represents a definition of default type, such as DEFINT A-Z.
@@ -34,11 +34,10 @@ impl DefType {
3434
// Letter ::= [a-zA-Z]
3535

3636
pub fn def_type_p() -> impl Parser<StringView, Output = DefType, Error = ParserError> {
37-
seq3(
37+
seq2(
3838
def_keyword_p(),
39-
whitespace_ignoring(),
40-
letter_ranges(),
41-
|l, _, r| DefType::new(l, r),
39+
demand_lead_ws(letter_ranges()),
40+
DefType::new,
4241
)
4342
}
4443

‎rusty_parser/src/core/do_loop.rs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ use crate::core::expression::ws_expr_pos_p;
44
use crate::core::statements::zero_or_more_statements;
55
use crate::input::StringView;
66
use crate::pc_specific::*;
7-
use crate::tokens::whitespace_ignoring;
87
use crate::{ParserError, *};
98

109
pub fn do_loop_p() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
@@ -19,7 +18,7 @@ pub fn do_loop_p() -> impl Parser<StringView, Output = Statement, Error = Parser
1918

2019
fn do_condition_top() -> impl Parser<StringView, Output = DoLoop, Error = ParserError> {
2120
seq4(
22-
whitespace_ignoring().and_keep_right(keyword_of!(Keyword::Until, Keyword::While)),
21+
lead_ws(keyword_of!(Keyword::Until, Keyword::While)),
2322
ws_expr_pos_p().or_expected("expression"),
2423
zero_or_more_statements!(Keyword::Loop),
2524
keyword(Keyword::Loop),

‎rusty_parser/src/core/expression.rs‎

Lines changed: 35 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -554,7 +554,6 @@ fn eager_expression_pos_p() -> impl Parser<StringView, Output = ExpressionPos, E
554554
}
555555

556556
mod single_or_double_literal {
557-
use rusty_pc::and::opt_and_tuple;
558557
use rusty_pc::*;
559558

560559
use crate::input::StringView;
@@ -570,36 +569,36 @@ mod single_or_double_literal {
570569
// TODO support more qualifiers besides '#'
571570

572571
pub fn parser() -> impl Parser<StringView, Output = ExpressionPos, Error = ParserError> {
573-
// TODO this is difficult to understand
574-
opt_and_tuple(
575-
// read integer digits optionally (might start with . e.g. `.123`)
576-
digits(),
572+
// read integer digits optionally (might start with . e.g. `.123`)
573+
digits()
574+
.to_option()
577575
// read dot and demand digits after decimal point
578576
// if dot is missing, the parser returns an empty result
579577
// the "deal breaker" is therefore the dot
580-
dot().and_keep_right(digits().to_fatal()),
581-
)
582-
// and parse optionally a type qualifier such as `#`
583-
.and_tuple(pound().to_option())
584-
// done parsing, flat map everything
585-
.and_then(|((opt_integer_digits, frac_digits), opt_pound)| {
586-
let left = opt_integer_digits
587-
.map(|token| token.to_string())
588-
.unwrap_or_else(|| "0".to_owned());
589-
let s = format!("{}.{}", left, frac_digits.as_str());
590-
if opt_pound.is_some() {
591-
match s.parse::<f64>() {
592-
Ok(f) => Ok(Expression::DoubleLiteral(f)),
593-
Err(err) => Err(err.into()),
594-
}
595-
} else {
596-
match s.parse::<f32>() {
597-
Ok(f) => Ok(Expression::SingleLiteral(f)),
598-
Err(err) => Err(err.into()),
578+
.and_keep_left(dot())
579+
// demand digits after decimal point
580+
.and_tuple(digits().to_fatal())
581+
// and parse optionally a type qualifier such as `#`
582+
.and_tuple(pound().to_option())
583+
// done parsing, flat map everything
584+
.and_then(|((opt_integer_digits, frac_digits), opt_pound)| {
585+
let left = opt_integer_digits
586+
.map(|token| token.to_string())
587+
.unwrap_or_else(|| "0".to_owned());
588+
let s = format!("{}.{}", left, frac_digits.as_str());
589+
if opt_pound.is_some() {
590+
match s.parse::<f64>() {
591+
Ok(f) => Ok(Expression::DoubleLiteral(f)),
592+
Err(err) => Err(err.into()),
593+
}
594+
} else {
595+
match s.parse::<f32>() {
596+
Ok(f) => Ok(Expression::SingleLiteral(f)),
597+
Err(err) => Err(err.into()),
598+
}
599599
}
600-
}
601-
})
602-
.with_pos()
600+
})
601+
.with_pos()
603602
}
604603
}
605604

@@ -998,16 +997,16 @@ mod built_in_function_call {
998997

999998
mod binary_expression {
1000999
use rusty_common::Positioned;
1001-
use rusty_pc::and::{TupleCombiner, opt_and_keep_right};
1000+
use rusty_pc::and::TupleCombiner;
10021001
use rusty_pc::*;
10031002

10041003
use super::{
10051004
built_in_function_call, expression_pos_p, guard, integer_or_long_literal, parenthesis, property, single_or_double_literal, string_literal, unary_expression
10061005
};
10071006
use crate::error::ParserError;
10081007
use crate::input::StringView;
1009-
use crate::pc_specific::{OrExpected, WithPos};
1010-
use crate::tokens::{TokenType, any_token, whitespace_ignoring};
1008+
use crate::pc_specific::{OrExpected, WithPos, lead_opt_ws, lead_ws};
1009+
use crate::tokens::{TokenType, any_token};
10111010
use crate::*;
10121011

10131012
// result ::= <non-bin-expr> <operator> <expr>
@@ -1078,14 +1077,9 @@ mod binary_expression {
10781077
-> impl Parser<StringView, bool, Output = Positioned<Operator>, Error = ParserError> {
10791078
IifParser::new(
10801079
// no whitespace needed
1081-
opt_and_keep_right(whitespace_ignoring(), operator_p()),
1080+
lead_opt_ws(operator_p()),
10821081
// whitespace needed
1083-
whitespace_ignoring()
1084-
.and_keep_right(operator_p())
1085-
.or(opt_and_keep_right(
1086-
whitespace_ignoring(),
1087-
symbol_operator_p(),
1088-
)),
1082+
lead_ws(operator_p()).or(lead_opt_ws(symbol_operator_p())),
10891083
)
10901084
}
10911085

@@ -1201,7 +1195,7 @@ pub mod file_handle {
12011195
use crate::error::ParserError;
12021196
use crate::input::StringView;
12031197
use crate::pc_specific::*;
1204-
use crate::tokens::{TokenType, any_token_of, pound, whitespace_ignoring};
1198+
use crate::tokens::{TokenType, any_token_of, pound};
12051199
use crate::*;
12061200

12071201
pub fn file_handle_p()
@@ -1230,7 +1224,7 @@ pub mod file_handle {
12301224
}
12311225

12321226
fn ws_file_handle() -> impl Parser<StringView, Output = ExpressionPos, Error = ParserError> {
1233-
whitespace_ignoring().and_keep_right(file_handle_as_expression_pos_p())
1227+
lead_ws(file_handle_as_expression_pos_p())
12341228
}
12351229
}
12361230

@@ -1239,8 +1233,8 @@ pub mod guard {
12391233

12401234
use crate::ParserError;
12411235
use crate::input::StringView;
1242-
use crate::pc_specific::WithExpected;
1243-
use crate::tokens::{any_symbol_of, any_token_of, whitespace_ignoring};
1236+
use crate::pc_specific::{WithExpected, whitespace_ignoring};
1237+
use crate::tokens::{any_symbol_of, any_token_of};
12441238

12451239
/// `result ::= " " | "("`
12461240
///

‎rusty_parser/src/core/for_loop.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use crate::core::statements::zero_or_more_statements;
66
use crate::error::ParserError;
77
use crate::input::StringView;
88
use crate::pc_specific::*;
9-
use crate::tokens::{equal_sign_ws, whitespace_ignoring};
9+
use crate::tokens::equal_sign_ws;
1010
use crate::*;
1111

1212
// FOR I = 0 TO 5 STEP 1
@@ -65,7 +65,7 @@ fn parse_for_p()
6565
}
6666

6767
fn next_counter_p() -> impl Parser<StringView, Output = ExpressionPos, Error = ParserError> {
68-
whitespace_ignoring().and_keep_right(property::parser())
68+
lead_ws(property::parser())
6969
}
7070

7171
#[cfg(test)]

‎rusty_parser/src/core/global_statement.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -159,7 +159,7 @@ fn main_program() -> impl Parser<StringView, Output = Program, Error = ParserErr
159159
}
160160

161161
fn next_statements() -> impl Parser<StringView, Output = Program, Error = ParserError> {
162-
next_statement().padded_by_ws().zero_or_more()
162+
padded_by_ws(next_statement()).zero_or_more()
163163
}
164164

165165
fn next_statement() -> impl Parser<StringView, Output = GlobalStatementPos, Error = ParserError> {

‎rusty_parser/src/core/go_sub.rs‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ use rusty_pc::*;
33
use crate::core::name::bare_name_p;
44
use crate::input::StringView;
55
use crate::pc_specific::*;
6-
use crate::tokens::whitespace_ignoring;
76
use crate::{Keyword, ParserError, Statement};
87

98
pub fn statement_go_sub_p() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
@@ -13,13 +12,9 @@ pub fn statement_go_sub_p() -> impl Parser<StringView, Output = Statement, Error
1312
}
1413

1514
pub fn statement_return_p() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
16-
seq2(
17-
keyword(Keyword::Return),
18-
whitespace_ignoring()
19-
.and_keep_right(bare_name_p())
20-
.to_option(),
21-
|_, name| Statement::Return(name),
22-
)
15+
keyword(Keyword::Return)
16+
.and_keep_right(lead_ws(bare_name_p()).to_option())
17+
.map(Statement::Return)
2318
}
2419

2520
#[cfg(test)]

‎rusty_parser/src/core/if_block.rs‎

Lines changed: 2 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
use rusty_pc::and::IgnoringBothCombiner;
21
use rusty_pc::*;
32

43
use crate::core::comment::comment_p;
@@ -9,7 +8,6 @@ use crate::core::single_line_statements::{
98
use crate::core::statements::zero_or_more_statements;
109
use crate::input::StringView;
1110
use crate::pc_specific::*;
12-
use crate::tokens::whitespace_ignoring;
1311
use crate::{ParserError, *};
1412

1513
pub fn if_block_p() -> impl Parser<StringView, Output = Statement, Error = ParserError> {
@@ -58,14 +56,11 @@ fn single_line_if_else_p() -> impl Parser<
5856
}
5957

6058
fn single_line_comment_p() -> impl Parser<StringView, Output = Statements, Error = ParserError> {
61-
whitespace_ignoring()
62-
.to_option()
63-
.and(comment_p().with_pos(), |_, s| vec![s])
59+
lead_opt_ws(comment_p().with_pos()).map(|s| vec![s])
6460
}
6561

6662
fn single_line_else_p() -> impl Parser<StringView, Output = Statements, Error = ParserError> {
67-
whitespace_ignoring()
68-
.and(keyword(Keyword::Else), IgnoringBothCombiner)
63+
lead_ws(keyword(Keyword::Else))
6964
.and_keep_right(single_line_statements_p().or_expected("Statements for single line ELSE"))
7065
}
7166

0 commit comments

Comments
 (0)