Initial Rust project setup and POC implementation of the Forthic tokenizer

Author: joshvfleming

Makefile

@@ -43,6 +43,12 @@ test-react:
 	@echo "============"
 	cd forthic-react/v1 && npm install && CI=1 npm run test
 
+test-rs:
+	@echo
+	@echo "Forthic Rust tests"
+	@echo "============"
+	cargo test --manifest-path tests/tests_rs/Cargo.toml
+
 
 test-all: test-py test-react test-js
 

forthic-rs/.gitignore

@@ -0,0 +1,2 @@
+target/
+.env

forthic-rs/Cargo.lock

@@ -0,0 +1,11 @@
+[package]
+name = "forthic-rs"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+enum_dispatch = "0.3.12"
+itertools = "0.12.0"
+lazy_static = "1.4"

forthic-rs/Cargo.toml

@@ -0,0 +1,17 @@
+#[derive(Debug)]
+pub struct InterpreterError {
+    pub message: String,
+    pub filename: String,
+    pub line: u32,
+}
+
+#[macro_export]
+macro_rules! interpreter_error {
+    ($msg:expr) => {
+        InterpreterError {
+            message: $msg.to_string(),
+            filename: file!().to_string(),
+            line: line!(),
+        }
+    };
+}

forthic-rs/src/errors.rs

@@ -0,0 +1,3 @@
+pub mod errors;
+pub mod token;
+pub mod tokenizer;

forthic-rs/src/lib.rs

@@ -0,0 +1,11 @@
+mod errors;
+mod token;
+mod tokenizer;
+
+use tokenizer::Tokenizer;
+
+fn main() {
+    let mut lexer = Tokenizer::new("Hello, world!");
+    let t = lexer.next_token().unwrap();
+    println!("{:?}", t);
+}

forthic-rs/src/main.rs

@@ -0,0 +1,14 @@
+#[derive(Debug, PartialEq, Clone)]
+pub enum Token {
+    String(String),
+    Comment(String),
+    StartArray,
+    EndArray,
+    StartModule(String),
+    EndModule,
+    StartDefinition(String),
+    EndDefinition,
+    StartMemo(String),
+    Word(String),
+    EOS,
+}