implemented lexer in minifier, added test

Author: Pamblam

jSQL.js

@@ -1069,6 +1069,7 @@ function jSQLCreateQuery(){
 function jSQLLexer(input) {
 	this.input = input;
 	this.pos = 0;
+	this.real_pos = 0;
 	this.tokens = [];
 	this.token_matches = [];
 }
@@ -1097,28 +1098,25 @@ jSQLLexer.prototype.getTokens = function(){
 			var r = matches[type_id][match_index];
 			if(r.index !== this.pos) continue;
 			var token = new jSQLToken(this.pos, r[0], type_id);
-			if(throwaway.indexOf(token.type) === -1) this.tokens.push(token);
 			this.pos += token.length;
-			type_id=0;
+			if(throwaway.indexOf(token.type) === -1) this.tokens.push(token);
+			type_id=-1;
 			break;
 		}
 	}
-	if(this.pos !== this.input.length)
-		return _throw(new jSQL_Lexer_Error(this.pos, this.input)); 
-	
+	if(this.pos !== this.input.length){
+		var pos;
+		if(this.tokens.length){
+			var lastToken = this.tokens[this.tokens.length-1];
+			pos = lastToken.input_pos + lastToken.length;
+		}else pos = 0;
+		return _throw(new jSQL_Lexer_Error(pos, this.input)); 
+	}
 	return this.tokens;
 };
 
 jSQLLexer.token_types = [
 
-	// WHITESPACE
-	{pattern: /[\n\r]/g,
-		type: 'WHITESPACE',
-		name: "LINEBREAK"},
-	{pattern: /[ \t]/g,
-		type: 'WHITESPACE',
-		name: "WHITESPACE"},
-	
 	// STRINGs
 	{pattern: /"(?:[^"\\]|\\.)*"/g,
 		type: 'STRING',
@@ -1135,6 +1133,14 @@ jSQLLexer.token_types = [
 		type: 'COMMENT',
 		name: "MULTI LINE COMMENT"},
 
+	// WHITESPACE
+	{pattern: /\r?\n|\r/g,
+		type: 'WHITESPACE',
+		name: "LINEBREAK"},
+	{pattern: /[ \t]/g,
+		type: 'WHITESPACE',
+		name: "WHITESPACE"},
+
 	// NUMBERs
 	{pattern: /\d+/g,
 		type: 'NUMBER',

jSQL.min.js

@@ -7,7 +7,7 @@
     "example": "tests"
   },
   "scripts": {
-    "test": "node ./tests/test1.js && node ./tests/test2.js && node ./tests/test3.js && node ./tests/test4.js && node ./tests/test5.js && node ./tests/test6.js"
+    "test": "node ./tests/test1.js && node ./tests/test2.js && node ./tests/test3.js && node ./tests/test4.js && node ./tests/test5.js && node ./tests/test6.js && node ./tests/test7.js"
   },
   "repository": {
     "type": "git",

package.json

@@ -3,6 +3,7 @@
 function jSQLLexer(input) {
 	this.input = input;
 	this.pos = 0;
+	this.real_pos = 0;
 	this.tokens = [];
 	this.token_matches = [];
 }
@@ -31,28 +32,25 @@ jSQLLexer.prototype.getTokens = function(){
 			var r = matches[type_id][match_index];
 			if(r.index !== this.pos) continue;
 			var token = new jSQLToken(this.pos, r[0], type_id);
-			if(throwaway.indexOf(token.type) === -1) this.tokens.push(token);
 			this.pos += token.length;
-			type_id=0;
+			if(throwaway.indexOf(token.type) === -1) this.tokens.push(token);
+			type_id=-1;
 			break;
 		}
 	}
-	if(this.pos !== this.input.length)
-		return _throw(new jSQL_Lexer_Error(this.pos, this.input)); 
-	
+	if(this.pos !== this.input.length){
+		var pos;
+		if(this.tokens.length){
+			var lastToken = this.tokens[this.tokens.length-1];
+			pos = lastToken.input_pos + lastToken.length;
+		}else pos = 0;
+		return _throw(new jSQL_Lexer_Error(pos, this.input)); 
+	}
 	return this.tokens;
 };
 
 jSQLLexer.token_types = [
 
-	// WHITESPACE
-	{pattern: /[\n\r]/g,
-		type: 'WHITESPACE',
-		name: "LINEBREAK"},
-	{pattern: /[ \t]/g,
-		type: 'WHITESPACE',
-		name: "WHITESPACE"},
-	
 	// STRINGs
 	{pattern: /"(?:[^"\\]|\\.)*"/g,
 		type: 'STRING',
@@ -69,6 +67,14 @@ jSQLLexer.token_types = [
 		type: 'COMMENT',
 		name: "MULTI LINE COMMENT"},
 
+	// WHITESPACE
+	{pattern: /\r?\n|\r/g,
+		type: 'WHITESPACE',
+		name: "LINEBREAK"},
+	{pattern: /[ \t]/g,
+		type: 'WHITESPACE',
+		name: "WHITESPACE"},
+
 	// NUMBERs
 	{pattern: /\d+/g,
 		type: 'NUMBER',

src/lexer/jSQLLexer.js

@@ -90,17 +90,5 @@ jSQL.load(function(){
 		}
 	}
 
-	console.log(jSQL.minify("\
- \
-insert into    `typetest` \
-values (\
-  11, '{\
-      \"greeting\":\"hello\"\
-}', 'function(){\
-  console.log(\"hello world\")\
-}', true, 3, '72', 'rob', \
-'Mon, 25 Dec 1995 13:30:00 +0430', \
-'scooby doo', \"hello\")"));
-	
 	console.log("\n\nsuccess");
 });