Skip to content

Commit e21d1f9

Browse files
author
Donald Steinert
committed
Merge pull request #191 from shift-js/feat/lexer
refactors and cleans up lexer.js and lexerFunctions.js
2 parents 29fb8dc + a16a0df commit e21d1f9

File tree

3 files changed

+115
-118
lines changed

3 files changed

+115
-118
lines changed

‎tests/lexerTests/lexerTestsThirdMilestone.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5614,7 +5614,7 @@ describe('Lexer: Third Milestone', function() {
56145614
expect(lexer(input)).to.deep.equal(output);
56155615
});
56165616

5617-
it('should handle the drop first method', function () {
5617+
it('should handle the drop last method', function () {
56185618
input = String.raw `var arr = [1,2,3,4,5]
56195619
var lessLast = arr.dropLast()
56205620
var lessLastThree = arr.dropLast(3)`;

‎transpiler/lexer/lexer.js

Lines changed: 27 additions & 95 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,15 @@ module.exports = function(code) {
2626
var insideTuple = [];
2727
var insideInvocation = [];
2828
var insideInitialization = [];
29-
// TODO - scope
3029

31-
// advances the position of i by specified number of positions
30+
// helper functions to advance the lexer's position in the input code
31+
// and clear the chunk
3232
var advance = function(positions) {
3333
i += positions;
3434
};
35-
3635
var clearChunk = function() {
3736
chunk = '';
3837
};
39-
40-
// advances the position of i by specified number of positions and clears chunk
4138
var advanceAndClear = function(positions) {
4239
i += positions;
4340
clearChunk();
@@ -51,24 +48,22 @@ module.exports = function(code) {
5148
nextCol = code[i + 1];
5249
nextNextCol = code[i + 2];
5350
var lastToken = tokens[tokens.length - 1];
54-
var lastCollectionIndex = insideCollection.length - 1;
55-
var lastCollection = insideCollection[lastCollectionIndex];
56-
var lastFunctionIndex = insideFunction.length - 1;
57-
var lastFunction = insideFunction[lastFunctionIndex];
51+
var lastCollection = insideCollection[insideCollection.length - 1];
52+
var lastFunction = insideFunction[insideFunction.length - 1];
5853

5954
// console.log(chunk);
6055
// console.log(currCol);
6156
// console.log(nextCol);
6257
// console.log(tokens);
6358
// console.log(emptyLine);
6459

65-
// newline handling
60+
// handles new lines
6661
if (lexerFunctions.handleNewLine(emptyLine, tokens, lastToken, currCol)) {
6762
advanceAndClear(1);
6863
continue
6964
}
7065

71-
// comment handling
66+
// handles comments
7267
if (lexerFunctions.checkForCommentStart(insideComment, chunk, tokens,
7368
currCol, nextCol)) {
7469
advanceAndClear(2);
@@ -83,50 +78,37 @@ module.exports = function(code) {
8378
continue;
8479
}
8580

86-
// ignoring whitespace
87-
if (chunk === ' ') {
81+
// ignores chunks that are solely whitespace
82+
if (lexerFunctions.checkForWhitespace(chunk)) {
8883
advanceAndClear(1);
8984
continue;
9085
}
9186

92-
// tracks state: whether inside a string
87+
// tracks whether inside a string
9388
if (currCol === '"' && insideString.status) {
9489
insideString.status = false;
9590
} else if (currCol === '"') {
9691
insideString.status = true;
9792
}
9893

99-
// number handling
94+
// handles numbers
10095
if (lexerFunctions.handleNumber(insideString, insideNumber, chunk, tokens, nextCol, nextNextCol) === true) {
10196
advanceAndClear(1);
10297
continue;
10398
} else if (lexerFunctions.handleNumber(insideString, insideNumber, chunk, tokens, nextCol, nextNextCol) === "skip"){
104-
advance(2);
10599
lexerFunctions.handleEndOfFile(nextCol, tokens);
100+
advance(2);
106101
continue;
107102
}
108103

109-
// handle ranges
110-
if (!insideString.status && !lexerFunctions.checkIfInsideComment(insideComment)) {
111-
if (currCol === '.' && nextCol === '.' && nextNextCol === '.') {
112-
if (insideFunction.length && insideFunction[insideFunction.length - 1].insideParams === true) {
113-
lexerFunctions.checkFor('FUNCTION_DECLARATION', '...', tokens);
114-
advanceAndClear(3);
115-
continue;
116-
} else {
117-
lexerFunctions.checkFor('RANGES', '...', tokens);
118-
advanceAndClear(3);
119-
continue;
120-
}
121-
}
122-
if (currCol === '.' && nextCol === '.' && nextNextCol === '<') {
123-
lexerFunctions.checkFor('RANGES', '..<', tokens);
124-
advanceAndClear(3);
125-
continue;
126-
}
104+
// handles ranges
105+
if (lexerFunctions.handleRange(insideString, insideFunction, insideComment,
106+
tokens, currCol, nextCol, nextNextCol)) {
107+
advanceAndClear(3);
108+
continue;
127109
}
128110

129-
// string interpolation handling
111+
// handles string interpolation
130112
if (lexerFunctions.checkForStringInterpolationStart(stringInterpolation,
131113
insideString, chunk, tokens, nextCol, nextNextCol)) {
132114
advanceAndClear(3);
@@ -196,15 +178,13 @@ module.exports = function(code) {
196178
lexerFunctions.handleEndOfFile(nextCol, tokens);
197179
continue;
198180
}
199-
200181
if (insideInvocation.length && chunk === '(' && (insideInvocation[insideInvocation.length - 1]).status) {
201182
lexerFunctions.checkFor('PUNCTUATION', chunk, tokens);
202183
var last = insideInvocation[insideInvocation.length - 1];
203184
last.parens++;
204185
advanceAndClear(1);
205186
continue;
206187
}
207-
208188
if (insideInvocation.length && chunk === ')' && (insideInvocation[insideInvocation.length - 1]).status) {
209189
lexerFunctions.checkFor('PUNCTUATION', chunk, tokens);
210190
var last = insideInvocation[insideInvocation.length - 1];
@@ -264,8 +244,8 @@ module.exports = function(code) {
264244
continue;
265245
}
266246

267-
if (tokens.length >= 2 && tokens[tokens.length - 2]['type'] === 'PUNCTUATION' &&
268-
tokens[tokens.length - 2]['value'] === '(' && lastFunction && lastFunction.insideReturnStatement === true) {
247+
if (tokens.length >= 2 && tokens[tokens.length - 2].type === 'PUNCTUATION' &&
248+
tokens[tokens.length - 2].value === '(' && lastFunction && lastFunction.insideReturnStatement === true) {
269249
tokens[tokens.length - 2].type = 'PARAMS_START';
270250
}
271251

@@ -304,7 +284,7 @@ module.exports = function(code) {
304284
continue;
305285
}
306286

307-
// collection initializer syntax handling
287+
// collection initializer handling
308288
if (tokens.length && currCol === '(' &&
309289
(lastToken.type === 'ARRAY_END' || lastToken.type === 'DICTIONARY_END')) {
310290
lexerFunctions.checkFor('FUNCTION_INVOCATION', currCol, tokens);
@@ -317,63 +297,17 @@ module.exports = function(code) {
317297
continue;
318298
}
319299

320-
/////////////////////////////////////////////
321-
// //
322-
// classes and structures handling //
323-
// //
324-
///////////////////////////////////////////////////////////////////////////
325-
326-
// handles inheritance operators
300+
// handles colons functioning as inheritance operators
327301
if (tokens.length > 2 && tokens[tokens.length - 2].value === ':' &&
328302
CLASS_NAMES[lastToken.value] && CLASS_NAMES[tokens[tokens.length - 3].value]) {
329303
tokens[tokens.length - 2].type = 'INHERITANCE_OPERATOR';
330304
}
331-
if (insideClass.length && insideClass[insideClass.length - 1].curly === 0 &&
332-
chunk === '{') {
333-
lexerFunctions.checkFor('CLASS_DEFINITION', chunk, tokens);
334-
insideClass[insideClass.length - 1].curly++;
335-
advanceAndClear(1);
336-
continue;
337-
}
338-
if (insideClass.length && insideClass[insideClass.length - 1].curly === 1 &&
339-
chunk === '}') {
340-
lexerFunctions.checkFor('CLASS_DEFINITION', chunk, tokens);
341-
insideClass.pop();
342-
advanceAndClear(1);
343-
lexerFunctions.handleEndOfFile(nextCol, tokens);
344-
continue;
345-
}
346-
if (insideStruct.length && insideStruct[insideStruct.length - 1].curly === 0 &&
347-
chunk === '{') {
348-
lexerFunctions.checkFor('STRUCT_DEFINITION', chunk, tokens);
349-
insideStruct[insideStruct.length - 1].curly++;
350-
advanceAndClear(1);
351-
continue;
352-
}
353-
if (insideStruct.length && insideStruct[insideStruct.length - 1].curly === 1 &&
354-
chunk === '}') {
355-
lexerFunctions.checkFor('STRUCT_DEFINITION', chunk, tokens);
356-
insideStruct.pop();
357-
advanceAndClear(1);
358-
lexerFunctions.handleEndOfFile(nextCol, tokens);
359-
continue;
360-
}
361-
if (tokens.length && (CLASS_NAMES[lastToken.value] ||
362-
STRUCT_NAMES[lastToken.value]) && chunk === '(') {
363-
lexerFunctions.checkFor('INITIALIZATION', chunk, tokens)
364-
var temp = {};
365-
temp.status = true;
366-
temp.parens = 1;
367-
insideInitialization.push(temp);
368-
advanceAndClear(1);
369-
continue;
370-
}
371-
if (chunk === ')' && insideInitialization.length &&
372-
insideInitialization[insideInitialization.length - 1].parens === 1) {
373-
lexerFunctions.checkFor('INITIALIZATION', chunk, tokens);
374-
insideInitialization.pop();
305+
306+
// handles classes and structs
307+
if (lexerFunctions.handleClassOrStruct(insideClass, insideStruct,
308+
insideInitialization, chunk, tokens, lastToken,
309+
nextCol, CLASS_NAMES, STRUCT_NAMES)) {
375310
advanceAndClear(1);
376-
lexerFunctions.handleEndOfFile(nextCol, tokens);
377311
continue;
378312
}
379313

@@ -396,8 +330,6 @@ module.exports = function(code) {
396330
continue;
397331
}
398332

399-
///////////////////////////////////////////////////////////////////////////
400-
401333
// main evaluation block
402334
if (!insideString.status && !insideNumber.status &&
403335
lexerFunctions.checkForEvaluationPoint(currCol, nextCol)) {
@@ -448,4 +380,4 @@ module.exports = function(code) {
448380
// console.log(tokens);
449381
return tokens;
450382

451-
};
383+
};

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy