Syntax errors in jinja

aelaguiz · aelaguiz · commit fdde76416be4 · 2014-06-29T20:51:38.000-05:00
diff --git a/html5lib/html5parser.py b/html5lib/html5parser.py
@@ -176,6 +176,7 @@ def mainLoop(self):
         for token in self.normalizedTokens():
             new_token = token
             while new_token is not None:
+                log.debug(u"Token {} Phase = {}".format(new_token, self.phase))
                 currentNode = self.tree.openElements[-1] if self.tree.openElements else None
                 currentNodeNamespace = currentNode.namespace if currentNode else None
                 currentNodeName = currentNode.name if currentNode else None
@@ -421,6 +422,7 @@ def resetInsertionMode(self):
                 new_phase = self.phases["inBody"]
                 break
 
+        log.debug(u"Changing phase to {}".format(new_phase))
         self.phase = new_phase
 
     def parseRCDataRawtext(self, token, contentType):
@@ -438,6 +440,7 @@ def parseRCDataRawtext(self, token, contentType):
 
         self.originalPhase = self.phase
 
+        log.debug(u"Changing phase to text")
         self.phase = self.phases["text"]
 
 
@@ -825,6 +828,8 @@ def startTagOther(self, token):
         def endTagHead(self, token):
             node = self.parser.tree.openElements.pop()
             assert node.name == "head", "Expected head got %s" % node.name
+            log = logging.getLogger(u"html5lib")
+            log.debug(u"Switching phase to afterHead")
             self.parser.phase = self.parser.phases["afterHead"]
 
         def endTagHtmlBodyBr(self, token):
@@ -835,6 +840,8 @@ def endTagOther(self, token):
             self.parser.parseError("unexpected-end-tag", {"name": token["name"]})
 
         def anythingElse(self):
+            log = logging.getLogger(u"html5lib")
+            log.debug(u"Implied end head tag")
             self.endTagHead(impliedTagToken("head"))
 
     # XXX If we implement a parser for which scripting is disabled we need to
@@ -905,6 +912,8 @@ def endTagOther(self, token):
 
         def anythingElse(self):
             self.tree.insertElement(impliedTagToken("body", "StartTag"))
+            log = logging.getLogger(u"html5lib")
+            log.debug(u"Changing phase to body")
             self.parser.phase = self.parser.phases["inBody"]
             self.parser.framesetOK = True
 
diff --git a/html5lib/tokenizer.py b/html5lib/tokenizer.py
@@ -324,6 +324,17 @@ def jinjaStatementEndState(self):
                 "selfClosing": False
             })
             self.state = self.dataState
+        elif data is EOF:
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+                                    "expected-jinja-statement-closing-tag-but-got-eof",
+                                    "datavars": {"data": data}})
+            self.state = self.dataState
+        else:
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+                                    "expected-jinja-statement-closing-tag-but-got-char",
+                                    "datavars": {"data": data}})
+            self.stream.unget(data)
+            self.state = self.bogusCommentState
 
         #self.state = self.dataState
         return True
@@ -339,6 +350,17 @@ def jinjaVariableEndState(self):
                 "selfClosing": False
             })
             self.state = self.dataState
+        elif data is EOF:
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+                                    "expected-jinja-variable-closing-tag-but-got-eof",
+                                    "datavars": {"data": data}})
+            self.state = self.dataState
+        else:
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+                                    "expected-jinja-variable-closing-tag-but-got-char",
+                                    "datavars": {"data": data}})
+            self.stream.unget(data)
+            self.state = self.bogusCommentState
 
         #self.state = self.dataState
         return True
@@ -349,8 +371,9 @@ def jinjaStatementState(self):
         if data == "%":
             self.state = self.jinjaStatementEndState
         elif data is EOF:
-            # Tokenization ends.
-            return False
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+                                    "eof-in-jinja-statement"})
+            self.state = self.dataState
         else:
             chars = self.stream.charsUntil(("%", "\u0000"))
             self.tokenQueue.append({"type": tokenTypes["JinjaStatementTag"], "data":
@@ -364,8 +387,9 @@ def jinjaVariableState(self):
         if data == "}":
             self.state = self.jinjaVariableEndState
         elif data is EOF:
-            # Tokenization ends.
-            return False
+            self.tokenQueue.append({"type": tokenTypes["ParseError"], "data":
+                                    "eof-in-jinja-variable"})
+            self.state = self.dataState
         else:
             chars = self.stream.charsUntil(("}", "\u0000"))
             self.tokenQueue.append({"type": tokenTypes["JinjaVariableTag"], "data":
