Tokenizer: use Python objects to represent tokens

html5lib/constants.py

@@ -2918,20 +2918,6 @@
     0x9F: "\u0178",
 }
 
-tokenTypes = {
-    "Doctype": 0,
-    "Characters": 1,
-    "SpaceCharacters": 2,
-    "StartTag": 3,
-    "EndTag": 4,
-    "EmptyTag": 5,
-    "Comment": 6,
-    "ParseError": 7
-}
-
-tagTokenTypes = frozenset([tokenTypes["StartTag"], tokenTypes["EndTag"],
-                           tokenTypes["EmptyTag"]])
-
 
 prefixes = {v: k for k, v in namespaces.items()}
 prefixes["http://www.w3.org/1998/Math/MathML"] = "math"

html5lib/tests/test_tokenizer2.py

@@ -4,13 +4,12 @@
 
 from six import unichr, text_type
 
-from html5lib._tokenizer import HTMLTokenizer
-from html5lib.constants import tokenTypes
+from html5lib._tokenizer import HTMLTokenizer, ParseError, StartTag
 
 
 def ignore_parse_errors(toks):
     for tok in toks:
-        if tok['type'] != tokenTypes['ParseError']:
+        if not isinstance(tok, ParseError):
             yield tok
 
 
@@ -23,9 +22,9 @@ def test_maintain_attribute_order():
     out = list(ignore_parse_errors(toks))
 
     assert len(out) == 1
-    assert out[0]['type'] == tokenTypes['StartTag']
+    assert isinstance(out[0], StartTag)
 
-    attrs_tok = out[0]['data']
+    attrs_tok = out[0].attributes
     assert len(attrs_tok) == len(attrs)
 
     for (in_name, in_value), (out_name, out_value) in zip(attrs, attrs_tok.items()):
@@ -40,9 +39,9 @@ def test_duplicate_attribute():
     out = list(ignore_parse_errors(toks))
 
     assert len(out) == 1
-    assert out[0]['type'] == tokenTypes['StartTag']
+    assert isinstance(out[0], StartTag)
 
-    attrs_tok = out[0]['data']
+    attrs_tok = out[0].attributes
     assert len(attrs_tok) == 1
     assert list(attrs_tok.items()) == [('a', '1')]
 
@@ -56,9 +55,9 @@ def test_maintain_duplicate_attribute_order():
     out = list(ignore_parse_errors(toks))
 
     assert len(out) == 1
-    assert out[0]['type'] == tokenTypes['StartTag']
+    assert isinstance(out[0], StartTag)
 
-    attrs_tok = out[0]['data']
+    attrs_tok = out[0].attributes
     assert len(attrs_tok) == len(attrs)
 
     for (in_name, in_value), (out_name, out_value) in zip(attrs, attrs_tok.items()):

html5lib/tests/tokenizer.py

@@ -8,7 +8,7 @@
 import pytest
 from six import unichr
 
-from html5lib._tokenizer import HTMLTokenizer
+from html5lib._tokenizer import HTMLTokenizer, StartTag
 from html5lib import constants, _utils
 
 
@@ -25,47 +25,44 @@ def parse(self, stream, encoding=None, innerHTML=False):
 
         tokenizer.state = getattr(tokenizer, self._state)
         if self._lastStartTag is not None:
-            tokenizer.currentToken = {"type": "startTag",
-                                      "name": self._lastStartTag}
+            tokenizer.currentToken = StartTag(name=self._lastStartTag)
 
-        types = {v: k for k, v in constants.tokenTypes.items()}
         for token in tokenizer:
-            getattr(self, 'process%s' % types[token["type"]])(token)
+            getattr(self, 'process%s' % token.__class__.__name__)(token)
 
         return self.outputTokens
 
     def processDoctype(self, token):
-        self.outputTokens.append(["DOCTYPE", token["name"], token["publicId"],
-                                  token["systemId"], token["correct"]])
+        self.outputTokens.append(["DOCTYPE", token.name, token.publicId,
+                                  token.systemId, token.correct])
 
     def processStartTag(self, token):
-        self.outputTokens.append(["StartTag", token["name"],
-                                  token["data"], token["selfClosing"]])
+        self.outputTokens.append(["StartTag", token.name,
+                                  token.attributes, token.self_closing])
 
     def processEmptyTag(self, token):
-        if token["name"] not in constants.voidElements:
+        if token.name not in constants.voidElements:
             self.outputTokens.append("ParseError")
-        self.outputTokens.append(["StartTag", token["name"], dict(token["data"][::-1])])
+        self.outputTokens.append(["StartTag", token.name, token.attributes])
 
     def processEndTag(self, token):
-        self.outputTokens.append(["EndTag", token["name"],
-                                  token["selfClosing"]])
+        self.outputTokens.append(["EndTag", token.name, token.self_closing])
 
     def processComment(self, token):
-        self.outputTokens.append(["Comment", token["data"]])
+        self.outputTokens.append(["Comment", token.data])
 
     def processSpaceCharacters(self, token):
-        self.outputTokens.append(["Character", token["data"]])
+        self.outputTokens.append(["Character", token.data])
         self.processSpaceCharacters = self.processCharacters
 
     def processCharacters(self, token):
-        self.outputTokens.append(["Character", token["data"]])
+        self.outputTokens.append(["Character", token.data])
 
     def processEOF(self, token):
         pass
 
     def processParseError(self, token):
-        self.outputTokens.append(["ParseError", token["data"]])
+        self.outputTokens.append(["ParseError", token.data])
 
 
 def concatenateCharacterTokens(tokens):

html5lib/treebuilders/base.py

@@ -2,6 +2,7 @@
 from six import text_type
 
 from ..constants import scopingElements, tableInsertModeElements, namespaces
+from .._tokenizer import StartTag
 
 # The scope markers are inserted when entering object elements,
 # marquees, table cells, and table captions, and are used to prevent formatting
@@ -249,10 +250,9 @@ def reconstructActiveFormattingElements(self):
             clone = entry.cloneNode()  # Mainly to get a new copy of the attributes
 
             # Step 9
-            element = self.insertElement({"type": "StartTag",
-                                          "name": clone.name,
-                                          "namespace": clone.namespace,
-                                          "data": clone.attributes})
+            tag = StartTag(name=clone.name, data=clone.attributes)
+            tag.namespace = clone.namespace
+            element = self.insertElement(tag)
 
             # Step 10
             self.activeFormattingElements[i] = element
@@ -286,24 +286,24 @@ def insertRoot(self, token):
         self.document.appendChild(element)
 
     def insertDoctype(self, token):
-        name = token["name"]
-        publicId = token["publicId"]
-        systemId = token["systemId"]
+        name = token.name
+        publicId = token.publicId
+        systemId = token.systemId
 
         doctype = self.doctypeClass(name, publicId, systemId)
         self.document.appendChild(doctype)
 
     def insertComment(self, token, parent=None):
         if parent is None:
             parent = self.openElements[-1]
-        parent.appendChild(self.commentClass(token["data"]))
+        parent.appendChild(self.commentClass(token.data))
 
     def createElement(self, token):
         """Create an element but don't insert it anywhere"""
-        name = token["name"]
-        namespace = token.get("namespace", self.defaultNamespace)
+        name = token.name
+        namespace = token.namespace or self.defaultNamespace
         element = self.elementClass(name, namespace)
-        element.attributes = token["data"]
+        element.attributes = token.attributes
         return element
 
     def _getInsertFromTable(self):
@@ -321,11 +321,11 @@ def _setInsertFromTable(self, value):
     insertFromTable = property(_getInsertFromTable, _setInsertFromTable)
 
     def insertElementNormal(self, token):
-        name = token["name"]
+        name = token.name
         assert isinstance(name, text_type), "Element %s not unicode" % name
-        namespace = token.get("namespace", self.defaultNamespace)
+        namespace = token.namespace or self.defaultNamespace
         element = self.elementClass(name, namespace)
-        element.attributes = token["data"]
+        element.attributes = token.attributes
         self.openElements[-1].appendChild(element)
         self.openElements.append(element)
         return element

html5lib/treebuilders/dom.py

@@ -126,9 +126,9 @@ def documentClass(self):
             return weakref.proxy(self)
 
         def insertDoctype(self, token):
-            name = token["name"]
-            publicId = token["publicId"]
-            systemId = token["systemId"]
+            name = token.name
+            publicId = token.publicId
+            systemId = token.systemId
 
             domimpl = Dom.getDOMImplementation()
             doctype = domimpl.createDocumentType(name, publicId, systemId)

html5lib/treebuilders/etree_lxml.py

@@ -308,9 +308,9 @@ def getFragment(self):
         return fragment
 
     def insertDoctype(self, token):
-        name = token["name"]
-        publicId = token["publicId"]
-        systemId = token["systemId"]
+        name = token.name
+        publicId = token.publicId
+        systemId = token.systemId
 
         if not name:
             warnings.warn("lxml cannot represent empty doctype", DataLossWarning)
@@ -359,23 +359,23 @@ def insertRoot(self, token):
                 else:
                     docStr += "''"
             docStr += ">"
-            if self.doctype.name != token["name"]:
+            if self.doctype.name != token.name:
                 warnings.warn("lxml cannot represent doctype with a different name to the root element", DataLossWarning)
         docStr += "<THIS_SHOULD_NEVER_APPEAR_PUBLICLY/>"
         root = etree.fromstring(docStr)
 
         # Append the initial comments:
         for comment_token in self.initial_comments:
-            comment = self.commentClass(comment_token["data"])
+            comment = self.commentClass(comment_token.data)
             root.addprevious(comment._element)
 
         # Create the root document and add the ElementTree to it
         self.document = self.documentClass()
         self.document._elementTree = root.getroottree()
 
         # Give the root element the right name
-        name = token["name"]
-        namespace = token.get("namespace", self.defaultNamespace)
+        name = token.name
+        namespace = token.namespace or self.defaultNamespace
         if namespace is None:
             etree_tag = name
         else: