Add HTML5Parser option

parsel/selector.py

@@ -6,6 +6,7 @@
 
 import six
 from lxml import etree, html
+from lxml.html import html5parser
 
 from .utils import flatten, iflatten, extract_regex
 from .csstranslator import HTMLTranslator, GenericTranslator
@@ -23,6 +24,10 @@ def __init__(self, *args, **kwargs):
     'xml': {'_parser': SafeXMLParser,
             '_csstranslator': GenericTranslator(),
             '_tostring_method': 'xml'},
+    'html5': {'_parser': html5parser.HTMLParser,
+              '_csstranslator': HTMLTranslator(),
+              '_tostring_method': 'html',
+    },
 }
 
 
@@ -39,8 +44,15 @@ def create_root_node(text, parser_cls, base_url=None):
     """Create root node for text using given parser class.
     """
     body = text.strip().replace('\x00', '').encode('utf8') or b'<html/>'
-    parser = parser_cls(recover=True, encoding='utf8')
-    root = etree.fromstring(body, parser=parser, base_url=base_url)
+    if parser_cls == html5parser.HTMLParser:
+        try:
+            parser = parser_cls(namespaceHTMLElements=False)
+            root = parser.parse(body, useChardet=False, override_encoding='utf8').getroot()
+        except ValueError:
+            raise TypeError('HTML5parser does not support control characters')
+    else:
+        parser = parser_cls(recover=True, encoding='utf8')
+        root = etree.fromstring(body, parser=parser, base_url=base_url)
     if root is None:
         root = etree.fromstring(b'<html/>', parser=parser, base_url=base_url)
     return root
@@ -158,7 +170,7 @@ class Selector(object):
 
     ``text`` is a ``unicode`` object in Python 2 or a ``str`` object in Python 3
 
-    ``type`` defines the selector type, it can be ``"html"``, ``"xml"`` or ``None`` (default).
+    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"html5"`` or ``None`` (default).
     If ``type`` is ``None``, the selector defaults to ``"html"``.
     """
 

setup.py

@@ -29,7 +29,8 @@ def has_environment_marker_platform_impl_support():
     'w3lib>=1.19.0',
     'lxml>=2.3',
     'six>=1.5.2',
-    'cssselect>=0.9'
+    'cssselect>=0.9',
+    'html5lib',
 ]
 extras_require = {}
 

tests/html_parser.json

@@ -0,0 +1,4 @@
+{
+  "html_parser": "html",
+  "html5_parser": "html5"
+}

tests/requirements.txt

@@ -1,2 +1,3 @@
 pytest
 pytest-cov
+ddt