diff --git a/CHANGES.md b/CHANGES.md index dcbcdd9..156700b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,7 +28,7 @@ * Update ConfigObj 5.1.0 (a68530a) to 5.1.0 (45fbf1b) * Update dateutil 2.8.0 (c90a30c) to 2.8.0 (27848c5) * Update DiskCache library 3.1.1 (2649ac9) to 4.0.0 (2c79bb9) -* Update feedparser 5.2.1 (2b11c80) to 5.2.1 (cbe18d0) +* Update feedparser 5.2.1 (2b11c80) to 6.0.0b1 (d12d3bd) * Update Fuzzywuzzy 0.15.1 to 0.17.0 (778162c) * Update Hachoir library 2.0a6 (c102cc7) to 2.0a6 (5b9e05a) * Update Js2Py 0.64 (efbfcca) to 0.64 (7858d1d) @@ -48,6 +48,7 @@ [develop changelog] * Update attr 19.2.0.dev0 (de84609) to 19.2.0.dev0 (154b4e5) +* Update feedparser 5.2.1 (2b11c80) to 5.2.1 (cbe18d0) * Update Requests library 2.22.0 (aeda65b) to 2.22.0 (0b6c110) * Update urllib3 release 1.25.2 (49eea80) to 1.25.3 (3387b20) * Update urllib3 release 1.25.3 (3387b20) to 1.25.3 (67715fd) diff --git a/lib/feedparser/__init__.py b/lib/feedparser/__init__.py index 916a61c..25e97fd 100644 --- a/lib/feedparser/__init__.py +++ b/lib/feedparser/__init__.py @@ -1,4 +1,4 @@ -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # @@ -25,7 +25,12 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE.""" -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals + +from .api import parse +from .datetimes import registerDateHandler +from .exceptions import * __author__ = 'Kurt McKee ' __license__ = 'BSD 2-clause' @@ -36,11 +41,6 @@ __version__ = '5.2.1' # change this to your application name and URL. USER_AGENT = "feedparser/%s +https://github.com/kurtmckee/feedparser/" % __version__ -from . import api -from .api import parse -from .datetimes import registerDateHandler -from .exceptions import * - # If you want feedparser to automatically resolve all relative URIs, set this # to 1. RESOLVE_RELATIVE_URIS = 1 diff --git a/lib/feedparser/api.py b/lib/feedparser/api.py index d2d97a6..7e552f2 100644 --- a/lib/feedparser/api.py +++ b/lib/feedparser/api.py @@ -1,5 +1,5 @@ # The public API for feedparser -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # @@ -26,13 +26,15 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals import xml.sax try: from io import BytesIO as _StringIO except ImportError: + # Python 2.7 try: from cStringIO import StringIO as _StringIO except ImportError: @@ -58,7 +60,7 @@ from .parsers.loose import _LooseFeedParser from .parsers.strict import _StrictFeedParser from .sanitizer import replace_doctype from .sgml import * -from .urls import _convert_to_idn, _makeSafeAbsoluteURI +from .urls import convert_to_idn, make_safe_absolute_uri from .util import FeedParserDict bytes_ = type(b'') @@ -96,6 +98,7 @@ SUPPORTED_VERSIONS = { 'cdf': 'CDF', } + def _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, handlers, request_headers, result): """URL, filename, or string --> stream @@ -158,15 +161,22 @@ def _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, h return url_file_stream_or_string.encode('utf-8') return url_file_stream_or_string -LooseFeedParser = type(str('LooseFeedParser'), ( - _LooseFeedParser, _FeedParserMixin, _BaseHTMLProcessor, object -), {}) -StrictFeedParser = type(str('StrictFeedParser'), ( - _StrictFeedParser, _FeedParserMixin, xml.sax.handler.ContentHandler, object -), {}) + +LooseFeedParser = type( + str('LooseFeedParser'), # `str()` call required for Python 2.7 + (_LooseFeedParser, _FeedParserMixin, _BaseHTMLProcessor, object), + {}, +) + +StrictFeedParser = type( + str('StrictFeedParser'), # `str()` call required for Python 2.7 + (_StrictFeedParser, _FeedParserMixin, xml.sax.handler.ContentHandler, object), + {}, +) + def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, referrer=None, handlers=None, request_headers=None, response_headers=None, resolve_relative_uris=None, sanitize_html=None): - '''Parse a feed from a URL, file, stream, or string. + """Parse a feed from a URL, file, stream, or string. :param url_file_stream_or_string: File-like object, URL, file path, or string. Both byte and text strings @@ -210,7 +220,8 @@ def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, refer :data:`feedparser.SANITIZE_HTML`, which is ``True``. :return: A :class:`FeedParserDict`. - ''' + """ + if not agent or sanitize_html is None or resolve_relative_uris is None: import feedparser if not agent: @@ -221,10 +232,10 @@ def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, refer resolve_relative_uris = feedparser.RESOLVE_RELATIVE_URIS result = FeedParserDict( - bozo = False, - entries = [], - feed = FeedParserDict(), - headers = {}, + bozo=False, + entries=[], + feed=FeedParserDict(), + headers={}, ) data = _open_resource(url_file_stream_or_string, etag, modified, agent, referrer, handlers, request_headers, result) @@ -243,7 +254,7 @@ def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, refer # Ensure that baseuri is an absolute URI using an acceptable URI scheme. contentloc = result['headers'].get('content-location', '') href = result.get('href', '') - baseuri = _makeSafeAbsoluteURI(href, contentloc) or _makeSafeAbsoluteURI(contentloc) or href + baseuri = make_safe_absolute_uri(href, contentloc) or make_safe_absolute_uri(contentloc) or href baselang = result['headers'].get('content-language', None) if isinstance(baselang, bytes_) and baselang is not None: @@ -281,5 +292,5 @@ def parse(url_file_stream_or_string, etag=None, modified=None, agent=None, refer result['feed'] = feedparser.feeddata result['entries'] = feedparser.entries result['version'] = result['version'] or feedparser.version - result['namespaces'] = feedparser.namespacesInUse + result['namespaces'] = feedparser.namespaces_in_use return result diff --git a/lib/feedparser/datetimes/__init__.py b/lib/feedparser/datetimes/__init__.py index 96b45e9..fc1b27d 100644 --- a/lib/feedparser/datetimes/__init__.py +++ b/lib/feedparser/datetimes/__init__.py @@ -1,3 +1,30 @@ +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + from __future__ import absolute_import from .asctime import _parse_date_asctime @@ -10,18 +37,21 @@ from .rfc822 import _parse_date_rfc822 from .w3dtf import _parse_date_w3dtf _date_handlers = [] + + def registerDateHandler(func): - '''Register a date handler function (takes string, returns 9-tuple date in GMT)''' + """Register a date handler function (takes string, returns 9-tuple date in GMT)""" _date_handlers.insert(0, func) -def _parse_date(dateString): - '''Parses a variety of date formats into a 9-tuple in GMT''' - if not dateString: + +def _parse_date(date_string): + """Parses a variety of date formats into a 9-tuple in GMT""" + if not date_string: return None for handler in _date_handlers: try: - date9tuple = handler(dateString) - except (KeyError, OverflowError, ValueError): + date9tuple = handler(date_string) + except (KeyError, OverflowError, ValueError, AttributeError): continue if not date9tuple: continue @@ -30,6 +60,7 @@ def _parse_date(dateString): return date9tuple return None + registerDateHandler(_parse_date_onblog) registerDateHandler(_parse_date_nate) registerDateHandler(_parse_date_greek) diff --git a/lib/feedparser/datetimes/asctime.py b/lib/feedparser/datetimes/asctime.py index 54fc8cb..b8ac3ab 100644 --- a/lib/feedparser/datetimes/asctime.py +++ b/lib/feedparser/datetimes/asctime.py @@ -1,9 +1,51 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals from .rfc822 import _parse_date_rfc822 -_months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', - 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] +_months = [ + 'jan', + 'feb', + 'mar', + 'apr', + 'may', + 'jun', + 'jul', + 'aug', + 'sep', + 'oct', + 'nov', + 'dec', +] + + def _parse_date_asctime(dt): """Parse asctime-style dates. diff --git a/lib/feedparser/datetimes/greek.py b/lib/feedparser/datetimes/greek.py index 08d8f8a..77894c9 100644 --- a/lib/feedparser/datetimes/greek.py +++ b/lib/feedparser/datetimes/greek.py @@ -1,35 +1,61 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import re from .rfc822 import _parse_date_rfc822 # Unicode strings for Greek date strings -_greek_months = \ - { \ - '\u0399\u03b1\u03bd': 'Jan', # c9e1ed in iso-8859-7 - '\u03a6\u03b5\u03b2': 'Feb', # d6e5e2 in iso-8859-7 - '\u039c\u03ac\u03ce': 'Mar', # ccdcfe in iso-8859-7 - '\u039c\u03b1\u03ce': 'Mar', # cce1fe in iso-8859-7 - '\u0391\u03c0\u03c1': 'Apr', # c1f0f1 in iso-8859-7 - '\u039c\u03ac\u03b9': 'May', # ccdce9 in iso-8859-7 - '\u039c\u03b1\u03ca': 'May', # cce1fa in iso-8859-7 - '\u039c\u03b1\u03b9': 'May', # cce1e9 in iso-8859-7 - '\u0399\u03bf\u03cd\u03bd': 'Jun', # c9effded in iso-8859-7 - '\u0399\u03bf\u03bd': 'Jun', # c9efed in iso-8859-7 - '\u0399\u03bf\u03cd\u03bb': 'Jul', # c9effdeb in iso-8859-7 - '\u0399\u03bf\u03bb': 'Jul', # c9f9eb in iso-8859-7 - '\u0391\u03cd\u03b3': 'Aug', # c1fde3 in iso-8859-7 - '\u0391\u03c5\u03b3': 'Aug', # c1f5e3 in iso-8859-7 - '\u03a3\u03b5\u03c0': 'Sep', # d3e5f0 in iso-8859-7 - '\u039f\u03ba\u03c4': 'Oct', # cfeaf4 in iso-8859-7 - '\u039d\u03bf\u03ad': 'Nov', # cdefdd in iso-8859-7 - '\u039d\u03bf\u03b5': 'Nov', # cdefe5 in iso-8859-7 - '\u0394\u03b5\u03ba': 'Dec', # c4e5ea in iso-8859-7 - } - -_greek_wdays = \ - { \ +_greek_months = { + '\u0399\u03b1\u03bd': 'Jan', # c9e1ed in iso-8859-7 + '\u03a6\u03b5\u03b2': 'Feb', # d6e5e2 in iso-8859-7 + '\u039c\u03ac\u03ce': 'Mar', # ccdcfe in iso-8859-7 + '\u039c\u03b1\u03ce': 'Mar', # cce1fe in iso-8859-7 + '\u0391\u03c0\u03c1': 'Apr', # c1f0f1 in iso-8859-7 + '\u039c\u03ac\u03b9': 'May', # ccdce9 in iso-8859-7 + '\u039c\u03b1\u03ca': 'May', # cce1fa in iso-8859-7 + '\u039c\u03b1\u03b9': 'May', # cce1e9 in iso-8859-7 + '\u0399\u03bf\u03cd\u03bd': 'Jun', # c9effded in iso-8859-7 + '\u0399\u03bf\u03bd': 'Jun', # c9efed in iso-8859-7 + '\u0399\u03bf\u03cd\u03bb': 'Jul', # c9effdeb in iso-8859-7 + '\u0399\u03bf\u03bb': 'Jul', # c9f9eb in iso-8859-7 + '\u0391\u03cd\u03b3': 'Aug', # c1fde3 in iso-8859-7 + '\u0391\u03c5\u03b3': 'Aug', # c1f5e3 in iso-8859-7 + '\u03a3\u03b5\u03c0': 'Sep', # d3e5f0 in iso-8859-7 + '\u039f\u03ba\u03c4': 'Oct', # cfeaf4 in iso-8859-7 + '\u039d\u03bf\u03ad': 'Nov', # cdefdd in iso-8859-7 + '\u039d\u03bf\u03b5': 'Nov', # cdefe5 in iso-8859-7 + '\u0394\u03b5\u03ba': 'Dec', # c4e5ea in iso-8859-7 +} + +_greek_wdays = { '\u039a\u03c5\u03c1': 'Sun', # caf5f1 in iso-8859-7 '\u0394\u03b5\u03c5': 'Mon', # c4e5f5 in iso-8859-7 '\u03a4\u03c1\u03b9': 'Tue', # d4f1e9 in iso-8859-7 @@ -37,20 +63,27 @@ _greek_wdays = \ '\u03a0\u03b5\u03bc': 'Thu', # d0e5ec in iso-8859-7 '\u03a0\u03b1\u03c1': 'Fri', # d0e1f1 in iso-8859-7 '\u03a3\u03b1\u03b2': 'Sat', # d3e1e2 in iso-8859-7 - } +} + +_greek_date_format_re = re.compile(r'([^,]+),\s+(\d{2})\s+([^\s]+)\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})\s+([^\s]+)') -_greek_date_format_re = \ - re.compile(r'([^,]+),\s+(\d{2})\s+([^\s]+)\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})\s+([^\s]+)') -def _parse_date_greek(dateString): - '''Parse a string according to a Greek 8-bit date format.''' - m = _greek_date_format_re.match(dateString) +def _parse_date_greek(date_string): + """Parse a string according to a Greek 8-bit date format.""" + m = _greek_date_format_re.match(date_string) if not m: return wday = _greek_wdays[m.group(1)] month = _greek_months[m.group(3)] rfc822date = '%(wday)s, %(day)s %(month)s %(year)s %(hour)s:%(minute)s:%(second)s %(zonediff)s' % \ - {'wday': wday, 'day': m.group(2), 'month': month, 'year': m.group(4),\ - 'hour': m.group(5), 'minute': m.group(6), 'second': m.group(7),\ - 'zonediff': m.group(8)} + { + 'wday': wday, + 'day': m.group(2), + 'month': month, + 'year': m.group(4), + 'hour': m.group(5), + 'minute': m.group(6), + 'second': m.group(7), + 'zonediff': m.group(8), + } return _parse_date_rfc822(rfc822date) diff --git a/lib/feedparser/datetimes/hungarian.py b/lib/feedparser/datetimes/hungarian.py index c066825..497cb0b 100644 --- a/lib/feedparser/datetimes/hungarian.py +++ b/lib/feedparser/datetimes/hungarian.py @@ -1,12 +1,39 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import re from .w3dtf import _parse_date_w3dtf # Unicode strings for Hungarian date strings -_hungarian_months = \ - { \ +_hungarian_months = { 'janu\u00e1r': '01', # e1 in iso-8859-2 'febru\u00e1ri': '02', # e1 in iso-8859-2 'm\u00e1rcius': '03', # e1 in iso-8859-2 @@ -19,14 +46,14 @@ _hungarian_months = \ 'okt\u00f3ber': '10', # f3 in iso-8859-2 'november': '11', 'december': '12', - } +} + +_hungarian_date_format_re = re.compile(r'(\d{4})-([^-]+)-(\d{,2})T(\d{,2}):(\d{2})((\+|-)(\d{,2}:\d{2}))') -_hungarian_date_format_re = \ - re.compile(r'(\d{4})-([^-]+)-(\d{,2})T(\d{,2}):(\d{2})((\+|-)(\d{,2}:\d{2}))') -def _parse_date_hungarian(dateString): - '''Parse a string according to a Hungarian 8-bit date format.''' - m = _hungarian_date_format_re.match(dateString) +def _parse_date_hungarian(date_string): + """Parse a string according to a Hungarian 8-bit date format.""" + m = _hungarian_date_format_re.match(date_string) if not m or m.group(2) not in _hungarian_months: return None month = _hungarian_months[m.group(2)] @@ -37,7 +64,12 @@ def _parse_date_hungarian(dateString): if len(hour) == 1: hour = '0' + hour w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s%(zonediff)s' % \ - {'year': m.group(1), 'month': month, 'day': day,\ - 'hour': hour, 'minute': m.group(5),\ - 'zonediff': m.group(6)} + { + 'year': m.group(1), + 'month': month, + 'day': day, + 'hour': hour, + 'minute': m.group(5), + 'zonediff': m.group(6), + } return _parse_date_w3dtf(w3dtfdate) diff --git a/lib/feedparser/datetimes/iso8601.py b/lib/feedparser/datetimes/iso8601.py index eb36e2d..bdf7652 100644 --- a/lib/feedparser/datetimes/iso8601.py +++ b/lib/feedparser/datetimes/iso8601.py @@ -1,4 +1,32 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import re import time @@ -12,12 +40,24 @@ import time # 0301-04-01), so we use templates instead. # Please note the order in templates is significant because we need a # greedy match. -_iso8601_tmpl = ['YYYY-?MM-?DD', 'YYYY-0MM?-?DD', 'YYYY-MM', 'YYYY-?OOO', - 'YY-?MM-?DD', 'YY-?OOO', 'YYYY', - '-YY-?MM', '-OOO', '-YY', - '--MM-?DD', '--MM', - '---DD', - 'CC', ''] +_iso8601_tmpl = [ + 'YYYY-?MM-?DD', + 'YYYY-0MM?-?DD', + 'YYYY-MM', + 'YYYY-?OOO', + 'YY-?MM-?DD', + 'YY-?OOO', + 'YYYY', + '-YY-?MM', + '-OOO', + '-YY', + '--MM-?DD', + '--MM', + '---DD', + 'CC', + '', +] + _iso8601_re = [ tmpl.replace( 'YYYY', r'(?P\d{4})').replace( @@ -41,11 +81,12 @@ try: except NameError: pass -def _parse_date_iso8601(dateString): - '''Parse a variety of ISO-8601-compatible formats like 20040105''' + +def _parse_date_iso8601(date_string): + """Parse a variety of ISO-8601-compatible formats like 20040105""" m = None for _iso8601_match in _iso8601_matches: - m = _iso8601_match(dateString) + m = _iso8601_match(date_string) if m: break if not m: diff --git a/lib/feedparser/datetimes/korean.py b/lib/feedparser/datetimes/korean.py index a2e481e..c0bcb56 100644 --- a/lib/feedparser/datetimes/korean.py +++ b/lib/feedparser/datetimes/korean.py @@ -1,24 +1,56 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import re from .w3dtf import _parse_date_w3dtf # 8-bit date handling routines written by ytrewq1. -_korean_year = '\ub144' # b3e2 in euc-kr +_korean_year = '\ub144' # b3e2 in euc-kr _korean_month = '\uc6d4' # bff9 in euc-kr -_korean_day = '\uc77c' # c0cf in euc-kr -_korean_am = '\uc624\uc804' # bfc0 c0fc in euc-kr -_korean_pm = '\uc624\ud6c4' # bfc0 c8c4 in euc-kr - -_korean_onblog_date_re = \ - re.compile(r'(\d{4})%s\s+(\d{2})%s\s+(\d{2})%s\s+(\d{2}):(\d{2}):(\d{2})' % \ - (_korean_year, _korean_month, _korean_day)) -_korean_nate_date_re = \ - re.compile(r'(\d{4})-(\d{2})-(\d{2})\s+(%s|%s)\s+(\d{,2}):(\d{,2}):(\d{,2})' % \ - (_korean_am, _korean_pm)) +_korean_day = '\uc77c' # c0cf in euc-kr +_korean_am = '\uc624\uc804' # bfc0 c0fc in euc-kr +_korean_pm = '\uc624\ud6c4' # bfc0 c8c4 in euc-kr + +_korean_onblog_date_re = re.compile( + r'(\d{4})%s\s+(\d{2})%s\s+(\d{2})%s\s+(\d{2}):(\d{2}):(\d{2})' + % (_korean_year, _korean_month, _korean_day) +) + +_korean_nate_date_re = re.compile( + r'(\d{4})-(\d{2})-(\d{2})\s+(%s|%s)\s+(\d{,2}):(\d{,2}):(\d{,2})' + % (_korean_am, _korean_pm)) + + def _parse_date_onblog(dateString): - '''Parse a string according to the OnBlog 8-bit date format''' + """Parse a string according to the OnBlog 8-bit date format""" m = _korean_onblog_date_re.match(dateString) if not m: return @@ -28,20 +60,27 @@ def _parse_date_onblog(dateString): 'zonediff': '+09:00'} return _parse_date_w3dtf(w3dtfdate) + def _parse_date_nate(dateString): - '''Parse a string according to the Nate 8-bit date format''' + """Parse a string according to the Nate 8-bit date format""" m = _korean_nate_date_re.match(dateString) if not m: return hour = int(m.group(5)) ampm = m.group(4) - if (ampm == _korean_pm): + if ampm == _korean_pm: hour += 12 hour = str(hour) if len(hour) == 1: hour = '0' + hour w3dtfdate = '%(year)s-%(month)s-%(day)sT%(hour)s:%(minute)s:%(second)s%(zonediff)s' % \ - {'year': m.group(1), 'month': m.group(2), 'day': m.group(3),\ - 'hour': hour, 'minute': m.group(6), 'second': m.group(7),\ - 'zonediff': '+09:00'} + { + 'year': m.group(1), + 'month': m.group(2), + 'day': m.group(3), + 'hour': hour, + 'minute': m.group(6), + 'second': m.group(7), + 'zonediff': '+09:00', + } return _parse_date_w3dtf(w3dtfdate) diff --git a/lib/feedparser/datetimes/perforce.py b/lib/feedparser/datetimes/perforce.py index f5b7d8c..6e9b036 100644 --- a/lib/feedparser/datetimes/perforce.py +++ b/lib/feedparser/datetimes/perforce.py @@ -1,4 +1,32 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals try: import rfc822 @@ -8,18 +36,18 @@ except ImportError: import re import time -def _parse_date_perforce(aDateString): + +def _parse_date_perforce(date_string): """parse a date in yyyy/mm/dd hh:mm:ss TTT format""" # Fri, 2006/09/15 08:19:53 EDT - _my_date_pattern = re.compile( \ - r'(\w{,3}), (\d{,4})/(\d{,2})/(\d{2}) (\d{,2}):(\d{2}):(\d{2}) (\w{,3})') + _my_date_pattern = re.compile(r'(\w{,3}), (\d{,4})/(\d{,2})/(\d{2}) (\d{,2}):(\d{2}):(\d{2}) (\w{,3})') - m = _my_date_pattern.search(aDateString) + m = _my_date_pattern.search(date_string) if m is None: return None dow, year, month, day, hour, minute, second, tz = m.groups() months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] - dateString = "%s, %s %s %s %s:%s:%s %s" % (dow, day, months[int(month) - 1], year, hour, minute, second, tz) - tm = rfc822.parsedate_tz(dateString) + new_date_string = "%s, %s %s %s %s:%s:%s %s" % (dow, day, months[int(month) - 1], year, hour, minute, second, tz) + tm = rfc822.parsedate_tz(new_date_string) if tm: return time.gmtime(rfc822.mktime_tz(tm)) diff --git a/lib/feedparser/datetimes/rfc822.py b/lib/feedparser/datetimes/rfc822.py index 88bc386..5cbd8f8 100644 --- a/lib/feedparser/datetimes/rfc822.py +++ b/lib/feedparser/datetimes/rfc822.py @@ -1,4 +1,32 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import datetime diff --git a/lib/feedparser/datetimes/w3dtf.py b/lib/feedparser/datetimes/w3dtf.py index 02bc712..e4ad451 100644 --- a/lib/feedparser/datetimes/w3dtf.py +++ b/lib/feedparser/datetimes/w3dtf.py @@ -1,4 +1,32 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import datetime @@ -17,6 +45,8 @@ timezonenames = { # Also supports MSSQL-style datetimes as defined at: # http://msdn.microsoft.com/en-us/library/ms186724.aspx # (basically, allow a space as a date/time/timezone separator) + + def _parse_date_w3dtf(datestr): if not datestr.strip(): return None @@ -57,8 +87,6 @@ def _parse_date_w3dtf(datestr): # Ensure that time has 3 elements. Using '0' means that the # minutes and seconds, if missing, will default to 0. time.extend(['0'] * (3 - len(time))) - tzhour = 0 - tzmin = 0 if parts[2][:1] in ('-', '+'): try: tzhour = int(parts[2][1:3]) @@ -70,6 +98,7 @@ def _parse_date_w3dtf(datestr): tzmin = tzmin * -1 else: tzhour = timezonenames.get(parts[2], 0) + tzmin = 0 try: hour, minute, second = [int(float(i)) for i in time] except ValueError: diff --git a/lib/feedparser/encodings.py b/lib/feedparser/encodings.py index b0fee8c..9889f5e 100644 --- a/lib/feedparser/encodings.py +++ b/lib/feedparser/encodings.py @@ -1,5 +1,5 @@ # Character encoding routines -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # @@ -26,7 +26,8 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals import cgi import codecs @@ -50,7 +51,9 @@ else: return chardet_encoding from .exceptions import ( - CharacterEncodingOverride, CharacterEncodingUnknown, NonXMLContentType, + CharacterEncodingOverride, + CharacterEncodingUnknown, + NonXMLContentType, ) bytes_ = type(b'') @@ -74,11 +77,12 @@ RE_XML_DECLARATION = re.compile(r'^<\?xml[^>]*?>') # Example: RE_XML_PI_ENCODING = re.compile(br'^<\?.*encoding=[\'"](.*?)[\'"].*\?>') + def convert_to_utf8(http_headers, data, result): - '''Detect and convert the character encoding to UTF-8. + """Detect and convert the character encoding to UTF-8. http_headers is a dictionary - data is a raw string (not Unicode)''' + data is a raw string (not Unicode)""" # This is so much trickier than it sounds, it's not even funny. # According to RFC 3023 ('XML Media Types'), if the HTTP Content-Type @@ -123,7 +127,6 @@ def convert_to_utf8(http_headers, data, result): bom_encoding = '' xml_encoding = '' - rfc3023_encoding = '' # Look at the first few bytes of the document to guess what # its encoding may be. We only need to decode enough of the @@ -197,14 +200,22 @@ def convert_to_utf8(http_headers, data, result): application_content_types = ('application/xml', 'application/xml-dtd', 'application/xml-external-parsed-entity') text_content_types = ('text/xml', 'text/xml-external-parsed-entity') - if (http_content_type in application_content_types) or \ - (http_content_type.startswith('application/') and - http_content_type.endswith('+xml')): + if ( + http_content_type in application_content_types + or ( + http_content_type.startswith('application/') + and http_content_type.endswith('+xml') + ) + ): acceptable_content_type = 1 rfc3023_encoding = http_encoding or xml_encoding or 'utf-8' - elif (http_content_type in text_content_types) or \ - (http_content_type.startswith('text/') and - http_content_type.endswith('+xml')): + elif ( + http_content_type in text_content_types + or ( + http_content_type.startswith('text/') + and http_content_type.endswith('+xml') + ) + ): acceptable_content_type = 1 rfc3023_encoding = http_encoding or 'us-ascii' elif http_content_type.startswith('text/'): diff --git a/lib/feedparser/exceptions.py b/lib/feedparser/exceptions.py index 9e57775..15f0e1c 100644 --- a/lib/feedparser/exceptions.py +++ b/lib/feedparser/exceptions.py @@ -1,5 +1,5 @@ # Exceptions used throughout feedparser -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # @@ -26,7 +26,8 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals __all__ = [ 'ThingsNobodyCaresAboutButMe', @@ -36,17 +37,22 @@ __all__ = [ 'UndeclaredNamespace', ] + class ThingsNobodyCaresAboutButMe(Exception): pass + class CharacterEncodingOverride(ThingsNobodyCaresAboutButMe): pass + class CharacterEncodingUnknown(ThingsNobodyCaresAboutButMe): pass + class NonXMLContentType(ThingsNobodyCaresAboutButMe): pass + class UndeclaredNamespace(Exception): pass diff --git a/lib/feedparser/html.py b/lib/feedparser/html.py index d16b3a8..d2da8b5 100644 --- a/lib/feedparser/html.py +++ b/lib/feedparser/html.py @@ -1,64 +1,117 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import re try: from html.entities import name2codepoint except ImportError: + # Python 2 + # noinspection PyUnresolvedReferences from htmlentitydefs import name2codepoint from .sgml import * _cp1252 = { - 128: '\u20ac', # euro sign - 130: '\u201a', # single low-9 quotation mark - 131: '\u0192', # latin small letter f with hook - 132: '\u201e', # double low-9 quotation mark - 133: '\u2026', # horizontal ellipsis - 134: '\u2020', # dagger - 135: '\u2021', # double dagger - 136: '\u02c6', # modifier letter circumflex accent - 137: '\u2030', # per mille sign - 138: '\u0160', # latin capital letter s with caron - 139: '\u2039', # single left-pointing angle quotation mark - 140: '\u0152', # latin capital ligature oe - 142: '\u017d', # latin capital letter z with caron - 145: '\u2018', # left single quotation mark - 146: '\u2019', # right single quotation mark - 147: '\u201c', # left double quotation mark - 148: '\u201d', # right double quotation mark - 149: '\u2022', # bullet - 150: '\u2013', # en dash - 151: '\u2014', # em dash - 152: '\u02dc', # small tilde - 153: '\u2122', # trade mark sign - 154: '\u0161', # latin small letter s with caron - 155: '\u203a', # single right-pointing angle quotation mark - 156: '\u0153', # latin small ligature oe - 158: '\u017e', # latin small letter z with caron - 159: '\u0178', # latin capital letter y with diaeresis + 128: '\u20ac', # euro sign + 130: '\u201a', # single low-9 quotation mark + 131: '\u0192', # latin small letter f with hook + 132: '\u201e', # double low-9 quotation mark + 133: '\u2026', # horizontal ellipsis + 134: '\u2020', # dagger + 135: '\u2021', # double dagger + 136: '\u02c6', # modifier letter circumflex accent + 137: '\u2030', # per mille sign + 138: '\u0160', # latin capital letter s with caron + 139: '\u2039', # single left-pointing angle quotation mark + 140: '\u0152', # latin capital ligature oe + 142: '\u017d', # latin capital letter z with caron + 145: '\u2018', # left single quotation mark + 146: '\u2019', # right single quotation mark + 147: '\u201c', # left double quotation mark + 148: '\u201d', # right double quotation mark + 149: '\u2022', # bullet + 150: '\u2013', # en dash + 151: '\u2014', # em dash + 152: '\u02dc', # small tilde + 153: '\u2122', # trade mark sign + 154: '\u0161', # latin small letter s with caron + 155: '\u203a', # single right-pointing angle quotation mark + 156: '\u0153', # latin small ligature oe + 158: '\u017e', # latin small letter z with caron + 159: '\u0178', # latin capital letter y with diaeresis } + class _BaseHTMLProcessor(sgmllib.SGMLParser, object): - special = re.compile('''[<>'"]''') + special = re.compile("""[<>'"]""") bare_ampersand = re.compile(r"&(?!#\d+;|#x[0-9a-fA-F]+;|\w+;)") - elements_no_end_tag = set([ - 'area', 'base', 'basefont', 'br', 'col', 'command', 'embed', 'frame', - 'hr', 'img', 'input', 'isindex', 'keygen', 'link', 'meta', 'param', - 'source', 'track', 'wbr' - ]) + elements_no_end_tag = { + 'area', + 'base', + 'basefont', + 'br', + 'col', + 'command', + 'embed', + 'frame', + 'hr', + 'img', + 'input', + 'isindex', + 'keygen', + 'link', + 'meta', + 'param', + 'source', + 'track', + 'wbr', + } def __init__(self, encoding=None, _type='application/xhtml+xml'): if encoding: self.encoding = encoding self._type = _type + self.pieces = [] super(_BaseHTMLProcessor, self).__init__() def reset(self): self.pieces = [] - sgmllib.SGMLParser.reset(self) + super(_BaseHTMLProcessor, self).reset() def _shorttag_replace(self, match): + """ + :type match: Match[str] + :rtype: str + """ + tag = match.group(1) if tag in self.elements_no_end_tag: return '<' + tag + ' />' @@ -71,68 +124,113 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser, object): # `tagfind` and `charref` regular expressions will be found as # they're declared above, not as they're declared in sgmllib. def goahead(self, i): - pass + raise NotImplementedError + + # Replace goahead with SGMLParser's goahead() code object. try: goahead.__code__ = sgmllib.SGMLParser.goahead.__code__ except AttributeError: + # Python 2 + # noinspection PyUnresolvedReferences goahead.func_code = sgmllib.SGMLParser.goahead.func_code def __parse_starttag(self, i): - pass + raise NotImplementedError + + # Replace __parse_starttag with SGMLParser's parse_starttag() code object. try: __parse_starttag.__code__ = sgmllib.SGMLParser.parse_starttag.__code__ except AttributeError: + # Python 2 + # noinspection PyUnresolvedReferences __parse_starttag.func_code = sgmllib.SGMLParser.parse_starttag.func_code - def parse_starttag(self,i): + def parse_starttag(self, i): j = self.__parse_starttag(i) if self._type == 'application/xhtml+xml': - if j>2 and self.rawdata[j-2:j]=='/>': + if j > 2 and self.rawdata[j-2:j] == '/>': self.unknown_endtag(self.lasttag) return j def feed(self, data): - data = re.compile(r'\s]+?)\s*/>', self._shorttag_replace, data) data = data.replace(''', "'") data = data.replace('"', '"') - sgmllib.SGMLParser.feed(self, data) - sgmllib.SGMLParser.close(self) + super(_BaseHTMLProcessor, self).feed(data) + super(_BaseHTMLProcessor, self).close() + + @staticmethod + def normalize_attrs(attrs): + """ + :type attrs: List[Tuple[str, str]] + :rtype: List[Tuple[str, str]] + """ - def normalize_attrs(self, attrs): if not attrs: return attrs # utility method to be called by descendants - attrs = dict([(k.lower(), v) for k, v in attrs]).items() - attrs = [(k, k in ('rel', 'type') and v.lower() or v) for k, v in attrs] + # Collapse any duplicate attribute names and values by converting + # *attrs* into a dictionary, then convert it back to a list. + attrs_d = {k.lower(): v for k, v in attrs} + attrs = [ + (k, k in ('rel', 'type') and v.lower() or v) + for k, v in attrs_d.items() + ] attrs.sort() return attrs def unknown_starttag(self, tag, attrs): - # called for each start tag + """ + :type tag: str + :type attrs: List[Tuple[str, str]] + :rtype: None + """ + + # Called for each start tag # attrs is a list of (attr, value) tuples # e.g. for 
, tag='pre', attrs=[('class', 'screen')]
         uattrs = []
-        strattrs=''
+        strattrs = ''
         if attrs:
             for key, value in attrs:
-                value=value.replace('>','>').replace('<','<').replace('"','"')
+                value = value.replace('>', '>')
+                value = value.replace('<', '<')
+                value = value.replace('"', '"')
                 value = self.bare_ampersand.sub("&", value)
                 uattrs.append((key, value))
-            strattrs = ''.join([' %s="%s"' % (key, value) for key, value in uattrs])
+            strattrs = ''.join(
+                ' %s="%s"' % (key, value)
+                for key, value in uattrs
+            )
         if tag in self.elements_no_end_tag:
             self.pieces.append('<%s%s />' % (tag, strattrs))
         else:
             self.pieces.append('<%s%s>' % (tag, strattrs))
 
     def unknown_endtag(self, tag):
-        # called for each end tag, e.g. for
, tag will be 'pre' + """ + :type tag: str + :rtype: None + """ + + # Called for each end tag, e.g. for , tag will be 'pre' # Reconstruct the original end tag. if tag not in self.elements_no_end_tag: self.pieces.append("" % tag) def handle_charref(self, ref): - # called for each character reference, e.g. for ' ', ref will be '160' + """ + :type ref: str + :rtype: None + """ + + # Called for each character reference, e.g. ' ' will extract '160' # Reconstruct the original character reference. ref = ref.lower() if ref.startswith('x'): @@ -146,7 +244,12 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser, object): self.pieces.append('&#%s;' % ref) def handle_entityref(self, ref): - # called for each entity reference, e.g. for '©', ref will be 'copy' + """ + :type ref: str + :rtype: None + """ + + # Called for each entity reference, e.g. '©' will extract 'copy' # Reconstruct the original entity reference. if ref in name2codepoint or ref == 'apos': self.pieces.append('&%s;' % ref) @@ -154,22 +257,42 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser, object): self.pieces.append('&%s' % ref) def handle_data(self, text): + """ + :type text: str + :rtype: None + """ + # called for each block of plain text, i.e. outside of any tag and # not containing any character or entity references # Store the original text verbatim. self.pieces.append(text) def handle_comment(self, text): - # called for each HTML comment, e.g. + """ + :type text: str + :rtype: None + """ + + # Called for HTML comments, e.g. # Reconstruct the original comment. self.pieces.append('' % text) def handle_pi(self, text): - # called for each processing instruction, e.g. + """ + :type text: str + :rtype: None + """ + + # Called for each processing instruction, e.g. # Reconstruct original processing instruction. self.pieces.append('' % text) def handle_decl(self, text): + """ + :type text: str + :rtype: None + """ + # called for the DOCTYPE, if present, e.g. # @@ -177,7 +300,14 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser, object): self.pieces.append('' % text) _new_declname_match = re.compile(r'[a-zA-Z][-_.a-zA-Z0-9:]*\s*').match + def _scan_name(self, i, declstartpos): + """ + :type i: int + :type declstartpos: int + :rtype: Tuple[Optional[str], int] + """ + rawdata = self.rawdata n = len(rawdata) if i == n: @@ -191,23 +321,44 @@ class _BaseHTMLProcessor(sgmllib.SGMLParser, object): return name.lower(), m.end() else: self.handle_data(rawdata) -# self.updatepos(declstartpos, i) + # self.updatepos(declstartpos, i) return None, -1 - def convert_charref(self, name): + @staticmethod + def convert_charref(name): + """ + :type name: str + :rtype: str + """ + return '&#%s;' % name - def convert_entityref(self, name): + @staticmethod + def convert_entityref(name): + """ + :type name: str + :rtype: str + """ + return '&%s;' % name def output(self): - '''Return processed HTML as a single string''' + """Return processed HTML as a single string. + + :rtype: str + """ + return ''.join(self.pieces) def parse_declaration(self, i): + """ + :type i: int + :rtype: int + """ + try: return sgmllib.SGMLParser.parse_declaration(self, i) except sgmllib.SGMLParseError: - # escape the doctype declaration and continue parsing + # Escape the doctype declaration and continue parsing. self.handle_data('<') return i+1 diff --git a/lib/feedparser/http.py b/lib/feedparser/http.py index 980dd2c..272faad 100644 --- a/lib/feedparser/http.py +++ b/lib/feedparser/http.py @@ -1,4 +1,32 @@ -from __future__ import absolute_import, unicode_literals, with_statement +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import datetime import gzip @@ -10,6 +38,7 @@ try: import urllib.parse import urllib.request except ImportError: + # Mock urllib sufficiently to work on Python 2.7 from urllib import splithost, splittype, splituser from urllib2 import build_opener, HTTPDigestAuthHandler, HTTPRedirectHandler, HTTPDefaultErrorHandler, Request from urlparse import urlparse @@ -20,6 +49,7 @@ except ImportError: splittype = staticmethod(splittype) splituser = staticmethod(splituser) urlparse = staticmethod(urlparse) + class request(object): build_opener = staticmethod(build_opener) HTTPDigestAuthHandler = HTTPDigestAuthHandler @@ -30,21 +60,20 @@ except ImportError: try: from io import BytesIO as _StringIO except ImportError: + # Python 2.7 try: from cStringIO import StringIO as _StringIO except ImportError: from StringIO import StringIO as _StringIO -try: - import base64, binascii -except ImportError: - base64 = binascii = None -else: - # Python 3.1 deprecated decodestring in favor of decodebytes - _base64decode = getattr(base64, 'decodebytes', base64.decodestring) +import base64 from .datetimes import _parse_date -from .urls import _convert_to_idn +from .urls import convert_to_idn + +# Python 3.1 deprecated decodestring in favor of decodebytes. +# This can be removed after Python 2.7 support is dropped. +_base64decode = getattr(base64, 'decodebytes', base64.decodestring) try: basestring @@ -57,6 +86,7 @@ bytes_ = type(b'') # want to send an Accept header, set this to None. ACCEPT_HEADER = "application/atom+xml,application/rdf+xml,application/rss+xml,application/x-netcdf,application/xml;q=0.9,text/xml;q=0.2,*/*;q=0.1" + class _FeedURLHandler(urllib.request.HTTPDigestAuthHandler, urllib.request.HTTPRedirectHandler, urllib.request.HTTPDefaultErrorHandler): def http_error_default(self, req, fp, code, msg, headers): # The default implementation just raises HTTPError. @@ -65,11 +95,11 @@ class _FeedURLHandler(urllib.request.HTTPDigestAuthHandler, urllib.request.HTTPR return fp def http_error_301(self, req, fp, code, msg, hdrs): - result = urllib.request.HTTPRedirectHandler.http_error_301(self, req, fp, - code, msg, hdrs) + result = urllib.request.HTTPRedirectHandler.http_error_301(self, req, fp, code, msg, hdrs) result.status = code result.newurl = result.geturl() return result + # The default implementations in urllib.request.HTTPRedirectHandler # are identical, so hardcoding a http_error_301 call above # won't affect anything @@ -89,8 +119,7 @@ class _FeedURLHandler(urllib.request.HTTPDigestAuthHandler, urllib.request.HTTPR # the request with the appropriate digest auth headers instead. # This evil genius hack has been brought to you by Aaron Swartz. host = urllib.parse.urlparse(req.get_full_url())[1] - if base64 is None or 'Authorization' not in req.headers \ - or 'WWW-Authenticate' not in headers: + if 'Authorization' not in req.headers or 'WWW-Authenticate' not in headers: return self.http_error_default(req, fp, code, msg, headers) auth = _base64decode(req.headers['Authorization'].split(' ')[1]) user, passw = auth.split(':') @@ -100,6 +129,7 @@ class _FeedURLHandler(urllib.request.HTTPDigestAuthHandler, urllib.request.HTTPR self.reset_retry_count() return retry + def _build_urllib2_request(url, agent, accept_header, etag, modified, referrer, auth, request_headers): request = urllib.request.Request(url) request.add_header('User-Agent', agent) @@ -119,14 +149,7 @@ def _build_urllib2_request(url, agent, accept_header, etag, modified, referrer, request.add_header('If-Modified-Since', '%s, %02d %s %04d %02d:%02d:%02d GMT' % (short_weekdays[modified[6]], modified[2], months[modified[1] - 1], modified[0], modified[3], modified[4], modified[5])) if referrer: request.add_header('Referer', referrer) - if gzip and zlib: - request.add_header('Accept-encoding', 'gzip, deflate') - elif gzip: - request.add_header('Accept-encoding', 'gzip') - elif zlib: - request.add_header('Accept-encoding', 'deflate') - else: - request.add_header('Accept-encoding', '') + request.add_header('Accept-encoding', 'gzip, deflate') if auth: request.add_header('Authorization', 'Basic %s' % auth) if accept_header: @@ -135,9 +158,10 @@ def _build_urllib2_request(url, agent, accept_header, etag, modified, referrer, # [('Cookie','Something'),('x-special-header','Another Value')] for header_name, header_value in request_headers.items(): request.add_header(header_name, header_value) - request.add_header('A-IM', 'feed') # RFC 3229 support + request.add_header('A-IM', 'feed') # RFC 3229 support return request + def get(url, etag=None, modified=None, agent=None, referrer=None, handlers=None, request_headers=None, result=None): if handlers is None: handlers = [] @@ -152,10 +176,11 @@ def get(url, etag=None, modified=None, agent=None, referrer=None, handlers=None, elif url.startswith('feed:'): url = 'http:' + url[5:] if not agent: + from . import USER_AGENT agent = USER_AGENT # Test for inline user:password credentials for HTTP basic auth auth = None - if base64 and not url.startswith('ftp:'): + if not url.startswith('ftp:'): urltype, rest = urllib.parse.splittype(url) realhost, rest = urllib.parse.splithost(rest) if realhost: @@ -166,18 +191,18 @@ def get(url, etag=None, modified=None, agent=None, referrer=None, handlers=None, # iri support if not isinstance(url, bytes_): - url = _convert_to_idn(url) + url = convert_to_idn(url) # try to open with urllib2 (to use optional headers) request = _build_urllib2_request(url, agent, ACCEPT_HEADER, etag, modified, referrer, auth, request_headers) opener = urllib.request.build_opener(*tuple(handlers + [_FeedURLHandler()])) - opener.addheaders = [] # RMK - must clear so we only send our custom User-Agent + opener.addheaders = [] # RMK - must clear so we only send our custom User-Agent f = opener.open(request) data = f.read() f.close() # lowercase all of the HTTP headers for comparisons per RFC 2616 - result['headers'] = dict((k.lower(), v) for k, v in f.headers.items()) + result['headers'] = {k.lower(): v for k, v in f.headers.items()} # if feed is gzip-compressed, decompress it if data and 'gzip' in result['headers'].get('content-encoding', ''): @@ -196,7 +221,7 @@ def get(url, etag=None, modified=None, agent=None, referrer=None, handlers=None, elif data and 'deflate' in result['headers'].get('content-encoding', ''): try: data = zlib.decompress(data) - except zlib.error as e: + except zlib.error: try: # The data may have no headers and no checksum. data = zlib.decompress(data, -15) diff --git a/lib/feedparser/mixin.py b/lib/feedparser/mixin.py index b55fdcf..1b0dc1a 100644 --- a/lib/feedparser/mixin.py +++ b/lib/feedparser/mixin.py @@ -1,12 +1,11 @@ -# Shared code that is common to the strict and loose feed parsers -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # # This file is a part of feedparser. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -26,39 +25,43 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals +import base64 +import binascii import copy import re - from xml.sax.saxutils import escape as _xmlescape -# base64 support for Atom feeds that contain embedded binary data -try: - import base64, binascii -except ImportError: - base64 = binascii = None -else: - # Python 3.1 deprecates decodestring in favor of decodebytes - _base64decode = getattr(base64, 'decodebytes', base64.decodestring) - try: from html.entities import name2codepoint, entitydefs except ImportError: + # Python 2 + # noinspection PyUnresolvedReferences from htmlentitydefs import name2codepoint, entitydefs from .html import _cp1252 from .namespaces import _base, cc, dc, georss, itunes, mediarss, psc -from .sanitizer import _sanitizeHTML, _HTMLSanitizer +from .sanitizer import _sanitize_html, _HTMLSanitizer from .util import FeedParserDict -from .urls import _urljoin, _makeSafeAbsoluteURI, _resolveRelativeURIs +from .urls import _urljoin, make_safe_absolute_uri, resolve_relative_uris + + +# Python 2.7 only offers "decodestring()". +# This name substitution can be removed when Python 2.7 support is dropped. +_base64decode = getattr(base64, 'decodebytes', base64.decodestring) + bytes_ = type(b'') try: + # Python 2 + # noinspection PyUnresolvedReferences,PyShadowingBuiltins chr = unichr except NameError: pass + class _FeedParserMixin( _base.Namespace, cc.Namespace, @@ -140,19 +143,56 @@ class _FeedParserMixin( } _matchnamespaces = {} - can_be_relative_uri = set(['link', 'id', 'wfw_comment', 'wfw_commentrss', 'docs', 'url', 'href', 'comments', 'icon', 'logo']) - can_contain_relative_uris = set(['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description']) - can_contain_dangerous_markup = set(['content', 'title', 'summary', 'info', 'tagline', 'subtitle', 'copyright', 'rights', 'description']) - html_types = ['text/html', 'application/xhtml+xml'] + can_be_relative_uri = { + 'comments', + 'docs', + 'href', + 'icon', + 'id', + 'link', + 'logo', + 'url', + 'wfw_comment', + 'wfw_commentrss', + } + + can_contain_relative_uris = { + 'content', + 'copyright', + 'description', + 'info', + 'rights', + 'subtitle', + 'summary', + 'tagline', + 'title', + } + + can_contain_dangerous_markup = { + 'content', + 'copyright', + 'description', + 'info', + 'rights', + 'subtitle', + 'summary', + 'tagline', + 'title', + } + + html_types = { + 'application/xhtml+xml', + 'text/html', + } def __init__(self): if not self._matchnamespaces: for k, v in self.namespaces.items(): self._matchnamespaces[k.lower()] = v - self.feeddata = FeedParserDict() # feed-level data - self.entries = [] # list of entry-level data - self.version = '' # feed type/version, see SUPPORTED_VERSIONS - self.namespacesInUse = {} # dictionary of namespaces defined by the feed + self.feeddata = FeedParserDict() # feed-level data + self.entries = [] # list of entry-level data + self.version = '' # feed type/version, see SUPPORTED_VERSIONS + self.namespaces_in_use = {} # dictionary of namespaces defined by the feed # the following are used internally to track state; # this is really out of control and should be refactored @@ -177,7 +217,7 @@ class _FeedParserMixin( self.title_depth = -1 self.depth = 0 if self.lang: - self.feeddata['language'] = self.lang.replace('_','-') + self.feeddata['language'] = self.lang.replace('_', '-') # A map of the following form: # { @@ -189,6 +229,9 @@ class _FeedParserMixin( self.property_depth_map = {} super(_FeedParserMixin, self).__init__() + def _normalize_attributes(self, kv): + raise NotImplementedError + def unknown_starttag(self, tag, attrs): # increment depth counter self.depth += 1 @@ -197,17 +240,17 @@ class _FeedParserMixin( attrs = [self._normalize_attributes(attr) for attr in attrs] # track xml:base and xml:lang - attrsD = dict(attrs) - baseuri = attrsD.get('xml:base', attrsD.get('base')) or self.baseuri + attrs_d = dict(attrs) + baseuri = attrs_d.get('xml:base', attrs_d.get('base')) or self.baseuri if isinstance(baseuri, bytes_): baseuri = baseuri.decode(self.encoding, 'ignore') # ensure that self.baseuri is always an absolute URI that # uses a whitelisted URI scheme (e.g. not `javscript:`) if self.baseuri: - self.baseuri = _makeSafeAbsoluteURI(self.baseuri, baseuri) or self.baseuri + self.baseuri = make_safe_absolute_uri(self.baseuri, baseuri) or self.baseuri else: self.baseuri = _urljoin(self.baseuri, baseuri) - lang = attrsD.get('xml:lang', attrsD.get('lang')) + lang = attrs_d.get('xml:lang', attrs_d.get('lang')) if lang == '': # xml:lang could be explicitly set to '', we need to capture that lang = None @@ -216,7 +259,7 @@ class _FeedParserMixin( lang = self.lang if lang: if tag in ('feed', 'rss', 'rdf:RDF'): - self.feeddata['language'] = lang.replace('_','-') + self.feeddata['language'] = lang.replace('_', '-') self.lang = lang self.basestack.append(self.baseuri) self.langstack.append(lang) @@ -224,24 +267,24 @@ class _FeedParserMixin( # track namespaces for prefix, uri in attrs: if prefix.startswith('xmlns:'): - self.trackNamespace(prefix[6:], uri) + self.track_namespace(prefix[6:], uri) elif prefix == 'xmlns': - self.trackNamespace(None, uri) + self.track_namespace(None, uri) # track inline content if self.incontent and not self.contentparams.get('type', 'xml').endswith('xml'): if tag in ('xhtml:div', 'div'): - return # typepad does this 10/2007 + return # typepad does this 10/2007 # element declared itself as escaped markup, but it isn't really self.contentparams['type'] = 'application/xhtml+xml' if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml': if tag.find(':') != -1: prefix, tag = tag.split(':', 1) - namespace = self.namespacesInUse.get(prefix, '') - if tag=='math' and namespace=='http://www.w3.org/1998/Math/MathML': - attrs.append(('xmlns',namespace)) - if tag=='svg' and namespace=='http://www.w3.org/2000/svg': - attrs.append(('xmlns',namespace)) + namespace = self.namespaces_in_use.get(prefix, '') + if tag == 'math' and namespace == 'http://www.w3.org/1998/Math/MathML': + attrs.append(('xmlns', namespace)) + if tag == 'svg' and namespace == 'http://www.w3.org/2000/svg': + attrs.append(('xmlns', namespace)) if tag == 'svg': self.svgOK += 1 return self.handle_data('<%s%s>' % (tag, self.strattrs(attrs)), escape=0) @@ -255,7 +298,8 @@ class _FeedParserMixin( if prefix: prefix = prefix + '_' - # special hack for better tracking of empty textinput/image elements in illformed feeds + # Special hack for better tracking of empty textinput/image elements in + # illformed feeds. if (not prefix) and tag not in ('title', 'link', 'description', 'name'): self.intextinput = 0 if (not prefix) and tag not in ('title', 'link', 'description', 'url', 'href', 'width', 'height'): @@ -265,17 +309,18 @@ class _FeedParserMixin( methodname = '_start_' + prefix + suffix try: method = getattr(self, methodname) - return method(attrsD) + return method(attrs_d) except AttributeError: - # Since there's no handler or something has gone wrong we explicitly add the element and its attributes + # Since there's no handler or something has gone wrong we + # explicitly add the element and its attributes. unknown_tag = prefix + suffix - if len(attrsD) == 0: + if len(attrs_d) == 0: # No attributes so merge it into the enclosing dictionary return self.push(unknown_tag, 1) else: # Has attributes so create it in its own dictionary - context = self._getContext() - context[unknown_tag] = attrsD + context = self._get_context() + context[unknown_tag] = attrs_d def unknown_endtag(self, tag): # match namespaces @@ -303,7 +348,7 @@ class _FeedParserMixin( if self.incontent and not self.contentparams.get('type', 'xml').endswith('xml'): # element declared itself as escaped markup, but it isn't really if tag in ('xhtml:div', 'div'): - return # typepad does this 10/2007 + return # typepad does this 10/2007 self.contentparams['type'] = 'application/xhtml+xml' if self.incontent and self.contentparams.get('type') == 'application/xhtml+xml': tag = tag.split(':')[-1] @@ -316,13 +361,13 @@ class _FeedParserMixin( self.baseuri = self.basestack[-1] if self.langstack: self.langstack.pop() - if self.langstack: # and (self.langstack[-1] is not None): + if self.langstack: # and (self.langstack[-1] is not None): self.lang = self.langstack[-1] self.depth -= 1 def handle_charref(self, ref): - # called for each character reference, e.g. for ' ', ref will be '160' + # Called for each character reference, e.g. for ' ', ref is '160' if not self.elementstack: return ref = ref.lower() @@ -337,7 +382,7 @@ class _FeedParserMixin( self.elementstack[-1][2].append(text) def handle_entityref(self, ref): - # called for each entity reference, e.g. for '©', ref will be 'copy' + # Called for each entity reference, e.g. for '©', ref is 'copy' if not self.elementstack: return if ref in ('lt', 'gt', 'quot', 'amp', 'apos'): @@ -356,7 +401,7 @@ class _FeedParserMixin( self.elementstack[-1][2].append(text) def handle_data(self, text, escape=1): - # called for each block of plain text, i.e. outside of any tag and + # Called for each block of plain text, i.e. outside of any tag and # not containing any character or entity references if not self.elementstack: return @@ -365,18 +410,18 @@ class _FeedParserMixin( self.elementstack[-1][2].append(text) def handle_comment(self, text): - # called for each comment, e.g. + # Called for each comment, e.g. pass def handle_pi(self, text): - # called for each processing instruction, e.g. + # Called for each processing instruction, e.g. pass def handle_decl(self, text): pass def parse_declaration(self, i): - # override internal declaration handler to handle CDATA blocks + # Override internal declaration handler to handle CDATA blocks. if self.rawdata[i:i+9] == '', i) if k == -1: @@ -393,17 +438,18 @@ class _FeedParserMixin( # We have an incomplete CDATA block. return k - def mapContentType(self, contentType): - contentType = contentType.lower() - if contentType == 'text' or contentType == 'plain': - contentType = 'text/plain' - elif contentType == 'html': - contentType = 'text/html' - elif contentType == 'xhtml': - contentType = 'application/xhtml+xml' - return contentType - - def trackNamespace(self, prefix, uri): + @staticmethod + def map_content_type(content_type): + content_type = content_type.lower() + if content_type == 'text' or content_type == 'plain': + content_type = 'text/plain' + elif content_type == 'html': + content_type = 'text/html' + elif content_type == 'xhtml': + content_type = 'application/xhtml+xml' + return content_type + + def track_namespace(self, prefix, uri): loweruri = uri.lower() if not self.version: if (prefix, loweruri) == (None, 'http://my.netscape.com/rdf/simple/0.9/'): @@ -418,29 +464,34 @@ class _FeedParserMixin( loweruri = uri if loweruri in self._matchnamespaces: self.namespacemap[prefix] = self._matchnamespaces[loweruri] - self.namespacesInUse[self._matchnamespaces[loweruri]] = uri + self.namespaces_in_use[self._matchnamespaces[loweruri]] = uri else: - self.namespacesInUse[prefix or ''] = uri + self.namespaces_in_use[prefix or ''] = uri - def resolveURI(self, uri): + def resolve_uri(self, uri): return _urljoin(self.baseuri or '', uri) - def decodeEntities(self, element, data): + @staticmethod + def decode_entities(element, data): return data - def strattrs(self, attrs): - return ''.join([' %s="%s"' % (t[0],_xmlescape(t[1],{'"':'"'})) for t in attrs]) + @staticmethod + def strattrs(attrs): + return ''.join( + ' %s="%s"' % (t[0], _xmlescape(t[1], {'"': '"'})) + for t in attrs + ) - def push(self, element, expectingText): - self.elementstack.append([element, expectingText, []]) + def push(self, element, expecting_text): + self.elementstack.append([element, expecting_text, []]) - def pop(self, element, stripWhitespace=1): + def pop(self, element, strip_whitespace=1): if not self.elementstack: return if self.elementstack[-1][0] != element: return - element, expectingText, pieces = self.elementstack.pop() + element, expecting_text, pieces = self.elementstack.pop() # Ensure each piece is a str for Python 3 for (i, v) in enumerate(pieces): @@ -452,11 +503,11 @@ class _FeedParserMixin( # only if all the remaining content is nested underneath it. # This means that the divs would be retained in the following: #
foo
bar
- while pieces and len(pieces)>1 and not pieces[-1].strip(): + while pieces and len(pieces) > 1 and not pieces[-1].strip(): del pieces[-1] - while pieces and len(pieces)>1 and not pieces[0].strip(): + while pieces and len(pieces) > 1 and not pieces[0].strip(): del pieces[0] - if pieces and (pieces[0] == '
' or pieces[0].startswith('
': + if pieces and (pieces[0] == '
' or pieces[0].startswith('
': depth = 0 for piece in pieces[:-1]: if piece.startswith('', s) or re.search(r'&#?\w+;', s)): - return + return False # all tags must be in a restricted subset of valid HTML tags if any((t for t in re.findall(r'', '') @@ -734,8 +793,8 @@ class _FeedParserMixin( if email: detail['email'] = email - def _addTag(self, term, scheme, label): - context = self._getContext() + def _add_tag(self, term, scheme, label): + context = self._get_context() tags = context.setdefault('tags', []) if (not term) and (not scheme) and (not label): return @@ -743,7 +802,7 @@ class _FeedParserMixin( if value not in tags: tags.append(value) - def _start_tags(self, attrsD): + def _start_tags(self, attrs_d): # This is a completely-made up element. Its semantics are determined # only by a single feed that precipitated bug report 392 on Google Code. # In short, this is junk code. @@ -751,4 +810,4 @@ class _FeedParserMixin( def _end_tags(self): for term in self.pop('tags').split(','): - self._addTag(term.strip(), None, None) + self._add_tag(term.strip(), None, None) diff --git a/lib/feedparser/namespaces/_base.py b/lib/feedparser/namespaces/_base.py index f25edb6..3b3b21c 100644 --- a/lib/feedparser/namespaces/_base.py +++ b/lib/feedparser/namespaces/_base.py @@ -1,5 +1,5 @@ # Support for the Atom, RSS, RDF, and CDF feed formats -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # @@ -26,14 +26,16 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals import copy -from ..datetimes import registerDateHandler, _parse_date -from ..urls import _makeSafeAbsoluteURI +from ..datetimes import _parse_date +from ..urls import make_safe_absolute_uri from ..util import FeedParserDict + class Namespace(object): """Support for the Atom, RSS, RDF, and CDF feed formats. @@ -58,16 +60,19 @@ class Namespace(object): 'http://purl.org/rss/1.0/modules/rss091#': '', } - def _start_rss(self, attrsD): - versionmap = {'0.91': 'rss091u', - '0.92': 'rss092', - '0.93': 'rss093', - '0.94': 'rss094'} - #If we're here then this is an RSS feed. - #If we don't have a version or have a version that starts with something - #other than RSS then there's been a mistake. Correct it. + def _start_rss(self, attrs_d): + versionmap = { + '0.91': 'rss091u', + '0.92': 'rss092', + '0.93': 'rss093', + '0.94': 'rss094', + } + + # If we're here then this is an RSS feed. + # If we don't have a version or have a version that starts with something + # other than RSS then there's been a mistake. Correct it. if not self.version or not self.version.startswith('rss'): - attr_version = attrsD.get('version', '') + attr_version = attrs_d.get('version', '') version = versionmap.get(attr_version) if version: self.version = version @@ -76,27 +81,27 @@ class Namespace(object): else: self.version = 'rss' - def _start_channel(self, attrsD): + def _start_channel(self, attrs_d): self.infeed = 1 - self._cdf_common(attrsD) + self._cdf_common(attrs_d) - def _cdf_common(self, attrsD): - if 'lastmod' in attrsD: + def _cdf_common(self, attrs_d): + if 'lastmod' in attrs_d: self._start_modified({}) - self.elementstack[-1][-1] = attrsD['lastmod'] + self.elementstack[-1][-1] = attrs_d['lastmod'] self._end_modified() - if 'href' in attrsD: + if 'href' in attrs_d: self._start_link({}) - self.elementstack[-1][-1] = attrsD['href'] + self.elementstack[-1][-1] = attrs_d['href'] self._end_link() - def _start_feed(self, attrsD): + def _start_feed(self, attrs_d): self.infeed = 1 versionmap = {'0.1': 'atom01', '0.2': 'atom02', '0.3': 'atom03'} if not self.version: - attr_version = attrsD.get('version') + attr_version = attrs_d.get('version') version = versionmap.get(attr_version) if version: self.version = version @@ -107,8 +112,8 @@ class Namespace(object): self.infeed = 0 _end_feed = _end_channel - def _start_image(self, attrsD): - context = self._getContext() + def _start_image(self, attrs_d): + context = self._get_context() if not self.inentry: context.setdefault('image', FeedParserDict()) self.inimage = 1 @@ -119,8 +124,8 @@ class Namespace(object): self.pop('image') self.inimage = 0 - def _start_textinput(self, attrsD): - context = self._getContext() + def _start_textinput(self, attrs_d): + context = self._get_context() context.setdefault('textinput', FeedParserDict()) self.intextinput = 1 self.title_depth = -1 @@ -132,11 +137,11 @@ class Namespace(object): self.intextinput = 0 _end_textInput = _end_textinput - def _start_author(self, attrsD): + def _start_author(self, attrs_d): self.inauthor = 1 self.push('author', 1) # Append a new FeedParserDict when expecting an author - context = self._getContext() + context = self._get_context() context.setdefault('authors', []) context['authors'].append(FeedParserDict()) _start_managingeditor = _start_author @@ -147,9 +152,9 @@ class Namespace(object): self._sync_author_detail() _end_managingeditor = _end_author - def _start_contributor(self, attrsD): + def _start_contributor(self, attrs_d): self.incontributor = 1 - context = self._getContext() + context = self._get_context() context.setdefault('contributors', []) context['contributors'].append(FeedParserDict()) self.push('contributor', 0) @@ -158,7 +163,7 @@ class Namespace(object): self.pop('contributor') self.incontributor = 0 - def _start_name(self, attrsD): + def _start_name(self, attrs_d): self.push('name', 0) def _end_name(self): @@ -170,10 +175,10 @@ class Namespace(object): elif self.incontributor: self._save_contributor('name', value) elif self.intextinput: - context = self._getContext() + context = self._get_context() context['name'] = value - def _start_width(self, attrsD): + def _start_width(self, attrs_d): self.push('width', 0) def _end_width(self): @@ -183,10 +188,10 @@ class Namespace(object): except ValueError: value = 0 if self.inimage: - context = self._getContext() + context = self._get_context() context['width'] = value - def _start_height(self, attrsD): + def _start_height(self, attrs_d): self.push('height', 0) def _end_height(self): @@ -196,10 +201,10 @@ class Namespace(object): except ValueError: value = 0 if self.inimage: - context = self._getContext() + context = self._get_context() context['height'] = value - def _start_url(self, attrsD): + def _start_url(self, attrs_d): self.push('href', 1) _start_homepage = _start_url _start_uri = _start_url @@ -213,7 +218,7 @@ class Namespace(object): _end_homepage = _end_url _end_uri = _end_url - def _start_email(self, attrsD): + def _start_email(self, attrs_d): self.push('email', 0) def _end_email(self): @@ -225,33 +230,33 @@ class Namespace(object): elif self.incontributor: self._save_contributor('email', value) - def _start_subtitle(self, attrsD): - self.pushContent('subtitle', attrsD, 'text/plain', 1) + def _start_subtitle(self, attrs_d): + self.push_content('subtitle', attrs_d, 'text/plain', 1) _start_tagline = _start_subtitle def _end_subtitle(self): - self.popContent('subtitle') + self.pop_content('subtitle') _end_tagline = _end_subtitle - def _start_rights(self, attrsD): - self.pushContent('rights', attrsD, 'text/plain', 1) + def _start_rights(self, attrs_d): + self.push_content('rights', attrs_d, 'text/plain', 1) _start_copyright = _start_rights def _end_rights(self): - self.popContent('rights') + self.pop_content('rights') _end_copyright = _end_rights - def _start_item(self, attrsD): + def _start_item(self, attrs_d): self.entries.append(FeedParserDict()) self.push('item', 0) self.inentry = 1 self.guidislink = 0 self.title_depth = -1 - id = self._getAttribute(attrsD, 'rdf:about') + id = self._get_attribute(attrs_d, 'rdf:about') if id: - context = self._getContext() + context = self._get_context() context['id'] = id - self._cdf_common(attrsD) + self._cdf_common(attrs_d) _start_entry = _start_item def _end_item(self): @@ -259,20 +264,20 @@ class Namespace(object): self.inentry = 0 _end_entry = _end_item - def _start_language(self, attrsD): + def _start_language(self, attrs_d): self.push('language', 1) def _end_language(self): self.lang = self.pop('language') - def _start_webmaster(self, attrsD): + def _start_webmaster(self, attrs_d): self.push('publisher', 1) def _end_webmaster(self): self.pop('publisher') self._sync_author_detail('publisher') - def _start_published(self, attrsD): + def _start_published(self, attrs_d): self.push('published', 1) _start_issued = _start_published _start_pubdate = _start_published @@ -283,7 +288,7 @@ class Namespace(object): _end_issued = _end_published _end_pubdate = _end_published - def _start_updated(self, attrsD): + def _start_updated(self, attrs_d): self.push('updated', 1) _start_modified = _start_updated _start_lastbuilddate = _start_updated @@ -295,24 +300,24 @@ class Namespace(object): _end_modified = _end_updated _end_lastbuilddate = _end_updated - def _start_created(self, attrsD): + def _start_created(self, attrs_d): self.push('created', 1) def _end_created(self): value = self.pop('created') self._save('created_parsed', _parse_date(value), overwrite=True) - def _start_expirationdate(self, attrsD): + def _start_expirationdate(self, attrs_d): self.push('expired', 1) def _end_expirationdate(self): self._save('expired_parsed', _parse_date(self.pop('expired')), overwrite=True) - def _start_category(self, attrsD): - term = attrsD.get('term') - scheme = attrsD.get('scheme', attrsD.get('domain')) - label = attrsD.get('label') - self._addTag(term, scheme, label) + def _start_category(self, attrs_d): + term = attrs_d.get('term') + scheme = attrs_d.get('scheme', attrs_d.get('domain')) + label = attrs_d.get('label') + self._add_tag(term, scheme, label) self.push('category', 1) _start_keywords = _start_category @@ -320,135 +325,137 @@ class Namespace(object): value = self.pop('category') if not value: return - context = self._getContext() + context = self._get_context() tags = context['tags'] if value and len(tags) and not tags[-1]['term']: tags[-1]['term'] = value else: - self._addTag(value, None, None) + self._add_tag(value, None, None) _end_keywords = _end_category - def _start_cloud(self, attrsD): - self._getContext()['cloud'] = FeedParserDict(attrsD) + def _start_cloud(self, attrs_d): + self._get_context()['cloud'] = FeedParserDict(attrs_d) - def _start_link(self, attrsD): - attrsD.setdefault('rel', 'alternate') - if attrsD['rel'] == 'self': - attrsD.setdefault('type', 'application/atom+xml') + def _start_link(self, attrs_d): + attrs_d.setdefault('rel', 'alternate') + if attrs_d['rel'] == 'self': + attrs_d.setdefault('type', 'application/atom+xml') else: - attrsD.setdefault('type', 'text/html') - context = self._getContext() - attrsD = self._itsAnHrefDamnIt(attrsD) - if 'href' in attrsD: - attrsD['href'] = self.resolveURI(attrsD['href']) - expectingText = self.infeed or self.inentry or self.insource + attrs_d.setdefault('type', 'text/html') + context = self._get_context() + attrs_d = self._enforce_href(attrs_d) + if 'href' in attrs_d: + attrs_d['href'] = self.resolve_uri(attrs_d['href']) + expecting_text = self.infeed or self.inentry or self.insource context.setdefault('links', []) if not (self.inentry and self.inimage): - context['links'].append(FeedParserDict(attrsD)) - if 'href' in attrsD: - expectingText = 0 - if (attrsD.get('rel') == 'alternate') and (self.mapContentType(attrsD.get('type')) in self.html_types): - context['link'] = attrsD['href'] + context['links'].append(FeedParserDict(attrs_d)) + if 'href' in attrs_d: + if ( + attrs_d.get('rel') == 'alternate' + and self.map_content_type(attrs_d.get('type')) in self.html_types + ): + context['link'] = attrs_d['href'] else: - self.push('link', expectingText) + self.push('link', expecting_text) def _end_link(self): - value = self.pop('link') + self.pop('link') - def _start_guid(self, attrsD): - self.guidislink = (attrsD.get('ispermalink', 'true') == 'true') + def _start_guid(self, attrs_d): + self.guidislink = (attrs_d.get('ispermalink', 'true') == 'true') self.push('id', 1) _start_id = _start_guid def _end_guid(self): value = self.pop('id') - self._save('guidislink', self.guidislink and 'link' not in self._getContext()) + self._save('guidislink', self.guidislink and 'link' not in self._get_context()) if self.guidislink: # guid acts as link, but only if 'ispermalink' is not present or is 'true', # and only if the item doesn't already have a link element self._save('link', value) _end_id = _end_guid - def _start_title(self, attrsD): + def _start_title(self, attrs_d): if self.svgOK: - return self.unknown_starttag('title', list(attrsD.items())) - self.pushContent('title', attrsD, 'text/plain', self.infeed or self.inentry or self.insource) + return self.unknown_starttag('title', list(attrs_d.items())) + self.push_content('title', attrs_d, 'text/plain', self.infeed or self.inentry or self.insource) def _end_title(self): if self.svgOK: return - value = self.popContent('title') + value = self.pop_content('title') if not value: return self.title_depth = self.depth - def _start_description(self, attrsD): - context = self._getContext() + def _start_description(self, attrs_d): + context = self._get_context() if 'summary' in context: self._summaryKey = 'content' - self._start_content(attrsD) + self._start_content(attrs_d) else: - self.pushContent('description', attrsD, 'text/html', self.infeed or self.inentry or self.insource) + self.push_content('description', attrs_d, 'text/html', self.infeed or self.inentry or self.insource) - def _start_abstract(self, attrsD): - self.pushContent('description', attrsD, 'text/plain', self.infeed or self.inentry or self.insource) + def _start_abstract(self, attrs_d): + self.push_content('description', attrs_d, 'text/plain', self.infeed or self.inentry or self.insource) def _end_description(self): if self._summaryKey == 'content': self._end_content() else: - value = self.popContent('description') + self.pop_content('description') self._summaryKey = None _end_abstract = _end_description - def _start_info(self, attrsD): - self.pushContent('info', attrsD, 'text/plain', 1) + def _start_info(self, attrs_d): + self.push_content('info', attrs_d, 'text/plain', 1) _start_feedburner_browserfriendly = _start_info def _end_info(self): - self.popContent('info') + self.pop_content('info') _end_feedburner_browserfriendly = _end_info - def _start_generator(self, attrsD): - if attrsD: - attrsD = self._itsAnHrefDamnIt(attrsD) - if 'href' in attrsD: - attrsD['href'] = self.resolveURI(attrsD['href']) - self._getContext()['generator_detail'] = FeedParserDict(attrsD) + def _start_generator(self, attrs_d): + if attrs_d: + attrs_d = self._enforce_href(attrs_d) + if 'href' in attrs_d: + attrs_d['href'] = self.resolve_uri(attrs_d['href']) + self._get_context()['generator_detail'] = FeedParserDict(attrs_d) self.push('generator', 1) def _end_generator(self): value = self.pop('generator') - context = self._getContext() + context = self._get_context() if 'generator_detail' in context: context['generator_detail']['name'] = value - def _start_summary(self, attrsD): - context = self._getContext() + def _start_summary(self, attrs_d): + context = self._get_context() if 'summary' in context: self._summaryKey = 'content' - self._start_content(attrsD) + self._start_content(attrs_d) else: self._summaryKey = 'summary' - self.pushContent(self._summaryKey, attrsD, 'text/plain', 1) + self.push_content(self._summaryKey, attrs_d, 'text/plain', 1) def _end_summary(self): if self._summaryKey == 'content': self._end_content() else: - self.popContent(self._summaryKey or 'summary') + self.pop_content(self._summaryKey or 'summary') self._summaryKey = None - def _start_enclosure(self, attrsD): - attrsD = self._itsAnHrefDamnIt(attrsD) - context = self._getContext() - attrsD['rel'] = 'enclosure' - context.setdefault('links', []).append(FeedParserDict(attrsD)) + def _start_enclosure(self, attrs_d): + attrs_d = self._enforce_href(attrs_d) + context = self._get_context() + attrs_d['rel'] = 'enclosure' + context.setdefault('links', []).append(FeedParserDict(attrs_d)) - def _start_source(self, attrsD): - if 'url' in attrsD: + def _start_source(self, attrs_d): + if 'url' in attrs_d: # This means that we're processing a source element from an RSS 2.0 feed - self.sourcedata['href'] = attrsD['url'] + self.sourcedata['href'] = attrs_d['url'] self.push('source', 1) self.insource = 1 self.title_depth = -1 @@ -458,27 +465,27 @@ class Namespace(object): value = self.pop('source') if value: self.sourcedata['title'] = value - self._getContext()['source'] = copy.deepcopy(self.sourcedata) + self._get_context()['source'] = copy.deepcopy(self.sourcedata) self.sourcedata.clear() - def _start_content(self, attrsD): - self.pushContent('content', attrsD, 'text/plain', 1) - src = attrsD.get('src') + def _start_content(self, attrs_d): + self.push_content('content', attrs_d, 'text/plain', 1) + src = attrs_d.get('src') if src: self.contentparams['src'] = src self.push('content', 1) - def _start_body(self, attrsD): - self.pushContent('content', attrsD, 'application/xhtml+xml', 1) + def _start_body(self, attrs_d): + self.push_content('content', attrs_d, 'application/xhtml+xml', 1) _start_xhtml_body = _start_body - def _start_content_encoded(self, attrsD): - self.pushContent('content', attrsD, 'text/html', 1) + def _start_content_encoded(self, attrs_d): + self.push_content('content', attrs_d, 'text/html', 1) _start_fullitem = _start_content_encoded def _end_content(self): - copyToSummary = self.mapContentType(self.contentparams.get('type')) in (['text/plain'] + self.html_types) - value = self.popContent('content') + copyToSummary = self.map_content_type(self.contentparams.get('type')) in ({'text/plain'} | self.html_types) + value = self.pop_content('content') if copyToSummary: self._save('summary', value) @@ -487,13 +494,13 @@ class Namespace(object): _end_content_encoded = _end_content _end_fullitem = _end_content - def _start_newlocation(self, attrsD): + def _start_newlocation(self, attrs_d): self.push('newlocation', 1) def _end_newlocation(self): url = self.pop('newlocation') - context = self._getContext() + context = self._get_context() # don't set newlocation if the context isn't right if context is not self.feeddata: return - context['newlocation'] = _makeSafeAbsoluteURI(self.baseuri, url.strip()) + context['newlocation'] = make_safe_absolute_uri(self.baseuri, url.strip()) diff --git a/lib/feedparser/namespaces/admin.py b/lib/feedparser/namespaces/admin.py index 7c663ae..c870cae 100644 --- a/lib/feedparser/namespaces/admin.py +++ b/lib/feedparser/namespaces/admin.py @@ -1,12 +1,12 @@ # Support for the administrative elements extension -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # # This file is a part of feedparser. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -26,10 +26,12 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals from ..util import FeedParserDict + class Namespace(object): # RDF Site Summary 1.0 Modules: Administrative # http://web.resource.org/rss/1.0/modules/admin/ @@ -38,18 +40,17 @@ class Namespace(object): 'http://webns.net/mvcb/': 'admin', } - def _start_admin_generatoragent(self, attrsD): + def _start_admin_generatoragent(self, attrs_d): self.push('generator', 1) - value = self._getAttribute(attrsD, 'rdf:resource') + value = self._get_attribute(attrs_d, 'rdf:resource') if value: self.elementstack[-1][2].append(value) self.pop('generator') - self._getContext()['generator_detail'] = FeedParserDict({'href': value}) + self._get_context()['generator_detail'] = FeedParserDict({'href': value}) - def _start_admin_errorreportsto(self, attrsD): + def _start_admin_errorreportsto(self, attrs_d): self.push('errorreportsto', 1) - value = self._getAttribute(attrsD, 'rdf:resource') + value = self._get_attribute(attrs_d, 'rdf:resource') if value: self.elementstack[-1][2].append(value) self.pop('errorreportsto') - diff --git a/lib/feedparser/namespaces/cc.py b/lib/feedparser/namespaces/cc.py index 6d8d393..e9e4240 100644 --- a/lib/feedparser/namespaces/cc.py +++ b/lib/feedparser/namespaces/cc.py @@ -1,12 +1,12 @@ # Support for the Creative Commons licensing extensions -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # # This file is a part of feedparser. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -26,10 +26,12 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals from ..util import FeedParserDict + class Namespace(object): supported_namespaces = { # RDF-based namespace @@ -45,27 +47,26 @@ class Namespace(object): 'http://backend.userland.com/creativeCommonsRssModule': 'creativecommons', } - def _start_cc_license(self, attrsD): - context = self._getContext() - value = self._getAttribute(attrsD, 'rdf:resource') - attrsD = FeedParserDict() - attrsD['rel'] = 'license' + def _start_cc_license(self, attrs_d): + context = self._get_context() + value = self._get_attribute(attrs_d, 'rdf:resource') + attrs_d = FeedParserDict() + attrs_d['rel'] = 'license' if value: - attrsD['href'] = value - context.setdefault('links', []).append(attrsD) + attrs_d['href'] = value + context.setdefault('links', []).append(attrs_d) - def _start_creativecommons_license(self, attrsD): + def _start_creativecommons_license(self, attrs_d): self.push('license', 1) _start_creativeCommons_license = _start_creativecommons_license def _end_creativecommons_license(self): value = self.pop('license') - context = self._getContext() - attrsD = FeedParserDict() - attrsD['rel'] = 'license' + context = self._get_context() + attrs_d = FeedParserDict() + attrs_d['rel'] = 'license' if value: - attrsD['href'] = value - context.setdefault('links', []).append(attrsD) + attrs_d['href'] = value + context.setdefault('links', []).append(attrs_d) del context['license'] _end_creativeCommons_license = _end_creativecommons_license - diff --git a/lib/feedparser/namespaces/dc.py b/lib/feedparser/namespaces/dc.py index 035bbad..f31f975 100644 --- a/lib/feedparser/namespaces/dc.py +++ b/lib/feedparser/namespaces/dc.py @@ -1,12 +1,12 @@ # Support for the Dublin Core metadata extensions -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # # This file is a part of feedparser. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -26,10 +26,12 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals -from ..util import FeedParserDict from ..datetimes import _parse_date +from ..util import FeedParserDict + class Namespace(object): supported_namespaces = { @@ -73,43 +75,43 @@ class Namespace(object): def _end_dcterms_modified(self): self._end_updated() - def _start_dc_author(self, attrsD): - self._start_author(attrsD) + def _start_dc_author(self, attrs_d): + self._start_author(attrs_d) - def _start_dc_creator(self, attrsD): - self._start_author(attrsD) + def _start_dc_creator(self, attrs_d): + self._start_author(attrs_d) - def _start_dc_date(self, attrsD): - self._start_updated(attrsD) + def _start_dc_date(self, attrs_d): + self._start_updated(attrs_d) - def _start_dc_description(self, attrsD): - self._start_description(attrsD) + def _start_dc_description(self, attrs_d): + self._start_description(attrs_d) - def _start_dc_language(self, attrsD): - self._start_language(attrsD) + def _start_dc_language(self, attrs_d): + self._start_language(attrs_d) - def _start_dc_publisher(self, attrsD): - self._start_webmaster(attrsD) + def _start_dc_publisher(self, attrs_d): + self._start_webmaster(attrs_d) - def _start_dc_rights(self, attrsD): - self._start_rights(attrsD) + def _start_dc_rights(self, attrs_d): + self._start_rights(attrs_d) - def _start_dc_subject(self, attrsD): - self._start_category(attrsD) + def _start_dc_subject(self, attrs_d): + self._start_category(attrs_d) - def _start_dc_title(self, attrsD): - self._start_title(attrsD) + def _start_dc_title(self, attrs_d): + self._start_title(attrs_d) - def _start_dcterms_created(self, attrsD): - self._start_created(attrsD) + def _start_dcterms_created(self, attrs_d): + self._start_created(attrs_d) - def _start_dcterms_issued(self, attrsD): - self._start_published(attrsD) + def _start_dcterms_issued(self, attrs_d): + self._start_published(attrs_d) - def _start_dcterms_modified(self, attrsD): - self._start_updated(attrsD) + def _start_dcterms_modified(self, attrs_d): + self._start_updated(attrs_d) - def _start_dcterms_valid(self, attrsD): + def _start_dcterms_valid(self, attrs_d): self.push('validity', 1) def _end_dcterms_valid(self): @@ -123,9 +125,9 @@ class Namespace(object): self._save('validity_end', value, overwrite=True) self._save('validity_end_parsed', _parse_date(value), overwrite=True) - def _start_dc_contributor(self, attrsD): + def _start_dc_contributor(self, attrs_d): self.incontributor = 1 - context = self._getContext() + context = self._get_context() context.setdefault('contributors', []) context['contributors'].append(FeedParserDict()) self.push('name', 0) @@ -133,4 +135,3 @@ class Namespace(object): def _end_dc_contributor(self): self._end_name() self.incontributor = 0 - diff --git a/lib/feedparser/namespaces/georss.py b/lib/feedparser/namespaces/georss.py index 8a5cf49..0a6d91f 100644 --- a/lib/feedparser/namespaces/georss.py +++ b/lib/feedparser/namespaces/georss.py @@ -1,12 +1,12 @@ # Support for the GeoRSS format -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # # This file is a part of feedparser. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -26,10 +26,12 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals from ..util import FeedParserDict + class Namespace(object): supported_namespaces = { 'http://www.w3.org/2003/01/geo/wgs84_pos#': 'geo', @@ -41,9 +43,9 @@ class Namespace(object): self.ingeometry = 0 super(Namespace, self).__init__() - def _start_georssgeom(self, attrsD): + def _start_georssgeom(self, attrs_d): self.push('geometry', 0) - context = self._getContext() + context = self._get_context() context['where'] = FeedParserDict() _start_georss_point = _start_georssgeom @@ -52,7 +54,7 @@ class Namespace(object): _start_georss_box = _start_georssgeom def _save_where(self, geometry): - context = self._getContext() + context = self._get_context() context['where'].update(geometry) def _end_georss_point(self): @@ -76,73 +78,73 @@ class Namespace(object): if geometry: self._save_where(geometry) - def _start_where(self, attrsD): + def _start_where(self, attrs_d): self.push('where', 0) - context = self._getContext() + context = self._get_context() context['where'] = FeedParserDict() _start_georss_where = _start_where - def _parse_srs_attrs(self, attrsD): - srsName = attrsD.get('srsname') + def _parse_srs_attrs(self, attrs_d): + srs_name = attrs_d.get('srsname') try: - srsDimension = int(attrsD.get('srsdimension', '2')) + srs_dimension = int(attrs_d.get('srsdimension', '2')) except ValueError: - srsDimension = 2 - context = self._getContext() - context['where']['srsName'] = srsName - context['where']['srsDimension'] = srsDimension + srs_dimension = 2 + context = self._get_context() + context['where']['srsName'] = srs_name + context['where']['srsDimension'] = srs_dimension - def _start_gml_point(self, attrsD): - self._parse_srs_attrs(attrsD) + def _start_gml_point(self, attrs_d): + self._parse_srs_attrs(attrs_d) self.ingeometry = 1 self.push('geometry', 0) - def _start_gml_linestring(self, attrsD): - self._parse_srs_attrs(attrsD) + def _start_gml_linestring(self, attrs_d): + self._parse_srs_attrs(attrs_d) self.ingeometry = 'linestring' self.push('geometry', 0) - def _start_gml_polygon(self, attrsD): - self._parse_srs_attrs(attrsD) + def _start_gml_polygon(self, attrs_d): + self._parse_srs_attrs(attrs_d) self.push('geometry', 0) - def _start_gml_exterior(self, attrsD): + def _start_gml_exterior(self, attrs_d): self.push('geometry', 0) - def _start_gml_linearring(self, attrsD): + def _start_gml_linearring(self, attrs_d): self.ingeometry = 'polygon' self.push('geometry', 0) - def _start_gml_pos(self, attrsD): + def _start_gml_pos(self, attrs_d): self.push('pos', 0) def _end_gml_pos(self): this = self.pop('pos') - context = self._getContext() - srsName = context['where'].get('srsName') - srsDimension = context['where'].get('srsDimension', 2) + context = self._get_context() + srs_name = context['where'].get('srsName') + srs_dimension = context['where'].get('srsDimension', 2) swap = True - if srsName and "EPSG" in srsName: - epsg = int(srsName.split(":")[-1]) + if srs_name and "EPSG" in srs_name: + epsg = int(srs_name.split(":")[-1]) swap = bool(epsg in _geogCS) - geometry = _parse_georss_point(this, swap=swap, dims=srsDimension) + geometry = _parse_georss_point(this, swap=swap, dims=srs_dimension) if geometry: self._save_where(geometry) - def _start_gml_poslist(self, attrsD): + def _start_gml_poslist(self, attrs_d): self.push('pos', 0) def _end_gml_poslist(self): this = self.pop('pos') - context = self._getContext() - srsName = context['where'].get('srsName') - srsDimension = context['where'].get('srsDimension', 2) + context = self._get_context() + srs_name = context['where'].get('srsName') + srs_dimension = context['where'].get('srsDimension', 2) swap = True - if srsName and "EPSG" in srsName: - epsg = int(srsName.split(":")[-1]) + if srs_name and "EPSG" in srs_name: + epsg = int(srs_name.split(":")[-1]) swap = bool(epsg in _geogCS) geometry = _parse_poslist( - this, self.ingeometry, swap=swap, dims=srsDimension) + this, self.ingeometry, swap=swap, dims=srs_dimension) if geometry: self._save_where(geometry) @@ -172,10 +174,11 @@ def _parse_poslist(value, geom_type, swap=True, dims=2): else: return None + def _gen_georss_coords(value, swap=True, dims=2): # A generator of (lon, lat) pairs from a string of encoded GeoRSS # coordinates. Converts to floats and swaps order. - latlons = ( float(ll) for ll in value.replace(',', ' ').split() ) + latlons = (float(ll) for ll in value.replace(',', ' ').split()) while True: try: t = [next(latlons), next(latlons)][::swap and -1 or 1] @@ -185,6 +188,7 @@ def _gen_georss_coords(value, swap=True, dims=2): except StopIteration: return + def _parse_georss_point(value, swap=True, dims=2): # A point contains a single latitude-longitude pair, separated by # whitespace. We'll also handle comma separators. @@ -194,6 +198,7 @@ def _parse_georss_point(value, swap=True, dims=2): except (IndexError, ValueError): return None + def _parse_georss_line(value, swap=True, dims=2): # A line contains a space separated list of latitude-longitude pairs in # WGS84 coordinate reference system, with each pair separated by @@ -204,6 +209,7 @@ def _parse_georss_line(value, swap=True, dims=2): except (IndexError, ValueError): return None + def _parse_georss_polygon(value, swap=True, dims=2): # A polygon contains a space separated list of latitude-longitude pairs, # with each pair separated by whitespace. There must be at least four @@ -217,6 +223,7 @@ def _parse_georss_polygon(value, swap=True, dims=2): return None return {'type': 'Polygon', 'coordinates': (ring,)} + def _parse_georss_box(value, swap=True, dims=2): # A bounding box is a rectangular region, often used to define the extents # of a map or a rough area of interest. A box contains two space separate @@ -228,40 +235,42 @@ def _parse_georss_box(value, swap=True, dims=2): except (IndexError, ValueError): return None + # The list of EPSG codes for geographic (latitude/longitude) coordinate # systems to support decoding of GeoRSS GML profiles. _geogCS = [ -3819, 3821, 3824, 3889, 3906, 4001, 4002, 4003, 4004, 4005, 4006, 4007, 4008, -4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016, 4018, 4019, 4020, 4021, 4022, -4023, 4024, 4025, 4027, 4028, 4029, 4030, 4031, 4032, 4033, 4034, 4035, 4036, -4041, 4042, 4043, 4044, 4045, 4046, 4047, 4052, 4053, 4054, 4055, 4075, 4081, -4120, 4121, 4122, 4123, 4124, 4125, 4126, 4127, 4128, 4129, 4130, 4131, 4132, -4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4143, 4144, 4145, -4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, -4159, 4160, 4161, 4162, 4163, 4164, 4165, 4166, 4167, 4168, 4169, 4170, 4171, -4172, 4173, 4174, 4175, 4176, 4178, 4179, 4180, 4181, 4182, 4183, 4184, 4185, -4188, 4189, 4190, 4191, 4192, 4193, 4194, 4195, 4196, 4197, 4198, 4199, 4200, -4201, 4202, 4203, 4204, 4205, 4206, 4207, 4208, 4209, 4210, 4211, 4212, 4213, -4214, 4215, 4216, 4218, 4219, 4220, 4221, 4222, 4223, 4224, 4225, 4226, 4227, -4228, 4229, 4230, 4231, 4232, 4233, 4234, 4235, 4236, 4237, 4238, 4239, 4240, -4241, 4242, 4243, 4244, 4245, 4246, 4247, 4248, 4249, 4250, 4251, 4252, 4253, -4254, 4255, 4256, 4257, 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, 4266, -4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, -4280, 4281, 4282, 4283, 4284, 4285, 4286, 4287, 4288, 4289, 4291, 4292, 4293, -4294, 4295, 4296, 4297, 4298, 4299, 4300, 4301, 4302, 4303, 4304, 4306, 4307, -4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315, 4316, 4317, 4318, 4319, 4322, -4324, 4326, 4463, 4470, 4475, 4483, 4490, 4555, 4558, 4600, 4601, 4602, 4603, -4604, 4605, 4606, 4607, 4608, 4609, 4610, 4611, 4612, 4613, 4614, 4615, 4616, -4617, 4618, 4619, 4620, 4621, 4622, 4623, 4624, 4625, 4626, 4627, 4628, 4629, -4630, 4631, 4632, 4633, 4634, 4635, 4636, 4637, 4638, 4639, 4640, 4641, 4642, -4643, 4644, 4645, 4646, 4657, 4658, 4659, 4660, 4661, 4662, 4663, 4664, 4665, -4666, 4667, 4668, 4669, 4670, 4671, 4672, 4673, 4674, 4675, 4676, 4677, 4678, -4679, 4680, 4681, 4682, 4683, 4684, 4685, 4686, 4687, 4688, 4689, 4690, 4691, -4692, 4693, 4694, 4695, 4696, 4697, 4698, 4699, 4700, 4701, 4702, 4703, 4704, -4705, 4706, 4707, 4708, 4709, 4710, 4711, 4712, 4713, 4714, 4715, 4716, 4717, -4718, 4719, 4720, 4721, 4722, 4723, 4724, 4725, 4726, 4727, 4728, 4729, 4730, -4731, 4732, 4733, 4734, 4735, 4736, 4737, 4738, 4739, 4740, 4741, 4742, 4743, -4744, 4745, 4746, 4747, 4748, 4749, 4750, 4751, 4752, 4753, 4754, 4755, 4756, -4757, 4758, 4759, 4760, 4761, 4762, 4763, 4764, 4765, 4801, 4802, 4803, 4804, -4805, 4806, 4807, 4808, 4809, 4810, 4811, 4813, 4814, 4815, 4816, 4817, 4818, -4819, 4820, 4821, 4823, 4824, 4901, 4902, 4903, 4904, 4979 ] + 3819, 3821, 3824, 3889, 3906, 4001, 4002, 4003, 4004, 4005, 4006, 4007, 4008, + 4009, 4010, 4011, 4012, 4013, 4014, 4015, 4016, 4018, 4019, 4020, 4021, 4022, + 4023, 4024, 4025, 4027, 4028, 4029, 4030, 4031, 4032, 4033, 4034, 4035, 4036, + 4041, 4042, 4043, 4044, 4045, 4046, 4047, 4052, 4053, 4054, 4055, 4075, 4081, + 4120, 4121, 4122, 4123, 4124, 4125, 4126, 4127, 4128, 4129, 4130, 4131, 4132, + 4133, 4134, 4135, 4136, 4137, 4138, 4139, 4140, 4141, 4142, 4143, 4144, 4145, + 4146, 4147, 4148, 4149, 4150, 4151, 4152, 4153, 4154, 4155, 4156, 4157, 4158, + 4159, 4160, 4161, 4162, 4163, 4164, 4165, 4166, 4167, 4168, 4169, 4170, 4171, + 4172, 4173, 4174, 4175, 4176, 4178, 4179, 4180, 4181, 4182, 4183, 4184, 4185, + 4188, 4189, 4190, 4191, 4192, 4193, 4194, 4195, 4196, 4197, 4198, 4199, 4200, + 4201, 4202, 4203, 4204, 4205, 4206, 4207, 4208, 4209, 4210, 4211, 4212, 4213, + 4214, 4215, 4216, 4218, 4219, 4220, 4221, 4222, 4223, 4224, 4225, 4226, 4227, + 4228, 4229, 4230, 4231, 4232, 4233, 4234, 4235, 4236, 4237, 4238, 4239, 4240, + 4241, 4242, 4243, 4244, 4245, 4246, 4247, 4248, 4249, 4250, 4251, 4252, 4253, + 4254, 4255, 4256, 4257, 4258, 4259, 4260, 4261, 4262, 4263, 4264, 4265, 4266, + 4267, 4268, 4269, 4270, 4271, 4272, 4273, 4274, 4275, 4276, 4277, 4278, 4279, + 4280, 4281, 4282, 4283, 4284, 4285, 4286, 4287, 4288, 4289, 4291, 4292, 4293, + 4294, 4295, 4296, 4297, 4298, 4299, 4300, 4301, 4302, 4303, 4304, 4306, 4307, + 4308, 4309, 4310, 4311, 4312, 4313, 4314, 4315, 4316, 4317, 4318, 4319, 4322, + 4324, 4326, 4463, 4470, 4475, 4483, 4490, 4555, 4558, 4600, 4601, 4602, 4603, + 4604, 4605, 4606, 4607, 4608, 4609, 4610, 4611, 4612, 4613, 4614, 4615, 4616, + 4617, 4618, 4619, 4620, 4621, 4622, 4623, 4624, 4625, 4626, 4627, 4628, 4629, + 4630, 4631, 4632, 4633, 4634, 4635, 4636, 4637, 4638, 4639, 4640, 4641, 4642, + 4643, 4644, 4645, 4646, 4657, 4658, 4659, 4660, 4661, 4662, 4663, 4664, 4665, + 4666, 4667, 4668, 4669, 4670, 4671, 4672, 4673, 4674, 4675, 4676, 4677, 4678, + 4679, 4680, 4681, 4682, 4683, 4684, 4685, 4686, 4687, 4688, 4689, 4690, 4691, + 4692, 4693, 4694, 4695, 4696, 4697, 4698, 4699, 4700, 4701, 4702, 4703, 4704, + 4705, 4706, 4707, 4708, 4709, 4710, 4711, 4712, 4713, 4714, 4715, 4716, 4717, + 4718, 4719, 4720, 4721, 4722, 4723, 4724, 4725, 4726, 4727, 4728, 4729, 4730, + 4731, 4732, 4733, 4734, 4735, 4736, 4737, 4738, 4739, 4740, 4741, 4742, 4743, + 4744, 4745, 4746, 4747, 4748, 4749, 4750, 4751, 4752, 4753, 4754, 4755, 4756, + 4757, 4758, 4759, 4760, 4761, 4762, 4763, 4764, 4765, 4801, 4802, 4803, 4804, + 4805, 4806, 4807, 4808, 4809, 4810, 4811, 4813, 4814, 4815, 4816, 4817, 4818, + 4819, 4820, 4821, 4823, 4824, 4901, 4902, 4903, 4904, 4979, +] diff --git a/lib/feedparser/namespaces/itunes.py b/lib/feedparser/namespaces/itunes.py index 37faa7e..e106c52 100644 --- a/lib/feedparser/namespaces/itunes.py +++ b/lib/feedparser/namespaces/itunes.py @@ -1,12 +1,12 @@ # Support for the iTunes format -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # # This file is a part of feedparser. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -26,10 +26,12 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals from ..util import FeedParserDict + class Namespace(object): supported_namespaces = { # Canonical namespace @@ -39,8 +41,8 @@ class Namespace(object): 'http://example.com/DTDs/PodCast-1.0.dtd': 'itunes', } - def _start_itunes_author(self, attrsD): - self._start_author(attrsD) + def _start_itunes_author(self, attrs_d): + self._start_author(attrs_d) def _end_itunes_author(self): self._end_author() @@ -48,31 +50,31 @@ class Namespace(object): def _end_itunes_category(self): self._end_category() - def _start_itunes_name(self, attrsD): - self._start_name(attrsD) + def _start_itunes_name(self, attrs_d): + self._start_name(attrs_d) def _end_itunes_name(self): self._end_name() - def _start_itunes_email(self, attrsD): - self._start_email(attrsD) + def _start_itunes_email(self, attrs_d): + self._start_email(attrs_d) def _end_itunes_email(self): self._end_email() - def _start_itunes_subtitle(self, attrsD): - self._start_subtitle(attrsD) + def _start_itunes_subtitle(self, attrs_d): + self._start_subtitle(attrs_d) def _end_itunes_subtitle(self): self._end_subtitle() - def _start_itunes_summary(self, attrsD): - self._start_summary(attrsD) + def _start_itunes_summary(self, attrs_d): + self._start_summary(attrs_d) def _end_itunes_summary(self): self._end_summary() - def _start_itunes_owner(self, attrsD): + def _start_itunes_owner(self, attrs_d): self.inpublisher = 1 self.push('publisher', 0) @@ -84,28 +86,27 @@ class Namespace(object): def _end_itunes_keywords(self): for term in self.pop('itunes_keywords').split(','): if term.strip(): - self._addTag(term.strip(), 'http://www.itunes.com/', None) + self._add_tag(term.strip(), 'http://www.itunes.com/', None) - def _start_itunes_category(self, attrsD): - self._addTag(attrsD.get('text'), 'http://www.itunes.com/', None) + def _start_itunes_category(self, attrs_d): + self._add_tag(attrs_d.get('text'), 'http://www.itunes.com/', None) self.push('category', 1) - def _start_itunes_image(self, attrsD): + def _start_itunes_image(self, attrs_d): self.push('itunes_image', 0) - if attrsD.get('href'): - self._getContext()['image'] = FeedParserDict({'href': attrsD.get('href')}) - elif attrsD.get('url'): - self._getContext()['image'] = FeedParserDict({'href': attrsD.get('url')}) + if attrs_d.get('href'): + self._get_context()['image'] = FeedParserDict({'href': attrs_d.get('href')}) + elif attrs_d.get('url'): + self._get_context()['image'] = FeedParserDict({'href': attrs_d.get('url')}) _start_itunes_link = _start_itunes_image def _end_itunes_block(self): value = self.pop('itunes_block', 0) - self._getContext()['itunes_block'] = (value == 'yes') and 1 or 0 + self._get_context()['itunes_block'] = (value == 'yes') and 1 or 0 def _end_itunes_explicit(self): value = self.pop('itunes_explicit', 0) # Convert 'yes' -> True, 'clean' to False, and any other value to None # False and None both evaluate as False, so the difference can be ignored # by applications that only need to know if the content is explicit. - self._getContext()['itunes_explicit'] = (None, False, True)[(value == 'yes' and 2) or value == 'clean' or 0] - + self._get_context()['itunes_explicit'] = (None, False, True)[(value == 'yes' and 2) or value == 'clean' or 0] diff --git a/lib/feedparser/namespaces/mediarss.py b/lib/feedparser/namespaces/mediarss.py index 878ed7c..f482371 100644 --- a/lib/feedparser/namespaces/mediarss.py +++ b/lib/feedparser/namespaces/mediarss.py @@ -1,12 +1,12 @@ # Support for the Media RSS format -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # # This file is a part of feedparser. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -26,10 +26,12 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals from ..util import FeedParserDict + class Namespace(object): supported_namespaces = { # Canonical namespace @@ -39,9 +41,9 @@ class Namespace(object): 'http://search.yahoo.com/mrss': 'media', } - def _start_media_category(self, attrsD): - attrsD.setdefault('scheme', 'http://search.yahoo.com/mrss/category_schema') - self._start_category(attrsD) + def _start_media_category(self, attrs_d): + attrs_d.setdefault('scheme', 'http://search.yahoo.com/mrss/category_schema') + self._start_category(attrs_d) def _end_media_category(self): self._end_category() @@ -49,94 +51,94 @@ class Namespace(object): def _end_media_keywords(self): for term in self.pop('media_keywords').split(','): if term.strip(): - self._addTag(term.strip(), None, None) + self._add_tag(term.strip(), None, None) - def _start_media_title(self, attrsD): - self._start_title(attrsD) + def _start_media_title(self, attrs_d): + self._start_title(attrs_d) def _end_media_title(self): title_depth = self.title_depth self._end_title() self.title_depth = title_depth - def _start_media_group(self, attrsD): + def _start_media_group(self, attrs_d): # don't do anything, but don't break the enclosed tags either pass - def _start_media_rating(self, attrsD): - context = self._getContext() - context.setdefault('media_rating', attrsD) + def _start_media_rating(self, attrs_d): + context = self._get_context() + context.setdefault('media_rating', attrs_d) self.push('rating', 1) def _end_media_rating(self): rating = self.pop('rating') if rating is not None and rating.strip(): - context = self._getContext() + context = self._get_context() context['media_rating']['content'] = rating - def _start_media_credit(self, attrsD): - context = self._getContext() + def _start_media_credit(self, attrs_d): + context = self._get_context() context.setdefault('media_credit', []) - context['media_credit'].append(attrsD) + context['media_credit'].append(attrs_d) self.push('credit', 1) def _end_media_credit(self): credit = self.pop('credit') - if credit != None and len(credit.strip()) != 0: - context = self._getContext() + if credit is not None and credit.strip(): + context = self._get_context() context['media_credit'][-1]['content'] = credit - def _start_media_description(self, attrsD): - self._start_description(attrsD) + def _start_media_description(self, attrs_d): + self._start_description(attrs_d) def _end_media_description(self): self._end_description() - def _start_media_restriction(self, attrsD): - context = self._getContext() - context.setdefault('media_restriction', attrsD) + def _start_media_restriction(self, attrs_d): + context = self._get_context() + context.setdefault('media_restriction', attrs_d) self.push('restriction', 1) def _end_media_restriction(self): restriction = self.pop('restriction') - if restriction != None and len(restriction.strip()) != 0: - context = self._getContext() + if restriction is not None and restriction.strip(): + context = self._get_context() context['media_restriction']['content'] = [cc.strip().lower() for cc in restriction.split(' ')] - def _start_media_license(self, attrsD): - context = self._getContext() - context.setdefault('media_license', attrsD) + def _start_media_license(self, attrs_d): + context = self._get_context() + context.setdefault('media_license', attrs_d) self.push('license', 1) def _end_media_license(self): - license = self.pop('license') - if license != None and len(license.strip()) != 0: - context = self._getContext() - context['media_license']['content'] = license + license_ = self.pop('license') + if license_ is not None and license_.strip(): + context = self._get_context() + context['media_license']['content'] = license_ - def _start_media_content(self, attrsD): - context = self._getContext() + def _start_media_content(self, attrs_d): + context = self._get_context() context.setdefault('media_content', []) - context['media_content'].append(attrsD) + context['media_content'].append(attrs_d) - def _start_media_thumbnail(self, attrsD): - context = self._getContext() + def _start_media_thumbnail(self, attrs_d): + context = self._get_context() context.setdefault('media_thumbnail', []) self.push('url', 1) # new - context['media_thumbnail'].append(attrsD) + context['media_thumbnail'].append(attrs_d) def _end_media_thumbnail(self): url = self.pop('url') - context = self._getContext() - if url != None and len(url.strip()) != 0: + context = self._get_context() + if url is not None and url.strip(): if 'url' not in context['media_thumbnail'][-1]: context['media_thumbnail'][-1]['url'] = url - def _start_media_player(self, attrsD): + def _start_media_player(self, attrs_d): self.push('media_player', 0) - self._getContext()['media_player'] = FeedParserDict(attrsD) + self._get_context()['media_player'] = FeedParserDict(attrs_d) def _end_media_player(self): value = self.pop('media_player') - context = self._getContext() + context = self._get_context() context['media_player']['content'] = value diff --git a/lib/feedparser/namespaces/psc.py b/lib/feedparser/namespaces/psc.py index c7dc843..2f4d5c7 100644 --- a/lib/feedparser/namespaces/psc.py +++ b/lib/feedparser/namespaces/psc.py @@ -1,12 +1,12 @@ # Support for the Podlove Simple Chapters format -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # # This file is a part of feedparser. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -26,12 +26,15 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals import datetime import re from .. import util + + class Namespace(object): supported_namespaces = { 'http://podlove.org/simple-chapters': 'psc', @@ -42,28 +45,30 @@ class Namespace(object): self.psc_chapters_flag = False super(Namespace, self).__init__() - def _start_psc_chapters(self, attrsD): - context = self._getContext() + def _start_psc_chapters(self, attrs_d): + context = self._get_context() if 'psc_chapters' not in context: self.psc_chapters_flag = True - attrsD['chapters'] = [] - context['psc_chapters'] = util.FeedParserDict(attrsD) + attrs_d['chapters'] = [] + context['psc_chapters'] = util.FeedParserDict(attrs_d) def _end_psc_chapters(self): self.psc_chapters_flag = False - def _start_psc_chapter(self, attrsD): + def _start_psc_chapter(self, attrs_d): if self.psc_chapters_flag: - start = self._getAttribute(attrsD, 'start') - attrsD['start_parsed'] = _parse_psc_chapter_start(start) + start = self._get_attribute(attrs_d, 'start') + attrs_d['start_parsed'] = _parse_psc_chapter_start(start) - context = self._getContext()['psc_chapters'] - context['chapters'].append(util.FeedParserDict(attrsD)) + context = self._get_context()['psc_chapters'] + context['chapters'].append(util.FeedParserDict(attrs_d)) -def _parse_psc_chapter_start(start): - FORMAT = r'^((\d{2}):)?(\d{2}):(\d{2})(\.(\d{3}))?$' - m = re.compile(FORMAT).match(start) +format_ = re.compile(r'^((\d{2}):)?(\d{2}):(\d{2})(\.(\d{3}))?$') + + +def _parse_psc_chapter_start(start): + m = format_.match(start) if m is None: return None diff --git a/lib/feedparser/parsers/loose.py b/lib/feedparser/parsers/loose.py index 597ce8b..1ded7a9 100644 --- a/lib/feedparser/parsers/loose.py +++ b/lib/feedparser/parsers/loose.py @@ -1,5 +1,5 @@ # The loose feed parser that interfaces with an SGML parsing library -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # @@ -26,26 +26,31 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals + class _LooseFeedParser(object): + contentparams = None + def __init__(self, baseuri=None, baselang=None, encoding=None, entities=None): self.baseuri = baseuri or '' self.lang = baselang or None - self.encoding = encoding or 'utf-8' # character encoding + self.encoding = encoding or 'utf-8' # character encoding self.entities = entities or {} super(_LooseFeedParser, self).__init__() - def _normalize_attributes(self, kv): + @staticmethod + def _normalize_attributes(kv): k = kv[0].lower() v = k in ('rel', 'type') and kv[1].lower() or kv[1] # the sgml parser doesn't handle entities in attributes, nor # does it pass the attribute values through as unicode, while # strict xml parsers do -- account for this difference v = v.replace('&', '&') - return (k, v) + return k, v - def decodeEntities(self, element, data): + def decode_entities(self, element, data): data = data.replace('<', '<') data = data.replace('<', '<') data = data.replace('<', '<') @@ -68,5 +73,9 @@ class _LooseFeedParser(object): data = data.replace('/', '/') return data - def strattrs(self, attrs): - return ''.join([' %s="%s"' % (n,v.replace('"','"')) for n,v in attrs]) + @staticmethod + def strattrs(attrs): + return ''.join( + ' %s="%s"' % (n, v.replace('"', '"')) + for n, v in attrs + ) diff --git a/lib/feedparser/parsers/strict.py b/lib/feedparser/parsers/strict.py index 29ad2be..911dd12 100644 --- a/lib/feedparser/parsers/strict.py +++ b/lib/feedparser/parsers/strict.py @@ -1,5 +1,5 @@ # The strict feed parser that interfaces with an XML parsing library -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # @@ -30,6 +30,7 @@ from __future__ import absolute_import, unicode_literals from ..exceptions import UndeclaredNamespace + class _StrictFeedParser(object): def __init__(self, baseuri, baselang, encoding): self.bozo = 0 @@ -40,17 +41,18 @@ class _StrictFeedParser(object): self.encoding = encoding super(_StrictFeedParser, self).__init__() - def _normalize_attributes(self, kv): + @staticmethod + def _normalize_attributes(kv): k = kv[0].lower() v = k in ('rel', 'type') and kv[1].lower() or kv[1] - return (k, v) + return k, v def startPrefixMapping(self, prefix, uri): if not uri: return # Jython uses '' instead of None; standardize on None prefix = prefix or None - self.trackNamespace(prefix, uri) + self.track_namespace(prefix, uri) if prefix and uri == 'http://www.w3.org/1999/xlink': self.decls['xmlns:' + prefix] = uri @@ -66,7 +68,7 @@ class _StrictFeedParser(object): else: givenprefix = None prefix = self._matchnamespaces.get(lowernamespace, givenprefix) - if givenprefix and (prefix == None or (prefix == '' and lowernamespace == '')) and givenprefix not in self.namespacesInUse: + if givenprefix and (prefix is None or (prefix == '' and lowernamespace == '')) and givenprefix not in self.namespaces_in_use: raise UndeclaredNamespace("'%s' is not associated with a namespace" % givenprefix) localname = str(localname).lower() @@ -78,15 +80,15 @@ class _StrictFeedParser(object): # at all). Thanks to MatejC for helping me test this and # tirelessly telling me that it didn't work yet. attrsD, self.decls = self.decls, {} - if localname=='math' and namespace=='http://www.w3.org/1998/Math/MathML': - attrsD['xmlns']=namespace - if localname=='svg' and namespace=='http://www.w3.org/2000/svg': - attrsD['xmlns']=namespace + if localname == 'math' and namespace == 'http://www.w3.org/1998/Math/MathML': + attrsD['xmlns'] = namespace + if localname == 'svg' and namespace == 'http://www.w3.org/2000/svg': + attrsD['xmlns'] = namespace if prefix: localname = prefix.lower() + ':' + localname - elif namespace and not qname: #Expat - for name,value in self.namespacesInUse.items(): + elif namespace and not qname: # Expat + for name, value in self.namespaces_in_use.items(): if name and value == namespace: localname = name + ':' + localname break @@ -115,8 +117,8 @@ class _StrictFeedParser(object): prefix = self._matchnamespaces.get(lowernamespace, givenprefix) if prefix: localname = prefix + ':' + localname - elif namespace and not qname: #Expat - for name,value in self.namespacesInUse.items(): + elif namespace and not qname: # Expat + for name, value in self.namespaces_in_use.items(): if name and value == namespace: localname = name + ':' + localname break diff --git a/lib/feedparser/sanitizer.py b/lib/feedparser/sanitizer.py index d6093b8..37c7d39 100644 --- a/lib/feedparser/sanitizer.py +++ b/lib/feedparser/sanitizer.py @@ -1,75 +1,394 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import re from .html import _BaseHTMLProcessor from .sgml import _SGML_AVAILABLE -from .urls import _makeSafeAbsoluteURI +from .urls import make_safe_absolute_uri + class _HTMLSanitizer(_BaseHTMLProcessor): - acceptable_elements = set(['a', 'abbr', 'acronym', 'address', 'area', - 'article', 'aside', 'audio', 'b', 'big', 'blockquote', 'br', 'button', - 'canvas', 'caption', 'center', 'cite', 'code', 'col', 'colgroup', - 'command', 'datagrid', 'datalist', 'dd', 'del', 'details', 'dfn', - 'dialog', 'dir', 'div', 'dl', 'dt', 'em', 'event-source', 'fieldset', - 'figcaption', 'figure', 'footer', 'font', 'form', 'header', 'h1', - 'h2', 'h3', 'h4', 'h5', 'h6', 'hr', 'i', 'img', 'input', 'ins', - 'keygen', 'kbd', 'label', 'legend', 'li', 'm', 'map', 'menu', 'meter', - 'multicol', 'nav', 'nextid', 'ol', 'output', 'optgroup', 'option', - 'p', 'pre', 'progress', 'q', 's', 'samp', 'section', 'select', - 'small', 'sound', 'source', 'spacer', 'span', 'strike', 'strong', - 'sub', 'sup', 'table', 'tbody', 'td', 'textarea', 'time', 'tfoot', - 'th', 'thead', 'tr', 'tt', 'u', 'ul', 'var', 'video', 'noscript']) - - acceptable_attributes = set(['abbr', 'accept', 'accept-charset', 'accesskey', - 'action', 'align', 'alt', 'autocomplete', 'autofocus', 'axis', - 'background', 'balance', 'bgcolor', 'bgproperties', 'border', - 'bordercolor', 'bordercolordark', 'bordercolorlight', 'bottompadding', - 'cellpadding', 'cellspacing', 'ch', 'challenge', 'char', 'charoff', - 'choff', 'charset', 'checked', 'cite', 'class', 'clear', 'color', 'cols', - 'colspan', 'compact', 'contenteditable', 'controls', 'coords', 'data', - 'datafld', 'datapagesize', 'datasrc', 'datetime', 'default', 'delay', - 'dir', 'disabled', 'draggable', 'dynsrc', 'enctype', 'end', 'face', 'for', - 'form', 'frame', 'galleryimg', 'gutter', 'headers', 'height', 'hidefocus', - 'hidden', 'high', 'href', 'hreflang', 'hspace', 'icon', 'id', 'inputmode', - 'ismap', 'keytype', 'label', 'leftspacing', 'lang', 'list', 'longdesc', - 'loop', 'loopcount', 'loopend', 'loopstart', 'low', 'lowsrc', 'max', - 'maxlength', 'media', 'method', 'min', 'multiple', 'name', 'nohref', - 'noshade', 'nowrap', 'open', 'optimum', 'pattern', 'ping', 'point-size', - 'poster', 'pqg', 'preload', 'prompt', 'radiogroup', 'readonly', 'rel', - 'repeat-max', 'repeat-min', 'replace', 'required', 'rev', 'rightspacing', - 'rows', 'rowspan', 'rules', 'scope', 'selected', 'shape', 'size', 'span', - 'src', 'start', 'step', 'summary', 'suppress', 'tabindex', 'target', - 'template', 'title', 'toppadding', 'type', 'unselectable', 'usemap', - 'urn', 'valign', 'value', 'variable', 'volume', 'vspace', 'vrml', - 'width', 'wrap', 'xml:lang']) - - unacceptable_elements_with_end_tag = set(['script', 'applet', 'style']) - - acceptable_css_properties = set(['azimuth', 'background-color', - 'border-bottom-color', 'border-collapse', 'border-color', - 'border-left-color', 'border-right-color', 'border-top-color', 'clear', - 'color', 'cursor', 'direction', 'display', 'elevation', 'float', 'font', - 'font-family', 'font-size', 'font-style', 'font-variant', 'font-weight', - 'height', 'letter-spacing', 'line-height', 'overflow', 'pause', - 'pause-after', 'pause-before', 'pitch', 'pitch-range', 'richness', - 'speak', 'speak-header', 'speak-numeral', 'speak-punctuation', - 'speech-rate', 'stress', 'text-align', 'text-decoration', 'text-indent', - 'unicode-bidi', 'vertical-align', 'voice-family', 'volume', - 'white-space', 'width']) + acceptable_elements = { + 'a', + 'abbr', + 'acronym', + 'address', + 'area', + 'article', + 'aside', + 'audio', + 'b', + 'big', + 'blockquote', + 'br', + 'button', + 'canvas', + 'caption', + 'center', + 'cite', + 'code', + 'col', + 'colgroup', + 'command', + 'datagrid', + 'datalist', + 'dd', + 'del', + 'details', + 'dfn', + 'dialog', + 'dir', + 'div', + 'dl', + 'dt', + 'em', + 'event-source', + 'fieldset', + 'figcaption', + 'figure', + 'font', + 'footer', + 'form', + 'h1', + 'h2', + 'h3', + 'h4', + 'h5', + 'h6', + 'header', + 'hr', + 'i', + 'img', + 'input', + 'ins', + 'kbd', + 'keygen', + 'label', + 'legend', + 'li', + 'm', + 'map', + 'menu', + 'meter', + 'multicol', + 'nav', + 'nextid', + 'noscript', + 'ol', + 'optgroup', + 'option', + 'output', + 'p', + 'pre', + 'progress', + 'q', + 's', + 'samp', + 'section', + 'select', + 'small', + 'sound', + 'source', + 'spacer', + 'span', + 'strike', + 'strong', + 'sub', + 'sup', + 'table', + 'tbody', + 'td', + 'textarea', + 'tfoot', + 'th', + 'thead', + 'time', + 'tr', + 'tt', + 'u', + 'ul', + 'var', + 'video', + } + + acceptable_attributes = { + 'abbr', + 'accept', + 'accept-charset', + 'accesskey', + 'action', + 'align', + 'alt', + 'autocomplete', + 'autofocus', + 'axis', + 'background', + 'balance', + 'bgcolor', + 'bgproperties', + 'border', + 'bordercolor', + 'bordercolordark', + 'bordercolorlight', + 'bottompadding', + 'cellpadding', + 'cellspacing', + 'ch', + 'challenge', + 'char', + 'charoff', + 'charset', + 'checked', + 'choff', + 'cite', + 'class', + 'clear', + 'color', + 'cols', + 'colspan', + 'compact', + 'contenteditable', + 'controls', + 'coords', + 'data', + 'datafld', + 'datapagesize', + 'datasrc', + 'datetime', + 'default', + 'delay', + 'dir', + 'disabled', + 'draggable', + 'dynsrc', + 'enctype', + 'end', + 'face', + 'for', + 'form', + 'frame', + 'galleryimg', + 'gutter', + 'headers', + 'height', + 'hidden', + 'hidefocus', + 'high', + 'href', + 'hreflang', + 'hspace', + 'icon', + 'id', + 'inputmode', + 'ismap', + 'keytype', + 'label', + 'lang', + 'leftspacing', + 'list', + 'longdesc', + 'loop', + 'loopcount', + 'loopend', + 'loopstart', + 'low', + 'lowsrc', + 'max', + 'maxlength', + 'media', + 'method', + 'min', + 'multiple', + 'name', + 'nohref', + 'noshade', + 'nowrap', + 'open', + 'optimum', + 'pattern', + 'ping', + 'point-size', + 'poster', + 'pqg', + 'preload', + 'prompt', + 'radiogroup', + 'readonly', + 'rel', + 'repeat-max', + 'repeat-min', + 'replace', + 'required', + 'rev', + 'rightspacing', + 'rows', + 'rowspan', + 'rules', + 'scope', + 'selected', + 'shape', + 'size', + 'span', + 'src', + 'start', + 'step', + 'summary', + 'suppress', + 'tabindex', + 'target', + 'template', + 'title', + 'toppadding', + 'type', + 'unselectable', + 'urn', + 'usemap', + 'valign', + 'value', + 'variable', + 'volume', + 'vrml', + 'vspace', + 'width', + 'wrap', + 'xml:lang', + } + + unacceptable_elements_with_end_tag = { + 'applet', + 'script', + 'style', + } + + acceptable_css_properties = { + 'azimuth', + 'background-color', + 'border-bottom-color', + 'border-collapse', + 'border-color', + 'border-left-color', + 'border-right-color', + 'border-top-color', + 'clear', + 'color', + 'cursor', + 'direction', + 'display', + 'elevation', + 'float', + 'font', + 'font-family', + 'font-size', + 'font-style', + 'font-variant', + 'font-weight', + 'height', + 'letter-spacing', + 'line-height', + 'overflow', + 'pause', + 'pause-after', + 'pause-before', + 'pitch', + 'pitch-range', + 'richness', + 'speak', + 'speak-header', + 'speak-numeral', + 'speak-punctuation', + 'speech-rate', + 'stress', + 'text-align', + 'text-decoration', + 'text-indent', + 'unicode-bidi', + 'vertical-align', + 'voice-family', + 'volume', + 'white-space', + 'width', + } # survey of common keywords found in feeds - acceptable_css_keywords = set(['auto', 'aqua', 'black', 'block', 'blue', - 'bold', 'both', 'bottom', 'brown', 'center', 'collapse', 'dashed', - 'dotted', 'fuchsia', 'gray', 'green', '!important', 'italic', 'left', - 'lime', 'maroon', 'medium', 'none', 'navy', 'normal', 'nowrap', 'olive', - 'pointer', 'purple', 'red', 'right', 'solid', 'silver', 'teal', 'top', - 'transparent', 'underline', 'white', 'yellow']) - - valid_css_values = re.compile(r'^(#[0-9a-f]+|rgb\(\d+%?,\d*%?,?\d*%?\)?|' + - r'\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?)$') - - mathml_elements = set([ + acceptable_css_keywords = { + '!important', + 'aqua', + 'auto', + 'black', + 'block', + 'blue', + 'bold', + 'both', + 'bottom', + 'brown', + 'center', + 'collapse', + 'dashed', + 'dotted', + 'fuchsia', + 'gray', + 'green', + 'italic', + 'left', + 'lime', + 'maroon', + 'medium', + 'navy', + 'none', + 'normal', + 'nowrap', + 'olive', + 'pointer', + 'purple', + 'red', + 'right', + 'silver', + 'solid', + 'teal', + 'top', + 'transparent', + 'underline', + 'white', + 'yellow', + } + + valid_css_values = re.compile( + r'^(' + r'#[0-9a-f]+' # Hex values + r'|rgb\(\d+%?,\d*%?,?\d*%?\)?' # RGB values + r'|\d{0,2}\.?\d{0,2}(cm|em|ex|in|mm|pc|pt|px|%|,|\))?' # Sizes/widths + r')$' + ) + + mathml_elements = { 'annotation', 'annotation-xml', 'maction', @@ -114,9 +433,9 @@ class _HTMLSanitizer(_BaseHTMLProcessor): 'munderover', 'none', 'semantics', - ]) + } - mathml_attributes = set([ + mathml_attributes = { 'accent', 'accentunder', 'actiontype', @@ -216,54 +535,214 @@ class _HTMLSanitizer(_BaseHTMLProcessor): 'xlink:type', 'xmlns', 'xmlns:xlink', - ]) + } # svgtiny - foreignObject + linearGradient + radialGradient + stop - svg_elements = set(['a', 'animate', 'animateColor', 'animateMotion', - 'animateTransform', 'circle', 'defs', 'desc', 'ellipse', 'foreignObject', - 'font-face', 'font-face-name', 'font-face-src', 'g', 'glyph', 'hkern', - 'linearGradient', 'line', 'marker', 'metadata', 'missing-glyph', 'mpath', - 'path', 'polygon', 'polyline', 'radialGradient', 'rect', 'set', 'stop', - 'svg', 'switch', 'text', 'title', 'tspan', 'use']) + svg_elements = { + 'a', + 'animate', + 'animateColor', + 'animateMotion', + 'animateTransform', + 'circle', + 'defs', + 'desc', + 'ellipse', + 'font-face', + 'font-face-name', + 'font-face-src', + 'foreignObject', + 'g', + 'glyph', + 'hkern', + 'line', + 'linearGradient', + 'marker', + 'metadata', + 'missing-glyph', + 'mpath', + 'path', + 'polygon', + 'polyline', + 'radialGradient', + 'rect', + 'set', + 'stop', + 'svg', + 'switch', + 'text', + 'title', + 'tspan', + 'use', + } # svgtiny + class + opacity + offset + xmlns + xmlns:xlink - svg_attributes = set(['accent-height', 'accumulate', 'additive', 'alphabetic', - 'arabic-form', 'ascent', 'attributeName', 'attributeType', - 'baseProfile', 'bbox', 'begin', 'by', 'calcMode', 'cap-height', - 'class', 'color', 'color-rendering', 'content', 'cx', 'cy', 'd', 'dx', - 'dy', 'descent', 'display', 'dur', 'end', 'fill', 'fill-opacity', - 'fill-rule', 'font-family', 'font-size', 'font-stretch', 'font-style', - 'font-variant', 'font-weight', 'from', 'fx', 'fy', 'g1', 'g2', - 'glyph-name', 'gradientUnits', 'hanging', 'height', 'horiz-adv-x', - 'horiz-origin-x', 'id', 'ideographic', 'k', 'keyPoints', 'keySplines', - 'keyTimes', 'lang', 'mathematical', 'marker-end', 'marker-mid', - 'marker-start', 'markerHeight', 'markerUnits', 'markerWidth', 'max', - 'min', 'name', 'offset', 'opacity', 'orient', 'origin', - 'overline-position', 'overline-thickness', 'panose-1', 'path', - 'pathLength', 'points', 'preserveAspectRatio', 'r', 'refX', 'refY', - 'repeatCount', 'repeatDur', 'requiredExtensions', 'requiredFeatures', - 'restart', 'rotate', 'rx', 'ry', 'slope', 'stemh', 'stemv', - 'stop-color', 'stop-opacity', 'strikethrough-position', - 'strikethrough-thickness', 'stroke', 'stroke-dasharray', - 'stroke-dashoffset', 'stroke-linecap', 'stroke-linejoin', - 'stroke-miterlimit', 'stroke-opacity', 'stroke-width', 'systemLanguage', - 'target', 'text-anchor', 'to', 'transform', 'type', 'u1', 'u2', - 'underline-position', 'underline-thickness', 'unicode', 'unicode-range', - 'units-per-em', 'values', 'version', 'viewBox', 'visibility', 'width', - 'widths', 'x', 'x-height', 'x1', 'x2', 'xlink:actuate', 'xlink:arcrole', - 'xlink:href', 'xlink:role', 'xlink:show', 'xlink:title', 'xlink:type', - 'xml:base', 'xml:lang', 'xml:space', 'xmlns', 'xmlns:xlink', 'y', 'y1', - 'y2', 'zoomAndPan']) + svg_attributes = { + 'accent-height', + 'accumulate', + 'additive', + 'alphabetic', + 'arabic-form', + 'ascent', + 'attributeName', + 'attributeType', + 'baseProfile', + 'bbox', + 'begin', + 'by', + 'calcMode', + 'cap-height', + 'class', + 'color', + 'color-rendering', + 'content', + 'cx', + 'cy', + 'd', + 'descent', + 'display', + 'dur', + 'dx', + 'dy', + 'end', + 'fill', + 'fill-opacity', + 'fill-rule', + 'font-family', + 'font-size', + 'font-stretch', + 'font-style', + 'font-variant', + 'font-weight', + 'from', + 'fx', + 'fy', + 'g1', + 'g2', + 'glyph-name', + 'gradientUnits', + 'hanging', + 'height', + 'horiz-adv-x', + 'horiz-origin-x', + 'id', + 'ideographic', + 'k', + 'keyPoints', + 'keySplines', + 'keyTimes', + 'lang', + 'marker-end', + 'marker-mid', + 'marker-start', + 'markerHeight', + 'markerUnits', + 'markerWidth', + 'mathematical', + 'max', + 'min', + 'name', + 'offset', + 'opacity', + 'orient', + 'origin', + 'overline-position', + 'overline-thickness', + 'panose-1', + 'path', + 'pathLength', + 'points', + 'preserveAspectRatio', + 'r', + 'refX', + 'refY', + 'repeatCount', + 'repeatDur', + 'requiredExtensions', + 'requiredFeatures', + 'restart', + 'rotate', + 'rx', + 'ry', + 'slope', + 'stemh', + 'stemv', + 'stop-color', + 'stop-opacity', + 'strikethrough-position', + 'strikethrough-thickness', + 'stroke', + 'stroke-dasharray', + 'stroke-dashoffset', + 'stroke-linecap', + 'stroke-linejoin', + 'stroke-miterlimit', + 'stroke-opacity', + 'stroke-width', + 'systemLanguage', + 'target', + 'text-anchor', + 'to', + 'transform', + 'type', + 'u1', + 'u2', + 'underline-position', + 'underline-thickness', + 'unicode', + 'unicode-range', + 'units-per-em', + 'values', + 'version', + 'viewBox', + 'visibility', + 'width', + 'widths', + 'x', + 'x-height', + 'x1', + 'x2', + 'xlink:actuate', + 'xlink:arcrole', + 'xlink:href', + 'xlink:role', + 'xlink:show', + 'xlink:title', + 'xlink:type', + 'xml:base', + 'xml:lang', + 'xml:space', + 'xmlns', + 'xmlns:xlink', + 'y', + 'y1', + 'y2', + 'zoomAndPan', + } svg_attr_map = None svg_elem_map = None - acceptable_svg_properties = set([ 'fill', 'fill-opacity', 'fill-rule', - 'stroke', 'stroke-width', 'stroke-linecap', 'stroke-linejoin', - 'stroke-opacity']) + acceptable_svg_properties = { + 'fill', + 'fill-opacity', + 'fill-rule', + 'stroke', + 'stroke-linecap', + 'stroke-linejoin', + 'stroke-opacity', + 'stroke-width', + } + + def __init__(self, encoding=None, _type='application/xhtml+xml'): + super(_HTMLSanitizer, self).__init__(encoding, _type) + + self.unacceptablestack = 0 + self.mathmlOK = 0 + self.svgOK = 0 def reset(self): - _BaseHTMLProcessor.reset(self) + super(_HTMLSanitizer, self).reset() self.unacceptablestack = 0 self.mathmlOK = 0 self.svgOK = 0 @@ -271,80 +750,80 @@ class _HTMLSanitizer(_BaseHTMLProcessor): def unknown_starttag(self, tag, attrs): acceptable_attributes = self.acceptable_attributes keymap = {} - if not tag in self.acceptable_elements or self.svgOK: + if tag not in self.acceptable_elements or self.svgOK: if tag in self.unacceptable_elements_with_end_tag: self.unacceptablestack += 1 # add implicit namespaces to html5 inline svg/mathml if self._type.endswith('html'): if not dict(attrs).get('xmlns'): - if tag=='svg': - attrs.append( ('xmlns','http://www.w3.org/2000/svg') ) - if tag=='math': - attrs.append( ('xmlns','http://www.w3.org/1998/Math/MathML') ) + if tag == 'svg': + attrs.append(('xmlns', 'http://www.w3.org/2000/svg')) + if tag == 'math': + attrs.append(('xmlns', 'http://www.w3.org/1998/Math/MathML')) # not otherwise acceptable, perhaps it is MathML or SVG? - if tag=='math' and ('xmlns','http://www.w3.org/1998/Math/MathML') in attrs: + if tag == 'math' and ('xmlns', 'http://www.w3.org/1998/Math/MathML') in attrs: self.mathmlOK += 1 - if tag=='svg' and ('xmlns','http://www.w3.org/2000/svg') in attrs: + if tag == 'svg' and ('xmlns', 'http://www.w3.org/2000/svg') in attrs: self.svgOK += 1 # chose acceptable attributes based on tag class, else bail - if self.mathmlOK and tag in self.mathml_elements: + if self.mathmlOK and tag in self.mathml_elements: acceptable_attributes = self.mathml_attributes elif self.svgOK and tag in self.svg_elements: - # for most vocabularies, lowercasing is a good idea. Many - # svg elements, however, are camel case + # For most vocabularies, lowercasing is a good idea. Many + # svg elements, however, are camel case. if not self.svg_attr_map: - lower=[attr.lower() for attr in self.svg_attributes] - mix=[a for a in self.svg_attributes if a not in lower] + lower = [attr.lower() for attr in self.svg_attributes] + mix = [a for a in self.svg_attributes if a not in lower] self.svg_attributes = lower - self.svg_attr_map = dict([(a.lower(),a) for a in mix]) + self.svg_attr_map = {a.lower(): a for a in mix} - lower=[attr.lower() for attr in self.svg_elements] - mix=[a for a in self.svg_elements if a not in lower] + lower = [attr.lower() for attr in self.svg_elements] + mix = [a for a in self.svg_elements if a not in lower] self.svg_elements = lower - self.svg_elem_map = dict([(a.lower(),a) for a in mix]) + self.svg_elem_map = {a.lower(): a for a in mix} acceptable_attributes = self.svg_attributes - tag = self.svg_elem_map.get(tag,tag) + tag = self.svg_elem_map.get(tag, tag) keymap = self.svg_attr_map - elif not tag in self.acceptable_elements: + elif tag not in self.acceptable_elements: return # declare xlink namespace, if needed if self.mathmlOK or self.svgOK: if any((a for a in attrs if a[0].startswith('xlink:'))): - if not ('xmlns:xlink','http://www.w3.org/1999/xlink') in attrs: - attrs.append(('xmlns:xlink','http://www.w3.org/1999/xlink')) + if not ('xmlns:xlink', 'http://www.w3.org/1999/xlink') in attrs: + attrs.append(('xmlns:xlink', 'http://www.w3.org/1999/xlink')) clean_attrs = [] for key, value in self.normalize_attrs(attrs): if key in acceptable_attributes: - key=keymap.get(key,key) + key = keymap.get(key, key) # make sure the uri uses an acceptable uri scheme if key == 'href': - value = _makeSafeAbsoluteURI(value) - clean_attrs.append((key,value)) - elif key=='style': + value = make_safe_absolute_uri(value) + clean_attrs.append((key, value)) + elif key == 'style': clean_value = self.sanitize_style(value) if clean_value: - clean_attrs.append((key,clean_value)) - _BaseHTMLProcessor.unknown_starttag(self, tag, clean_attrs) + clean_attrs.append((key, clean_value)) + super(_HTMLSanitizer, self).unknown_starttag(tag, clean_attrs) def unknown_endtag(self, tag): - if not tag in self.acceptable_elements: + if tag not in self.acceptable_elements: if tag in self.unacceptable_elements_with_end_tag: self.unacceptablestack -= 1 if self.mathmlOK and tag in self.mathml_elements: if tag == 'math' and self.mathmlOK: self.mathmlOK -= 1 elif self.svgOK and tag in self.svg_elements: - tag = self.svg_elem_map.get(tag,tag) + tag = self.svg_elem_map.get(tag, tag) if tag == 'svg' and self.svgOK: self.svgOK -= 1 else: return - _BaseHTMLProcessor.unknown_endtag(self, tag) + super(_HTMLSanitizer, self).unknown_endtag(tag) def handle_pi(self, text): pass @@ -354,29 +833,32 @@ class _HTMLSanitizer(_BaseHTMLProcessor): def handle_data(self, text): if not self.unacceptablestack: - _BaseHTMLProcessor.handle_data(self, text) + super(_HTMLSanitizer, self).handle_data(text) def sanitize_style(self, style): # disallow urls - style=re.compile(r'url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ',style) + style = re.compile(r'url\s*\(\s*[^\s)]+?\s*\)\s*').sub(' ', style) # gauntlet if not re.match(r"""^([:,;#%.\sa-zA-Z0-9!]|\w-\w|'[\s\w]+'|"[\s\w]+"|\([\d,\s]+\))*$""", style): return '' - # This replaced a regexp that used re.match and was prone to pathological back-tracking. + # This replaced a regexp that used re.match and was prone to + # pathological back-tracking. if re.sub(r"\s*[-\w]+\s*:\s*[^:;]*;?", '', style).strip(): return '' clean = [] - for prop,value in re.findall(r"([-\w]+)\s*:\s*([^:;]*)",style): + for prop, value in re.findall(r"([-\w]+)\s*:\s*([^:;]*)", style): if not value: continue if prop.lower() in self.acceptable_css_properties: clean.append(prop + ': ' + value + ';') - elif prop.split('-')[0].lower() in ['background','border','margin','padding']: + elif prop.split('-')[0].lower() in ['background', 'border', 'margin', 'padding']: for keyword in value.split(): - if not keyword in self.acceptable_css_keywords and \ - not self.valid_css_values.match(keyword): + if ( + keyword not in self.acceptable_css_keywords + and not self.valid_css_values.match(keyword) + ): break else: clean.append(prop + ': ' + value + ';') @@ -386,7 +868,7 @@ class _HTMLSanitizer(_BaseHTMLProcessor): return ' '.join(clean) def parse_comment(self, i, report=1): - ret = _BaseHTMLProcessor.parse_comment(self, i, report) + ret = super(_HTMLSanitizer, self).parse_comment(i, report) if ret >= 0: return ret # if ret == -1, this may be a malicious attempt to circumvent @@ -398,16 +880,17 @@ class _HTMLSanitizer(_BaseHTMLProcessor): return len(self.rawdata) -def _sanitizeHTML(htmlSource, encoding, _type): +def _sanitize_html(html_source, encoding, _type): if not _SGML_AVAILABLE: - return htmlSource + return html_source p = _HTMLSanitizer(encoding, _type) - htmlSource = htmlSource.replace(' RE_ENTITY_PATTERN = re.compile(br'^\s*]*?)>', re.MULTILINE) @@ -424,12 +907,13 @@ RE_DOCTYPE_PATTERN = re.compile(br'^\s*]*?)>', re.MULTILINE) # Forbidden: explode1 "&explode2;&explode2;" RE_SAFE_ENTITY_PATTERN = re.compile(br'\s+(\w+)\s+"(&#\w+;|[^&"]*)"') + def replace_doctype(data): - '''Strips and replaces the DOCTYPE, returns (rss_version, stripped_data) + """Strips and replaces the DOCTYPE, returns (rss_version, stripped_data) rss_version may be 'rss091n' or None stripped_data is the same XML document with a replaced DOCTYPE - ''' + """ # Divide the document into two groups by finding the location # of the first element that doesn't begin with '\n +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + from __future__ import absolute_import import re @@ -27,11 +54,30 @@ except ImportError: # Mock sgmllib enough to allow subclassing later on class sgmllib(object): + SGMLParseError = EnvironmentError + class SGMLParser(object): + lasttag = None + rawdata = None + + def close(self): + pass + + def feed(self, data): + pass + def goahead(self, i): pass + + def parse_declaration(self, i): + pass + def parse_starttag(self, i): pass + + def reset(self): + pass + else: _SGML_AVAILABLE = 1 @@ -44,8 +90,8 @@ else: charref = re.compile(r'&#(\d+|[xX][0-9a-fA-F]+);') tagfind = re.compile(r'[a-zA-Z][-_.:a-zA-Z0-9]*') attrfind = re.compile( - r'\s*([a-zA-Z_][-:.a-zA-Z_0-9]*)[$]?(\s*=\s*' - r'(\'[^\']*\'|"[^"]*"|[][\-a-zA-Z0-9./,:;+*%?!&$\(\)_#=~\'"@]*))?' + r"""\s*([a-zA-Z_][-:.a-zA-Z_0-9]*)[$]?(\s*=\s*""" + r"""('[^']*'|"[^"]*"|[][\-a-zA-Z0-9./,:;+*%?!&$()_#=~'"@]*))?""" ) # Unfortunately, these must be copied over to prevent NameError exceptions @@ -56,11 +102,20 @@ else: shorttagopen = sgmllib.shorttagopen starttagopen = sgmllib.starttagopen + class _EndBracketRegEx: def __init__(self): # Overriding the built-in sgmllib.endbracket regex allows the # parser to find angle brackets embedded in element attributes. - self.endbracket = re.compile(r'''([^'"<>]|"[^"]*"(?=>|/|\s|\w+=)|'[^']*'(?=>|/|\s|\w+=))*(?=[<>])|.*?(?=[<>])''') + self.endbracket = re.compile( + r'(' + r"""[^'"<>]""" + r"""|"[^"]*"(?=>|/|\s|\w+=)""" + r"""|'[^']*'(?=>|/|\s|\w+=))*(?=[<>])""" + r"""|.*?(?=[<>]""" + r')' + ) + def search(self, target, index=0): match = self.endbracket.match(target, index) if match is not None: @@ -68,9 +123,14 @@ else: # resolves a thread-safety. return EndBracketMatch(match) return None + + class EndBracketMatch: def __init__(self, match): self.match = match + def start(self, n): return self.match.end(n) + + endbracket = _EndBracketRegEx() diff --git a/lib/feedparser/urls.py b/lib/feedparser/urls.py index 85b8fd1..1d1b6c3 100644 --- a/lib/feedparser/urls.py +++ b/lib/feedparser/urls.py @@ -1,4 +1,32 @@ -from __future__ import absolute_import, unicode_literals +# Copyright 2010-2019 Kurt McKee +# Copyright 2002-2008 Mark Pilgrim +# All rights reserved. +# +# This file is a part of feedparser. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 'AS IS' +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. + +from __future__ import absolute_import +from __future__ import unicode_literals import re @@ -24,9 +52,10 @@ ACCEPTABLE_URI_SCHEMES = ( 'aim', 'callto', 'cvs', 'facetime', 'feed', 'git', 'gtalk', 'irc', 'ircs', 'irc6', 'itms', 'mms', 'msnim', 'skype', 'ssh', 'smb', 'svn', 'ymsg', ) -#ACCEPTABLE_URI_SCHEMES = () _urifixer = re.compile('^([A-Za-z][A-Za-z0-9+-.]*://)(/*)(.*?)') + + def _urljoin(base, uri): uri = _urifixer.sub(r'\1\3', uri) try: @@ -35,7 +64,8 @@ def _urljoin(base, uri): uri = '' return uri -def _convert_to_idn(url): + +def convert_to_idn(url): """Convert a URL to IDN notation""" # this function should only be called with a unicode string # strategy: if the host cannot be encoded in ascii, then @@ -59,7 +89,8 @@ def _convert_to_idn(url): else: return url -def _makeSafeAbsoluteURI(base, rel=None): + +def make_safe_absolute_uri(base, rel=None): # bail if ACCEPTABLE_URI_SCHEMES is empty if not ACCEPTABLE_URI_SCHEMES: return _urljoin(base, rel or '') @@ -78,54 +109,54 @@ def _makeSafeAbsoluteURI(base, rel=None): return '' return uri -class _RelativeURIResolver(_BaseHTMLProcessor): - relative_uris = set([('a', 'href'), - ('applet', 'codebase'), - ('area', 'href'), - ('audio', 'src'), - ('blockquote', 'cite'), - ('body', 'background'), - ('del', 'cite'), - ('form', 'action'), - ('frame', 'longdesc'), - ('frame', 'src'), - ('iframe', 'longdesc'), - ('iframe', 'src'), - ('head', 'profile'), - ('img', 'longdesc'), - ('img', 'src'), - ('img', 'usemap'), - ('input', 'src'), - ('input', 'usemap'), - ('ins', 'cite'), - ('link', 'href'), - ('object', 'classid'), - ('object', 'codebase'), - ('object', 'data'), - ('object', 'usemap'), - ('q', 'cite'), - ('script', 'src'), - ('source', 'src'), - ('video', 'poster'), - ('video', 'src')]) + +class RelativeURIResolver(_BaseHTMLProcessor): + relative_uris = { + ('a', 'href'), + ('applet', 'codebase'), + ('area', 'href'), + ('audio', 'src'), + ('blockquote', 'cite'), + ('body', 'background'), + ('del', 'cite'), + ('form', 'action'), + ('frame', 'longdesc'), + ('frame', 'src'), + ('iframe', 'longdesc'), + ('iframe', 'src'), + ('head', 'profile'), + ('img', 'longdesc'), + ('img', 'src'), + ('img', 'usemap'), + ('input', 'src'), + ('input', 'usemap'), + ('ins', 'cite'), + ('link', 'href'), + ('object', 'classid'), + ('object', 'codebase'), + ('object', 'data'), + ('object', 'usemap'), + ('q', 'cite'), + ('script', 'src'), + ('source', 'src'), + ('video', 'poster'), + ('video', 'src'), + } def __init__(self, baseuri, encoding, _type): _BaseHTMLProcessor.__init__(self, encoding, _type) self.baseuri = baseuri - def resolveURI(self, uri): - return _makeSafeAbsoluteURI(self.baseuri, uri.strip()) + def resolve_uri(self, uri): + return make_safe_absolute_uri(self.baseuri, uri.strip()) def unknown_starttag(self, tag, attrs): attrs = self.normalize_attrs(attrs) - attrs = [(key, ((tag, key) in self.relative_uris) and self.resolveURI(value) or value) for key, value in attrs] - _BaseHTMLProcessor.unknown_starttag(self, tag, attrs) + attrs = [(key, ((tag, key) in self.relative_uris) and self.resolve_uri(value) or value) for key, value in attrs] + super(RelativeURIResolver, self).unknown_starttag(tag, attrs) -def _resolveRelativeURIs(htmlSource, baseURI, encoding, _type): - # if not _SGML_AVAILABLE: - # return htmlSource - p = _RelativeURIResolver(baseURI, encoding, _type) - p.feed(htmlSource) +def resolve_relative_uris(html_source, base_uri, encoding, type_): + p = RelativeURIResolver(base_uri, encoding, type_) + p.feed(html_source) return p.output() - diff --git a/lib/feedparser/util.py b/lib/feedparser/util.py index f7c02c0..260d3c5 100644 --- a/lib/feedparser/util.py +++ b/lib/feedparser/util.py @@ -1,11 +1,11 @@ -# Copyright 2010-2015 Kurt McKee +# Copyright 2010-2019 Kurt McKee # Copyright 2002-2008 Mark Pilgrim # All rights reserved. # # This file is a part of feedparser. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -25,65 +25,84 @@ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import +from __future__ import unicode_literals import warnings + class FeedParserDict(dict): - keymap = {'channel': 'feed', - 'items': 'entries', - 'guid': 'id', - 'date': 'updated', - 'date_parsed': 'updated_parsed', - 'description': ['summary', 'subtitle'], - 'description_detail': ['summary_detail', 'subtitle_detail'], - 'url': ['href'], - 'modified': 'updated', - 'modified_parsed': 'updated_parsed', - 'issued': 'published', - 'issued_parsed': 'published_parsed', - 'copyright': 'rights', - 'copyright_detail': 'rights_detail', - 'tagline': 'subtitle', - 'tagline_detail': 'subtitle_detail'} + keymap = { + 'channel': 'feed', + 'items': 'entries', + 'guid': 'id', + 'date': 'updated', + 'date_parsed': 'updated_parsed', + 'description': ['summary', 'subtitle'], + 'description_detail': ['summary_detail', 'subtitle_detail'], + 'url': ['href'], + 'modified': 'updated', + 'modified_parsed': 'updated_parsed', + 'issued': 'published', + 'issued_parsed': 'published_parsed', + 'copyright': 'rights', + 'copyright_detail': 'rights_detail', + 'tagline': 'subtitle', + 'tagline_detail': 'subtitle_detail', + } + def __getitem__(self, key): - ''' + """ :return: A :class:`FeedParserDict`. - ''' + """ + if key == 'category': try: return dict.__getitem__(self, 'tags')[0]['term'] except IndexError: raise KeyError("object doesn't have key 'category'") elif key == 'enclosures': - norel = lambda link: FeedParserDict([(name,value) for (name,value) in link.items() if name!='rel']) - return [norel(link) for link in dict.__getitem__(self, 'links') if link['rel']=='enclosure'] + norel = lambda link: FeedParserDict([(name, value) for (name, value) in link.items() if name != 'rel']) + return [ + norel(link) + for link in dict.__getitem__(self, 'links') + if link['rel'] == 'enclosure' + ] elif key == 'license': for link in dict.__getitem__(self, 'links'): - if link['rel']=='license' and 'href' in link: + if link['rel'] == 'license' and 'href' in link: return link['href'] elif key == 'updated': # Temporarily help developers out by keeping the old # broken behavior that was reported in issue 310. # This fix was proposed in issue 328. - if not dict.__contains__(self, 'updated') and \ - dict.__contains__(self, 'published'): - warnings.warn("To avoid breaking existing software while " + if ( + not dict.__contains__(self, 'updated') + and dict.__contains__(self, 'published') + ): + warnings.warn( + "To avoid breaking existing software while " "fixing issue 310, a temporary mapping has been created " "from `updated` to `published` if `updated` doesn't " "exist. This fallback will be removed in a future version " - "of feedparser.", DeprecationWarning) + "of feedparser.", + DeprecationWarning, + ) return dict.__getitem__(self, 'published') return dict.__getitem__(self, 'updated') elif key == 'updated_parsed': - if not dict.__contains__(self, 'updated_parsed') and \ - dict.__contains__(self, 'published_parsed'): - warnings.warn("To avoid breaking existing software while " + if ( + not dict.__contains__(self, 'updated_parsed') + and dict.__contains__(self, 'published_parsed') + ): + warnings.warn( + "To avoid breaking existing software while " "fixing issue 310, a temporary mapping has been created " "from `updated_parsed` to `published_parsed` if " "`updated_parsed` doesn't exist. This fallback will be " "removed in a future version of feedparser.", - DeprecationWarning) + DeprecationWarning, + ) return dict.__getitem__(self, 'published_parsed') return dict.__getitem__(self, 'updated_parsed') else: @@ -112,9 +131,10 @@ class FeedParserDict(dict): has_key = __contains__ def get(self, key, default=None): - ''' + """ :return: A :class:`FeedParserDict`. - ''' + """ + try: return self.__getitem__(key) except KeyError: @@ -126,11 +146,11 @@ class FeedParserDict(dict): key = key[0] return dict.__setitem__(self, key, value) - def setdefault(self, key, value): - if key not in self: - self[key] = value - return value - return self[key] + def setdefault(self, k, default): + if k not in self: + self[k] = default + return default + return self[k] def __getattr__(self, key): # __getattribute__() is called first; this will be called @@ -141,4 +161,6 @@ class FeedParserDict(dict): raise AttributeError("object has no attribute '%s'" % key) def __hash__(self): + # This is incorrect behavior -- dictionaries shouldn't be hashable. + # Note to self: remove this behavior in the future. return id(self)