Remove Torrentshack.

Change improve provider title processing.
Change improve handling erroneous JSON responses.
Change improve find show with unicode characters.
Change improve result for providers Omgwtf, SpeedCD, Transmithenet, Zoogle.
Change validate .torrent files that contain optional header data.
Fix case where an episode status was not restored on failure.
Add raise log error if no wanted qualities are found.
Change add un/pw to Config/Media providers/Options for BTN API graceful fallback (can remove Api key for security).
Change only download torrent once when using blackhole.
Add Cloudflare module 1.6.8 (be0a536) to mitigate CloudFlare (IUAM) access validator.
Add Js2Py 0.43 (c1442f1) Cloudflare dependency.
Add pyjsparser 2.4.5 (cd5b829) Js2Py dependency.

CHANGES.md

@@ -2,6 +2,19 @@
 
 * Change skip episodes that have no wanted qualities
 * Change download picked .nzb file on demand and not before
+* Change improve provider title processing
+* Change improve handling erroneous JSON responses
+* Change improve find show with unicode characters
+* Change improve results for providers Omgwtf, SpeedCD, Transmithenet, Zoogle
+* Change validate .torrent files that contain optional header data
+* Fix case where an episode status was not restored on failure
+* Add raise log error if no wanted qualities are found
+* Change add un/pw to Config/Media providers/Options for BTN API graceful fallback (can remove Api key for security)
+* Change only download torrent once when using blackhole
+* Add Cloudflare module 1.6.8 (be0a536) to handle specific CF connections
+* Add Js2Py 0.43 (c1442f1) Cloudflare dependency
+* Add pyjsparser 2.4.5 (cd5b829) Js2Py dependency
+* Remove Torrentshack
 
 
 ### 0.12.5 (2017-01-16 16:22:00 UTC)

HACKS.txt

@@ -8,6 +8,7 @@ Libs with customisations...
 /lib/hachoir_metadata/metadata.py
 /lib/hachoir_metadata/riff.py
 /lib/hachoir_parser/guess.py
+/lib/hachoir_parser/misc/torrent.py
 /lib/lockfile/mkdirlockfile.py
 /lib/pynma/pynma.py
 /lib/requests/packages/urllib3/connectionpool.py

SickBeard.py

@@ -420,9 +420,8 @@ class SickGear(object):
         startup_background_tasks = threading.Thread(name='FETCH-XEMDATA', target=sickbeard.scene_exceptions.get_xem_ids)
         startup_background_tasks.start()
 
-        # sure, why not?
         if sickbeard.USE_FAILED_DOWNLOADS:
-            failed_history.trimHistory()
+            failed_history.remove_old_history()
 
         # Start an update if we're supposed to
         if self.force_update or sickbeard.UPDATE_SHOWS_ON_START:

gui/slick/js/newShow.js

@@ -35,7 +35,6 @@ $(document).ready(function () {
 			? text
 			.replace(/["]/g, '"')
 			: text
-			.replace(/Pokémon/, 'Pokemon')
 			.replace(/(?:["]|")/g, '')
 		);
 	}

lib/cfscrape.py

@@ -0,0 +1,161 @@
+from time import sleep
+import logging
+import random
+import re
+from requests.sessions import Session
+import js2py
+from copy import deepcopy
+
+try:
+    from urlparse import urlparse
+except ImportError:
+    from urllib.parse import urlparse
+
+DEFAULT_USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:46.0) Gecko/20100101 Firefox/46.0",
+    "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:41.0) Gecko/20100101 Firefox/41.0"
+]
+
+DEFAULT_USER_AGENT = random.choice(DEFAULT_USER_AGENTS)
+
+
+class CloudflareScraper(Session):
+    def __init__(self, *args, **kwargs):
+        super(CloudflareScraper, self).__init__(*args, **kwargs)
+
+        if "requests" in self.headers["User-Agent"]:
+            # Spoof Firefox on Linux if no custom User-Agent has been set
+            self.headers["User-Agent"] = DEFAULT_USER_AGENT
+
+    def request(self, method, url, *args, **kwargs):
+        resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs)
+
+        # Check if Cloudflare anti-bot is on
+        if ( resp.status_code == 503
+             and resp.headers.get("Server") == "cloudflare-nginx"
+             and b"jschl_vc" in resp.content
+             and b"jschl_answer" in resp.content
+        ):
+            return self.solve_cf_challenge(resp, **kwargs)
+
+        # Otherwise, no Cloudflare anti-bot detected
+        return resp
+
+    def solve_cf_challenge(self, resp, **original_kwargs):
+        sleep(5)  # Cloudflare requires a delay before solving the challenge
+
+        body = resp.text
+        parsed_url = urlparse(resp.url)
+        domain = urlparse(resp.url).netloc
+        submit_url = "%s://%s/cdn-cgi/l/chk_jschl" % (parsed_url.scheme, domain)
+
+        cloudflare_kwargs = deepcopy(original_kwargs)
+        params = cloudflare_kwargs.setdefault("params", {})
+        headers = cloudflare_kwargs.setdefault("headers", {})
+        headers["Referer"] = resp.url
+
+        try:
+            params["jschl_vc"] = re.search(r'name="jschl_vc" value="(\w+)"', body).group(1)
+            params["pass"] = re.search(r'name="pass" value="(.+?)"', body).group(1)
+
+            # Extract the arithmetic operation
+            js = self.extract_js(body)
+
+        except Exception:
+            # Something is wrong with the page.
+            # This may indicate Cloudflare has changed their anti-bot
+            # technique. If you see this and are running the latest version,
+            # please open a GitHub issue so I can update the code accordingly.
+            logging.error("[!] Unable to parse Cloudflare anti-bots page. "
+                          "Try upgrading cloudflare-scrape, or submit a bug report "
+                          "if you are running the latest version. Please read "
+                          "https://github.com/Anorov/cloudflare-scrape#updates "
+                          "before submitting a bug report.")
+            raise
+
+        # Safely evaluate the Javascript expression
+        params["jschl_answer"] = str(int(js2py.eval_js(js)) + len(domain))
+
+        # Requests transforms any request into a GET after a redirect,
+        # so the redirect has to be handled manually here to allow for
+        # performing other types of requests even as the first request.
+        method = resp.request.method
+        cloudflare_kwargs["allow_redirects"] = False
+        redirect = self.request(method, submit_url, **cloudflare_kwargs)
+        return self.request(method, redirect.headers["Location"], **original_kwargs)
+
+    def extract_js(self, body):
+        js = re.search(r"setTimeout\(function\(\){\s+(var "
+                        "s,t,o,p,b,r,e,a,k,i,n,g,f.+?\r?\n[\s\S]+?a\.value =.+?)\r?\n", body).group(1)
+        js = re.sub(r"a\.value = (parseInt\(.+?\)).+", r"\1", js)
+        js = re.sub(r"\s{3,}[a-z](?: = |\.).+", "", js)
+
+        # Strip characters that could be used to exit the string context
+        # These characters are not currently used in Cloudflare's arithmetic snippet
+        js = re.sub(r"[\n\\']", "", js)
+
+        return js
+
+    @classmethod
+    def create_scraper(cls, sess=None, **kwargs):
+        """
+        Convenience function for creating a ready-to-go requests.Session (subclass) object.
+        """
+        scraper = cls()
+
+        if sess:
+            attrs = ["auth", "cert", "cookies", "headers", "hooks", "params", "proxies", "data"]
+            for attr in attrs:
+                val = getattr(sess, attr, None)
+                if val:
+                    setattr(scraper, attr, val)
+
+        return scraper
+
+
+    ## Functions for integrating cloudflare-scrape with other applications and scripts
+
+    @classmethod
+    def get_tokens(cls, url, user_agent=None, **kwargs):
+        scraper = cls.create_scraper()
+        if user_agent:
+            scraper.headers["User-Agent"] = user_agent
+
+        try:
+            resp = scraper.get(url)
+            resp.raise_for_status()
+        except Exception as e:
+            logging.error("'%s' returned an error. Could not collect tokens." % url)
+            raise
+
+        domain = urlparse(resp.url).netloc
+        cookie_domain = None
+
+        for d in scraper.cookies.list_domains():
+            if d.startswith(".") and d in ("." + domain):
+                cookie_domain = d
+                break
+        else:
+            raise ValueError("Unable to find Cloudflare cookies. Does the site actually have Cloudflare IUAM (\"I'm Under Attack Mode\") enabled?")
+
+        return ({
+                    "__cfduid": scraper.cookies.get("__cfduid", "", domain=cookie_domain),
+                    "cf_clearance": scraper.cookies.get("cf_clearance", "", domain=cookie_domain)
+                },
+                scraper.headers["User-Agent"]
+               )
+
+    @classmethod
+    def get_cookie_string(cls, url, user_agent=None, **kwargs):
+        """
+        Convenience function for building a Cookie HTTP header value.
+        """
+        tokens, user_agent = cls.get_tokens(url, user_agent=user_agent)
+        return "; ".join("=".join(pair) for pair in tokens.items()), user_agent
+
+create_scraper = CloudflareScraper.create_scraper
+get_tokens = CloudflareScraper.get_tokens
+get_cookie_string = CloudflareScraper.get_cookie_string

lib/hachoir_parser/misc/torrent.py

@@ -143,6 +143,7 @@ def Entry(parent, name):
 class TorrentFile(Parser):
     endian = LITTLE_ENDIAN
     MAGIC = "d8:announce"
+    MAGIC_EXTENSION = "d13:announce-list"
     PARSER_TAGS = {
         "id": "torrent",
         "category": "misc",
@@ -154,7 +155,8 @@ class TorrentFile(Parser):
     }
 
     def validate(self):
-        if self.stream.readBytes(0, len(self.MAGIC)) != self.MAGIC:
+        if self.stream.readBytes(0, len(self.MAGIC)) != self.MAGIC and \
+                        self.stream.readBytes(0, len(self.MAGIC_EXTENSION)) != self.MAGIC_EXTENSION:
             return "Invalid magic"
         return True
 

lib/js2py/__init__.py

@@ -0,0 +1,71 @@
+# The MIT License
+#
+# Copyright 2014, 2015 Piotr Dabkowski
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the 'Software'),
+# to deal in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so, subject
+# to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+# LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+#  OR THE USE OR OTHER DEALINGS IN THE SOFTWARE
+
+""" This module allows you to translate and execute Javascript in pure python.
+    Basically its implementation of ECMAScript 5.1 in pure python.
+
+    Use eval_js method to execute javascript code and get resulting python object (builtin if possible).
+
+    EXAMPLE:
+    >>> import js2py
+    >>> add = js2py.eval_js('function add(a, b) {return a + b}')
+    >>> add(1, 2) + 3
+    6
+    >>> add('1', 2, 3)
+    u'12'
+    >>> add.constructor
+    function Function() { [python code] }
+
+
+    Or use EvalJs to execute many javascript code fragments under same context - you would be able to get any
+    variable from the context!
+
+    >>> js = js2py.EvalJs()
+    >>> js.execute('var a = 10; function f(x) {return x*x};')
+    >>> js.f(9)
+    81
+    >>> js.a
+    10
+
+    Also you can use its console method to play with interactive javascript console.
+
+
+    Use parse_js to parse (syntax tree is just like in esprima.js) and translate_js to trasnlate JavaScript.
+
+    Finally, you can use pyimport statement from inside JS code to import and use python libraries.
+
+    >>> js2py.eval_js('pyimport urllib; urllib.urlopen("https://www.google.com")')
+
+    NOTE: This module is still not fully finished:
+
+          Date and JSON builtin objects are not implemented
+          Array prototype is not fully finished (will be soon)
+
+    Other than that everything should work fine.
+
+"""
+
+__author__ = 'Piotr Dabkowski'
+__all__  = ['EvalJs', 'translate_js', 'import_js', 'eval_js', 'parse_js', 'translate_file',
+            'run_file', 'disable_pyimport', 'eval_js6', 'translate_js6', 'PyJsException', 'get_file_contents', 'write_file_contents']
+from .base import PyJsException
+from .evaljs import *
+from .translators import parse as parse_js
+

lib/js2py/constructors/__init__.py

@@ -0,0 +1 @@
+__author__ = 'Piotr Dabkowski'

lib/js2py/constructors/jsarray.py

@@ -0,0 +1,38 @@
+from ..base import *
+
+@Js
+def Array():
+    if len(arguments)==0 or len(arguments)>1:
+        return arguments.to_list()
+    a = arguments[0]
+    if isinstance(a, PyJsNumber):
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js([])
+        temp.put('length', a)
+        return temp
+    return [a]
+
+Array.create = Array
+Array.own['length']['value'] = Js(1)
+
+@Js
+def isArray(arg):
+    return arg.Class=='Array'
+
+
+Array.define_own_property('isArray', {'value': isArray,
+                                         'enumerable': False,
+                                         'writable': True,
+                                         'configurable': True})
+
+Array.define_own_property('prototype', {'value': ArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+ArrayPrototype.define_own_property('constructor', {'value': Array,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})

lib/js2py/constructors/jsarraybuffer.py

@@ -0,0 +1,34 @@
+# this is based on jsarray.py
+
+# todo check everything :)
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+
+@Js
+def ArrayBuffer():
+    a = arguments[0]
+    if isinstance(a, PyJsNumber):
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(bytearray([0]*length))
+        return temp
+    return Js(bytearray([0]))
+
+ArrayBuffer.create = ArrayBuffer
+ArrayBuffer.own['length']['value'] = Js(None)
+
+ArrayBuffer.define_own_property('prototype', {'value': ArrayBufferPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+ArrayBufferPrototype.define_own_property('constructor', {'value': ArrayBuffer,
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': True})

lib/js2py/constructors/jsboolean.py

@@ -0,0 +1,11 @@
+from ..base import *
+
+BooleanPrototype.define_own_property('constructor', {'value': Boolean,
+                                                     'enumerable': False,
+                                                     'writable': True,
+                                                     'configurable': True})
+
+Boolean.define_own_property('prototype', {'value': BooleanPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})

lib/js2py/constructors/jsdate.py

@@ -0,0 +1,368 @@
+from ..base import *
+from .time_helpers import *
+
+TZ_OFFSET = (time.altzone//3600)
+ABS_OFFSET = abs(TZ_OFFSET)
+TZ_NAME = time.tzname[1]
+ISO_FORMAT = '%s-%s-%sT%s:%s:%s.%sZ'
+@Js
+def Date(year, month, date, hours, minutes, seconds, ms):
+    return now().to_string()
+
+Date.Class = 'Date'
+
+def now():
+    return PyJsDate(int(time.time()*1000), prototype=DatePrototype)
+
+
+@Js
+def UTC(year, month, date, hours, minutes, seconds, ms): # todo complete this
+    args = arguments
+    y = args[0].to_number()
+    m = args[1].to_number()
+    l = len(args)
+    dt = args[2].to_number() if l>2 else Js(1)
+    h = args[3].to_number() if l>3 else Js(0)
+    mi = args[4].to_number() if l>4 else Js(0)
+    sec = args[5].to_number() if l>5 else Js(0)
+    mili = args[6].to_number() if l>6 else Js(0)
+    if not y.is_nan() and 0<=y.value<=99:
+        y = y + Js(1900)
+    t = TimeClip(MakeDate(MakeDay(y, m, dt), MakeTime(h, mi, sec, mili)))
+    return PyJsDate(t, prototype=DatePrototype)
+
+@Js
+def parse(string):
+    return PyJsDate(TimeClip(parse_date(string.to_string().value)), prototype=DatePrototype)
+
+
+
+Date.define_own_property('now', {'value': Js(now),
+                                 'enumerable': False,
+                                 'writable': True,
+                                 'configurable': True})
+
+Date.define_own_property('parse', {'value': parse,
+                                 'enumerable': False,
+                                 'writable': True,
+                                 'configurable': True})
+
+Date.define_own_property('UTC', {'value': UTC,
+                                 'enumerable': False,
+                                 'writable': True,
+                                 'configurable': True})
+
+class PyJsDate(PyJs):
+    Class = 'Date'
+    extensible = True
+    def __init__(self, value, prototype=None):
+        self.value = value
+        self.own = {}
+        self.prototype = prototype
+
+    # todo fix this problematic datetime part
+    def to_local_dt(self):
+        return datetime.datetime.utcfromtimestamp(UTCToLocal(self.value)//1000)
+
+    def to_utc_dt(self):
+        return datetime.datetime.utcfromtimestamp(self.value//1000)
+
+    def local_strftime(self, pattern):
+        if self.value is NaN:
+            return 'Invalid Date'
+        try:
+            dt = self.to_local_dt()
+        except:
+            raise MakeError('TypeError', 'unsupported date range. Will fix in future versions')
+        try:
+            return dt.strftime(pattern)
+        except:
+            raise MakeError('TypeError', 'Could not generate date string from this date (limitations of python.datetime)')
+
+    def utc_strftime(self, pattern):
+        if self.value is NaN:
+            return 'Invalid Date'
+        try:
+            dt = self.to_utc_dt()
+        except:
+            raise MakeError('TypeError', 'unsupported date range. Will fix in future versions')
+        try:
+            return dt.strftime(pattern)
+        except:
+            raise MakeError('TypeError', 'Could not generate date string from this date (limitations of python.datetime)')
+
+
+
+
+def parse_date(py_string):   # todo support all date string formats
+    try:
+        dt = datetime.datetime.strptime(py_string, "%Y-%m-%dT%H:%M:%S.%fZ")
+        return MakeDate(MakeDay(Js(dt.year), Js(dt.month-1), Js(dt.day)), MakeTime(Js(dt.hour), Js(dt.minute), Js(dt.second), Js(dt.microsecond//1000)))
+    except:
+        raise MakeError('TypeError', 'Could not parse date %s - unsupported date format. Currently only supported format is RFC3339 utc. Sorry!' % py_string)
+
+
+
+def date_constructor(*args):
+    if len(args)>=2:
+        return date_constructor2(*args)
+    elif len(args)==1:
+        return date_constructor1(args[0])
+    else:
+        return date_constructor0()
+
+
+def date_constructor0():
+    return now()
+
+
+def date_constructor1(value):
+    v = value.to_primitive()
+    if v._type()=='String':
+        v = parse_date(v.value)
+    else:
+        v = v.to_int()
+    return PyJsDate(TimeClip(v), prototype=DatePrototype)
+
+
+def date_constructor2(*args):
+    y = args[0].to_number()
+    m = args[1].to_number()
+    l = len(args)
+    dt = args[2].to_number() if l>2 else Js(1)
+    h = args[3].to_number() if l>3 else Js(0)
+    mi = args[4].to_number() if l>4 else Js(0)
+    sec = args[5].to_number() if l>5 else Js(0)
+    mili = args[6].to_number() if l>6 else Js(0)
+    if not y.is_nan() and 0<=y.value<=99:
+        y = y + Js(1900)
+    t = TimeClip(LocalToUTC(MakeDate(MakeDay(y, m, dt), MakeTime(h, mi, sec, mili))))
+    return PyJsDate(t, prototype=DatePrototype)
+
+Date.create = date_constructor
+
+DatePrototype = PyJsDate(float('nan'), prototype=ObjectPrototype)
+
+def check_date(obj):
+    if obj.Class!='Date':
+        raise MakeError('TypeError', 'this is not a Date object')
+
+
+class DateProto:
+    def toString():
+        check_date(this)
+        if this.value is NaN:
+            return 'Invalid Date'
+        offset = (UTCToLocal(this.value) - this.value)//msPerHour
+        return this.local_strftime('%a %b %d %Y %H:%M:%S GMT') + '%s00 (%s)' % (pad(offset, 2, True), GetTimeZoneName(this.value))
+
+    def toDateString():
+        check_date(this)
+        return this.local_strftime('%d %B %Y')
+
+    def toTimeString():
+        check_date(this)
+        return this.local_strftime('%H:%M:%S')
+
+    def toLocaleString():
+        check_date(this)
+        return this.local_strftime('%d %B %Y %H:%M:%S')
+
+    def toLocaleDateString():
+        check_date(this)
+        return this.local_strftime('%d %B %Y')
+
+    def toLocaleTimeString():
+        check_date(this)
+        return this.local_strftime('%H:%M:%S')
+
+    def valueOf():
+        check_date(this)
+        return this.value
+
+    def getTime():
+        check_date(this)
+        return this.value
+
+    def getFullYear():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return YearFromTime(UTCToLocal(this.value))
+
+    def getUTCFullYear():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return YearFromTime(this.value)
+
+    def getMonth():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return MonthFromTime(UTCToLocal(this.value))
+
+    def getDate():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return DateFromTime(UTCToLocal(this.value))
+
+    def getUTCMonth():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return MonthFromTime(this.value)
+
+    def getUTCDate():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return DateFromTime(this.value)
+
+    def getDay():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return WeekDay(UTCToLocal(this.value))
+
+    def getUTCDay():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return WeekDay(this.value)
+
+    def getHours():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return HourFromTime(UTCToLocal(this.value))
+
+    def getUTCHours():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return HourFromTime(this.value)
+
+    def getMinutes():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return MinFromTime(UTCToLocal(this.value))
+
+    def getUTCMinutes():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return MinFromTime(this.value)
+
+    def getSeconds():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return SecFromTime(UTCToLocal(this.value))
+
+    def getUTCSeconds():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return SecFromTime(this.value)
+
+    def getMilliseconds():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return msFromTime(UTCToLocal(this.value))
+
+    def getUTCMilliseconds():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return msFromTime(this.value)
+
+    def getTimezoneOffset():
+        check_date(this)
+        if this.value is NaN:
+            return NaN
+        return (this.value - UTCToLocal(this.value))//60000
+
+
+    def setTime(time):
+        check_date(this)
+        this.value = TimeClip(time.to_number().to_int())
+        return this.value
+
+    def setMilliseconds(ms):
+        check_date(this)
+        t = UTCToLocal(this.value)
+        tim = MakeTime(HourFromTime(t), MinFromTime(t), SecFromTime(t), ms.to_int())
+        u = TimeClip(LocalToUTC(MakeDate(Day(t), tim)))
+        this.value = u
+        return u
+
+    def setUTCMilliseconds(ms):
+        check_date(this)
+        t = this.value
+        tim = MakeTime(HourFromTime(t), MinFromTime(t), SecFromTime(t), ms.to_int())
+        u = TimeClip(MakeDate(Day(t), tim))
+        this.value = u
+        return u
+
+    # todo Complete all setters!
+
+    def toUTCString():
+        check_date(this)
+        return this.utc_strftime('%d %B %Y %H:%M:%S')
+
+    def toISOString():
+        check_date(this)
+        t = this.value
+        year = YearFromTime(t)
+        month, day, hour, minute, second, milli = pad(MonthFromTime(t)+1), pad(DateFromTime(t)), pad(HourFromTime(t)), pad(MinFromTime(t)), pad(SecFromTime(t)), pad(msFromTime(t))
+        return ISO_FORMAT % (unicode(year) if 0<=year<=9999 else pad(year, 6, True), month, day, hour, minute, second, milli)
+
+    def toJSON(key):
+        o = this.to_object()
+        tv = o.to_primitive('Number')
+        if tv.Class=='Number' and not tv.is_finite():
+            return this.null
+        toISO = o.get('toISOString')
+        if not toISO.is_callable():
+            raise this.MakeError('TypeError', 'toISOString is not callable')
+        return toISO.call(o, ())
+
+
+def pad(num, n=2, sign=False):
+    '''returns n digit string representation of the num'''
+    s = unicode(abs(num))
+    if len(s)<n:
+        s = '0'*(n-len(s)) + s
+    if not sign:
+        return s
+    if num>=0:
+        return '+'+s
+    else:
+        return '-'+s
+
+
+
+
+
+
+
+
+
+
+fill_prototype(DatePrototype, DateProto, default_attrs)
+
+
+
+Date.define_own_property('prototype', {'value': DatePrototype,
+                                 'enumerable': False,
+                                 'writable': False,
+                                 'configurable': False})
+
+DatePrototype.define_own_property('constructor', {'value': Date,
+                                  'enumerable': False,
+                                  'writable': True,
+                                  'configurable': True})

lib/js2py/constructors/jsfloat32array.py

@@ -0,0 +1,69 @@
+# this is based on jsarray.py
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+
+@Js
+def Float32Array():
+    TypedArray = (PyJsInt8Array,PyJsUint8Array,PyJsUint8ClampedArray,PyJsInt16Array,PyJsUint16Array,PyJsInt32Array,PyJsUint32Array,PyJsFloat32Array,PyJsFloat64Array)
+    a = arguments[0]
+    if isinstance(a, PyJsNumber): # length
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(numpy.full(length, 0, dtype=numpy.float32))
+        temp.put('length', a)
+        return temp
+    elif isinstance(a, PyJsString): # object (string)
+        temp = Js(numpy.array(list(a.value), dtype=numpy.float32))
+        temp.put('length', Js(len(list(a.value))))
+        return temp
+    elif isinstance(a, PyJsArray) or isinstance(a,TypedArray) or isinstance(a,PyJsArrayBuffer): # object (Array, TypedArray)
+        array = a.to_list()
+        array = [(int(item.value) if item.value != None else 0) for item in array]
+        temp = Js(numpy.array(array, dtype=numpy.float32))
+        temp.put('length', Js(len(array)))
+        return temp
+    elif isinstance(a,PyObjectWrapper): # object (ArrayBuffer, etc)
+        if len(a.obj) % 4 != 0:
+            raise MakeError('RangeError', 'Byte length of Float32Array should be a multiple of 4')
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+            if offset % 4 != 0:
+                raise MakeError('RangeError', 'Start offset of Float32Array should be a multiple of 4')
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = int((len(a.obj)-offset)/4)
+        array = numpy.frombuffer(a.obj, dtype=numpy.float32, count=length, offset=offset)
+        temp = Js(array)
+        temp.put('length', Js(length))
+        temp.buff = array
+        return temp
+    temp = Js(numpy.full(0, 0, dtype=numpy.float32))
+    temp.put('length', Js(0))
+    return temp
+
+Float32Array.create = Float32Array
+Float32Array.own['length']['value'] = Js(3)
+
+Float32Array.define_own_property('prototype', {'value': Float32ArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+Float32ArrayPrototype.define_own_property('constructor', {'value': Float32Array,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})
+
+Float32ArrayPrototype.define_own_property('BYTES_PER_ELEMENT', {'value': Js(4),
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': False})

lib/js2py/constructors/jsfloat64array.py

@@ -0,0 +1,69 @@
+# this is based on jsarray.py
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+
+@Js
+def Float64Array():
+    TypedArray = (PyJsInt8Array,PyJsUint8Array,PyJsUint8ClampedArray,PyJsInt16Array,PyJsUint16Array,PyJsInt32Array,PyJsUint32Array,PyJsFloat32Array,PyJsFloat64Array)
+    a = arguments[0]
+    if isinstance(a, PyJsNumber): # length
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(numpy.full(length, 0, dtype=numpy.float64))
+        temp.put('length', a)
+        return temp
+    elif isinstance(a, PyJsString): # object (string)
+        temp = Js(numpy.array(list(a.value), dtype=numpy.float64))
+        temp.put('length', Js(len(list(a.value))))
+        return temp
+    elif isinstance(a, PyJsArray) or isinstance(a,TypedArray) or isinstance(a,PyJsArrayBuffer): # object (Array, TypedArray)
+        array = a.to_list()
+        array = [(int(item.value) if item.value != None else 0) for item in array]
+        temp = Js(numpy.array(array, dtype=numpy.float64))
+        temp.put('length', Js(len(array)))
+        return temp
+    elif isinstance(a,PyObjectWrapper): # object (ArrayBuffer, etc)
+        if len(a.obj) % 8 != 0:
+            raise MakeError('RangeError', 'Byte length of Float64Array should be a multiple of 8')
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+            if offset % 8 != 0:
+                raise MakeError('RangeError', 'Start offset of Float64Array should be a multiple of 8')
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = int((len(a.obj)-offset)/8)
+        array = numpy.frombuffer(a.obj, dtype=numpy.float64, count=length, offset=offset)
+        temp = Js(array)
+        temp.put('length', Js(length))
+        temp.buff = array
+        return temp
+    temp = Js(numpy.full(0, 0, dtype=numpy.float64))
+    temp.put('length', Js(0))
+    return temp
+
+Float64Array.create = Float64Array
+Float64Array.own['length']['value'] = Js(3)
+
+Float64Array.define_own_property('prototype', {'value': Float64ArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+Float64ArrayPrototype.define_own_property('constructor', {'value': Float64Array,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})
+
+Float64ArrayPrototype.define_own_property('BYTES_PER_ELEMENT', {'value': Js(8),
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': False})

lib/js2py/constructors/jsfunction.py

@@ -0,0 +1,49 @@
+from ..base import *
+try:
+    from ..translators.translator import translate_js
+except:
+    pass
+
+
+@Js
+def Function():
+    # convert arguments to python list of strings
+    a = [e.to_string().value for e in arguments.to_list()]
+    body = ';'
+    args = ()
+    if len(a):
+        body = '%s;' % a[-1]
+        args = a[:-1]
+    # translate this function to js inline function
+    js_func = '(function (%s) {%s})' % (','.join(args), body)
+    # now translate js inline to python function
+    py_func =  translate_js(js_func, '')
+    # add set func scope to global scope
+    # a but messy solution but works :)
+    globals()['var'] = PyJs.GlobalObject
+    # define py function and return it
+    temp = executor(py_func, globals())
+    temp.source = '{%s}'%body
+    temp.func_name = 'anonymous'
+    return temp
+
+def executor(f, glob):
+    exec(f, globals())
+    return globals()['PyJs_anonymous_0_']
+
+
+#new statement simply calls Function
+Function.create = Function
+
+#set constructor property inside FunctionPrototype
+
+fill_in_props(FunctionPrototype, {'constructor':Function}, default_attrs)
+
+#attach prototype to Function constructor
+Function.define_own_property('prototype', {'value': FunctionPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+#Fix Function length (its 0 and should be 1)
+Function.own['length']['value'] = Js(1)
+

lib/js2py/constructors/jsint16array.py

@@ -0,0 +1,68 @@
+# this is based on jsarray.py
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+@Js
+def Int16Array():
+    TypedArray = (PyJsInt8Array,PyJsUint8Array,PyJsUint8ClampedArray,PyJsInt16Array,PyJsUint16Array,PyJsInt32Array,PyJsUint32Array,PyJsFloat32Array,PyJsFloat64Array)
+    a = arguments[0]
+    if isinstance(a, PyJsNumber): # length
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(numpy.full(length, 0, dtype=numpy.int16))
+        temp.put('length', a)
+        return temp
+    elif isinstance(a, PyJsString): # object (string)
+        temp = Js(numpy.array(list(a.value), dtype=numpy.int16))
+        temp.put('length', Js(len(list(a.value))))
+        return temp
+    elif isinstance(a, PyJsArray) or isinstance(a,TypedArray) or isinstance(a,PyJsArrayBuffer): # object (Array, TypedArray)
+        array = a.to_list()
+        array = [(int(item.value) if item.value != None else 0) for item in array]
+        temp = Js(numpy.array(array, dtype=numpy.int16))
+        temp.put('length', Js(len(array)))
+        return temp
+    elif isinstance(a,PyObjectWrapper): # object (ArrayBuffer, etc)
+        if len(a.obj) % 2 != 0:
+            raise MakeError('RangeError', 'Byte length of Int16Array should be a multiple of 2')
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+            if offset % 2 != 0:
+                raise MakeError('RangeError', 'Start offset of Int16Array should be a multiple of 2')
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = int((len(a.obj)-offset)/2)
+        array = numpy.frombuffer(a.obj, dtype=numpy.int16, count=length, offset=offset)
+        temp = Js(array)
+        temp.put('length', Js(length))
+        temp.buff = array
+        return temp
+    temp = Js(numpy.full(0, 0, dtype=numpy.int16))
+    temp.put('length', Js(0))
+    return temp
+
+Int16Array.create = Int16Array
+Int16Array.own['length']['value'] = Js(3)
+
+Int16Array.define_own_property('prototype', {'value': Int16ArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+Int16ArrayPrototype.define_own_property('constructor', {'value': Int16Array,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})
+
+Int16ArrayPrototype.define_own_property('BYTES_PER_ELEMENT', {'value': Js(2),
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': False})

lib/js2py/constructors/jsint32array.py

@@ -0,0 +1,69 @@
+# this is based on jsarray.py
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+
+@Js
+def Int32Array():
+    TypedArray = (PyJsInt8Array,PyJsUint8Array,PyJsUint8ClampedArray,PyJsInt16Array,PyJsUint16Array,PyJsInt32Array,PyJsUint32Array,PyJsFloat32Array,PyJsFloat64Array)
+    a = arguments[0]
+    if isinstance(a, PyJsNumber): # length
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(numpy.full(length, 0, dtype=numpy.int32))
+        temp.put('length', a)
+        return temp
+    elif isinstance(a, PyJsString): # object (string)
+        temp = Js(numpy.array(list(a.value), dtype=numpy.int32))
+        temp.put('length', Js(len(list(a.value))))
+        return temp
+    elif isinstance(a, PyJsArray) or isinstance(a,TypedArray) or isinstance(a,PyJsArrayBuffer): # object (Array, TypedArray)
+        array = a.to_list()
+        array = [(int(item.value) if item.value != None else 0) for item in array]
+        temp = Js(numpy.array(array, dtype=numpy.int32))
+        temp.put('length', Js(len(array)))
+        return temp
+    elif isinstance(a,PyObjectWrapper): # object (ArrayBuffer, etc)
+        if len(a.obj) % 4 != 0:
+            raise MakeError('RangeError', 'Byte length of Int32Array should be a multiple of 4')
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+            if offset % 4 != 0:
+                raise MakeError('RangeError', 'Start offset of Int32Array should be a multiple of 4')
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = int((len(a.obj)-offset)/4)
+        array = numpy.frombuffer(a.obj, dtype=numpy.int32, count=length, offset=offset)
+        temp = Js(array)
+        temp.put('length', Js(length))
+        temp.buff = array
+        return temp
+    temp = Js(numpy.full(0, 0, dtype=numpy.int32))
+    temp.put('length', Js(0))
+    return temp
+
+Int32Array.create = Int32Array
+Int32Array.own['length']['value'] = Js(3)
+
+Int32Array.define_own_property('prototype', {'value': Int32ArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+Int32ArrayPrototype.define_own_property('constructor', {'value': Int32Array,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})
+
+Int32ArrayPrototype.define_own_property('BYTES_PER_ELEMENT', {'value': Js(4),
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': False})

lib/js2py/constructors/jsint8array.py

@@ -0,0 +1,64 @@
+# this is based on jsarray.py
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+@Js
+def Int8Array():
+    TypedArray = (PyJsInt8Array,PyJsUint8Array,PyJsUint8ClampedArray,PyJsInt16Array,PyJsUint16Array,PyJsInt32Array,PyJsUint32Array,PyJsFloat32Array,PyJsFloat64Array)
+    a = arguments[0]
+    if isinstance(a, PyJsNumber): # length
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(numpy.full(length, 0, dtype=numpy.int8))
+        temp.put('length', a)
+        return temp
+    elif isinstance(a, PyJsString): # object (string)
+        temp = Js(numpy.array(list(a.value), dtype=numpy.int8))
+        temp.put('length', Js(len(list(a.value))))
+        return temp
+    elif isinstance(a, PyJsArray) or isinstance(a,TypedArray) or isinstance(a,PyJsArrayBuffer): # object (Array, TypedArray)
+        array = a.to_list()
+        array = [(int(item.value) if item.value != None else 0) for item in array]
+        temp = Js(numpy.array(array, dtype=numpy.int8))
+        temp.put('length', Js(len(array)))
+        return temp
+    elif isinstance(a,PyObjectWrapper): # object (ArrayBuffer, etc)
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = int(len(a.obj)-offset)
+        array = numpy.frombuffer(a.obj, dtype=numpy.int8, count=length, offset=offset)
+        temp = Js(array)
+        temp.put('length', Js(length))
+        temp.buff = array
+        return temp
+    temp = Js(numpy.full(0, 0, dtype=numpy.int8))
+    temp.put('length', Js(0))
+    return temp
+
+Int8Array.create = Int8Array
+Int8Array.own['length']['value'] = Js(3)
+
+Int8Array.define_own_property('prototype', {'value': Int8ArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+Int8ArrayPrototype.define_own_property('constructor', {'value': Int8Array,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})
+
+Int8ArrayPrototype.define_own_property('BYTES_PER_ELEMENT', {'value': Js(1),
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': False})

lib/js2py/constructors/jsmath.py

@@ -0,0 +1,151 @@
+from ..base import *
+import math
+import random
+
+Math = PyJsObject(prototype=ObjectPrototype)
+Math.Class = 'Math'
+
+CONSTANTS = {'E': 2.7182818284590452354,
+             'LN10': 2.302585092994046,
+             'LN2': 0.6931471805599453,
+             'LOG2E': 1.4426950408889634,
+             'LOG10E': 0.4342944819032518,
+             'PI': 3.1415926535897932,
+             'SQRT1_2': 0.7071067811865476,
+             'SQRT2': 1.4142135623730951}
+
+for constant, value in CONSTANTS.items():
+    Math.define_own_property(constant, {'value': Js(value),
+                                        'writable': False,
+                                        'enumerable': False,
+                                        'configurable': False})
+
+class MathFunctions:
+    def abs(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        return abs(a)
+
+    def acos(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        try:
+            return math.acos(a)
+        except:
+            return NaN
+
+    def asin(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        try:
+            return math.asin(a)
+        except:
+            return NaN
+
+    def atan(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        return math.atan(a)
+
+    def atan2(y, x):
+        a = x.to_number().value
+        b = y.to_number().value
+        if a!=a or b!=b: # it must be a nan
+            return NaN
+        return math.atan2(b, a)
+
+    def ceil(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        return math.ceil(a)
+
+    def floor(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        return math.floor(a)
+
+    def round(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        return round(a)
+
+    def sin(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        return math.sin(a)
+
+    def cos(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        return  math.cos(a)
+
+    def tan(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        return math.tan(a)
+
+    def log(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        try:
+            return math.log(a)
+        except:
+            return NaN
+
+    def exp(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        return math.exp(a)
+
+    def pow(x, y):
+        a = x.to_number().value
+        b = y.to_number().value
+        if a!=a or b!=b: # it must be a nan
+            return NaN
+        try:
+            return a**b
+        except:
+            return NaN
+
+    def sqrt(x):
+        a = x.to_number().value
+        if a!=a: # it must be a nan
+            return NaN
+        try:
+            return a**0.5
+        except:
+            return NaN
+
+    def min():
+        if not len(arguments):
+            return -Infinity
+        lis = tuple(e.to_number().value for e in arguments.to_list())
+        if any(e!=e for e in lis): # we dont want NaNs
+            return NaN
+        return min(*lis)
+
+    def max():
+        if not len(arguments):
+            return -Infinity
+        lis = tuple(e.to_number().value for e in arguments.to_list())
+        if any(e!=e for e in lis): # we dont want NaNs
+            return NaN
+        return max(*lis)
+
+    def random():
+        return random.random()
+
+
+fill_prototype(Math, MathFunctions, default_attrs)

lib/js2py/constructors/jsnumber.py

@@ -0,0 +1,18 @@
+from ..base import *
+
+
+CONSTS = {'prototype': NumberPrototype,
+          'MAX_VALUE':1.7976931348623157e308,
+          'MIN_VALUE': 5.0e-324,
+          'NaN': NaN,
+          'NEGATIVE_INFINITY': float('-inf'),
+          'POSITIVE_INFINITY': float('inf')}
+
+fill_in_props(Number, CONSTS,  {'enumerable': False,
+                                'writable': False,
+                                'configurable': False})
+
+NumberPrototype.define_own_property('constructor', {'value': Number,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})

lib/js2py/constructors/jsobject.py

@@ -0,0 +1,172 @@
+from ..base import *
+import six
+
+#todo Double check everything is OK
+
+@Js
+def Object():
+    val = arguments.get('0')
+    if val.is_null() or val.is_undefined():
+        return PyJsObject(prototype=ObjectPrototype)
+    return val.to_object()
+
+
+@Js
+def object_constructor():
+    if len(arguments):
+        val = arguments.get('0')
+        if val.TYPE=='Object':
+            #Implementation dependent, but my will simply return :)
+            return val
+        elif val.TYPE in ('Number', 'String', 'Boolean'):
+            return val.to_object()
+    return PyJsObject(prototype=ObjectPrototype)
+
+Object.create = object_constructor
+Object.own['length']['value'] = Js(1)
+
+
+class ObjectMethods:
+    def getPrototypeOf(obj):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.getPrototypeOf called on non-object')
+        return null if obj.prototype is None else obj.prototype
+
+    def getOwnPropertyDescriptor (obj, prop):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.getOwnPropertyDescriptor called on non-object')
+        return obj.own.get(prop.to_string().value) # will return undefined if we dont have this prop
+
+    def getOwnPropertyNames(obj):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.getOwnPropertyDescriptor called on non-object')
+        return obj.own.keys()
+
+    def create(obj):
+        if not (obj.is_object() or obj.is_null()):
+            raise MakeError('TypeError', 'Object prototype may only be an Object or null')
+        temp = PyJsObject(prototype=(None if obj.is_null() else obj))
+        if len(arguments)>1 and not arguments[1].is_undefined():
+            if six.PY2:
+                ObjectMethods.defineProperties.__func__(temp, arguments[1])
+            else:
+                ObjectMethods.defineProperties(temp, arguments[1])
+        return temp
+
+    def defineProperty(obj, prop, attrs):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.defineProperty called on non-object')
+        name = prop.to_string().value
+        if not obj.define_own_property(name, ToPropertyDescriptor(attrs)):
+            raise MakeError('TypeError', 'Cannot redefine property: %s' % name)
+        return obj
+
+    def defineProperties(obj, properties):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.defineProperties called on non-object')
+        props = properties.to_object()
+        for name in props:
+            desc = ToPropertyDescriptor(props.get(name.value))
+            if not obj.define_own_property(name.value, desc):
+                raise MakeError('TypeError', 'Failed to define own property: %s'%name.value)
+        return obj
+
+    def seal(obj):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.seal called on non-object')
+        for desc in obj.own.values():
+            desc['configurable'] = False
+        obj.extensible = False
+        return obj
+
+    def freeze(obj):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.freeze called on non-object')
+        for desc in obj.own.values():
+            desc['configurable'] = False
+            if is_data_descriptor(desc):
+                desc['writable'] = False
+        obj.extensible = False
+        return obj
+
+    def preventExtensions(obj):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.preventExtensions on non-object')
+        obj.extensible = False
+        return obj
+
+    def isSealed(obj):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.isSealed called on non-object')
+        if obj.extensible:
+            return False
+        for desc in obj.own.values():
+            if desc['configurable']:
+                return False
+        return True
+
+    def isFrozen(obj):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.isFrozen called on non-object')
+        if obj.extensible:
+            return False
+        for desc in obj.own.values():
+            if desc['configurable']:
+                return False
+            if is_data_descriptor(desc) and desc['writable']:
+                return False
+        return True
+
+    def isExtensible(obj):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.isExtensible called on non-object')
+        return obj.extensible
+
+    def keys(obj):
+        if not obj.is_object():
+            raise MakeError('TypeError', 'Object.keys called on non-object')
+        return [e for e,d in six.iteritems(obj.own) if d.get('enumerable')]
+
+
+# add methods attached to Object constructor
+fill_prototype(Object, ObjectMethods, default_attrs)
+# add constructor to prototype
+fill_in_props(ObjectPrototype, {'constructor':Object}, default_attrs)
+# add prototype property to the constructor.
+Object.define_own_property('prototype', {'value': ObjectPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+
+
+# some utility functions:
+
+def ToPropertyDescriptor(obj):  # page 38 (50 absolute)
+    if obj.TYPE!='Object':
+        raise MakeError('TypeError', 'Can\'t convert non-object to property descriptor')
+    desc = {}
+    if obj.has_property('enumerable'):
+        desc['enumerable'] = obj.get('enumerable').to_boolean().value
+    if obj.has_property('configurable'):
+        desc['configurable'] = obj.get('configurable').to_boolean().value
+    if obj.has_property('value'):
+        desc['value'] = obj.get('value')
+    if obj.has_property('writable'):
+        desc['writable'] = obj.get('writable').to_boolean().value
+    if obj.has_property('get'):
+        cand =  obj.get('get')
+        if not (cand.is_undefined() or cand.is_callable()):
+            raise MakeError('TypeError', 'Invalid getter (it has to be a function or undefined)')
+        desc['get'] = cand
+    if obj.has_property('set'):
+        cand =  obj.get('set')
+        if not (cand.is_undefined() or cand.is_callable()):
+            raise MakeError('TypeError', 'Invalid setter (it has to be a function or undefined)')
+        desc['set'] = cand
+    if ('get' in desc or 'set' in desc) and ('value' in desc or 'writable' in desc):
+        raise MakeError('TypeError', 'Invalid property.  A property cannot both have accessors and be writable or have a value.')
+    return desc
+
+
+

lib/js2py/constructors/jsregexp.py

@@ -0,0 +1,11 @@
+from ..base import *
+
+RegExpPrototype.define_own_property('constructor', {'value': RegExp,
+                                                     'enumerable': False,
+                                                     'writable': True,
+                                                     'configurable': True})
+
+RegExp.define_own_property('prototype', {'value': RegExpPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})

lib/js2py/constructors/jsstring.py

@@ -0,0 +1,30 @@
+from ..base import *
+# python 3 support
+import six
+if six.PY3:
+    unichr = chr
+
+@Js
+def fromCharCode():
+    args = arguments.to_list()
+    res = u''
+    for e in args:
+        res +=unichr(e.to_uint16())
+    return this.Js(res)
+
+fromCharCode.own['length']['value'] = Js(1)
+
+String.define_own_property('fromCharCode', {'value': fromCharCode,
+                                         'enumerable': False,
+                                         'writable': True,
+                                         'configurable': True})
+
+String.define_own_property('prototype', {'value': StringPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+StringPrototype.define_own_property('constructor', {'value': String,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})

lib/js2py/constructors/jsuint16array.py

@@ -0,0 +1,68 @@
+# this is based on jsarray.py
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+@Js
+def Uint16Array():
+    TypedArray = (PyJsInt8Array,PyJsUint8Array,PyJsUint8ClampedArray,PyJsInt16Array,PyJsUint16Array,PyJsInt32Array,PyJsUint32Array,PyJsFloat32Array,PyJsFloat64Array)
+    a = arguments[0]
+    if isinstance(a, PyJsNumber): # length
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(numpy.full(length, 0, dtype=numpy.uint16))
+        temp.put('length', a)
+        return temp
+    elif isinstance(a, PyJsString): # object (string)
+        temp = Js(numpy.array(list(a.value), dtype=numpy.uint16))
+        temp.put('length', Js(len(list(a.value))))
+        return temp
+    elif isinstance(a, PyJsArray) or isinstance(a,TypedArray) or isinstance(a,PyJsArrayBuffer): # object (Array, TypedArray)
+        array = a.to_list()
+        array = [(int(item.value) if item.value != None else 0) for item in array]
+        temp = Js(numpy.array(array, dtype=numpy.uint16))
+        temp.put('length', Js(len(array)))
+        return temp
+    elif isinstance(a,PyObjectWrapper): # object (ArrayBuffer, etc)
+        if len(a.obj) % 2 != 0:
+            raise MakeError('RangeError', 'Byte length of Uint16Array should be a multiple of 2')
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+            if offset % 2 != 0:
+                raise MakeError('RangeError', 'Start offset of Uint16Array should be a multiple of 2')
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = int((len(a.obj)-offset)/2)
+        array = numpy.frombuffer(a.obj, dtype=numpy.uint16, count=length, offset=offset)
+        temp = Js(array)
+        temp.put('length', Js(length))
+        temp.buff = array
+        return temp
+    temp = Js(numpy.full(0, 0, dtype=numpy.uint16))
+    temp.put('length', Js(0))
+    return temp
+
+Uint16Array.create = Uint16Array
+Uint16Array.own['length']['value'] = Js(3)
+
+Uint16Array.define_own_property('prototype', {'value': Uint16ArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+Uint16ArrayPrototype.define_own_property('constructor', {'value': Uint16Array,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})
+
+Uint16ArrayPrototype.define_own_property('BYTES_PER_ELEMENT', {'value': Js(2),
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': False})

lib/js2py/constructors/jsuint32array.py

@@ -0,0 +1,74 @@
+# this is based on jsarray.py
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+@Js
+def Uint32Array():
+    TypedArray = (PyJsInt8Array,PyJsUint8Array,PyJsUint8ClampedArray,PyJsInt16Array,PyJsUint16Array,PyJsInt32Array,PyJsUint32Array,PyJsFloat32Array,PyJsFloat64Array)
+    a = arguments[0]
+    if isinstance(a, PyJsNumber): # length
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(numpy.full(length, 0, dtype=numpy.uint32))
+        temp.put('length', a)
+        return temp
+    elif isinstance(a, PyJsString): # object (string)
+        temp = Js(numpy.array(list(a.value), dtype=numpy.uint32))
+        temp.put('length', Js(len(list(a.value))))
+        return temp
+    elif isinstance(a, PyJsArray) or isinstance(a,TypedArray) or isinstance(a,PyJsArrayBuffer): # object (Array, TypedArray)
+        array = a.to_list()
+        array = [(int(item.value) if item.value != None else 0) for item in array]
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = len(array)-offset
+        temp = Js(numpy.array(array[offset:offset+length], dtype=numpy.uint32))
+        temp.put('length', Js(length))
+        return temp
+    elif isinstance(a,PyObjectWrapper): # object (ArrayBuffer, etc)
+        if len(a.obj) % 4 != 0:
+            raise MakeError('RangeError', 'Byte length of Uint32Array should be a multiple of 4')
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+            if offset % 4 != 0:
+                raise MakeError('RangeError', 'Start offset of Uint32Array should be a multiple of 4')
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = int((len(a.obj)-offset)/4)
+        temp = Js(numpy.frombuffer(a.obj, dtype=numpy.uint32, count=length, offset=offset))
+        temp.put('length', Js(length))
+        return temp
+    temp = Js(numpy.full(0, 0, dtype=numpy.uint32))
+    temp.put('length', Js(0))
+    return temp
+
+Uint32Array.create = Uint32Array
+Uint32Array.own['length']['value'] = Js(3)
+
+Uint32Array.define_own_property('prototype', {'value': Uint32ArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+Uint32ArrayPrototype.define_own_property('constructor', {'value': Uint32Array,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})
+
+Uint32ArrayPrototype.define_own_property('BYTES_PER_ELEMENT', {'value': Js(4),
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': False})

lib/js2py/constructors/jsuint8array.py

@@ -0,0 +1,65 @@
+# this is based on jsarray.py
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+
+@Js
+def Uint8Array():
+    TypedArray = (PyJsInt8Array,PyJsUint8Array,PyJsUint8ClampedArray,PyJsInt16Array,PyJsUint16Array,PyJsInt32Array,PyJsUint32Array,PyJsFloat32Array,PyJsFloat64Array)
+    a = arguments[0]
+    if isinstance(a, PyJsNumber): # length
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(numpy.full(length, 0, dtype=numpy.uint8))
+        temp.put('length', a)
+        return temp
+    elif isinstance(a, PyJsString): # object (string)
+        temp = Js(numpy.array(list(a.value), dtype=numpy.uint8))
+        temp.put('length', Js(len(list(a.value))))
+        return temp
+    elif isinstance(a, PyJsArray) or isinstance(a,TypedArray) or isinstance(a,PyJsArrayBuffer): # object (Array, TypedArray)
+        array = a.to_list()
+        array = [(int(item.value) if item.value != None else 0) for item in array]
+        temp = Js(numpy.array(array, dtype=numpy.uint8))
+        temp.put('length', Js(len(array)))
+        return temp
+    elif isinstance(a,PyObjectWrapper): # object (ArrayBuffer, etc)
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = int(len(a.obj)-offset)
+        array = numpy.frombuffer(a.obj, dtype=numpy.uint8, count=length, offset=offset)
+        temp = Js(array)
+        temp.put('length', Js(length))
+        temp.buff = array
+        return temp
+    temp = Js(numpy.full(0, 0, dtype=numpy.uint8))
+    temp.put('length', Js(0))
+    return temp
+
+Uint8Array.create = Uint8Array
+Uint8Array.own['length']['value'] = Js(3)
+
+Uint8Array.define_own_property('prototype', {'value': Uint8ArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+Uint8ArrayPrototype.define_own_property('constructor', {'value': Uint8Array,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})
+
+Uint8ArrayPrototype.define_own_property('BYTES_PER_ELEMENT', {'value': Js(1),
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': False})

lib/js2py/constructors/jsuint8clampedarray.py

@@ -0,0 +1,65 @@
+# this is based on jsarray.py
+
+from ..base import *
+try:
+    import numpy
+except:
+    pass
+
+
+@Js
+def Uint8ClampedArray():
+    TypedArray = (PyJsInt8Array,PyJsUint8Array,PyJsUint8ClampedArray,PyJsInt16Array,PyJsUint16Array,PyJsInt32Array,PyJsUint32Array,PyJsFloat32Array,PyJsFloat64Array)
+    a = arguments[0]
+    if isinstance(a, PyJsNumber): # length
+        length = a.to_uint32()
+        if length!=a.value:
+            raise MakeError('RangeError', 'Invalid array length')
+        temp = Js(numpy.full(length, 0, dtype=numpy.uint8), Clamped=True)
+        temp.put('length', a)
+        return temp
+    elif isinstance(a, PyJsString): # object (string)
+        temp = Js(numpy.array(list(a.value), dtype=numpy.uint8), Clamped=True)
+        temp.put('length', Js(len(list(a.value))))
+        return temp
+    elif isinstance(a, PyJsArray) or isinstance(a,TypedArray) or isinstance(a,PyJsArrayBuffer): # object (Array, TypedArray)
+        array = a.to_list()
+        array = [(int(item.value) if item.value != None else 0) for item in array]
+        temp = Js(numpy.array(array, dtype=numpy.uint8), Clamped=True)
+        temp.put('length', Js(len(array)))
+        return temp
+    elif isinstance(a,PyObjectWrapper): # object (ArrayBuffer, etc)
+        if len(arguments) > 1:
+            offset = int(arguments[1].value)
+        else:
+            offset = 0
+        if len(arguments) > 2:
+            length = int(arguments[2].value)
+        else:
+            length = int(len(a.obj)-offset)
+        array = numpy.frombuffer(a.obj, dtype=numpy.uint8, count=length, offset=offset)
+        temp = Js(array, Clamped=True)
+        temp.put('length', Js(length))
+        temp.buff = array
+        return temp
+    temp = Js(numpy.full(0, 0, dtype=numpy.uint8), Clamped=True)
+    temp.put('length', Js(0))
+    return temp
+
+Uint8ClampedArray.create = Uint8ClampedArray
+Uint8ClampedArray.own['length']['value'] = Js(3)
+
+Uint8ClampedArray.define_own_property('prototype', {'value': Uint8ClampedArrayPrototype,
+                                         'enumerable': False,
+                                         'writable': False,
+                                         'configurable': False})
+
+Uint8ClampedArrayPrototype.define_own_property('constructor', {'value': Uint8ClampedArray,
+                                                    'enumerable': False,
+                                                    'writable': True,
+                                                    'configurable': True})
+
+Uint8ClampedArrayPrototype.define_own_property('BYTES_PER_ELEMENT', {'value': Js(1),
+                                                    'enumerable': False,
+                                                    'writable': False,
+                                                    'configurable': False})

lib/js2py/constructors/time_helpers.py

@@ -0,0 +1,184 @@
+# NOTE: t must be INT!!!
+import time
+import datetime
+import warnings
+
+try:
+    from tzlocal import get_localzone
+    LOCAL_ZONE = get_localzone()
+except: # except all problems...
+    warnings.warn('Please install or fix tzlocal library (pip install tzlocal) in order to make Date object work better. Otherwise I will assume DST is in effect all the time')
+    class LOCAL_ZONE:
+        @staticmethod
+        def dst(*args):
+            return 1
+
+from js2py.base import MakeError
+CUM = (0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365)
+msPerDay = 86400000
+msPerYear = int(86400000*365.242)
+msPerSecond = 1000
+msPerMinute = 60000
+msPerHour = 3600000
+HoursPerDay = 24
+MinutesPerHour = 60
+SecondsPerMinute = 60
+NaN = float('nan')
+LocalTZA = - time.timezone * msPerSecond
+
+
+
+
+
+def DaylightSavingTA(t):
+    if t is NaN:
+        return t
+    try:
+        return int(LOCAL_ZONE.dst(datetime.datetime.utcfromtimestamp(t//1000)).seconds)*1000
+    except:
+        warnings.warn('Invalid datetime date, assumed DST time, may be inaccurate...', Warning)
+        return 1
+        #raise MakeError('TypeError', 'date not supported by python.datetime. I will solve it in future versions')
+
+def GetTimeZoneName(t):
+    return time.tzname[DaylightSavingTA(t)>0]
+
+def LocalToUTC(t):
+    return t - LocalTZA - DaylightSavingTA(t - LocalTZA)
+
+def UTCToLocal(t):
+    return t + LocalTZA + DaylightSavingTA(t)
+
+
+def Day(t):
+    return t//86400000
+
+
+def TimeWithinDay(t):
+    return t%86400000
+
+def DaysInYear(y):
+    if y%4:
+        return 365
+    elif y%100:
+        return 366
+    elif y%400:
+        return 365
+    else:
+        return 366
+
+
+def DayFromYear(y):
+    return 365 * (y-1970) + (y-1969)//4 -(y-1901)//100 + (y-1601)//400
+
+def TimeFromYear(y):
+    return 86400000 * DayFromYear(y)
+
+def YearFromTime(t):
+    guess = 1970 - t//31556908800  # msPerYear
+    gt = TimeFromYear(guess)
+    if gt<=t:
+        while gt<=t:
+            guess += 1
+            gt = TimeFromYear(guess)
+        return guess-1
+    else:
+        while gt>t:
+            guess -= 1
+            gt = TimeFromYear(guess)
+        return guess
+
+
+def DayWithinYear(t):
+    return Day(t) - DayFromYear(YearFromTime(t))
+
+def InLeapYear(t):
+    y = YearFromTime(t)
+    if y%4:
+        return 0
+    elif y%100:
+        return 1
+    elif y%400:
+        return 0
+    else:
+        return 1
+
+def MonthFromTime(t):
+    day = DayWithinYear(t)
+    leap = InLeapYear(t)
+    if day<31:
+        return 0
+    day -= leap
+    if day<59:
+        return 1
+    elif day<90:
+        return 2
+    elif day<120:
+        return 3
+    elif day<151:
+        return 4
+    elif day<181:
+        return 5
+    elif day<212:
+        return 6
+    elif day<243:
+        return 7
+    elif day<273:
+        return 8
+    elif day<304:
+        return 9
+    elif day<334:
+        return 10
+    else:
+        return 11
+
+def DateFromTime(t):
+    mon = MonthFromTime(t)
+    day = DayWithinYear(t)
+    return day-CUM[mon] - (1 if InLeapYear(t) and mon>=2 else 0) + 1
+
+def WeekDay(t):
+    # 0 == sunday
+    return (Day(t) + 4) % 7
+
+def msFromTime(t):
+    return t % 1000
+
+def SecFromTime(t):
+    return (t//1000) % 60
+
+def MinFromTime(t):
+    return (t//60000) % 60
+
+def HourFromTime(t):
+    return (t//3600000) % 24
+
+def MakeTime (hour, Min, sec, ms):
+    # takes PyJs objects and returns t
+    if not (hour.is_finite() and Min.is_finite() and sec.is_finite() and ms.is_finite()):
+        return NaN
+    h, m, s, milli = hour.to_int(), Min.to_int(), sec.to_int(), ms.to_int()
+    return h*3600000 + m*60000 + s*1000 + milli
+
+
+def MakeDay(year, month, date):
+    # takes PyJs objects and returns t
+    if not (year.is_finite() and month.is_finite() and date.is_finite()):
+        return NaN
+    y, m, dt = year.to_int(), month.to_int(), date.to_int()
+    y += m//12
+    mn = m % 12
+    d = DayFromYear(y) + CUM[mn] + dt - 1 + (1 if DaysInYear(y)==366 and mn>=2 else 0)
+    return  d # ms per day
+
+def MakeDate(day, time):
+    return 86400000*day + time
+
+
+def TimeClip(t):
+    if t!=t or abs(t)==float('inf'):
+        return NaN
+    if abs(t) > 8.64 *10**15:
+        return NaN
+    return int(t)
+

lib/js2py/es6/__init__.py

@@ -0,0 +1,32 @@
+INITIALISED = False
+babel = None
+babelPresetEs2015 = None
+
+def js6_to_js5(code):
+    global INITIALISED, babel, babelPresetEs2015
+    if not INITIALISED:
+        import signal, warnings, time
+        warnings.warn('\nImporting babel.py for the first time - this can take some time. \nPlease note that currently Javascript 6 in Js2Py is unstable and slow. Use only for tiny scripts!')
+
+        from .babel import babel as _babel
+        babel = _babel.Object.babel
+        babelPresetEs2015 = _babel.Object.babelPresetEs2015
+
+        # very weird hack. Somehow this helps babel to initialise properly!
+        try:
+            babel.transform('warmup', {'presets': {}})
+            signal.alarm(2)
+            def kill_it(a,b): raise KeyboardInterrupt('Better work next time!')
+            signal.signal(signal.SIGALRM, kill_it)
+            babel.transform('stuckInALoop', {'presets': babelPresetEs2015}).code
+            for n in range(3):
+                time.sleep(1)
+        except:
+            print("Initialised babel!")
+        INITIALISED = True
+    return babel.transform(code, {'presets': babelPresetEs2015}).code
+
+if __name__=='__main__':
+    print(js6_to_js5('obj={}; obj.x = function() {return () => this}'))
+    print()
+    print(js6_to_js5('const a = 1;'))

lib/js2py/es6/babel.js

@@ -0,0 +1,6 @@
+// run buildBabel in this folder to convert this code to python!
+var babel = require("babel-core");
+var babelPresetEs2015 = require("babel-preset-es2015");
+
+Object.babelPresetEs2015 = babelPresetEs2015;
+Object.babel = babel;

lib/js2py/es6/buildBabel

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+
+# install all the dependencies and pack everything into one file babel_bundle.js (converted to es2015).
+npm install babel-core babel-cli babel-preset-es2015 browserify
+browserify babel.js -o babel_bundle.js -t [ babelify --presets [ es2015 ] ]
+
+# translate babel_bundle.js using js2py -> generates babel.py
+echo "Generating babel.py..."
+python -c "import js2py;js2py.translate_file('babel_bundle.js', 'babel.py');"
+rm babel_bundle.js
+

lib/js2py/evaljs.py

@@ -0,0 +1,258 @@
+# coding=utf-8
+from .translators import translate_js, DEFAULT_HEADER
+from .es6 import js6_to_js5
+import sys
+import time
+import json
+import six
+import os
+import hashlib
+import codecs
+
+__all__  = ['EvalJs', 'translate_js', 'import_js', 'eval_js', 'translate_file', 'eval_js6', 'translate_js6', 'run_file', 'disable_pyimport', 'get_file_contents', 'write_file_contents']
+DEBUG = False
+
+def disable_pyimport():
+    import pyjsparser.parser
+    pyjsparser.parser.ENABLE_PYIMPORT = False
+
+def path_as_local(path):
+    if os.path.isabs(path):
+        return path
+    # relative to cwd
+    return os.path.join(os.getcwd(), path)
+
+def import_js(path, lib_name, globals):
+    """Imports from javascript source file.
+      globals is your globals()"""
+    with codecs.open(path_as_local(path), "r", "utf-8") as f:
+        js = f.read()
+    e = EvalJs()
+    e.execute(js)
+    var = e.context['var']
+    globals[lib_name] = var.to_python()
+
+
+def get_file_contents(path_or_file):
+    if hasattr(path_or_file, 'read'):
+        js = path_or_file.read()
+    else:
+        with codecs.open(path_as_local(path_or_file), "r", "utf-8") as f:
+            js = f.read()
+    return js
+
+
+def write_file_contents(path_or_file, contents):
+    if hasattr(path_or_file, 'write'):
+        path_or_file.write(contents)
+    else:
+        with open(path_as_local(path_or_file), 'w') as f:
+            f.write(contents)
+
+def translate_file(input_path, output_path):
+    '''
+    Translates input JS file to python and saves the it to the output path.
+    It appends some convenience code at the end so that it is easy to import JS objects.
+
+    For example we have a file 'example.js' with:   var a = function(x) {return x}
+    translate_file('example.js', 'example.py')
+
+    Now example.py can be easily importend and used:
+    >>> from example import example
+    >>> example.a(30)
+    30
+    '''
+    js = get_file_contents(input_path)
+
+    py_code = translate_js(js)
+    lib_name = os.path.basename(output_path).split('.')[0]
+    head = '__all__ = [%s]\n\n# Don\'t look below, you will not understand this Python code :) I don\'t.\n\n' % repr(lib_name)
+    tail = '\n\n# Add lib to the module scope\n%s = var.to_python()' % lib_name
+    out = head + py_code + tail
+    write_file_contents(output_path, out)
+
+
+
+
+
+
+def run_file(path_or_file, context=None):
+    ''' Context must be EvalJS object. Runs given path as a JS program. Returns (eval_value, context).
+    '''
+    if context is None:
+        context = EvalJs()
+    if not isinstance(context, EvalJs):
+        raise TypeError('context must be the instance of EvalJs')
+    eval_value = context.eval(get_file_contents(path_or_file))
+    return eval_value, context
+
+
+
+def eval_js(js):
+    """Just like javascript eval. Translates javascript to python,
+       executes and returns python object.
+       js is javascript source code
+
+       EXAMPLE:
+        >>> import js2py
+        >>> add = js2py.eval_js('function add(a, b) {return a + b}')
+        >>> add(1, 2) + 3
+        6
+        >>> add('1', 2, 3)
+        u'12'
+        >>> add.constructor
+        function Function() { [python code] }
+
+       NOTE: For Js Number, String, Boolean and other base types returns appropriate python BUILTIN type.
+       For Js functions and objects, returns Python wrapper - basically behaves like normal python object.
+       If you really want to convert object to python dict you can use to_dict method.
+       """
+    e = EvalJs()
+    return e.eval(js)
+
+def eval_js6(js):
+    return eval_js(js6_to_js5(js))
+
+
+def translate_js6(js):
+    return translate_js(js6_to_js5(js))
+
+
+
+class EvalJs(object):
+    """This class supports continuous execution of javascript under same context.
+
+        >>> js = EvalJs()
+        >>> js.execute('var a = 10;function f(x) {return x*x};')
+        >>> js.f(9)
+        81
+        >>> js.a
+        10
+
+        context is a python dict or object that contains python variables that should be available to JavaScript
+        For example:
+        >>> js = EvalJs({'a': 30})
+        >>> js.execute('var x = a')
+        >>> js.x
+        30
+
+       You can run interactive javascript console with console method!"""
+    def __init__(self, context={}):
+        self.__dict__['_context'] = {}
+        exec(DEFAULT_HEADER, self._context)
+        self.__dict__['_var'] = self._context['var'].to_python()
+        if not isinstance(context, dict):
+            try:
+                context = context.__dict__
+            except:
+                raise TypeError('context has to be either a dict or have __dict__ attr')
+        for k, v in six.iteritems(context):
+            setattr(self._var, k, v)
+
+    def execute(self, js=None, use_compilation_plan=False):
+        """executes javascript js in current context
+
+        During initial execute() the converted js is cached for re-use. That means next time you
+        run the same javascript snippet you save many instructions needed to parse and convert the
+        js code to python code.
+
+        This cache causes minor overhead (a cache dicts is updated) but the Js=>Py conversion process
+        is typically expensive compared to actually running the generated python code.
+
+        Note that the cache is just a dict, it has no expiration or cleanup so when running this
+        in automated situations with vast amounts of snippets it might increase memory usage.
+        """
+        try:
+            cache = self.__dict__['cache']
+        except KeyError:
+            cache = self.__dict__['cache'] = {}
+        hashkey = hashlib.md5(js.encode('utf-8')).digest()
+        try:
+            compiled = cache[hashkey]
+        except KeyError:
+            code = translate_js(js, '', use_compilation_plan=use_compilation_plan)
+            compiled = cache[hashkey] = compile(code, '<EvalJS snippet>', 'exec')
+        exec(compiled, self._context)
+
+    def eval(self, expression, use_compilation_plan=False):
+        """evaluates expression in current context and returns its value"""
+        code = 'PyJsEvalResult = eval(%s)'%json.dumps(expression)
+        self.execute(code, use_compilation_plan=use_compilation_plan)
+        return self['PyJsEvalResult']
+
+    def execute_debug(self, js):
+        """executes javascript js in current context
+        as opposed to the (faster) self.execute method, you can use your regular debugger
+        to set breakpoints and inspect the generated python code
+        """
+        code = translate_js(js, '')
+        # make sure you have a temp folder:
+        filename = 'temp' + os.sep + '_' + hashlib.md5(code).hexdigest() + '.py'
+        try:
+            with open(filename, mode='w') as f:
+                f.write(code)
+            execfile(filename, self._context)
+        except Exception as err:
+            raise err
+        finally:
+            os.remove(filename)
+            try:
+                os.remove(filename + 'c')
+            except:
+                pass
+
+    def eval_debug(self, expression):
+        """evaluates expression in current context and returns its value
+        as opposed to the (faster) self.execute method, you can use your regular debugger
+        to set breakpoints and inspect the generated python code
+        """
+        code = 'PyJsEvalResult = eval(%s)'%json.dumps(expression)
+        self.execute_debug(code)
+        return self['PyJsEvalResult']
+
+    def __getattr__(self, var):
+        return getattr(self._var, var)
+
+    def __getitem__(self, var):
+        return getattr(self._var, var)
+
+    def __setattr__(self, var, val):
+        return setattr(self._var, var, val)
+
+    def __setitem__(self, var, val):
+        return setattr(self._var, var, val)
+
+    def console(self):
+        """starts to interact (starts interactive console) Something like code.InteractiveConsole"""
+        while True:
+            if six.PY2:
+                code = raw_input('>>> ')
+            else:
+                code = input('>>>')
+            try:
+                print(self.eval(code))
+            except KeyboardInterrupt:
+                break
+            except Exception as e:
+                import traceback
+                if DEBUG:
+                    sys.stderr.write(traceback.format_exc())
+                else:
+                    sys.stderr.write('EXCEPTION: '+str(e)+'\n')
+                time.sleep(0.01)
+
+
+
+
+#print x
+
+
+
+if __name__=='__main__':
+    #with open('C:\Users\Piotrek\Desktop\esprima.js', 'rb') as f:
+    #    x = f.read()
+    e = EvalJs()
+    e.execute('square(x)')
+    #e.execute(x)
+    e.console()
+

lib/js2py/host/console.py

@@ -0,0 +1,11 @@
+from ..base import *
+
+@Js
+def console():
+    pass
+
+@Js
+def log():
+    print(arguments[0])
+
+console.put('log', log)

lib/js2py/host/dom/constants.py

@@ -0,0 +1,47 @@
+from js2py.base import *
+
+def _get_conts(idl):
+    def is_valid(c):
+        try:
+            exec(c)
+            return 1
+        except:
+            pass
+    return '\n'.join(filter(is_valid, (' '.join(e.strip(' ;').split()[-3:]) for e in idl.splitlines())))
+
+
+default_attrs = {'writable':True, 'enumerable':True, 'configurable':True}
+
+
+def compose_prototype(Class, attrs=default_attrs):
+    prototype = Class()
+    for i in dir(Class):
+        e = getattr(Class, i)
+        if hasattr(e, '__func__'):
+            temp = PyJsFunction(e.__func__, FunctionPrototype)
+            attrs = {k:v for k,v in attrs.iteritems()}
+            attrs['value'] = temp
+            prototype.define_own_property(i, attrs)
+    return prototype
+
+
+# Error codes
+
+INDEX_SIZE_ERR = 1
+DOMSTRING_SIZE_ERR = 2
+HIERARCHY_REQUEST_ERR = 3
+WRONG_DOCUMENT_ERR = 4
+INVALID_CHARACTER_ERR = 5
+NO_DATA_ALLOWED_ERR = 6
+NO_MODIFICATION_ALLOWED_ERR = 7
+NOT_FOUND_ERR = 8
+NOT_SUPPORTED_ERR = 9
+INUSE_ATTRIBUTE_ERR = 10
+INVALID_STATE_ERR = 11
+SYNTAX_ERR = 12
+INVALID_MODIFICATION_ERR = 13
+NAMESPACE_ERR = 14
+INVALID_ACCESS_ERR = 15
+VALIDATION_ERR = 16
+TYPE_MISMATCH_ERR = 17
+

lib/js2py/host/dom/interface.py

@@ -0,0 +1,73 @@
+from StringIO import StringIO
+from constants import *
+from bs4 import BeautifulSoup
+from js2py.base import *
+try:
+    import lxml
+    def parse(source):
+        return BeautifulSoup(source, 'lxml')
+except:
+    def parse(source):
+        return BeautifulSoup(source)
+
+
+
+
+
+
+
+x = '''<table>
+  <tbody>
+    <tr>
+      <td>Shady Grove</td>
+      <td>Aeolian</td>
+    </tr>
+    <tr>
+      <td>Over the River, Charlie</td>
+      <td>Dorian</td>
+    </tr>
+  </tbody>
+</table>'''
+
+
+
+class DOM(PyJs):
+    prototype = ObjectPrototype
+    def __init__(self):
+        self.own = {}
+
+    def readonly(self, name, val):
+        self.define_own_property(name, {'writable':False, 'enumerable':False, 'configurable':False, 'value': Js(val)})
+
+
+
+# DOMStringList
+
+class DOMStringListPrototype(DOM):
+    Class = 'DOMStringListPrototype'
+
+    def contains(element):
+        return element.to_string().value in this._string_list
+
+    def item(index):
+        return this._string_list[index.to_int()] if 0<=index.to_int()<len(this._string_list) else index.null
+
+
+class DOMStringList(DOM):
+    Class = 'DOMStringList'
+    prototype = compose_prototype(DOMStringListPrototype)
+    def __init__(self, _string_list):
+        self.own = {}
+
+        self._string_list = _string_list
+
+
+# NameList
+
+class NameListPrototype(DOM):
+
+
+
+
+
+

lib/js2py/host/jseval.py

@@ -0,0 +1,50 @@
+from ..base import *
+import inspect
+try:
+    from js2py.translators.translator import translate_js
+except:
+    pass
+
+@Js
+def Eval(code):
+    local_scope = inspect.stack()[3][0].f_locals['var']
+    global_scope = this.GlobalObject
+    # todo fix scope - we have to behave differently if called through variable other than eval
+    # we will use local scope (default)
+    globals()['var'] = local_scope
+    try:
+        py_code = translate_js(code.to_string().value, '')
+    except SyntaxError as syn_err:
+        raise MakeError('SyntaxError', str(syn_err))
+    lines = py_code.split('\n')
+    # a simple way to return value from eval. Will not work in complex cases.
+    has_return = False
+    for n in xrange(len(lines)):
+        line = lines[len(lines)-n-1]
+        if line.strip():
+            if line.startswith(' '):
+                break
+            elif line.strip()=='pass':
+                continue
+            elif any(line.startswith(e) for e in ['return ', 'continue ', 'break', 'raise ']):
+                break
+            else:
+                has_return = True
+                cand = 'EVAL_RESULT = (%s)\n'%line
+                try:
+                    compile(cand, '', 'exec')
+                except SyntaxError:
+                    break
+                lines[len(lines)-n-1] = cand
+                py_code = '\n'.join(lines)
+                break
+    #print py_code
+    executor(py_code)
+    if has_return:
+        return globals()['EVAL_RESULT']
+
+
+
+def executor(code):
+    exec(code, globals())
+

lib/js2py/host/jsfunctions.py

@@ -0,0 +1,85 @@
+from ..base import *
+
+RADIX_CHARS = {'1': 1, '0': 0, '3': 3, '2': 2, '5': 5, '4': 4, '7': 7, '6': 6, '9': 9, '8': 8, 'a': 10, 'c': 12,
+               'b': 11, 'e': 14, 'd': 13, 'g': 16, 'f': 15, 'i': 18, 'h': 17, 'k': 20, 'j': 19, 'm': 22, 'l': 21,
+               'o': 24, 'n': 23, 'q': 26, 'p': 25, 's': 28, 'r': 27, 'u': 30, 't': 29, 'w': 32, 'v': 31, 'y': 34,
+               'x': 33, 'z': 35, 'A': 10, 'C': 12, 'B': 11, 'E': 14, 'D': 13, 'G': 16, 'F': 15, 'I': 18, 'H': 17,
+               'K': 20, 'J': 19, 'M': 22, 'L': 21, 'O': 24, 'N': 23, 'Q': 26, 'P': 25, 'S': 28, 'R': 27, 'U': 30,
+               'T': 29, 'W': 32, 'V': 31, 'Y': 34, 'X': 33, 'Z': 35}
+@Js
+def parseInt (string , radix):
+    string = string.to_string().value.lstrip()
+    sign = 1
+    if string and string[0] in ('+', '-'):
+        if string[0]=='-':
+            sign = -1
+        string = string[1:]
+    r = radix.to_int32()
+    strip_prefix = True
+    if r:
+        if r<2 or r>36:
+            return NaN
+        if r!=16:
+            strip_prefix = False
+    else:
+        r = 10
+    if strip_prefix:
+        if len(string)>=2 and string[:2] in ('0x', '0X'):
+            string = string[2:]
+            r = 16
+    n = 0
+    num  = 0
+    while n<len(string):
+        cand = RADIX_CHARS.get(string[n])
+        if cand is None or not cand < r:
+            break
+        num = cand + num*r
+        n += 1
+    if not n:
+        return NaN
+    return sign*num
+
+@Js
+def parseFloat(string):
+    string = string.to_string().value.strip()
+    sign = 1
+    if string and string[0] in ('+', '-'):
+        if string[0]=='-':
+            sign = -1
+        string = string[1:]
+    num = None
+    length = 1
+    max_len = None
+    failed = 0
+    while length<=len(string):
+        try:
+            num = float(string[:length])
+            max_len = length
+            failed = 0
+        except:
+            failed += 1
+            if failed>4: # cant be a number anymore
+                break
+        length += 1
+    if num is None:
+        return NaN
+    return sign*float(string[:max_len])
+
+@Js
+def isNaN(number):
+    if number.to_number().is_nan():
+        return true
+    return false
+
+@Js
+def isFinite(number):
+    num = number.to_number()
+    if num.is_nan() or num.is_infinity():
+        return false
+    return true
+
+
+#todo URI handling!
+
+
+

lib/js2py/prototypes/__init__.py

@@ -0,0 +1 @@
+__author__ = 'Piotr Dabkowski'

lib/js2py/prototypes/jsarray.py

@@ -0,0 +1,458 @@
+import six
+
+if six.PY3:
+    xrange = range
+    import functools
+
+def to_arr(this):
+    """Returns Python array from Js array"""
+    return [this.get(str(e)) for e in xrange(len(this))]
+
+
+ARR_STACK = set({})
+
+class ArrayPrototype:
+    def toString():
+        # this function is wrong but I will leave it here fore debugging purposes.
+        func = this.get('join')
+        if not func.is_callable():
+            @this.Js
+            def func():
+                return '[object %s]'%this.Class
+        return func.call(this, ())
+
+    def toLocaleString():
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        # separator is simply a comma ','
+        if not arr_len:
+            return ''
+        res = []
+        for i in xrange(arr_len):
+            element = array[str(i)]
+            if element.is_undefined() or element.is_null():
+                res.append('')
+            else:
+                cand = element.to_object()
+                str_func = element.get('toLocaleString')
+                if not str_func.is_callable():
+                    raise this.MakeError('TypeError', 'toLocaleString method of item at index %d is not callable'%i)
+                res.append(element.callprop('toLocaleString').value)
+        return ','.join(res)
+
+    def concat():
+        array = this.to_object()
+        A = this.Js([])
+        items = [array]
+        items.extend(to_arr(arguments))
+        n = 0
+        for E in items:
+            if E.Class=='Array':
+                k = 0
+                e_len = len(E)
+                while k<e_len:
+                    if E.has_property(str(k)):
+                        A.put(str(n), E.get(str(k)))
+                    n+=1
+                    k+=1
+            else:
+                A.put(str(n), E)
+                n+=1
+        return A
+
+    def join(separator):
+        ARR_STACK.add(this)
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        separator = ',' if separator.is_undefined() else separator.to_string().value
+        elems = []
+        for e in xrange(arr_len):
+            elem = array.get(str(e))
+            if elem in ARR_STACK:
+                s = ''
+            else:
+                s = elem.to_string().value
+            elems.append(s if not (elem.is_undefined() or elem.is_null()) else '')
+        res =  separator.join(elems)
+        ARR_STACK.remove(this)
+        return res
+
+    def pop(): #todo check
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if not arr_len:
+            array.put('length', this.Js(arr_len))
+            return None
+        ind = str(arr_len-1)
+        element = array.get(ind)
+        array.delete(ind)
+        array.put('length', this.Js(arr_len-1))
+        return element
+
+
+    def push(item): # todo check
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        to_put = arguments.to_list()
+        i = arr_len
+        for i, e in enumerate(to_put, arr_len):
+            array.put(str(i), e)
+        if to_put:
+            i+=1
+            array.put('length', this.Js(i))
+        return i
+
+
+    def reverse():
+        array = this.to_object() # my own algorithm
+        vals = to_arr(array)
+        has_props = [array.has_property(str(e)) for e in xrange(len(array))]
+        vals.reverse()
+        has_props.reverse()
+        for i, val in enumerate(vals):
+            if has_props[i]:
+                array.put(str(i), val)
+            else:
+                array.delete(str(i))
+        return array
+
+
+    def shift():  #todo check
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if not arr_len:
+            array.put('length', this.Js(0))
+            return None
+        first = array.get('0')
+        for k in xrange(1, arr_len):
+            from_s, to_s = str(k), str(k-1)
+            if array.has_property(from_s):
+                array.put(to_s, array.get(from_s))
+            else:
+                array.delete(to)
+        array.delete(str(arr_len-1))
+        array.put('length', this.Js(str(arr_len-1)))
+        return first
+
+    def slice(start, end): # todo check
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        relative_start = start.to_int()
+        k = max((arr_len + relative_start), 0) if relative_start<0 else  min(relative_start, arr_len)
+        relative_end = arr_len if end.is_undefined() else end.to_int()
+        final =  max((arr_len + relative_end), 0) if relative_end<0 else min(relative_end, arr_len)
+        res = []
+        n = 0
+        while k<final:
+            pk = str(k)
+            if array.has_property(pk):
+                res.append(array.get(pk))
+            k += 1
+            n += 1
+        return res
+
+    def sort(cmpfn):
+        if not this.Class in ('Array', 'Arguments'):
+            return this.to_object() # do nothing
+        arr = []
+        for i in xrange(len(this)):
+            arr.append(this.get(six.text_type(i)))
+
+        if not arr:
+            return this
+        if not cmpfn.is_callable():
+            cmpfn = None
+        cmp = lambda a,b: sort_compare(a, b, cmpfn)
+        if six.PY3:
+            key = functools.cmp_to_key(cmp)
+            arr.sort(key=key)
+        else:
+            arr.sort(cmp=cmp)
+        for i in xrange(len(arr)):
+            this.put(six.text_type(i), arr[i])
+
+        return this
+
+    def splice(start, deleteCount):
+        # 1-8
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        relative_start = start.to_int()
+        actual_start = max((arr_len + relative_start),0) if relative_start<0 else min(relative_start, arr_len)
+        actual_delete_count =  min(max(deleteCount.to_int(),0 ), arr_len - actual_start)
+        k = 0
+        A = this.Js([])
+        # 9
+        while k<actual_delete_count:
+            if array.has_property(str(actual_start+k)):
+                A.put(str(k), array.get(str(actual_start+k)))
+            k += 1
+        # 10-11
+        items = to_arr(arguments)[2:]
+        items_len = len(items)
+        # 12
+        if items_len<actual_delete_count:
+            k = actual_start
+            while k < (arr_len-actual_delete_count):
+                fr = str(k+actual_delete_count)
+                to = str(k+items_len)
+                if array.has_property(fr):
+                    array.put(to, array.get(fr))
+                else:
+                    array.delete(to)
+                k += 1
+            k = arr_len
+            while k > (arr_len - actual_delete_count + items_len):
+                array.delete(str(k-1))
+                k -= 1
+        # 13
+        elif items_len>actual_delete_count:
+            k = arr_len - actual_delete_count
+            while k>actual_start:
+                fr = str(k + actual_delete_count - 1)
+                to = str(k + items_len - 1)
+                if array.has_property(fr):
+                    array.put(to, array.get(fr))
+                else:
+                    array.delete(to)
+                k -= 1
+        # 14-17
+        k = actual_start
+        while items:
+            E = items.pop(0)
+            array.put(str(k), E)
+            k += 1
+        array.put('length', this.Js(arr_len - actual_delete_count + items_len))
+        return A
+
+    def unshift():
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        argCount = len(arguments)
+        k = arr_len
+        while k > 0:
+            fr = str(k - 1)
+            to = str(k + argCount - 1)
+            if array.has_property(fr):
+                array.put(to, array.get(fr))
+            else:
+                array.delete(to)
+            k -= 1
+        j = 0
+        items = to_arr(arguments)
+        while items:
+            E = items.pop(0)
+            array.put(str(j), E)
+            j += 1
+        array.put('length', this.Js(arr_len + argCount))
+        return arr_len + argCount
+
+    def indexOf(searchElement):
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if arr_len == 0:
+            return -1
+        if len(arguments)>1:
+            n = arguments[1].to_int()
+        else:
+            n = 0
+        if n >= arr_len:
+            return -1
+        if n >= 0:
+            k = n
+        else:
+            k = arr_len - abs(n)
+            if k < 0:
+                k = 0
+        while k < arr_len:
+            if array.has_property(str(k)):
+                elementK = array.get(str(k))
+                if searchElement.strict_equality_comparison(elementK):
+                    return k
+            k += 1
+        return -1
+
+    def lastIndexOf(searchElement):
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if arr_len == 0:
+            return -1
+        if len(arguments)>1:
+            n = arguments[1].to_int()
+        else:
+            n = arr_len - 1
+        if n >= 0:
+            k = min(n, arr_len-1)
+        else:
+            k = arr_len - abs(n)
+        while k >= 0:
+            if array.has_property(str(k)):
+                elementK = array.get(str(k))
+                if searchElement.strict_equality_comparison(elementK):
+                    return k
+            k -= 1
+        return -1
+
+
+    def every(callbackfn):
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        k = 0
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                if not callbackfn.call(T, (kValue, this.Js(k), array)).to_boolean().value:
+                    return False
+            k += 1
+        return True
+
+
+    def some(callbackfn):
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        k = 0
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                if callbackfn.call(T, (kValue, this.Js(k), array)).to_boolean().value:
+                    return True
+            k += 1
+        return False
+
+    def forEach(callbackfn):
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        k = 0
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                callbackfn.call(T, (kValue, this.Js(k), array))
+            k+=1
+
+    def map(callbackfn):
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        A = this.Js([])
+        k = 0
+        while k<arr_len:
+            Pk = str(k)
+            if array.has_property(Pk):
+                kValue = array.get(Pk)
+                mappedValue = callbackfn.call(T, (kValue, this.Js(k), array))
+                A.define_own_property(Pk, {'value': mappedValue, 'writable': True,
+                    'enumerable': True, 'configurable': True})
+            k += 1
+        return A
+
+    def filter(callbackfn):
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        res = []
+        k = 0
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                if callbackfn.call(T, (kValue, this.Js(k), array)).to_boolean().value:
+                    res.append(kValue)
+            k += 1
+        return res # converted to js array automatically
+
+    def reduce(callbackfn):
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        if not arr_len and len(arguments)<2:
+            raise this.MakeError('TypeError', 'Reduce of empty array with no initial value')
+        k = 0
+        if len(arguments)>1: # initial value present
+            accumulator = arguments[1]
+        else:
+            kPresent = False
+            while not kPresent and k<arr_len:
+                kPresent = array.has_property(str(k))
+                if kPresent:
+                    accumulator = array.get(str(k))
+                k += 1
+            if not kPresent:
+                raise this.MakeError('TypeError', 'Reduce of empty array with no initial value')
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                accumulator = callbackfn.call(this.undefined, (accumulator, kValue, this.Js(k), array))
+            k += 1
+        return accumulator
+
+
+    def reduceRight(callbackfn):
+        array = this.to_object()
+        arr_len = array.get('length').to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        if not arr_len and len(arguments)<2:
+            raise this.MakeError('TypeError', 'Reduce of empty array with no initial value')
+        k = arr_len - 1
+        if len(arguments)>1: # initial value present
+            accumulator = arguments[1]
+        else:
+            kPresent = False
+            while not kPresent and k>=0:
+                kPresent = array.has_property(str(k))
+                if kPresent:
+                    accumulator = array.get(str(k))
+                k -= 1
+            if not kPresent:
+                raise this.MakeError('TypeError', 'Reduce of empty array with no initial value')
+        while k>=0:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                accumulator = callbackfn.call(this.undefined, (accumulator, kValue, this.Js(k), array))
+            k -= 1
+        return accumulator
+
+
+def sort_compare(a, b, comp):
+    if a is None:
+        if b is None:
+            return 0
+        return 1
+    if b is None:
+        if a is None:
+            return 0
+        return -1
+    if a.is_undefined():
+        if b.is_undefined():
+            return 0
+        return 1
+    if b.is_undefined():
+        if a.is_undefined():
+            return 0
+        return -1
+    if comp is not None:
+        res = comp.call(a.undefined, (a, b))
+        return res.to_int()
+    x, y = a.to_string(), b.to_string()
+    if x<y:
+        return -1
+    elif x>y:
+        return 1
+    return 0
+
+
+
+

lib/js2py/prototypes/jsarraybuffer.py

@@ -0,0 +1,17 @@
+# this is based on jsarray.py
+
+import six
+
+if six.PY3:
+    xrange = range
+    import functools
+
+def to_arr(this):
+    """Returns Python array from Js array"""
+    return [this.get(str(e)) for e in xrange(len(this))]
+
+
+ARR_STACK = set({})
+
+class ArrayBufferPrototype:
+    pass

lib/js2py/prototypes/jsboolean.py

@@ -0,0 +1,16 @@
+
+
+class BooleanPrototype:
+    def toString():
+        if this.Class!='Boolean':
+            raise this.Js(TypeError)('this must be a boolean')
+        return 'true' if this.value else 'false'
+
+    def valueOf():
+        if this.Class!='Boolean':
+            raise this.Js(TypeError)('this must be a boolean')
+        return this.value
+
+
+
+

lib/js2py/prototypes/jserror.py

@@ -0,0 +1,10 @@
+
+class ErrorPrototype:
+    def toString():
+        if this.TYPE!='Object':
+            raise this.MakeError('TypeError', 'Error.prototype.toString called on non-object')
+        name = this.get('name')
+        name = 'Error' if name.is_undefined() else name.to_string().value
+        msg = this.get('message')
+        msg = '' if msg.is_undefined() else msg.to_string().value
+        return name + (name and msg and ': ') + msg

lib/js2py/prototypes/jsfunction.py

@@ -0,0 +1,53 @@
+# python 3 support
+import six
+if six.PY3:
+    basestring = str
+    long = int
+    xrange = range
+    unicode = str
+
+
+# todo fix apply and bind
+
+class FunctionPrototype:            
+    def toString():
+        if not this.is_callable():
+            raise TypeError('toString is not generic!')
+        args = ', '.join(this.code.__code__.co_varnames[:this.argcount])
+        return 'function %s(%s) '%(this.func_name, args)+this.source
+    
+    def call():
+        arguments_ = arguments
+        if not len(arguments):
+            obj = this.Js(None)
+        else:
+            obj = arguments[0]
+        if len(arguments)<=1:
+            args = () 
+        else:
+            args = tuple([arguments_[e] for e in xrange(1, len(arguments_))])
+        return this.call(obj, args)
+    
+    def apply():
+        if not len(arguments):
+            obj = this.Js(None)
+        else:
+            obj = arguments[0]
+        if len(arguments)<=1:
+            args = () 
+        else:
+            appl = arguments[1]
+            args = tuple([appl[e] for e in xrange(len(appl))])
+        return this.call(obj, args)
+
+    def bind(thisArg):
+        target = this
+        if not target.is_callable():
+            raise this.MakeError('Object must be callable in order to be used with bind method')
+        if len(arguments) <= 1:
+            args = ()
+        else:
+            args = tuple([arguments[e] for e in xrange(1, len(arguments))])
+        return this.PyJsBoundFunction(target, thisArg, args)
+
+

lib/js2py/prototypes/jsjson.py

@@ -0,0 +1,210 @@
+import json
+from ..base import Js
+indent = ''
+# python 3 support
+import six
+if six.PY3:
+    basestring = str
+    long = int
+    xrange = range
+    unicode = str
+
+
+def parse(text):
+    reviver = arguments[1]
+    s = text.to_string().value
+    try:
+        unfiltered = json.loads(s)
+    except:
+        raise this.MakeError('SyntaxError', 'Could not parse JSON string - Invalid syntax')
+    unfiltered = to_js(this, unfiltered)
+    if reviver.is_callable():
+        root = this.Js({'': unfiltered})
+        walk(root, '', reviver)
+    else:
+        return unfiltered
+
+
+def stringify(value, replacer, space):
+    global indent
+    stack = set([])
+    indent = ''
+    property_list = replacer_function = this.undefined
+    if replacer.is_object():
+        if replacer.is_callable():
+            replacer_function = replacer
+        elif replacer.Class=='Array':
+            property_list = []
+            for e in replacer:
+                v = replacer[e]
+                item = this.undefined
+                if v._type()=='Number':
+                    item = v.to_string()
+                elif v._type()=='String':
+                    item = v
+                elif v.is_object():
+                    if v.Class in ('String', 'Number'):
+                        item = v.to_string()
+                if not item.is_undefined() and item.value not in property_list:
+                    property_list.append(item.value)
+    if space.is_object():
+        if space.Class=='Number':
+            space = space.to_number()
+        elif space.Class=='String':
+            space = space.to_string()
+    if space._type()=='Number':
+        space = this.Js(min(10, space.to_int()))
+        gap = max(int(space.value), 0)* ' '
+    elif space._type()=='String':
+        gap = space.value[:10]
+    else:
+        gap = ''
+    return this.Js(Str('', this.Js({'':value}), replacer_function, property_list, gap, stack, space))
+
+
+
+def Str(key, holder, replacer_function, property_list, gap, stack, space):
+    value = holder[key]
+    if value.is_object():
+        to_json = value.get('toJSON')
+        if to_json.is_callable():
+            value = to_json.call(value, (key,))
+    if not replacer_function.is_undefined():
+        value = replacer_function.call(holder, (key, value))
+
+    if value.is_object():
+        if value.Class=='String':
+            value = value.to_string()
+        elif value.Class=='Number':
+            value = value.to_number()
+        elif value.Class=='Boolean':
+            value = value.to_boolean()
+    if value.is_null():
+        return 'null'
+    elif value.Class=='Boolean':
+        return 'true' if value.value else 'false'
+    elif value._type()=='String':
+        return Quote(value)
+    elif value._type()=='Number':
+        if not value.is_infinity():
+            return value.to_string()
+        return 'null'
+    if value.is_object() and not value.is_callable():
+        if value.Class=='Array':
+            return ja(value, stack, gap, property_list, replacer_function, space)
+        else:
+            return jo(value, stack, gap, property_list, replacer_function, space)
+    return None # undefined
+
+
+
+def jo(value, stack, gap, property_list, replacer_function, space):
+    global indent
+    if value in stack:
+        raise value.MakeError('TypeError', 'Converting circular structure to JSON')
+    stack.add(value)
+    stepback = indent
+    indent += gap
+    if not property_list.is_undefined():
+        k = property_list
+    else:
+        k = [e.value for e in value]
+    partial = []
+    for p in k:
+        str_p = value.Js(Str(p, value, replacer_function, property_list, gap, stack, space))
+        if not str_p.is_undefined():
+            member = json.dumps(p) + ':' + (' ' if gap else '')  + str_p.value # todo not sure here - what space character?
+            partial.append(member)
+    if not partial:
+        final = '{}'
+    else:
+        if not gap:
+            final = '{%s}' % ','.join(partial)
+        else:
+            sep = ',\n'+indent
+            properties = sep.join(partial)
+            final = '{\n'+indent+properties+'\n'+stepback+'}'
+    stack.remove(value)
+    indent = stepback
+    return final
+
+
+def ja(value, stack, gap, property_list, replacer_function, space):
+    global indent
+    if value in stack:
+        raise value.MakeError('TypeError', 'Converting circular structure to JSON')
+    stack.add(value)
+    stepback = indent
+    indent += gap
+    partial = []
+    length = len(value)
+    for index in xrange(length):
+        index = str(index)
+        str_index = value.Js(Str(index, value, replacer_function, property_list, gap, stack, space))
+        if str_index.is_undefined():
+            partial.append('null')
+        else:
+            partial.append(str_index.value)
+    if not partial:
+        final = '[]'
+    else:
+        if not gap:
+            final = '[%s]' % ','.join(partial)
+        else:
+            sep = ',\n'+indent
+            properties = sep.join(partial)
+            final = '[\n'+indent +properties+'\n'+stepback+']'
+    stack.remove(value)
+    indent = stepback
+    return final
+
+
+
+
+
+def Quote(string):
+    return string.Js(json.dumps(string.value))
+
+
+def to_js(this, d):
+    if isinstance(d, dict):
+        return this.Js(dict((k,this.Js(v)) for k, v in six.iteritems(d)))
+    return this.Js(d)
+
+
+
+def walk(holder, name, reviver):
+    val = holder.get(name)
+    if val.Class=='Array':
+        for i in xrange(len(val)):
+            i = unicode(i)
+            new_element = walk(val, i, reviver)
+            if new_element.is_undefined():
+                val.delete(i)
+            else:
+                new_element.put(i, new_element)
+    elif val.is_object():
+        for key in val:
+            new_element = walk(val, key, reviver)
+            if new_element.is_undefined():
+                val.delete(key)
+            else:
+                val.put(key, new_element)
+    return reviver.call(holder, (name, val))
+
+
+
+
+
+
+JSON = Js({})
+
+JSON.define_own_property('parse', {'value': Js(parse),
+                                   'enumerable': False,
+                                   'writable': True,
+                                   'configurable': True})
+
+JSON.define_own_property('stringify', {'value': Js(stringify),
+                                       'enumerable': False,
+                                       'writable': True,
+                                       'configurable': True})

lib/js2py/prototypes/jsnumber.py

@@ -0,0 +1,100 @@
+import six
+if six.PY3:
+    basestring = str
+    long = int
+    xrange = range
+    unicode = str
+
+
+RADIX_SYMBOLS = {0: '0', 1: '1', 2: '2', 3: '3', 4: '4', 5: '5', 6: '6', 7: '7', 8: '8', 9: '9',
+                 10: 'a', 11: 'b', 12: 'c', 13: 'd', 14: 'e', 15: 'f', 16: 'g', 17: 'h', 18: 'i', 19: 'j', 20: 'k',
+                 21: 'l', 22: 'm', 23: 'n', 24: 'o', 25: 'p', 26: 'q', 27: 'r', 28: 's', 29: 't', 30: 'u', 31: 'v',
+                 32: 'w', 33: 'x', 34: 'y', 35: 'z'}
+
+
+def to_str_rep(num):
+    if num.is_nan():
+        return num.Js('NaN')
+    elif num.is_infinity():
+        sign = '-' if num.value<0 else ''
+        return num.Js(sign+'Infinity')
+    elif isinstance(num.value, (long, int)) or num.value.is_integer():  # dont print .0
+        return num.Js(unicode(int(num.value)))
+    return num.Js(unicode(num.value)) # accurate enough
+
+
+class NumberPrototype:
+    def toString(radix):
+        if this.Class!='Number':
+            raise this.MakeError('TypeError', 'Number.prototype.valueOf is not generic')
+        if radix.is_undefined():
+            return to_str_rep(this)
+        r = radix.to_int()
+        if r==10:
+            return to_str_rep(this)
+        if r not in xrange(2, 37): 
+            raise this.MakeError('RangeError', 'Number.prototype.toString() radix argument must be between 2 and 36')
+        num = this.to_int()
+        if num < 0:
+            num = -num
+            sign = '-'
+        else:
+            sign = ''
+        res = ''
+        while num:
+            s = RADIX_SYMBOLS[num % r]
+            num = num // r
+            res = s + res
+        return sign + (res if res else '0')
+
+    def valueOf():
+        if this.Class!='Number':
+            raise this.MakeError('TypeError', 'Number.prototype.valueOf is not generic')
+        return this.value
+
+    def toLocaleString():
+        return this.to_string()
+
+    def toFixed (fractionDigits):
+        if this.Class!='Number':
+            raise this.MakeError('TypeError', 'Number.prototype.toFixed called on incompatible receiver')
+        digs = fractionDigits.to_int()
+        if digs<0 or digs>20:
+            raise this.MakeError('RangeError', 'toFixed() digits argument must be between 0 and 20')
+        elif this.is_infinity():
+            return 'Infinity' if this.value>0 else '-Infinity'
+        elif this.is_nan():
+            return 'NaN'
+        return format(this.value, '-.%df'%digs)
+
+
+    def toExponential (fractionDigits):
+        if this.Class!='Number':
+            raise this.MakeError('TypeError', 'Number.prototype.toExponential called on incompatible receiver')
+        digs = fractionDigits.to_int()
+        if digs<0 or digs>20:
+            raise this.MakeError('RangeError', 'toFixed() digits argument must be between 0 and 20')
+        elif this.is_infinity():
+            return 'Infinity' if this.value>0 else '-Infinity'
+        elif this.is_nan():
+            return 'NaN'
+        return format(this.value, '-.%de'%digs)
+
+    def toPrecision (precision):
+        if this.Class!='Number':
+            raise this.MakeError('TypeError', 'Number.prototype.toPrecision called on incompatible receiver')
+        if precision.is_undefined():
+            return this.to_string()
+        prec = precision.to_int()
+        if this.is_nan():
+            return 'NaN'
+        elif this.is_infinity():
+            return 'Infinity' if this.value>0 else '-Infinity'
+        digs = prec - len(str(int(this.value)))
+        if digs>=0:
+            return format(this.value, '-.%df'%digs)
+        else:
+            return format(this.value, '-.%df'%(prec-1))
+
+
+

lib/js2py/prototypes/jsobject.py

@@ -0,0 +1,36 @@
+
+class ObjectPrototype:
+    def toString():
+        return '[object %s]'%this.Class
+
+    def valueOf():
+        return this.to_object()
+
+
+    def toLocaleString():
+        return this.callprop('toString')
+
+    def hasOwnProperty(prop):
+        return this.get_own_property(prop.to_string().value) is not None
+
+    def isPrototypeOf(obj):
+        #a bit stupid specification but well
+        # for example Object.prototype.isPrototypeOf.call((5).__proto__, 5) gives false
+        if not obj.is_object():
+            return False
+        while 1:
+            obj = obj.prototype
+            if obj is None or obj.is_null():
+                return False
+            if obj is this:
+                return True
+
+    def propertyIsEnumerable(prop):
+        cand = this.own.get(prop.to_string().value)
+        return cand is not None and cand.get('enumerable')
+
+
+
+
+
+

lib/js2py/prototypes/jsregexp.py

@@ -0,0 +1,46 @@
+
+class RegExpPrototype:
+    def toString():
+        flags = u''
+        try:
+            if this.glob:
+                flags += u'g'
+            if this.ignore_case:
+                flags += u'i'
+            if this.multiline:
+                flags += u'm'
+        except:
+            pass
+        v = this.value if this.value else '(?:)'
+        return u'/%s/'%v + flags
+
+    def test(string):
+        return Exec(this, string) is not this.null
+
+    def exec2(string):  # will be changed to exec in base.py. cant name it exec here
+        return Exec(this, string)
+
+
+
+
+def Exec(this, string):
+    if this.Class!='RegExp':
+        raise this.MakeError('TypeError', 'RegExp.prototype.exec is not generic!')
+    string = string.to_string()
+    length = len(string)
+    i = this.get('lastIndex').to_int() if this.glob else 0
+    matched = False
+    while not matched:
+        if i < 0 or i > length:
+            this.put('lastIndex', this.Js(0))
+            return this.null
+        matched = this.match(string.value, i)
+        i += 1
+    start, end = matched.span()#[0]+i-1, matched.span()[1]+i-1
+    if this.glob:
+        this.put('lastIndex', this.Js(end))
+    arr = this.Js([this.Js(e) for e in [matched.group()]+list(matched.groups())])
+    arr.put('index', this.Js(start))
+    arr.put('input', string)
+    return arr
+

lib/js2py/prototypes/jsstring.py

@@ -0,0 +1,307 @@
+# -*- coding: utf-8 -*-
+from .jsregexp import Exec
+import re
+DIGS = set('0123456789')
+WHITE = u"\u0009\u000A\u000B\u000C\u000D\u0020\u00A0\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF"
+
+def replacement_template(rep, source, span, npar):
+    """Takes the replacement template and some info about the match and returns filled template
+       """
+    n = 0
+    res = ''
+    while n < len(rep)-1:
+        char = rep[n]
+        if char=='$':
+            if rep[n+1]=='$':
+                res += '$'
+                n += 2
+                continue
+            elif rep[n+1]=='`':
+                # replace with string that is BEFORE match
+                res += source[:span[0]]
+                n += 2
+                continue
+            elif rep[n+1]=='\'':
+                # replace with string that is AFTER match
+                res += source[span[1]:]
+                n += 2
+                continue
+            elif rep[n+1] in DIGS:
+                dig = rep[n+1]
+                if n+2<len(rep) and rep[n+2] in DIGS:
+                    dig += rep[n+2]
+                num = int(dig)
+                # we will not do any replacements if we dont have this npar or dig is 0
+                if not num or num>len(npar):
+                    res += '$'+dig
+                else:
+                    # None - undefined has to be replaced with ''
+                    res += npar[num-1] if npar[num-1] else ''
+                n += 1 + len(dig)
+                continue
+        res += char
+        n += 1
+    if n<len(rep):
+        res += rep[-1]
+    return res
+
+
+###################################################
+
+class StringPrototype:
+    def toString():
+        if this.Class!='String':
+            raise this.MakeError('TypeError', 'String.prototype.toString is not generic')
+        return this.value
+
+    def valueOf():
+        if this.Class!='String':
+            raise this.MakeError('TypeError', 'String.prototype.valueOf is not generic')
+        return this.value
+
+    def charAt(pos):
+        this.cok()
+        pos = pos.to_int()
+        s = this.to_string()
+        if 0<= pos < len(s.value):
+            char = s.value[pos]
+            if char not in s.CHAR_BANK:
+                s.Js(char) # add char to char bank
+            return s.CHAR_BANK[char]
+        return s.CHAR_BANK['']
+
+    def charCodeAt(pos):
+        this.cok()
+        pos = pos.to_int()
+        s = this.to_string()
+        if 0<= pos < len(s.value):
+            return s.Js(ord(s.value[pos]))
+        return s.NaN
+
+    def concat():
+        this.cok()
+        s = this.to_string()
+        res = s.value
+        for e in arguments.to_list():
+            res += e.to_string().value
+        return res
+
+    def indexOf(searchString, position):
+        this.cok()
+        s = this.to_string().value
+        search = searchString.to_string().value
+        pos = position.to_int()
+        return this.Js(s.find(search, min(max(pos, 0), len(s))) )
+
+    def lastIndexOf(searchString, position):
+        this.cok()
+        s = this.to_string().value
+        search = searchString.to_string().value
+        pos = position.to_number()
+        pos = 10**15 if pos.is_nan() else pos.to_int()
+        return s.rfind(search, 0, min(max(pos, 0)+1, len(s)))
+
+    def localeCompare(that):
+        this.cok()
+        s = this.to_string()
+        that = that.to_string()
+        if s<that:
+            return this.Js(-1)
+        elif s>that:
+            return this.Js(1)
+        return this.Js(0)
+
+    def match(regexp):
+        this.cok()
+        s = this.to_string()
+        r = this.RegExp(regexp) if regexp.Class!='RegExp' else regexp
+        if not r.glob:
+            return Exec(r, s)
+        r.put('lastIndex', this.Js(0))
+        found = []
+        previous_last_index = 0
+        last_match = True
+        while last_match:
+            result = Exec(r, s)
+            if result.is_null():
+                last_match=False
+            else:
+                this_index = r.get('lastIndex').value
+                if this_index==previous_last_index:
+                    r.put('lastIndex', this.Js(this_index+1))
+                    previous_last_index += 1
+                else:
+                    previous_last_index = this_index
+                matchStr = result.get('0')
+                found.append(matchStr)
+        if not found:
+            return this.null
+        return found
+
+
+    def replace(searchValue, replaceValue):
+        # VERY COMPLICATED. to check again.
+        this.cok()
+        string = this.to_string()
+        s = string.value
+        res = ''
+        if not replaceValue.is_callable():
+            replaceValue = replaceValue.to_string().value
+            func = False
+        else:
+            func = True
+        # Replace all  ( global )
+        if searchValue.Class == 'RegExp' and searchValue.glob:
+            last = 0
+            for e in re.finditer(searchValue.pat, s):
+                res += s[last:e.span()[0]]
+                if func:
+                    # prepare arguments for custom func (replaceValue)
+                    args = (e.group(),) +  e.groups() + (e.span()[1], string)
+                    # convert all types to JS
+                    args = map(this.Js, args)
+                    res += replaceValue(*args).to_string().value
+                else:
+                    res += replacement_template(replaceValue, s, e.span(), e.groups())
+                last = e.span()[1]
+            res += s[last:]
+            return this.Js(res)
+        elif searchValue.Class=='RegExp':
+            e = re.search(searchValue.pat, s)
+            if e is None:
+                return string
+            span = e.span()
+            pars = e.groups()
+            match = e.group()
+        else:
+            match = searchValue.to_string().value
+            ind =  s.find(match)
+            if ind==-1:
+                return string
+            span = ind, ind + len(match)
+            pars = ()
+        res = s[:span[0]]
+        if func:
+            args = (match,) +  pars + (span[1], string)
+            # convert all types to JS
+            this_ = this
+            args = tuple([this_.Js(x) for x in args])
+            res += replaceValue(*args).to_string().value
+        else:
+            res += replacement_template(replaceValue, s, span, pars)
+        res += s[span[1]:]
+        return res
+
+    def search(regexp):
+        this.cok()
+        string = this.to_string()
+        if regexp.Class=='RegExp':
+            rx = regexp
+        else:
+            rx = this.RegExp(regexp)
+        res = re.search(rx.pat, string.value)
+        if res is not None:
+            return this.Js(res.span()[0])
+        return -1
+
+    def slice(start, end):
+        this.cok()
+        s = this.to_string()
+        start = start.to_int()
+        length = len(s.value)
+        end = length if  end.is_undefined() else end.to_int()
+        #From = max(length+start, 0) if start<0 else min(length, start)
+        #To = max(length+end, 0) if end<0 else min(length, end)
+        return s.value[start:end]
+
+
+    def split (separator, limit):
+        # its a bit different that re.split!
+        this.cok()
+        S = this.to_string()
+        s = S.value
+        lim = 2**32-1 if limit.is_undefined() else limit.to_uint32()
+        if not lim:
+            return []
+        if separator.is_undefined():
+            return [s]
+        len_s = len(s)
+        res = []
+        R = separator if separator.Class=='RegExp' else separator.to_string()
+        if not len_s:
+            if SplitMatch(s, 0, R) is None:
+                return [S]
+            return []
+        p = q = 0
+        while q!=len_s:
+            e, cap = SplitMatch(s, q, R)
+            if e is None or e==p:
+                q += 1
+                continue
+            res.append(s[p:q])
+            p = q = e
+            if len(res)==lim:
+                return res
+            for element in cap:
+                res.append(this.Js(element))
+                if len(res)==lim:
+                    return res
+        res.append(s[p:])
+        return res
+
+
+    def substring (start, end):
+        this.cok()
+        s = this.to_string().value
+        start = start.to_int()
+        length = len(s)
+        end = length if end.is_undefined() else end.to_int()
+        fstart = min(max(start, 0), length)
+        fend = min(max(end, 0), length)
+        return this.Js(s[min(fstart, fend):max(fstart, fend)])
+
+    def substr(start, length):
+        #I hate this function and its description in specification
+        r1 = this.to_string().value
+        r2 = start.to_int()
+        r3 = 10**20 if  length.is_undefined() else length.to_int()
+        r4 = len(r1)
+        r5 = r2 if r2>=0 else max(0, r2+r4)
+        r6 = min(max(r3 ,0), r4 - r5)
+        if r6<=0:
+            return ''
+        return r1[r5:r5+r6]
+
+    def toLowerCase():
+        this.cok()
+        return this.Js(this.to_string().value.lower())
+
+    def toLocaleLowerCase():
+        this.cok()
+        return this.Js(this.to_string().value.lower())
+
+    def toUpperCase():
+        this.cok()
+        return this.Js(this.to_string().value.upper())
+
+    def toLocaleUpperCase():
+        this.cok()
+        return this.Js(this.to_string().value.upper())
+
+    def trim():
+        this.cok()
+        return this.Js(this.to_string().value.strip(WHITE))
+
+
+
+
+def SplitMatch(s, q, R):
+    # s is Py String to match, q is the py int match start and R is Js RegExp or String.
+    if R.Class=='RegExp':
+        res = R.match(s, q)
+        return (None, ()) if res is None else (res.span()[1], res.groups())
+    # R is just a string
+    if s[q:].startswith(R.value):
+        return q+len(R.value), ()
+    return None, ()
+

lib/js2py/prototypes/jstypedarray.py

@@ -0,0 +1,324 @@
+# this is based on jsarray.py
+
+import six
+try:
+    import numpy
+except:
+    pass
+
+if six.PY3:
+    xrange = range
+    import functools
+
+def to_arr(this):
+    """Returns Python array from Js array"""
+    return [this.get(str(e)) for e in xrange(len(this))]
+
+
+ARR_STACK = set({})
+
+class TypedArrayPrototype:
+
+    def toString():
+        # this function is wrong
+        func = this.get('join')
+        if not func.is_callable():
+            @this.Js
+            def func():
+                return '[object %s]'%this.Class
+        return func.call(this, ())
+
+    def toLocaleString(locales=None,options=None):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        # separator is simply a comma ','
+        if not arr_len:
+            return ''
+        res = []
+        for i in xrange(arr_len):
+            element = array[str(i)]
+            if element.is_undefined() or element.is_null():
+                res.append('')
+            else:
+                cand = element.to_object()
+                str_func = element.get('toLocaleString')
+                if not str_func.is_callable():
+                    raise this.MakeError('TypeError', 'toLocaleString method of item at index %d is not callable'%i)
+                res.append(element.callprop('toLocaleString').value)
+        return ','.join(res)
+
+    def join(separator):
+        ARR_STACK.add(this)
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        separator = ',' if separator.is_undefined() else separator.to_string().value
+        elems = []
+        for e in xrange(arr_len):
+            elem = array.get(str(e))
+            if elem in ARR_STACK:
+                s = ''
+            else:
+                s = elem.to_string().value
+            elems.append(s if not (elem.is_undefined() or elem.is_null()) else '')
+        res =  separator.join(elems)
+        ARR_STACK.remove(this)
+        return res
+
+    def reverse():
+        array = this.to_object() # my own algorithm
+        vals = to_arr(array)
+        has_props = [array.has_property(str(e)) for e in xrange(len(array))]
+        vals.reverse()
+        has_props.reverse()
+        for i, val in enumerate(vals):
+            if has_props[i]:
+                array.put(str(i), val)
+            else:
+                array.delete(str(i))
+        return array
+
+    def slice(start, end): # todo check
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        relative_start = start.to_int()
+        k = max((arr_len + relative_start), 0) if relative_start<0 else  min(relative_start, arr_len)
+        relative_end = arr_len if end.is_undefined() else end.to_int()
+        final =  max((arr_len + relative_end), 0) if relative_end<0 else min(relative_end, arr_len)
+        res = []
+        n = 0
+        while k<final:
+            pk = str(k)
+            if array.has_property(pk):
+                res.append(array.get(pk))
+            k += 1
+            n += 1
+        return res
+
+    def sort(cmpfn):
+        if not this.Class in ('Array', 'Arguments'):
+            return this.to_object() # do nothing
+        arr = []
+        for i in xrange(len(this)):
+            arr.append(this.get(six.text_type(i)))
+
+        if not arr:
+            return this
+        if not cmpfn.is_callable():
+            cmpfn = None
+        cmp = lambda a,b: sort_compare(a, b, cmpfn)
+        if six.PY3:
+            key = functools.cmp_to_key(cmp)
+            arr.sort(key=key)
+        else:
+            arr.sort(cmp=cmp)
+        for i in xrange(len(arr)):
+            this.put(six.text_type(i), arr[i])
+
+        return this
+
+    def indexOf(searchElement):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        if arr_len == 0:
+            return -1
+        if len(arguments)>1:
+            n = arguments[1].to_int()
+        else:
+            n = 0
+        if n >= arr_len:
+            return -1
+        if n >= 0:
+            k = n
+        else:
+            k = arr_len - abs(n)
+            if k < 0:
+                k = 0
+        while k < arr_len:
+            if array.has_property(str(k)):
+                elementK = array.get(str(k))
+                if searchElement.strict_equality_comparison(elementK):
+                    return k
+            k += 1
+        return -1
+
+    def lastIndexOf(searchElement):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        if arr_len == 0:
+            return -1
+        if len(arguments)>1:
+            n = arguments[1].to_int()
+        else:
+            n = arr_len - 1
+        if n >= 0:
+            k = min(n, arr_len-1)
+        else:
+            k = arr_len - abs(n)
+        while k >= 0:
+            if array.has_property(str(k)):
+                elementK = array.get(str(k))
+                if searchElement.strict_equality_comparison(elementK):
+                    return k
+            k -= 1
+        return -1
+
+    def every(callbackfn):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        k = 0
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                if not callbackfn.call(T, (kValue, this.Js(k), array)).to_boolean().value:
+                    return False
+            k += 1
+        return True
+
+    def some(callbackfn):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        k = 0
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                if callbackfn.call(T, (kValue, this.Js(k), array)).to_boolean().value:
+                    return True
+            k += 1
+        return False
+
+    def forEach(callbackfn):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        k = 0
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                callbackfn.call(T, (kValue, this.Js(k), array))
+            k+=1
+
+    def map(callbackfn):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        A = this.Js([])
+        k = 0
+        while k<arr_len:
+            Pk = str(k)
+            if array.has_property(Pk):
+                kValue = array.get(Pk)
+                mappedValue = callbackfn.call(T, (kValue, this.Js(k), array))
+                A.define_own_property(Pk, {'value': mappedValue, 'writable': True,
+                    'enumerable': True, 'configurable': True})
+            k += 1
+        return A
+
+    def filter(callbackfn):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        T = arguments[1]
+        res = []
+        k = 0
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                if callbackfn.call(T, (kValue, this.Js(k), array)).to_boolean().value:
+                    res.append(kValue)
+            k += 1
+        return res # converted to js array automatically
+
+    def reduce(callbackfn):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        if not arr_len and len(arguments)<2:
+            raise this.MakeError('TypeError', 'Reduce of empty array with no initial value')
+        k = 0
+        if len(arguments)>1: # initial value present
+            accumulator = arguments[1]
+        else:
+            kPresent = False
+            while not kPresent and k<arr_len:
+                kPresent = array.has_property(str(k))
+                if kPresent:
+                    accumulator = array.get(str(k))
+                k += 1
+            if not kPresent:
+                raise this.MakeError('TypeError', 'Reduce of empty array with no initial value')
+        while k<arr_len:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                accumulator = callbackfn.call(this.undefined, (accumulator, kValue, this.Js(k), array))
+            k += 1
+        return accumulator
+
+
+    def reduceRight(callbackfn):
+        array = this.to_object()
+        arr_len = array.get("length").to_uint32()
+        if not callbackfn.is_callable():
+            raise this.MakeError('TypeError', 'callbackfn must be a function')
+        if not arr_len and len(arguments)<2:
+            raise this.MakeError('TypeError', 'Reduce of empty array with no initial value')
+        k = arr_len - 1
+        if len(arguments)>1: # initial value present
+            accumulator = arguments[1]
+        else:
+            kPresent = False
+            while not kPresent and k>=0:
+                kPresent = array.has_property(str(k))
+                if kPresent:
+                    accumulator = array.get(str(k))
+                k -= 1
+            if not kPresent:
+                raise this.MakeError('TypeError', 'Reduce of empty array with no initial value')
+        while k>=0:
+            if array.has_property(str(k)):
+                kValue = array.get(str(k))
+                accumulator = callbackfn.call(this.undefined, (accumulator, kValue, this.Js(k), array))
+            k -= 1
+        return accumulator
+
+def sort_compare(a, b, comp):
+    if a is None:
+        if b is None:
+            return 0
+        return 1
+    if b is None:
+        if a is None:
+            return 0
+        return -1
+    if a.is_undefined():
+        if b.is_undefined():
+            return 0
+        return 1
+    if b.is_undefined():
+        if a.is_undefined():
+            return 0
+        return -1
+    if comp is not None:
+        res = comp.call(a.undefined, (a, b))
+        return res.to_int()
+    x, y = a.to_string(), b.to_string()
+    if x<y:
+        return -1
+    elif x>y:
+        return 1
+    return 0
+
+
+
+

lib/js2py/pyjs.py

@@ -0,0 +1,66 @@
+from .base import *
+from .constructors.jsmath import Math
+from .constructors.jsdate import Date
+from .constructors.jsobject import Object
+from .constructors.jsfunction import Function
+from .constructors.jsstring import String
+from .constructors.jsnumber import Number
+from .constructors.jsboolean import Boolean
+from .constructors.jsregexp import RegExp
+from .constructors.jsarray import Array
+from .constructors.jsarraybuffer import ArrayBuffer
+from .constructors.jsint8array import Int8Array
+from .constructors.jsuint8array import Uint8Array
+from .constructors.jsuint8clampedarray import Uint8ClampedArray
+from .constructors.jsint16array import Int16Array
+from .constructors.jsuint16array import Uint16Array
+from .constructors.jsint32array import Int32Array
+from .constructors.jsuint32array import Uint32Array
+from .constructors.jsfloat32array import Float32Array
+from .constructors.jsfloat64array import Float64Array
+from .prototypes.jsjson import JSON
+from .host.console import console
+from .host.jseval import Eval
+from .host.jsfunctions import parseFloat, parseInt, isFinite, isNaN
+
+# Now we have all the necessary items to create global environment for script
+__all__ = ['Js', 'PyJsComma', 'PyJsStrictEq', 'PyJsStrictNeq',
+           'PyJsException', 'PyJsBshift', 'Scope', 'PyExceptionToJs',
+           'JsToPyException', 'JS_BUILTINS', 'appengine', 'set_global_object',
+           'JsRegExp', 'PyJsException', 'PyExceptionToJs', 'JsToPyException', 'PyJsSwitchException']
+
+
+# these were defined in base.py
+builtins = ('true','false','null','undefined','Infinity',
+            'NaN', 'console', 'String', 'Number', 'Boolean', 'RegExp',
+            'Math', 'Date', 'Object', 'Function', 'Array',
+            'Int8Array', 'Uint8Array', 'Uint8ClampedArray',
+            'Int16Array','Uint16Array',
+            'Int32Array', 'Uint32Array',
+            'Float32Array', 'Float64Array',
+            'ArrayBuffer',
+            'parseFloat', 'parseInt', 'isFinite', 'isNaN')
+            #Array, Function, JSON,   Error is done later :)
+            # also some built in functions like eval...
+
+def set_global_object(obj):
+    obj.IS_CHILD_SCOPE = False
+    this = This({})
+    this.own = obj.own
+    this.prototype = obj.prototype
+    PyJs.GlobalObject = this
+    # make this available
+    obj.register('this')
+    obj.put('this', this)
+
+
+
+scope = dict(zip(builtins, [globals()[e] for e in builtins]))
+# Now add errors:
+for name, error in ERRORS.items():
+    scope[name] = error
+#add eval
+scope['eval'] = Eval
+scope['JSON'] = JSON
+JS_BUILTINS = dict((k,v) for k,v in scope.items())
+

lib/js2py/translators/__init__.py

@@ -0,0 +1,38 @@
+# The MIT License
+#
+# Copyright 2014, 2015 Piotr Dabkowski
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the 'Software'),
+# to deal in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so, subject
+# to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+# LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+#  OR THE USE OR OTHER DEALINGS IN THE SOFTWARE
+
+__all__ = ['PyJsParser', 'Node', 'WrappingNode', 'node_to_dict', 'parse', 'translate_js', 'translate', 'syntax_tree_translate',
+           'DEFAULT_HEADER']
+__author__ = 'Piotr Dabkowski'
+__version__ = '2.2.0'
+from pyjsparser import PyJsParser
+from .translator import translate_js, trasnlate, syntax_tree_translate, DEFAULT_HEADER
+
+
+def parse(javascript_code):
+    """Returns syntax tree of javascript_code.
+
+    Syntax tree has the same structure as syntax tree produced by esprima.js
+
+       Same as PyJsParser().parse  For your convenience :) """
+    p = PyJsParser()
+    return p.parse(javascript_code)
+
+

lib/js2py/translators/friendly_nodes.py

@@ -0,0 +1,327 @@
+import binascii
+
+from pyjsparser import PyJsParser
+import six
+if six.PY3:
+    basestring = str
+    long = int
+    xrange = range
+    unicode = str
+
+REGEXP_CONVERTER = PyJsParser()
+
+def to_hex(s):
+    return binascii.hexlify(s.encode('utf8')).decode('utf8')  # fucking python 3, I hate it so much
+                                                              # wtf was wrong with s.encode('hex') ???
+def indent(lines, ind=4):
+    return ind*' '+lines.replace('\n', '\n'+ind*' ').rstrip(' ')
+
+def inject_before_lval(source, lval, code):
+    if source.count(lval)>1:
+        print()
+        print(lval)
+        raise RuntimeError('To many lvals (%s)' % lval)
+    elif not source.count(lval):
+        print()
+        print(lval)
+        assert lval not in source
+        raise RuntimeError('No lval found "%s"' % lval)
+    end = source.index(lval)
+    inj = source.rfind('\n', 0, end)
+    ind = inj
+    while source[ind+1]==' ':
+        ind+=1
+    ind -= inj
+    return source[:inj+1]+ indent(code, ind) + source[inj+1:]
+
+
+def get_continue_label(label):
+    return CONTINUE_LABEL%to_hex(label)
+
+def get_break_label(label):
+    return BREAK_LABEL%to_hex(label)
+
+
+def is_valid_py_name(name):
+    try:
+        compile(name+' =  11', 'a','exec')
+    except:
+        return False
+    return True
+
+def indent(lines, ind=4):
+    return ind*' '+lines.replace('\n', '\n'+ind*' ').rstrip(' ')
+
+def compose_regex(val):
+    reg, flags = val
+    #reg = REGEXP_CONVERTER._unescape_string(reg)
+    return u'/%s/%s' % (reg, flags)
+
+def float_repr(f):
+    if int(f)==f:
+        return repr(int(f))
+    return repr(f)
+
+def argsplit(args, sep=','):
+    """used to split JS args (it is not that simple as it seems because
+       sep can be inside brackets).
+
+       pass args *without* brackets!
+
+       Used also to parse array and object elements, and more"""
+    parsed_len  = 0
+    last = 0
+    splits = []
+    for e in bracket_split(args, brackets=['()', '[]', '{}']):
+        if e[0] not in ('(', '[', '{'):
+            for i, char in enumerate(e):
+                if char==sep:
+                    splits.append(args[last:parsed_len+i])
+                    last = parsed_len + i + 1
+        parsed_len += len(e)
+    splits.append(args[last:])
+    return splits
+
+
+def bracket_split(source, brackets=('()','{}','[]'), strip=False):
+    """DOES NOT RETURN EMPTY STRINGS (can only return empty bracket content if strip=True)"""
+    starts = [e[0] for e in brackets]
+    in_bracket = 0
+    n = 0
+    last = 0
+    while n<len(source):
+        e = source[n]
+        if not in_bracket and e in starts:
+            in_bracket = 1
+            start = n
+            b_start, b_end = brackets[starts.index(e)]
+        elif in_bracket:
+            if e==b_start:
+                in_bracket += 1
+            elif e==b_end:
+                in_bracket -= 1
+                if not in_bracket:
+                    if source[last:start]:
+                        yield source[last:start]
+                    last = n+1
+                    yield source[start+strip:n+1-strip]
+        n+=1
+    if source[last:]:
+        yield source[last:]
+
+
+def js_comma(a, b):
+    return 'PyJsComma('+a+','+b+')'
+
+def js_or(a, b):
+    return '('+a+' or '+b+')'
+
+def js_bor(a, b):
+    return '('+a+'|'+b+')'
+
+def js_bxor(a, b):
+    return '('+a+'^'+b+')'
+
+def js_band(a, b):
+    return '('+a+'&'+b+')'
+
+def js_and(a, b):
+    return '('+a+' and '+b+')'
+def js_strict_eq(a, b):
+    return 'PyJsStrictEq('+a+','+b+')'
+
+def js_strict_neq(a, b):
+    return 'PyJsStrictNeq('+a+','+b+')'
+
+#Not handled by python in the same way like JS. For example 2==2==True returns false.
+# In JS above would return true so we need brackets.
+def js_abstract_eq(a, b):
+    return '('+a+'=='+b+')'
+
+#just like ==
+def js_abstract_neq(a, b):
+    return '('+a+'!='+b+')'
+
+def js_lt(a, b):
+    return '('+a+'<'+b+')'
+
+def js_le(a, b):
+    return '('+a+'<='+b+')'
+
+def js_ge(a, b):
+    return '('+a+'>='+b+')'
+
+def js_gt(a, b):
+    return '('+a+'>'+b+')'
+
+def js_in(a, b):
+    return b+'.contains('+a+')'
+
+def js_instanceof(a, b):
+    return a+'.instanceof('+b+')'
+
+def js_lshift(a, b):
+    return '('+a+'<<'+b+')'
+
+def js_rshift(a, b):
+    return '('+a+'>>'+b+')'
+
+def js_shit(a, b):
+    return 'PyJsBshift('+a+','+b+')'
+
+def js_add(a, b):  # To simplify later process of converting unary operators + and ++
+    return '(%s+%s)'%(a, b)
+
+def js_sub(a, b):  # To simplify
+    return '(%s-%s)'%(a, b)
+
+def js_mul(a, b):
+    return '('+a+'*'+b+')'
+
+def js_div(a, b):
+    return '('+a+'/'+b+')'
+
+def js_mod(a, b):
+    return '('+a+'%'+b+')'
+
+def js_typeof(a):
+    cand = list(bracket_split(a, ('()',)))
+    if len(cand)==2 and cand[0]=='var.get':
+        return cand[0]+cand[1][:-1]+',throw=False).typeof()'
+    return a+'.typeof()'
+
+def js_void(a):
+    # eval and return undefined
+    return 'PyJsComma(%s, Js(None))' % a
+
+def js_new(a):
+    cands = list(bracket_split(a, ('()',)))
+    lim = len(cands)
+    if lim < 2:
+        return a + '.create()'
+    n = 0
+    while n < lim:
+        c = cands[n]
+        if c[0]=='(':
+            if cands[n-1].endswith('.get') and n+1>=lim:  # last get operation.
+                return a + '.create()'
+            elif cands[n-1][0]=='(':
+                return ''.join(cands[:n])+'.create' + c + ''.join(cands[n+1:])
+            elif cands[n-1]=='.callprop':
+                 beg = ''.join(cands[:n-1])
+                 args = argsplit(c[1:-1],',')
+                 prop = args[0]
+                 new_args = ','.join(args[1:])
+                 create = '.get(%s).create(%s)' % (prop, new_args)
+                 return beg + create +  ''.join(cands[n+1:])
+        n+=1
+    return a + '.create()'
+
+
+def js_delete(a):
+    #replace last get with delete.
+    c = list(bracket_split(a, ['()']))
+    beg, arglist = ''.join(c[:-1]).strip(), c[-1].strip()  #strips just to make sure... I will remove it later
+    if beg[-4:]!='.get':
+        print(a)
+        raise SyntaxError('Invalid delete operation')
+    return beg[:-3]+'delete'+arglist
+
+
+def js_neg(a):
+    return '(-'+a+')'
+
+def js_pos(a):
+    return '(+'+a+')'
+
+def js_inv(a):
+    return '(~'+a+')'
+
+def js_not(a):
+    return a+'.neg()'
+
+def js_postfix(a, inc, post):
+    bra = list(bracket_split(a, ('()',)))
+    meth = bra[-2]
+    if not meth.endswith('get'):
+        raise SyntaxError('Invalid ++ or -- operation.')
+    bra[-2] = bra[-2][:-3] + 'put'
+    bra[-1] = '(%s,Js(%s.to_number())%sJs(1))' % (bra[-1][1:-1], a, '+' if inc else '-')
+    res = ''.join(bra)
+    return res if not post else '(%s%sJs(1))' % (res, '-' if inc else '+')
+
+def js_pre_inc(a):
+    return js_postfix(a, True, False)
+
+def js_post_inc(a):
+    return js_postfix(a, True, True)
+
+def js_pre_dec(a):
+    return js_postfix(a, False, False)
+
+def js_post_dec(a):
+    return js_postfix(a, False, True)
+
+
+
+CONTINUE_LABEL = 'JS_CONTINUE_LABEL_%s'
+BREAK_LABEL = 'JS_BREAK_LABEL_%s'
+PREPARE = '''HOLDER = var.own.get(NAME)\nvar.force_own_put(NAME, PyExceptionToJs(PyJsTempException))\n'''
+RESTORE = '''if HOLDER is not None:\n    var.own[NAME] = HOLDER\nelse:\n    del var.own[NAME]\ndel HOLDER\n'''
+TRY_CATCH = '''%stry:\nBLOCKfinally:\n%s''' % (PREPARE, indent(RESTORE))
+
+
+
+OR = {'||': js_or}
+AND = {'&&': js_and}
+BOR = {'|': js_bor}
+BXOR = {'^': js_bxor}
+BAND = {'&': js_band}
+
+EQS = {'===': js_strict_eq,
+       '!==': js_strict_neq,
+       '==': js_abstract_eq, # we need == and != too. Read a note above method
+       '!=': js_abstract_neq}
+
+#Since JS does not have chained comparisons we need to implement all cmp methods.
+COMPS = {'<': js_lt,
+         '<=': js_le,
+         '>=': js_ge,
+         '>': js_gt,
+         'instanceof': js_instanceof,  #todo change to validitate
+         'in': js_in}
+
+BSHIFTS = {'<<': js_lshift,
+           '>>': js_rshift,
+           '>>>': js_shit}
+
+ADDS = {'+': js_add,
+        '-': js_sub}
+
+MULTS = {'*': js_mul,
+         '/': js_div,
+         '%': js_mod}
+BINARY = {}
+BINARY.update(ADDS)
+BINARY.update(MULTS)
+BINARY.update(BSHIFTS)
+BINARY.update(COMPS)
+BINARY.update(EQS)
+BINARY.update(BAND)
+BINARY.update(BXOR)
+BINARY.update(BOR)
+BINARY.update(AND)
+BINARY.update(OR)
+#Note they dont contain ++ and -- methods because they both have 2 different methods
+# correct method will be found automatically in translate function
+UNARY = {'typeof': js_typeof,
+         'void': js_void,
+         'new': js_new,
+         'delete': js_delete,
+         '!': js_not,
+         '-': js_neg,
+         '+': js_pos,
+         '~': js_inv,
+         '++': None,
+         '--': None
+         }

lib/js2py/translators/jsregexps.py

@@ -0,0 +1,219 @@
+from pyjsparser.pyjsparserdata import *
+
+REGEXP_SPECIAL_SINGLE = {'\\', '^', '$', '*', '+', '?', '.'}
+
+NOT_PATTERN_CHARS = {'^', '$', '\\', '.', '*', '+', '?', '(', ')', '[', ']',  '|'}  # what about '{', '}',  ???
+
+CHAR_CLASS_ESCAPE = {'d', 'D', 's', 'S', 'w', 'W'}
+CONTROL_ESCAPE_CHARS = {'f', 'n', 'r', 't', 'v'}
+CONTROL_LETTERS = {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't',
+                   'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N',
+                   'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'}
+
+def SpecialChar(char):
+    return {'type': 'SpecialChar',
+            'content': char}
+
+
+def isPatternCharacter(char):
+    return char not in NOT_PATTERN_CHARS
+
+class JsRegExpParser:
+    def __init__(self, source, flags):
+        self.source = source
+        self.flags = flags
+        self.index = 0
+        self.length = len(source)
+        self.lineNumber = 0
+        self.lineStart = 0
+
+
+    def parsePattern(self):
+        '''Perform sctring escape - for regexp literals'''
+        return {'type': 'Pattern',
+                'contents': self.parseDisjunction()}
+
+    def parseDisjunction(self):
+        alternatives = []
+        while True:
+            alternatives.append(self.parseAlternative())
+            if not self.isEOF():
+                self.expect_character('|')
+            else:
+                break
+        return {'type': 'Disjunction',
+                'contents': alternatives}
+
+    def isEOF(self):
+        if self.index>=self.length:
+            return True
+        return False
+
+    def expect_character(self, character):
+        if self.source[self.index]!=character:
+            self.throwUnexpected(character)
+        self.index += 1
+
+    def parseAlternative(self):
+        contents = []
+        while not self.isEOF() and self.source[self.index]!='|':
+            contents.append(self.parseTerm())
+        return {'type': 'Alternative',
+                'contents': contents}
+
+    def follows(self, chars):
+        for i, c in enumerate(chars):
+            if self.index+i>=self.length or self.source[self.index+i] != c:
+                return False
+        return True
+
+    def parseTerm(self):
+        assertion = self.parseAssertion()
+        if assertion:
+            return assertion
+        else:
+            return {'type': 'Term',
+                    'contents': self.parseAtom()}  # quantifier will go inside atom!
+
+
+    def parseAssertion(self):
+        if self.follows('$'):
+            content = SpecialChar('$')
+            self.index += 1
+        elif self.follows('^'):
+            content = SpecialChar('^')
+            self.index += 1
+        elif self.follows('\\b'):
+            content = SpecialChar('\\b')
+            self.index += 2
+        elif self.follows('\\B'):
+            content = SpecialChar('\\B')
+            self.index += 2
+        elif self.follows('(?='):
+            self.index += 3
+            dis = self.parseDisjunction()
+            self.expect_character(')')
+            content = {'type': 'Lookached',
+                       'contents': dis,
+                       'negated': False}
+        elif self.follows('(?!'):
+            self.index += 3
+            dis = self.parseDisjunction()
+            self.expect_character(')')
+            content = {'type': 'Lookached',
+                       'contents': dis,
+                       'negated': True}
+        else:
+            return None
+        return {'type': 'Assertion',
+                'content': content}
+
+    def parseAtom(self):
+        if self.follows('.'):
+            content =  SpecialChar('.')
+            self.index += 1
+        elif self.follows('\\'):
+            self.index += 1
+            content = self.parseAtomEscape()
+        elif self.follows('['):
+            content = self.parseCharacterClass()
+        elif self.follows('(?:'):
+            self.index += 3
+            dis = self.parseDisjunction()
+            self.expect_character(')')
+            content = 'idk'
+        elif self.follows('('):
+            self.index += 1
+            dis = self.parseDisjunction()
+            self.expect_character(')')
+            content = 'idk'
+        elif isPatternCharacter(self.source[self.index]):
+            content = self.source[self.index]
+            self.index += 1
+        else:
+            return None
+        quantifier = self.parseQuantifier()
+        return {'type': 'Atom',
+                'content': content,
+                'quantifier': quantifier}
+
+    def parseQuantifier(self):
+        prefix = self.parseQuantifierPrefix()
+        if not prefix:
+            return None
+        greedy = True
+        if self.follows('?'):
+            self.index += 1
+            greedy = False
+        return {'type': 'Quantifier',
+                'contents': prefix,
+                'greedy': greedy}
+
+    def parseQuantifierPrefix(self):
+        if self.isEOF():
+            return None
+        if self.follows('+'):
+            content = '+'
+            self.index += 1
+        elif self.follows('?'):
+            content = '?'
+            self.index += 1
+        elif self.follows('*'):
+            content = '*'
+            self.index += 1
+        elif self.follows('{'): # try matching otherwise return None and restore the state
+            i = self.index
+            self.index += 1
+            digs1 = self.scanDecimalDigs()
+            # if no minimal number of digs provided then return no quantifier
+            if not digs1:
+                self.index = i
+                return None
+            # scan char limit if provided
+            if self.follows(','):
+                self.index += 1
+                digs2 = self.scanDecimalDigs()
+            else:
+                digs2 = ''
+            # must be valid!
+            if not self.follows('}'):
+                self.index = i
+                return None
+            else:
+                self.expect_character('}')
+                content = int(digs1), int(digs2) if digs2 else None
+        else:
+            return None
+        return content
+
+
+    def parseAtomEscape(self):
+        ch = self.source[self.index]
+        if isDecimalDigit(ch) and ch!=0:
+            digs = self.scanDecimalDigs()
+        elif ch in CHAR_CLASS_ESCAPE:
+            self.index += 1
+            return SpecialChar('\\' + ch)
+        else:
+            return self.parseCharacterEscape()
+
+    def parseCharacterEscape(self):
+        ch = self.source[self.index]
+        if ch in CONTROL_ESCAPE_CHARS:
+            return SpecialChar('\\' + ch)
+        if ch=='c':
+            'ok, fuck this shit.'
+
+
+    def scanDecimalDigs(self):
+        s = self.index
+        while not self.isEOF() and isDecimalDigit(self.source[self.index]):
+            self.index += 1
+        return self.source[s:self.index]
+
+
+
+
+
+a = JsRegExpParser('a(?=x)', '')
+print(a.parsePattern())

lib/js2py/translators/translating_nodes.py

@@ -0,0 +1,641 @@
+from __future__ import unicode_literals
+from pyjsparser.pyjsparserdata import *
+from .friendly_nodes import *
+import random
+import six
+
+if six.PY3:
+    from functools import reduce
+    xrange = range
+    unicode = str
+# number of characters above which expression will be split to multiple lines in order to avoid python parser stack overflow
+# still experimental so I suggest to set it to 400 in order to avoid common errors
+# set it to smaller value only if you have problems with parser stack overflow
+LINE_LEN_LIMIT = 400   #  200  # or any other value - the larger the smaller probability of errors :)
+
+class ForController:
+    def __init__(self):
+        self.inside = [False]
+        self.update = ''
+
+    def enter_for(self, update):
+        self.inside.append(True)
+        self.update = update
+
+    def leave_for(self):
+        self.inside.pop()
+
+    def enter_other(self):
+        self.inside.append(False)
+
+    def leave_other(self):
+        self.inside.pop()
+
+    def is_inside(self):
+        return self.inside[-1]
+
+
+
+class InlineStack:
+    NAME = 'PyJs_%s_%d_'
+    def __init__(self):
+        self.reps = {}
+        self.names = []
+
+    def inject_inlines(self, source):
+        for lval in self.names: # first in first out! Its important by the way
+            source = inject_before_lval(source, lval, self.reps[lval])
+        return source
+
+    def require(self, typ):
+        name = self.NAME % (typ, len(self.names))
+        self.names.append(name)
+        return name
+
+    def define(self, name, val):
+        self.reps[name] = val
+
+    def reset(self):
+        self.rel = {}
+        self.names = []
+
+
+class ContextStack:
+    def __init__(self):
+        self.to_register = set([])
+        self.to_define = {}
+
+    def reset(self):
+        self.to_register = set([])
+        self.to_define = {}
+
+    def register(self, var):
+        self.to_register.add(var)
+
+    def define(self, name, code):
+        self.to_define[name] = code
+        self.register(name)
+
+    def get_code(self):
+        code = 'var.registers([%s])\n' % ', '.join(repr(e) for e in self.to_register)
+        for name, func_code in six.iteritems(self.to_define):
+            code += func_code
+        return code
+
+
+
+def clean_stacks():
+    global Context, inline_stack
+    Context = ContextStack()
+    inline_stack = InlineStack()
+
+
+
+
+def to_key(literal_or_identifier):
+    ''' returns string representation of this object'''
+    if literal_or_identifier['type']=='Identifier':
+        return literal_or_identifier['name']
+    elif literal_or_identifier['type']=='Literal':
+        k = literal_or_identifier['value']
+        if isinstance(k, float):
+            return unicode(float_repr(k))
+        elif 'regex' in literal_or_identifier:
+            return compose_regex(k)
+        elif isinstance(k, bool):
+            return 'true' if k else 'false'
+        elif k is None:
+            return 'null'
+        else:
+            return unicode(k)
+
+def trans(ele, standard=False):
+    """Translates esprima syntax tree to python by delegating to appropriate translating node"""
+    try:
+        node = globals().get(ele['type'])
+        if not node:
+            raise NotImplementedError('%s is not supported!' % ele['type'])
+        if standard:
+            node = node.__dict__['standard'] if 'standard' in node.__dict__ else node
+        return node(**ele)
+    except:
+        #print ele
+        raise
+
+
+def limited(func):
+    '''Decorator limiting resulting line length in order to avoid python parser stack overflow -
+      If expression longer than LINE_LEN_LIMIT characters then it will be moved to upper line
+     USE ONLY ON EXPRESSIONS!!! '''
+    def f(standard=False, **args):
+        insert_pos = len(inline_stack.names)  # in case line is longer than limit we will have to insert the lval at current position
+                                              # this is because calling func will change inline_stack.
+                                              # we cant use inline_stack.require here because we dont know whether line overflows yet
+        res = func(**args)
+        if len(res)>LINE_LEN_LIMIT:
+            name = inline_stack.require('LONG')
+            inline_stack.names.pop()
+            inline_stack.names.insert(insert_pos, name)
+            res = 'def %s(var=var):\n    return %s\n' % (name, res)
+            inline_stack.define(name, res)
+            return name+'()'
+        else:
+            return res
+    f.__dict__['standard'] = func
+    return f
+
+
+
+
+
+
+# ==== IDENTIFIERS AND LITERALS  =======
+
+
+inf = float('inf')
+
+
+def Literal(type, value, raw, regex=None):
+    if regex: # regex
+        return 'JsRegExp(%s)' % repr(compose_regex(value))
+    elif value is None:  # null
+        return 'var.get(u"null")'
+    # Todo template
+    # String, Bool, Float
+    return 'Js(%s)' % repr(value) if value!=inf else 'Js(float("inf"))'
+
+def Identifier(type, name):
+    return 'var.get(%s)' % repr(name)
+
+@limited
+def MemberExpression(type, computed, object, property):
+    far_left = trans(object)
+    if computed:  # obj[prop] type accessor
+        # may be literal which is the same in every case so we can save some time on conversion
+        if property['type'] == 'Literal':
+            prop = repr(to_key(property))
+        else: # worst case
+            prop = trans(property)
+    else: # always the same since not computed (obj.prop accessor)
+        prop = repr(to_key(property))
+    return far_left + '.get(%s)' % prop
+
+
+def ThisExpression(type):
+    return 'var.get(u"this")'
+
+@limited
+def CallExpression(type, callee, arguments):
+    arguments = [trans(e) for e in arguments]
+    if callee['type']=='MemberExpression':
+        far_left = trans(callee['object'])
+        if callee['computed']:  # obj[prop] type accessor
+            # may be literal which is the same in every case so we can save some time on conversion
+            if callee['property']['type'] == 'Literal':
+                prop = repr(to_key(callee['property']))
+            else: # worst case
+                prop = trans(callee['property'])  # its not a string literal! so no repr
+        else: # always the same since not computed (obj.prop accessor)
+            prop = repr(to_key(callee['property']))
+        arguments.insert(0, prop)
+        return far_left + '.callprop(%s)' % ', '.join(arguments)
+    else: # standard call
+        return trans(callee) + '(%s)' % ', '.join(arguments)
+
+
+
+# ========== ARRAYS ============
+
+
+def ArrayExpression(type, elements):  # todo fix null inside problem
+    return 'Js([%s])' % ', '.join(trans(e) if e else 'None' for e in elements)
+
+
+
+# ========== OBJECTS =============
+
+def ObjectExpression(type, properties):
+    name = inline_stack.require('Object')
+    elems = []
+    after = ''
+    for p in properties:
+        if p['kind']=='init':
+            elems.append('%s:%s' % Property(**p))
+        elif p['kind']=='set':
+            k, setter = Property(**p)  # setter is just a lval referring to that function, it will be defined in InlineStack automatically
+            after += '%s.define_own_property(%s, {"set":%s, "configurable":True, "enumerable":True})\n' % (name, k, setter)
+        elif p['kind']=='get':
+            k, getter = Property(**p)
+            after += '%s.define_own_property(%s, {"get":%s, "configurable":True, "enumerable":True})\n' % (name, k, getter)
+        else:
+            raise RuntimeError('Unexpected object propery kind')
+    obj = '%s = Js({%s})\n' % (name, ','.join(elems))
+    inline_stack.define(name, obj+after)
+    return name
+
+
+
+def Property(type, kind, key, computed, value, method, shorthand):
+    if shorthand or computed:
+        raise NotImplementedError('Shorthand and Computed properties not implemented!')
+    k = to_key(key)
+    if k is None:
+        raise SyntaxError('Invalid key in dictionary! Or bug in Js2Py')
+    v = trans(value)
+    return repr(k), v
+
+
+# ========== EXPRESSIONS ============
+
+
+@limited
+def UnaryExpression(type, operator, argument, prefix):
+    a = trans(argument, standard=True) # unary involve some complex operations so we cant use line shorteners here
+    if operator=='delete':
+        if argument['type'] in ('Identifier', 'MemberExpression'):
+            # means that operation is valid
+            return js_delete(a)
+        return 'PyJsComma(%s, Js(True))' % a   # otherwise not valid, just perform expression and return true.
+    elif operator=='typeof':
+        return js_typeof(a)
+    return UNARY[operator](a)
+
+@limited
+def BinaryExpression(type, operator, left, right):
+    a = trans(left)
+    b = trans(right)
+    # delegate to our friends
+    return BINARY[operator](a,b)
+
+@limited
+def UpdateExpression(type, operator, argument, prefix):
+    a = trans(argument, standard=True)  # also complex operation involving parsing of the result so no line length reducing here
+    return js_postfix(a, operator=='++', not prefix)
+
+@limited
+def AssignmentExpression(type, operator, left, right):
+    operator = operator[:-1]
+    if left['type']=='Identifier':
+        if operator:
+            return 'var.put(%s, %s, %s)' % (repr(to_key(left)), trans(right), repr(operator))
+        else:
+            return 'var.put(%s, %s)' % (repr(to_key(left)), trans(right))
+    elif left['type']=='MemberExpression':
+        far_left = trans(left['object'])
+        if left['computed']:  # obj[prop] type accessor
+            # may be literal which is the same in every case so we can save some time on conversion
+            if left['property']['type'] == 'Literal':
+                prop = repr(to_key(left['property']))
+            else: # worst case
+                prop = trans(left['property'])   # its not a string literal! so no repr
+        else: # always the same since not computed (obj.prop accessor)
+            prop = repr(to_key(left['property']))
+        if operator:
+            return far_left + '.put(%s, %s, %s)' % (prop, trans(right), repr(operator))
+        else:
+            return far_left + '.put(%s, %s)' % (prop, trans(right))
+    else:
+        raise SyntaxError('Invalid left hand side in assignment!')
+six
+@limited
+def SequenceExpression(type, expressions):
+    return reduce(js_comma, (trans(e) for e in expressions))
+
+@limited
+def NewExpression(type, callee, arguments):
+    return trans(callee) + '.create(%s)' % ', '.join(trans(e) for e in arguments)
+
+@limited
+def ConditionalExpression(type, test, consequent, alternate): # caused plenty of problems in my home-made translator :)
+    return '(%s if %s else %s)' % (trans(consequent), trans(test), trans(alternate))
+
+
+
+# ===========  STATEMENTS =============
+
+
+def BlockStatement(type, body):
+    return StatementList(body) # never returns empty string! In the worst case returns pass\n
+
+
+def ExpressionStatement(type, expression):
+    return trans(expression) + '\n'  # end expression space with new line
+
+
+def BreakStatement(type, label):
+    if label:
+        return 'raise %s("Breaked")\n' % (get_break_label(label['name']))
+    else:
+        return 'break\n'
+
+
+def ContinueStatement(type, label):
+    if label:
+        return 'raise %s("Continued")\n' % (get_continue_label(label['name']))
+    else:
+        return 'continue\n'
+
+def ReturnStatement(type, argument):
+    return 'return %s\n' % (trans(argument) if argument else "var.get('undefined')")
+
+
+def EmptyStatement(type):
+    return 'pass\n'
+
+
+def DebuggerStatement(type):
+    return 'pass\n'
+
+
+def DoWhileStatement(type, body, test):
+    inside = trans(body) + 'if not %s:\n' % trans(test) + indent('break\n')
+    result = 'while 1:\n' + indent(inside)
+    return result
+
+
+
+def ForStatement(type, init, test, update, body):
+    update = indent(trans(update)) if update else ''
+    init = trans(init)  if init else ''
+    if not init.endswith('\n'):
+        init += '\n'
+    test = trans(test) if test else '1'
+    if not update:
+        result = '#for JS loop\n%swhile %s:\n%s%s\n' % (init, test, indent(trans(body)), update)
+    else:
+        result = '#for JS loop\n%swhile %s:\n' % (init, test)
+        body = 'try:\n%sfinally:\n    %s\n' % (indent(trans(body)), update)
+        result += indent(body)
+    return result
+
+
+def ForInStatement(type, left, right, body, each):
+    res =  'for PyJsTemp in %s:\n' % trans(right)
+    if left['type']=="VariableDeclaration":
+        addon = trans(left) # make sure variable is registered
+        if addon != 'pass\n':
+            res = addon + res # we have to execute this expression :(
+        # now extract the name
+        try:
+            name = left['declarations'][0]['id']['name']
+        except:
+            raise RuntimeError('Unusual ForIn loop')
+    elif left['type']=='Identifier':
+        name = left['name']
+    else:
+        raise RuntimeError('Unusual ForIn loop')
+    res += indent('var.put(%s, PyJsTemp)\n' % repr(name) + trans(body))
+    return res
+
+
+def IfStatement(type, test, consequent, alternate):
+    # NOTE we cannot do elif because function definition inside elif statement would not be possible!
+    IF = 'if %s:\n' % trans(test)
+    IF += indent(trans(consequent))
+    if not alternate:
+        return IF
+    ELSE = 'else:\n' + indent(trans(alternate))
+    return IF + ELSE
+
+
+def LabeledStatement(type, label, body):
+    # todo consider using smarter approach!
+    inside = trans(body)
+    defs = ''
+    if inside.startswith('while ') or inside.startswith('for ') or inside.startswith('#for'):
+        # we have to add contine label as well...
+        # 3 or 1 since #for loop type has more lines before real for.
+        sep = 1 if not inside.startswith('#for') else 3
+        cont_label = get_continue_label(label['name'])
+        temp = inside.split('\n')
+        injected = 'try:\n'+'\n'.join(temp[sep:])
+        injected += 'except %s:\n    pass\n'%cont_label
+        inside = '\n'.join(temp[:sep])+'\n'+indent(injected)
+        defs += 'class %s(Exception): pass\n'%cont_label
+    break_label = get_break_label(label['name'])
+    inside = 'try:\n%sexcept %s:\n    pass\n'% (indent(inside), break_label)
+    defs += 'class %s(Exception): pass\n'%break_label
+    return defs + inside
+
+
+def StatementList(lis):
+    if lis:  # ensure we don't return empty string because it may ruin indentation!
+        code = ''.join(trans(e) for e in lis)
+        return code if code else 'pass\n'
+    else:
+        return 'pass\n'
+
+def PyimportStatement(type, imp):
+    lib = imp['name']
+    jlib = 'PyImport_%s' % lib
+    code = 'import %s as %s\n' % (lib, jlib)
+    #check whether valid lib name...
+    try:
+        compile(code, '', 'exec')
+    except:
+        raise SyntaxError('Invalid Python module name (%s) in pyimport statement'%lib)
+    # var.pyimport will handle module conversion to PyJs object
+    code += 'var.pyimport(%s, %s)\n' % (repr(lib), jlib)
+    return code
+
+def SwitchStatement(type, discriminant, cases):
+    #TODO there will be a problem with continue in a switch statement.... FIX IT
+    code = 'while 1:\n' + indent('SWITCHED = False\nCONDITION = (%s)\n')
+    code = code % trans(discriminant)
+    for case in cases:
+        case_code = None
+        if case['test']: # case (x):
+            case_code = 'if SWITCHED or PyJsStrictEq(CONDITION, %s):\n' % (trans(case['test']))
+        else:  # default:
+            case_code = 'if True:\n'
+        case_code += indent('SWITCHED = True\n')
+        case_code += indent(StatementList(case['consequent']))
+        # one more indent for whole
+        code += indent(case_code)
+    # prevent infinite loop and sort out nested switch...
+    code += indent('SWITCHED = True\nbreak\n')
+    return code
+
+
+def ThrowStatement(type, argument):
+    return 'PyJsTempException = JsToPyException(%s)\nraise PyJsTempException\n' % trans(argument)
+
+
+def TryStatement(type, block, handler, handlers, guardedHandlers, finalizer):
+    result = 'try:\n%s' % indent(trans(block))
+    # complicated catch statement...
+    if handler:
+        identifier = handler['param']['name']
+        holder = 'PyJsHolder_%s_%d'%(to_hex(identifier), random.randrange(1e8))
+        identifier = repr(identifier)
+        result += 'except PyJsException as PyJsTempException:\n'
+        # fill in except ( catch ) block and remember to recover holder variable to its previous state
+        result += indent(TRY_CATCH.replace('HOLDER', holder).replace('NAME', identifier).replace('BLOCK', indent(trans(handler['body']))))
+    # translate finally statement if present
+    if finalizer:
+        result += 'finally:\n%s' % indent(trans(finalizer))
+    return result
+
+
+def LexicalDeclaration(type, declarations, kind):
+    raise NotImplementedError('let and const not implemented yet but they will be soon! Check github for updates.')
+
+
+def VariableDeclarator(type, id, init):
+    name = id['name']
+    # register the name if not already registered
+    Context.register(name)
+    if init:
+        return 'var.put(%s, %s)\n' % (repr(name), trans(init))
+    return ''
+
+
+def VariableDeclaration(type, declarations, kind):
+    code = ''.join(trans(d) for d in declarations)
+    return code if code else 'pass\n'
+
+
+def WhileStatement(type, test, body):
+    result = 'while %s:\n'%trans(test) + indent(trans(body))
+    return result
+
+
+def WithStatement(type, object, body):
+    raise NotImplementedError('With statement not implemented!')
+
+
+def Program(type, body):
+    inline_stack.reset()
+    code = ''.join(trans(e) for e in body)
+    # here add hoisted elements (register variables and define functions)
+    code = Context.get_code() + code
+    # replace all inline variables
+    code = inline_stack.inject_inlines(code)
+    return code
+
+
+
+# ======== FUNCTIONS ============
+
+def FunctionDeclaration(type, id, params, defaults, body, generator, expression):
+    if generator:
+        raise NotImplementedError('Generators not supported')
+    if defaults:
+        raise NotImplementedError('Defaults not supported')
+    if not id:
+        return FunctionExpression(type, id, params, defaults, body, generator, expression) + '\n'
+    JsName = id['name']
+    PyName = 'PyJsHoisted_%s_' % JsName
+    PyName = PyName if is_valid_py_name(PyName) else 'PyJsHoistedNonPyName'
+    # this is quite complicated
+    global Context
+    previous_context = Context
+    # change context to the context of this function
+    Context = ContextStack()
+    # translate body within current context
+    code = trans(body)
+    # get arg names
+    vars = [v['name'] for v in params]
+    # args are automaticaly registered variables
+    Context.to_register.update(vars)
+    # add all hoisted elements inside function
+    code = Context.get_code() + code
+    # check whether args are valid python names:
+    used_vars = []
+    for v in vars:
+        if is_valid_py_name(v):
+            used_vars.append(v)
+        else: # invalid arg in python, for example $, replace with alternatice arg
+            used_vars.append('PyJsArg_%s_' % to_hex(v))
+    header = '@Js\n'
+    header+= 'def %s(%sthis, arguments, var=var):\n' % (PyName, ', '.join(used_vars) +(', ' if vars else ''))
+    # transfer names from Py scope to Js scope
+    arg_map = dict(zip(vars, used_vars))
+    arg_map.update({'this':'this', 'arguments':'arguments'})
+    arg_conv = 'var = Scope({%s}, var)\n' % ', '.join(repr(k)+':'+v for k,v in six.iteritems(arg_map))
+    # and finally set the name of the function to its real name:
+    footer = '%s.func_name = %s\n' % (PyName, repr(JsName))
+    footer+= 'var.put(%s, %s)\n' % (repr(JsName), PyName)
+    whole_code = header + indent(arg_conv+code) + footer
+    # restore context
+    Context = previous_context
+    # define in upper context
+    Context.define(JsName, whole_code)
+    return 'pass\n'
+
+
+def FunctionExpression(type, id, params, defaults, body, generator, expression):
+    if generator:
+        raise NotImplementedError('Generators not supported')
+    if defaults:
+        raise NotImplementedError('Defaults not supported')
+    JsName = id['name'] if id else 'anonymous'
+    if not is_valid_py_name(JsName):
+        ScriptName = 'InlineNonPyName'
+    else:
+        ScriptName = JsName
+    PyName = inline_stack.require(ScriptName)  # this is unique
+
+    # again quite complicated
+    global Context
+    previous_context = Context
+    # change context to the context of this function
+    Context = ContextStack()
+    # translate body within current context
+    code = trans(body)
+    # get arg names
+    vars = [v['name'] for v in params]
+    # args are automaticaly registered variables
+    Context.to_register.update(vars)
+    # add all hoisted elements inside function
+    code = Context.get_code() + code
+    # check whether args are valid python names:
+    used_vars = []
+    for v in vars:
+        if is_valid_py_name(v):
+            used_vars.append(v)
+        else: # invalid arg in python, for example $, replace with alternatice arg
+            used_vars.append('PyJsArg_%s_' % to_hex(v))
+    header = '@Js\n'
+    header+= 'def %s(%sthis, arguments, var=var):\n' % (PyName, ', '.join(used_vars) +(', ' if vars else ''))
+    # transfer names from Py scope to Js scope
+    arg_map = dict(zip(vars, used_vars))
+    arg_map.update({'this':'this', 'arguments':'arguments'})
+    if id: # make self available from inside...
+        if id['name'] not in arg_map:
+            arg_map[id['name']] = PyName
+    arg_conv = 'var = Scope({%s}, var)\n' % ', '.join(repr(k)+':'+v for k,v in six.iteritems(arg_map))
+    # and finally set the name of the function to its real name:
+    footer = '%s._set_name(%s)\n' % (PyName, repr(JsName))
+    whole_code = header + indent(arg_conv+code) + footer
+    # restore context
+    Context = previous_context
+    # define in upper context
+    inline_stack.define(PyName, whole_code)
+    return PyName
+
+
+LogicalExpression = BinaryExpression
+PostfixExpression = UpdateExpression
+
+clean_stacks()
+
+if __name__=='__main__':
+    import codecs
+    import time
+    import pyjsparser
+
+    c = None#'''`ijfdij`'''
+    if not c:
+        with codecs.open("esp.js", "r", "utf-8") as f:
+            c = f.read()
+
+    print('Started')
+    t = time.time()
+    res = trans(pyjsparser.PyJsParser().parse(c))
+    dt = time.time() - t+ 0.000000001
+    print('Translated everyting in', round(dt,5), 'seconds.')
+    print('Thats %d characters per second' % int(len(c)/dt))
+    with open('res.py', 'w') as f:
+        f.write(res)
+

lib/js2py/translators/translator.py

@@ -0,0 +1,179 @@
+import pyjsparser
+import pyjsparser.parser
+from . import translating_nodes
+
+import hashlib
+import re
+
+
+# Enable Js2Py exceptions and pyimport in parser
+pyjsparser.parser.ENABLE_JS2PY_ERRORS = True
+pyjsparser.parser.ENABLE_PYIMPORT = True
+
+# the re below is how we'll recognise numeric constants.
+# it finds any 'simple numeric that is not preceded with an alphanumeric character
+# the numeric can be a float (so a dot is found) but
+# it does not recognise notation such as 123e5, 0xFF, infinity or NaN
+CP_NUMERIC_RE = re.compile(r'(?<![a-zA-Z0-9_"\'])([0-9\.]+)')
+CP_NUMERIC_PLACEHOLDER = '__PyJsNUM_%i_PyJsNUM__'
+CP_NUMERIC_PLACEHOLDER_REVERSE_RE = re.compile(
+    CP_NUMERIC_PLACEHOLDER.replace('%i', '([0-9\.]+)')
+    )
+
+# the re below is how we'll recognise string constants
+# it finds a ' or ", then reads until the next matching ' or "
+# this re only services simple cases, it can not be used when
+# there are escaped quotes in the expression
+
+#CP_STRING_1 = re.compile(r'(["\'])(.*?)\1') # this is how we'll recognise string constants
+
+CP_STRING = '"([^\\\\"]+|\\\\([bfnrtv\'"\\\\]|[0-3]?[0-7]{1,2}|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}))*"|\'([^\\\\\']+|\\\\([bfnrtv\'"\\\\]|[0-3]?[0-7]{1,2}|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}))*\''
+CP_STRING_RE = re.compile(CP_STRING) # this is how we'll recognise string constants
+CP_STRING_PLACEHOLDER = '__PyJsSTR_%i_PyJsSTR__'
+CP_STRING_PLACEHOLDER_REVERSE_RE = re.compile(
+    CP_STRING_PLACEHOLDER.replace('%i', '([0-9\.]+)')
+    )
+
+cache = {}
+
+# This crap is still needed but I removed it for speed reasons. Have to think ofa  better idea
+# import js2py.pyjs, sys
+# # Redefine builtin objects... Do you have a better idea?
+# for m in list(sys.modules):
+# 	if m.startswith('js2py'):
+# 		del sys.modules[m]
+# del js2py.pyjs
+# del js2py
+
+DEFAULT_HEADER = u'''from js2py.pyjs import *
+# setting scope
+var = Scope( JS_BUILTINS )
+set_global_object(var)
+
+# Code follows:
+'''
+
+
+def dbg(x):
+    """does nothing, legacy dummy function"""
+    return ''
+
+def translate_js(js, HEADER=DEFAULT_HEADER, use_compilation_plan=False):
+    """js has to be a javascript source code.
+       returns equivalent python code."""
+    if use_compilation_plan and not '//' in js and not '/*' in js:
+        return translate_js_with_compilation_plan(js, HEADER=HEADER)
+    parser = pyjsparser.PyJsParser()
+    parsed = parser.parse(js) # js to esprima syntax tree
+    # Another way of doing that would be with my auto esprima translation but its much slower and causes import problems:
+    # parsed = esprima.parse(js).to_dict()
+    translating_nodes.clean_stacks()
+    return HEADER + translating_nodes.trans(parsed)  # syntax tree to python code
+
+class match_unumerator(object):
+    """This class ise used """
+    matchcount = -1
+
+    def __init__(self, placeholder_mask):
+        self.placeholder_mask = placeholder_mask
+        self.matches = []
+
+    def __call__(self, match):
+        self.matchcount += 1
+        self.matches.append(match.group(0))
+        return self.placeholder_mask%self.matchcount
+
+    def __repr__(self):
+        return '\n'.join(self.placeholder_mask%counter + '=' + match for counter, match in enumerate(self.matches))
+
+    def wrap_up(self, output):
+        for counter, value in enumerate(self.matches):
+            output = output.replace("u'" + self.placeholder_mask%(counter) + "'", value, 1)
+        return output
+
+def get_compilation_plan(js):
+    match_increaser_str = match_unumerator(CP_STRING_PLACEHOLDER)
+    compilation_plan = re.sub(
+        CP_STRING, match_increaser_str, js
+    )
+
+    match_increaser_num = match_unumerator(CP_NUMERIC_PLACEHOLDER)
+    compilation_plan = re.sub(CP_NUMERIC_RE, match_increaser_num, compilation_plan)
+    # now put quotes, note that just patching string replaces is somewhat faster than
+    # using another re:
+    compilation_plan = compilation_plan.replace('__PyJsNUM_', '"__PyJsNUM_').replace('_PyJsNUM__', '_PyJsNUM__"')
+    compilation_plan = compilation_plan.replace('__PyJsSTR_', '"__PyJsSTR_').replace('_PyJsSTR__', '_PyJsSTR__"')
+
+    return match_increaser_str, match_increaser_num, compilation_plan
+
+def translate_js_with_compilation_plan(js, HEADER=DEFAULT_HEADER):
+    """js has to be a javascript source code.
+       returns equivalent python code.
+
+       compile plans only work with the following restrictions:
+       - only enabled for oneliner expressions
+       - when there are comments in the js code string substitution is disabled
+       - when there nested escaped quotes string substitution is disabled, so
+
+       cacheable:
+       Q1 == 1 && name == 'harry'
+
+       not cacheable:
+       Q1 == 1 && name == 'harry' // some comment
+
+       not cacheable:
+       Q1 == 1 && name == 'o\'Reilly'
+
+       not cacheable:
+       Q1 == 1 && name /* some comment */ == 'o\'Reilly'
+       """
+
+    match_increaser_str, match_increaser_num, compilation_plan = get_compilation_plan(js)
+
+    cp_hash = hashlib.md5(compilation_plan.encode('utf-8')).digest()
+    try:
+        python_code = cache[cp_hash]['proto_python_code']
+    except:
+        parser = pyjsparser.PyJsParser()
+        parsed = parser.parse(compilation_plan) # js to esprima syntax tree
+        # Another way of doing that would be with my auto esprima translation but its much slower and causes import problems:
+        # parsed = esprima.parse(js).to_dict()
+        translating_nodes.clean_stacks()
+        python_code = translating_nodes.trans(parsed)  # syntax tree to python code
+        cache[cp_hash] = {
+            'compilation_plan': compilation_plan,
+            'proto_python_code': python_code,
+        }
+
+    python_code = match_increaser_str.wrap_up(python_code)
+    python_code = match_increaser_num.wrap_up(python_code)
+
+    return HEADER + python_code
+
+def trasnlate(js, HEADER=DEFAULT_HEADER):
+    """js has to be a javascript source code.
+       returns equivalent python code.
+
+       Equivalent to translate_js"""
+    return translate_js(js, HEADER)
+
+
+syntax_tree_translate = translating_nodes.trans
+
+if __name__=='__main__':
+    PROFILE = False
+    import js2py
+    import codecs
+    def main():
+        with codecs.open("esprima.js", "r", "utf-8") as f:
+            d = f.read()
+            r = js2py.translate_js(d)
+
+            with open('res.py','wb') as f2:
+                f2.write(r)
+            exec(r, {})
+    if PROFILE:
+        import cProfile
+        cProfile.run('main()', sort='tottime')
+    else:
+        main()

lib/js2py/utils/injector.py

@@ -0,0 +1,239 @@
+__all__ = ['fix_js_args']
+
+import types
+from collections import namedtuple
+import opcode
+import six
+import sys
+import dis
+
+if six.PY3:
+    xrange = range
+    chr = lambda x: x
+
+# Opcode constants used for comparison and replacecment
+LOAD_FAST = opcode.opmap['LOAD_FAST']
+LOAD_GLOBAL = opcode.opmap['LOAD_GLOBAL']
+STORE_FAST = opcode.opmap['STORE_FAST']
+
+def fix_js_args(func):
+    '''Use this function when unsure whether func takes this and arguments as its last 2 args.
+       It will append 2 args if it does not.'''
+    fcode = six.get_function_code(func)
+    fargs = fcode.co_varnames[fcode.co_argcount-2:fcode.co_argcount]
+    if fargs==('this', 'arguments') or fargs==('arguments', 'var'):
+        return func
+    code = append_arguments(six.get_function_code(func), ('this','arguments'))
+
+    return types.FunctionType(code, six.get_function_globals(func), func.__name__, closure=six.get_function_closure(func))
+
+    
+def append_arguments(code_obj, new_locals):
+    co_varnames = code_obj.co_varnames   # Old locals
+    co_names = code_obj.co_names   # Old globals
+    co_names+=tuple(e for e in new_locals if e not in co_names)
+    co_argcount = code_obj.co_argcount     # Argument count
+    co_code = code_obj.co_code         # The actual bytecode as a string
+
+    # Make one pass over the bytecode to identify names that should be
+    # left in code_obj.co_names.
+    not_removed = set(opcode.hasname) - set([LOAD_GLOBAL])
+    saved_names = set()
+    for inst in instructions(code_obj):
+        if inst[0] in not_removed:
+            saved_names.add(co_names[inst[1]])
+
+    # Build co_names for the new code object. This should consist of
+    # globals that were only accessed via LOAD_GLOBAL
+    names = tuple(name for name in co_names
+                  if name not in set(new_locals) - saved_names)
+
+    # Build a dictionary that maps the indices of the entries in co_names
+    # to their entry in the new co_names
+    name_translations = dict((co_names.index(name), i)
+                             for i, name in enumerate(names))
+
+    # Build co_varnames for the new code object. This should consist of
+    # the entirety of co_varnames with new_locals spliced in after the
+    # arguments
+    new_locals_len = len(new_locals)
+    varnames = (co_varnames[:co_argcount] + new_locals +
+                co_varnames[co_argcount:])
+
+    # Build the dictionary that maps indices of entries in the old co_varnames
+    # to their indices in the new co_varnames
+    range1, range2 = xrange(co_argcount), xrange(co_argcount, len(co_varnames))
+    varname_translations = dict((i, i) for i in range1)
+    varname_translations.update((i, i + new_locals_len) for i in range2)
+
+    # Build the dictionary that maps indices of deleted entries of co_names
+    # to their indices in the new co_varnames
+    names_to_varnames = dict((co_names.index(name), varnames.index(name))
+                             for name in new_locals)
+
+    # Now we modify the actual bytecode
+    modified = []
+    for inst in instructions(code_obj):
+        op, arg = inst.opcode, inst.arg
+        # If the instruction is a LOAD_GLOBAL, we have to check to see if
+        # it's one of the globals that we are replacing. Either way,
+        # update its arg using the appropriate dict.
+        if inst.opcode == LOAD_GLOBAL:
+            if inst.arg in names_to_varnames:
+                op = LOAD_FAST
+                arg = names_to_varnames[inst.arg]
+            elif inst.arg in name_translations:
+                arg = name_translations[inst.arg]
+            else:
+                raise ValueError("a name was lost in translation")
+        # If it accesses co_varnames or co_names then update its argument.
+        elif inst.opcode in opcode.haslocal:
+            arg = varname_translations[inst.arg]
+        elif inst.opcode in opcode.hasname:
+            arg = name_translations[inst.arg]
+        modified.extend(write_instruction(op, arg))
+    if six.PY2:
+        code = ''.join(modified)
+        args = (co_argcount + new_locals_len,
+                              code_obj.co_nlocals + new_locals_len,
+                              code_obj.co_stacksize,
+                              code_obj.co_flags,
+                              code,
+                              code_obj.co_consts,
+                              names,
+                              varnames,
+                              code_obj.co_filename,
+                              code_obj.co_name,
+                              code_obj.co_firstlineno,
+                              code_obj.co_lnotab,
+                              code_obj.co_freevars,
+                              code_obj.co_cellvars)
+    else:
+        code = bytes(modified)
+        args = (co_argcount + new_locals_len,
+                0,
+                code_obj.co_nlocals + new_locals_len,
+                code_obj.co_stacksize,
+                code_obj.co_flags,
+                code,
+                code_obj.co_consts,
+                names,
+                varnames,
+                code_obj.co_filename,
+                code_obj.co_name,
+                code_obj.co_firstlineno,
+                code_obj.co_lnotab,
+                code_obj.co_freevars,
+                code_obj.co_cellvars)
+
+    # Done modifying codestring - make the code object
+    return types.CodeType(*args)
+
+
+def instructions(code_obj):
+    # easy for python 3.4+
+    if sys.version_info >= (3, 4):
+        for inst in dis.Bytecode(code_obj):
+            yield inst
+    else:
+        # otherwise we have to manually parse
+        code = code_obj.co_code
+        NewInstruction = namedtuple('Instruction', ('opcode', 'arg'))
+        if six.PY2:
+            code = map(ord, code)
+        i, L = 0, len(code)
+        extended_arg = 0
+        while i < L:
+            op = code[i]
+            i+= 1
+            if op < opcode.HAVE_ARGUMENT:
+                yield NewInstruction(op, None)
+                continue
+            oparg = code[i] + (code[i+1] << 8) + extended_arg
+            extended_arg = 0
+            i += 2
+            if op == opcode.EXTENDED_ARG:
+                extended_arg = oparg << 16
+                continue
+            yield NewInstruction(op, oparg)
+
+def write_instruction(op, arg):
+    if sys.version_info < (3, 6):
+        if arg is None:
+            return [chr(op)]
+        elif arg <= 65536:
+            return [chr(op), chr(arg & 255), chr((arg >> 8) & 255)]
+        elif arg <= 4294967296:
+            return [chr(opcode.EXTENDED_ARG),
+                    chr((arg >> 16) & 255),
+                    chr((arg >> 24) & 255),
+                    chr(op),
+                    chr(arg & 255),
+                    chr((arg >> 8) & 255)]
+        else:
+            raise ValueError("Invalid oparg: {0} is too large".format(oparg))
+    else:  # python 3.6+ uses wordcode instead of bytecode and they already supply all the EXTENDEND_ARG ops :)
+        if arg is None:
+            return [chr(op), 0]
+        return [chr(op), arg & 255]
+        # the code below is for case when extended args are to be determined automatically
+        # if op == opcode.EXTENDED_ARG:
+        #     return []  # this will be added automatically
+        # elif arg < 1 << 8:
+        #     return [chr(op), arg]
+        # elif arg < 1 << 32:
+        #     subs = [1<<24, 1<<16, 1<<8]  # allowed op extension sizes
+        #     for sub in subs:
+        #         if arg >= sub:
+        #             fit = int(arg / sub)
+        #             return [chr(opcode.EXTENDED_ARG), fit]  + write_instruction(op, arg - fit * sub)
+        # else:
+        #     raise ValueError("Invalid oparg: {0} is too large".format(oparg))
+
+
+def check(code_obj):
+    old_bytecode = code_obj.co_code
+    insts = list(instructions(code_obj))
+
+    pos_to_inst = {}
+    bytelist = []
+
+    for inst in insts:
+        pos_to_inst[len(bytelist)] = inst
+        bytelist.extend(write_instruction(inst.opcode, inst.arg))
+    if six.PY2:
+        new_bytecode = ''.join(bytelist)
+    else:
+        new_bytecode = bytes(bytelist)
+    if new_bytecode != old_bytecode:
+        print(new_bytecode)
+        print(old_bytecode)
+        for i in range(min(len(new_bytecode), len(old_bytecode))):
+            if old_bytecode[i] != new_bytecode[i]:
+                while 1:
+                    if i in pos_to_inst:
+                        print(pos_to_inst[i])
+                        print(pos_to_inst[i-2])
+                        print(list(map(chr, old_bytecode))[i-4:i+8])
+                        print(bytelist[i-4:i+8])
+                        break
+            raise RuntimeError('Your python version made changes to the bytecode')
+
+
+check(six.get_function_code(check))
+
+
+
+if __name__=='__main__':
+    x = 'Wrong'
+    dick = 3000
+    def func(a):
+        print(x,y,z, a)
+        print(dick)
+        d = (x,)
+        for e in  (e for e in x):
+            print(e)
+        return x, y, z
+    func2 =types.FunctionType(append_arguments(six.get_function_code(func), ('x', 'y', 'z')), six.get_function_globals(func), func.__name__, closure=six.get_function_closure(func))
+    args = (2,2,3,4),3,4
+    assert func2(1, *args) == args

lib/pyjsparser/__init__.py

@@ -0,0 +1,4 @@
+__all__ = ['PyJsParser', 'parse', 'JsSyntaxError']
+__author__ = 'Piotr Dabkowski'
+__version__ = '2.2.0'
+from .parser import PyJsParser, parse, JsSyntaxError

lib/pyjsparser/pyjsparserdata.py

@@ -0,0 +1,303 @@
+# The MIT License
+#
+# Copyright 2014, 2015 Piotr Dabkowski
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the 'Software'),
+# to deal in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so, subject
+# to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or
+# substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT
+# LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE
+#  OR THE USE OR OTHER DEALINGS IN THE SOFTWARE
+from __future__ import unicode_literals
+
+import sys
+import unicodedata
+from collections import defaultdict
+
+PY3 = sys.version_info >= (3,0)
+
+if PY3:
+    unichr = chr
+    xrange = range
+    unicode = str
+
+token = {
+        'BooleanLiteral': 1,
+        'EOF': 2,
+        'Identifier': 3,
+        'Keyword': 4,
+        'NullLiteral': 5,
+        'NumericLiteral': 6,
+        'Punctuator': 7,
+        'StringLiteral': 8,
+        'RegularExpression': 9,
+        'Template': 10
+    }
+
+
+TokenName = dict((v,k) for k,v in token.items())
+
+FnExprTokens = ['(', '{', '[', 'in', 'typeof', 'instanceof', 'new',
+                    'return', 'case', 'delete', 'throw', 'void',
+                    # assignment operators
+                    '=', '+=', '-=', '*=', '/=', '%=', '<<=', '>>=', '>>>=',
+                    '&=', '|=', '^=', ',',
+                    # binary/unary operators
+                    '+', '-', '*', '/', '%', '++', '--', '<<', '>>', '>>>', '&',
+                    '|', '^', '!', '~', '&&', '||', '?', ':', '===', '==', '>=',
+                    '<=', '<', '>', '!=', '!==']
+
+syntax= set(('AssignmentExpression',
+         'AssignmentPattern',
+         'ArrayExpression',
+         'ArrayPattern',
+         'ArrowFunctionExpression',
+         'BlockStatement',
+         'BinaryExpression',
+         'BreakStatement',
+         'CallExpression',
+         'CatchClause',
+         'ClassBody',
+         'ClassDeclaration',
+         'ClassExpression',
+         'ConditionalExpression',
+         'ContinueStatement',
+         'DoWhileStatement',
+         'DebuggerStatement',
+         'EmptyStatement',
+         'ExportAllDeclaration',
+         'ExportDefaultDeclaration',
+         'ExportNamedDeclaration',
+         'ExportSpecifier',
+         'ExpressionStatement',
+         'ForStatement',
+         'ForInStatement',
+         'FunctionDeclaration',
+         'FunctionExpression',
+         'Identifier',
+         'IfStatement',
+         'ImportDeclaration',
+         'ImportDefaultSpecifier',
+         'ImportNamespaceSpecifier',
+         'ImportSpecifier',
+         'Literal',
+         'LabeledStatement',
+         'LogicalExpression',
+         'MemberExpression',
+         'MethodDefinition',
+         'NewExpression',
+         'ObjectExpression',
+         'ObjectPattern',
+         'Program',
+         'Property',
+         'RestElement',
+         'ReturnStatement',
+         'SequenceExpression',
+         'SpreadElement',
+         'Super',
+         'SwitchCase',
+         'SwitchStatement',
+         'TaggedTemplateExpression',
+         'TemplateElement',
+         'TemplateLiteral',
+         'ThisExpression',
+         'ThrowStatement',
+         'TryStatement',
+         'UnaryExpression',
+         'UpdateExpression',
+         'VariableDeclaration',
+         'VariableDeclarator',
+         'WhileStatement',
+         'WithStatement'))
+
+
+# Error messages should be identical to V8.
+messages = {
+        'UnexpectedToken': 'Unexpected token %s',
+        'UnexpectedNumber': 'Unexpected number',
+        'UnexpectedString': 'Unexpected string',
+        'UnexpectedIdentifier': 'Unexpected identifier',
+        'UnexpectedReserved': 'Unexpected reserved word',
+        'UnexpectedTemplate': 'Unexpected quasi %s',
+        'UnexpectedEOS': 'Unexpected end of input',
+        'NewlineAfterThrow': 'Illegal newline after throw',
+        'InvalidRegExp': 'Invalid regular expression',
+        'UnterminatedRegExp': 'Invalid regular expression: missing /',
+        'InvalidLHSInAssignment': 'Invalid left-hand side in assignment',
+        'InvalidLHSInForIn': 'Invalid left-hand side in for-in',
+        'MultipleDefaultsInSwitch': 'More than one default clause in switch statement',
+        'NoCatchOrFinally': 'Missing catch or finally after try',
+        'UnknownLabel': 'Undefined label \'%s\'',
+        'Redeclaration': '%s \'%s\' has already been declared',
+        'IllegalContinue': 'Illegal continue statement',
+        'IllegalBreak': 'Illegal break statement',
+        'IllegalReturn': 'Illegal return statement',
+        'StrictModeWith': 'Strict mode code may not include a with statement',
+        'StrictCatchVariable': 'Catch variable may not be eval or arguments in strict mode',
+        'StrictVarName': 'Variable name may not be eval or arguments in strict mode',
+        'StrictParamName': 'Parameter name eval or arguments is not allowed in strict mode',
+        'StrictParamDupe': 'Strict mode function may not have duplicate parameter names',
+        'StrictFunctionName': 'Function name may not be eval or arguments in strict mode',
+        'StrictOctalLiteral': 'Octal literals are not allowed in strict mode.',
+        'StrictDelete': 'Delete of an unqualified identifier in strict mode.',
+        'StrictLHSAssignment': 'Assignment to eval or arguments is not allowed in strict mode',
+        'StrictLHSPostfix': 'Postfix increment/decrement may not have eval or arguments operand in strict mode',
+        'StrictLHSPrefix': 'Prefix increment/decrement may not have eval or arguments operand in strict mode',
+        'StrictReservedWord': 'Use of future reserved word in strict mode',
+        'TemplateOctalLiteral': 'Octal literals are not allowed in template strings.',
+        'ParameterAfterRestParameter': 'Rest parameter must be last formal parameter',
+        'DefaultRestParameter': 'Unexpected token =',
+        'ObjectPatternAsRestParameter': 'Unexpected token {',
+        'DuplicateProtoProperty': 'Duplicate __proto__ fields are not allowed in object literals',
+        'ConstructorSpecialMethod': 'Class constructor may not be an accessor',
+        'DuplicateConstructor': 'A class may only have one constructor',
+        'StaticPrototype': 'Classes may not have static property named prototype',
+        'MissingFromClause': 'Unexpected token',
+        'NoAsAfterImportNamespace': 'Unexpected token',
+        'InvalidModuleSpecifier': 'Unexpected token',
+        'IllegalImportDeclaration': 'Unexpected token',
+        'IllegalExportDeclaration': 'Unexpected token'}
+
+PRECEDENCE = {'||':1,
+             '&&':2,
+             '|':3,
+             '^':4,
+             '&':5,
+             '==':6,
+             '!=':6,
+             '===':6,
+             '!==':6,
+             '<':7,
+             '>':7,
+             '<=':7,
+             '>=':7,
+             'instanceof':7,
+             'in':7,
+             '<<':8,
+             '>>':8,
+             '>>>':8,
+             '+':9,
+             '-':9,
+             '*':11,
+             '/':11,
+             '%':11}
+
+class Token: pass
+class Syntax: pass
+class Messages: pass
+class PlaceHolders:
+    ArrowParameterPlaceHolder = 'ArrowParameterPlaceHolder'
+
+for k,v in token.items():
+    setattr(Token, k, v)
+
+for e in syntax:
+    setattr(Syntax, e, e)
+
+for k,v in messages.items():
+    setattr(Messages, k, v)
+
+#http://stackoverflow.com/questions/14245893/efficiently-list-all-characters-in-a-given-unicode-category
+BOM = u'\uFEFF'
+ZWJ = u'\u200D'
+ZWNJ = u'\u200C'
+TAB = u'\u0009'
+VT = u'\u000B'
+FF = u'\u000C'
+SP = u'\u0020'
+NBSP = u'\u00A0'
+LF = u'\u000A'
+CR = u'\u000D'
+LS = u'\u2028'
+PS = u'\u2029'
+
+U_CATEGORIES = defaultdict(list)
+for c in map(unichr, range(sys.maxunicode + 1)):
+    U_CATEGORIES[unicodedata.category(c)].append(c)
+UNICODE_LETTER = set(U_CATEGORIES['Lu']+U_CATEGORIES['Ll']+
+                     U_CATEGORIES['Lt']+U_CATEGORIES['Lm']+
+                     U_CATEGORIES['Lo']+U_CATEGORIES['Nl'])
+UNICODE_COMBINING_MARK = set(U_CATEGORIES['Mn']+U_CATEGORIES['Mc'])
+UNICODE_DIGIT = set(U_CATEGORIES['Nd'])
+UNICODE_CONNECTOR_PUNCTUATION = set(U_CATEGORIES['Pc'])
+IDENTIFIER_START = UNICODE_LETTER.union(set(('$','_', '\\'))) # and some fucking unicode escape sequence
+IDENTIFIER_PART = IDENTIFIER_START.union(UNICODE_COMBINING_MARK).union(UNICODE_DIGIT)\
+    .union(UNICODE_CONNECTOR_PUNCTUATION).union(set((ZWJ, ZWNJ)))
+
+WHITE_SPACE = set((0x20, 0x09, 0x0B, 0x0C, 0xA0, 0x1680,
+               0x180E, 0x2000, 0x2001, 0x2002, 0x2003,
+                0x2004, 0x2005, 0x2006, 0x2007, 0x2008,
+                0x2009, 0x200A, 0x202F, 0x205F, 0x3000,
+                0xFEFF))
+
+LINE_TERMINATORS = set((0x0A, 0x0D, 0x2028, 0x2029))
+
+def isIdentifierStart(ch):
+    return (ch if isinstance(ch, unicode) else unichr(ch))  in IDENTIFIER_START
+
+def isIdentifierPart(ch):
+    return (ch if isinstance(ch, unicode) else unichr(ch))  in IDENTIFIER_PART
+
+def isWhiteSpace(ch):
+    return (ord(ch) if isinstance(ch, unicode) else ch) in WHITE_SPACE
+
+def isLineTerminator(ch):
+    return (ord(ch) if isinstance(ch, unicode) else ch)  in LINE_TERMINATORS
+
+OCTAL = set(('0', '1', '2', '3', '4', '5', '6', '7'))
+DEC = set(('0', '1', '2', '3', '4', '5', '6', '7', '8', '9'))
+HEX = set('0123456789abcdefABCDEF')
+HEX_CONV = dict(('0123456789abcdef'[n],n) for n in xrange(16))
+for i,e in enumerate('ABCDEF', 10):
+    HEX_CONV[e] = i
+
+
+def isDecimalDigit(ch):
+    return (ch if isinstance(ch, unicode) else unichr(ch)) in DEC
+
+def isHexDigit(ch):
+    return (ch if isinstance(ch, unicode) else unichr(ch))  in HEX
+
+def isOctalDigit(ch):
+    return (ch if isinstance(ch, unicode) else unichr(ch))  in OCTAL
+
+def isFutureReservedWord(w):
+    return w in ('enum', 'export', 'import', 'super')
+
+
+RESERVED_WORD = set(('implements', 'interface', 'package', 'private', 'protected', 'public', 'static', 'yield', 'let'))
+def isStrictModeReservedWord(w):
+    return w in RESERVED_WORD
+
+def isRestrictedWord(w):
+    return w in  ('eval', 'arguments')
+
+
+KEYWORDS = set(('if', 'in', 'do', 'var', 'for', 'new', 'try', 'let', 'this', 'else', 'case',
+                     'void', 'with', 'enum', 'while', 'break', 'catch', 'throw', 'const', 'yield',
+                     'class', 'super', 'return', 'typeof', 'delete', 'switch', 'export', 'import',
+                     'default', 'finally', 'extends', 'function', 'continue', 'debugger', 'instanceof', 'pyimport'))
+def isKeyword(w):
+        # 'const' is specialized as Keyword in V8.
+        # 'yield' and 'let' are for compatibility with SpiderMonkey and ES.next.
+        # Some others are from future reserved words.
+        return w in KEYWORDS
+
+
+class JsSyntaxError(Exception): pass
+
+if __name__=='__main__':
+    assert isLineTerminator('\n')
+    assert isLineTerminator(0x0A)
+    assert isIdentifierStart('$')
+    assert isIdentifierStart(100)
+    assert isWhiteSpace(' ')

lib/pyjsparser/std_nodes.py

@@ -0,0 +1,471 @@
+from .pyjsparserdata import *
+
+
+class BaseNode:
+    def finish(self):
+        pass
+
+    def finishArrayExpression(self, elements):
+        self.type = Syntax.ArrayExpression
+        self.elements = elements
+        self.finish()
+        return self
+
+    def finishArrayPattern(self, elements):
+        self.type = Syntax.ArrayPattern
+        self.elements = elements
+        self.finish()
+        return self
+
+    def finishArrowFunctionExpression(self, params, defaults, body, expression):
+        self.type = Syntax.ArrowFunctionExpression
+        self.id = None
+        self.params = params
+        self.defaults = defaults
+        self.body = body
+        self.generator = False
+        self.expression = expression
+        self.finish()
+        return self
+
+    def finishAssignmentExpression(self, operator, left, right):
+        self.type = Syntax.AssignmentExpression
+        self.operator = operator
+        self.left = left
+        self.right = right
+        self.finish()
+        return self
+
+    def finishAssignmentPattern(self, left, right):
+        self.type = Syntax.AssignmentPattern
+        self.left = left
+        self.right = right
+        self.finish()
+        return self
+
+    def finishBinaryExpression(self, operator, left, right):
+        self.type = Syntax.LogicalExpression if (operator == '||' or operator == '&&') else Syntax.BinaryExpression
+        self.operator = operator
+        self.left = left
+        self.right = right
+        self.finish()
+        return self
+
+    def finishBlockStatement(self, body):
+        self.type = Syntax.BlockStatement
+        self.body = body
+        self.finish()
+        return self
+
+    def finishBreakStatement(self, label):
+        self.type = Syntax.BreakStatement
+        self.label = label
+        self.finish()
+        return self
+
+    def finishCallExpression(self, callee, args):
+        self.type = Syntax.CallExpression
+        self.callee = callee
+        self.arguments = args
+        self.finish()
+        return self
+
+    def finishCatchClause(self, param, body):
+        self.type = Syntax.CatchClause
+        self.param = param
+        self.body = body
+        self.finish()
+        return self
+
+    def finishClassBody(self, body):
+        self.type = Syntax.ClassBody
+        self.body = body
+        self.finish()
+        return self
+
+    def finishClassDeclaration(self, id, superClass, body):
+        self.type = Syntax.ClassDeclaration
+        self.id = id
+        self.superClass = superClass
+        self.body = body
+        self.finish()
+        return self
+
+    def finishClassExpression(self, id, superClass, body):
+        self.type = Syntax.ClassExpression
+        self.id = id
+        self.superClass = superClass
+        self.body = body
+        self.finish()
+        return self
+
+    def finishConditionalExpression(self, test, consequent, alternate):
+        self.type = Syntax.ConditionalExpression
+        self.test = test
+        self.consequent = consequent
+        self.alternate = alternate
+        self.finish()
+        return self
+
+    def finishContinueStatement(self, label):
+        self.type = Syntax.ContinueStatement
+        self.label = label
+        self.finish()
+        return self
+
+    def finishDebuggerStatement(self, ):
+        self.type = Syntax.DebuggerStatement
+        self.finish()
+        return self
+
+    def finishDoWhileStatement(self, body, test):
+        self.type = Syntax.DoWhileStatement
+        self.body = body
+        self.test = test
+        self.finish()
+        return self
+
+    def finishEmptyStatement(self, ):
+        self.type = Syntax.EmptyStatement
+        self.finish()
+        return self
+
+    def finishExpressionStatement(self, expression):
+        self.type = Syntax.ExpressionStatement
+        self.expression = expression
+        self.finish()
+        return self
+
+    def finishForStatement(self, init, test, update, body):
+        self.type = Syntax.ForStatement
+        self.init = init
+        self.test = test
+        self.update = update
+        self.body = body
+        self.finish()
+        return self
+
+    def finishForInStatement(self, left, right, body):
+        self.type = Syntax.ForInStatement
+        self.left = left
+        self.right = right
+        self.body = body
+        self.each = False
+        self.finish()
+        return self
+
+    def finishFunctionDeclaration(self, id, params, defaults, body):
+        self.type = Syntax.FunctionDeclaration
+        self.id = id
+        self.params = params
+        self.defaults = defaults
+        self.body = body
+        self.generator = False
+        self.expression = False
+        self.finish()
+        return self
+
+    def finishFunctionExpression(self, id, params, defaults, body):
+        self.type = Syntax.FunctionExpression
+        self.id = id
+        self.params = params
+        self.defaults = defaults
+        self.body = body
+        self.generator = False
+        self.expression = False
+        self.finish()
+        return self
+
+    def finishIdentifier(self, name):
+        self.type = Syntax.Identifier
+        self.name = name
+        self.finish()
+        return self
+
+    def finishIfStatement(self, test, consequent, alternate):
+        self.type = Syntax.IfStatement
+        self.test = test
+        self.consequent = consequent
+        self.alternate = alternate
+        self.finish()
+        return self
+
+    def finishLabeledStatement(self, label, body):
+        self.type = Syntax.LabeledStatement
+        self.label = label
+        self.body = body
+        self.finish()
+        return self
+
+    def finishLiteral(self, token):
+        self.type = Syntax.Literal
+        self.value = token['value']
+        self.raw = None  # todo fix it?
+        if token.get('regex'):
+            self.regex = token['regex']
+        self.finish()
+        return self
+
+    def finishMemberExpression(self, accessor, object, property):
+        self.type = Syntax.MemberExpression
+        self.computed = accessor == '['
+        self.object = object
+        self.property = property
+        self.finish()
+        return self
+
+    def finishNewExpression(self, callee, args):
+        self.type = Syntax.NewExpression
+        self.callee = callee
+        self.arguments = args
+        self.finish()
+        return self
+
+    def finishObjectExpression(self, properties):
+        self.type = Syntax.ObjectExpression
+        self.properties = properties
+        self.finish()
+        return self
+
+    def finishObjectPattern(self, properties):
+        self.type = Syntax.ObjectPattern
+        self.properties = properties
+        self.finish()
+        return self
+
+    def finishPostfixExpression(self, operator, argument):
+        self.type = Syntax.UpdateExpression
+        self.operator = operator
+        self.argument = argument
+        self.prefix = False
+        self.finish()
+        return self
+
+    def finishProgram(self, body):
+        self.type = Syntax.Program
+        self.body = body
+        self.finish()
+        return self
+
+    def finishPyimport(self, imp):
+        self.type = 'PyimportStatement'
+        self.imp = imp
+        self.finish()
+        return self
+
+    def finishProperty(self, kind, key, computed, value, method, shorthand):
+        self.type = Syntax.Property
+        self.key = key
+        self.computed = computed
+        self.value = value
+        self.kind = kind
+        self.method = method
+        self.shorthand = shorthand
+        self.finish()
+        return self
+
+    def finishRestElement(self, argument):
+        self.type = Syntax.RestElement
+        self.argument = argument
+        self.finish()
+        return self
+
+    def finishReturnStatement(self, argument):
+        self.type = Syntax.ReturnStatement
+        self.argument = argument
+        self.finish()
+        return self
+
+    def finishSequenceExpression(self, expressions):
+        self.type = Syntax.SequenceExpression
+        self.expressions = expressions
+        self.finish()
+        return self
+
+    def finishSpreadElement(self, argument):
+        self.type = Syntax.SpreadElement
+        self.argument = argument
+        self.finish()
+        return self
+
+    def finishSwitchCase(self, test, consequent):
+        self.type = Syntax.SwitchCase
+        self.test = test
+        self.consequent = consequent
+        self.finish()
+        return self
+
+    def finishSuper(self, ):
+        self.type = Syntax.Super
+        self.finish()
+        return self
+
+    def finishSwitchStatement(self, discriminant, cases):
+        self.type = Syntax.SwitchStatement
+        self.discriminant = discriminant
+        self.cases = cases
+        self.finish()
+        return self
+
+    def finishTaggedTemplateExpression(self, tag, quasi):
+        self.type = Syntax.TaggedTemplateExpression
+        self.tag = tag
+        self.quasi = quasi
+        self.finish()
+        return self
+
+    def finishTemplateElement(self, value, tail):
+        self.type = Syntax.TemplateElement
+        self.value = value
+        self.tail = tail
+        self.finish()
+        return self
+
+    def finishTemplateLiteral(self, quasis, expressions):
+        self.type = Syntax.TemplateLiteral
+        self.quasis = quasis
+        self.expressions = expressions
+        self.finish()
+        return self
+
+    def finishThisExpression(self, ):
+        self.type = Syntax.ThisExpression
+        self.finish()
+        return self
+
+    def finishThrowStatement(self, argument):
+        self.type = Syntax.ThrowStatement
+        self.argument = argument
+        self.finish()
+        return self
+
+    def finishTryStatement(self, block, handler, finalizer):
+        self.type = Syntax.TryStatement
+        self.block = block
+        self.guardedHandlers = []
+        self.handlers = [handler] if handler else []
+        self.handler = handler
+        self.finalizer = finalizer
+        self.finish()
+        return self
+
+    def finishUnaryExpression(self, operator, argument):
+        self.type = Syntax.UpdateExpression if (operator == '++' or operator == '--') else Syntax.UnaryExpression
+        self.operator = operator
+        self.argument = argument
+        self.prefix = True
+        self.finish()
+        return self
+
+    def finishVariableDeclaration(self, declarations):
+        self.type = Syntax.VariableDeclaration
+        self.declarations = declarations
+        self.kind = 'var'
+        self.finish()
+        return self
+
+    def finishLexicalDeclaration(self, declarations, kind):
+        self.type = Syntax.VariableDeclaration
+        self.declarations = declarations
+        self.kind = kind
+        self.finish()
+        return self
+
+    def finishVariableDeclarator(self, id, init):
+        self.type = Syntax.VariableDeclarator
+        self.id = id
+        self.init = init
+        self.finish()
+        return self
+
+    def finishWhileStatement(self, test, body):
+        self.type = Syntax.WhileStatement
+        self.test = test
+        self.body = body
+        self.finish()
+        return self
+
+    def finishWithStatement(self, object, body):
+        self.type = Syntax.WithStatement
+        self.object = object
+        self.body = body
+        self.finish()
+        return self
+
+    def finishExportSpecifier(self, local, exported):
+        self.type = Syntax.ExportSpecifier
+        self.exported = exported or local
+        self.local = local
+        self.finish()
+        return self
+
+    def finishImportDefaultSpecifier(self, local):
+        self.type = Syntax.ImportDefaultSpecifier
+        self.local = local
+        self.finish()
+        return self
+
+    def finishImportNamespaceSpecifier(self, local):
+        self.type = Syntax.ImportNamespaceSpecifier
+        self.local = local
+        self.finish()
+        return self
+
+    def finishExportNamedDeclaration(self, declaration, specifiers, src):
+        self.type = Syntax.ExportNamedDeclaration
+        self.declaration = declaration
+        self.specifiers = specifiers
+        self.source = src
+        self.finish()
+        return self
+
+    def finishExportDefaultDeclaration(self, declaration):
+        self.type = Syntax.ExportDefaultDeclaration
+        self.declaration = declaration
+        self.finish()
+        return self
+
+    def finishExportAllDeclaration(self, src):
+        self.type = Syntax.ExportAllDeclaration
+        self.source = src
+        self.finish()
+        return self
+
+    def finishImportSpecifier(self, local, imported):
+        self.type = Syntax.ImportSpecifier
+        self.local = local or imported
+        self.imported = imported
+        self.finish()
+        return self
+
+    def finishImportDeclaration(self, specifiers, src):
+        self.type = Syntax.ImportDeclaration
+        self.specifiers = specifiers
+        self.source = src
+        self.finish()
+        return self
+
+    def __getitem__(self, item):
+        return getattr(self, item)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+
+class Node(BaseNode):
+    pass
+
+
+class WrappingNode(BaseNode):
+    def __init__(self, startToken=None):
+        pass
+
+
+def node_to_dict(node):  # extremely important for translation speed
+    if isinstance(node, list):
+        return [node_to_dict(e) for e in node]
+    elif isinstance(node, dict):
+        return dict((k, node_to_dict(v)) for k, v in node.items())
+    elif not isinstance(node, BaseNode):
+        return node
+    return dict((k, node_to_dict(v)) for k, v in node.__dict__.items())

sickbeard/classes.py

@@ -21,6 +21,7 @@ import datetime
 import sickbeard
 from lib.dateutil import parser
 from sickbeard.common import Quality
+from unidecode import unidecode
 
 try:
     from collections import OrderedDict
@@ -133,35 +134,36 @@ class AllShowsListUI:
         self.log = log
 
     def selectSeries(self, allSeries):
-        searchResults = []
-        seriesnames = []
+        search_results = []
 
         # get all available shows
         if allSeries:
-            if 'searchterm' in self.config:
-                searchterm = self.config['searchterm']
+            search_term = self.config.get('searchterm', '').lower()
+            if search_term:
                 # try to pick a show that's in my show list
-                for curShow in allSeries:
-                    if curShow in searchResults:
+                for cur_show in allSeries:
+                    if cur_show in search_results:
                         continue
 
-                    if 'seriesname' in curShow:
-                        seriesnames.append(curShow['seriesname'])
-                    if 'aliasnames' in curShow:
-                        seriesnames.extend(curShow['aliasnames'].split('|'))
-
-                    for name in seriesnames:
-                        if searchterm.lower() in name.lower():
-                            if 'firstaired' not in curShow:
-                                curShow['firstaired'] = str(datetime.date.fromordinal(1))
-                                curShow['firstaired'] = re.sub('([-]0{2}){1,}', '', curShow['firstaired'])
-                                fixDate = parser.parse(curShow['firstaired'], fuzzy=True).date()
-                                curShow['firstaired'] = fixDate.strftime('%Y-%m-%d')
-
-                            if curShow not in searchResults:
-                                searchResults += [curShow]
-
-        return searchResults
+                    seriesnames = []
+                    if 'seriesname' in cur_show:
+                        name = cur_show['seriesname'].lower()
+                        seriesnames += [name, unidecode(name.encode('utf-8').decode('utf-8'))]
+                    if 'aliasnames' in cur_show:
+                        name = cur_show['aliasnames'].lower()
+                        seriesnames += name.split('|') + unidecode(name.encode('utf-8').decode('utf-8')).split('|')
+
+                    if search_term in set(seriesnames):
+                        if 'firstaired' not in cur_show:
+                            cur_show['firstaired'] = str(datetime.date.fromordinal(1))
+                            cur_show['firstaired'] = re.sub('([-]0{2})+', '', cur_show['firstaired'])
+                            fix_date = parser.parse(cur_show['firstaired'], fuzzy=True).date()
+                            cur_show['firstaired'] = fix_date.strftime('%Y-%m-%d')
+
+                        if cur_show not in search_results:
+                            search_results += [cur_show]
+
+        return search_results
 
 
 class ShowListUI:

sickbeard/failed_history.py

@@ -1,5 +1,4 @@
 # Author: Tyler Fenby <tylerfenby@gmail.com>
-# URL: http://code.google.com/p/sickbeard/
 #
 # This file is part of SickGear.
 #
@@ -16,26 +15,41 @@
 # You should have received a copy of the GNU General Public License
 # along with SickGear.  If not, see <http://www.gnu.org/licenses/>.
 
+import datetime
 import re
 import urllib
-import datetime
 
-from sickbeard import db
-from sickbeard import logger
-from sickbeard.exceptions import ex, EpisodeNotFoundException
+from sickbeard import db, logger
+from sickbeard.common import FAILED, WANTED, Quality, statusStrings
+from sickbeard.exceptions import EpisodeNotFoundException, ex
 from sickbeard.history import dateFormat
-from sickbeard.common import Quality
-from sickbeard.common import WANTED, FAILED
 
 
-def prepareFailedName(release):
+def db_cmd(sql, params, select=True):
+
+    my_db = db.DBConnection('failed.db')
+    sql_result = select and my_db.select(sql, params) or my_db.action(sql, params)
+    return sql_result
+
+
+def db_select(sql, params):
+
+    return db_cmd(sql, params)
+
+
+def db_action(sql, params=None):
+
+    return db_cmd(sql, params, False)
+
+
+def prepare_failed_name(release):
     """Standardizes release name for failed DB"""
 
     fixed = urllib.unquote(release)
-    if (fixed.endswith(".nzb")):
-        fixed = fixed.rpartition(".")[0]
+    if fixed.endswith('.nzb'):
+        fixed = fixed.rpartition('.')[0]
 
-    fixed = re.sub("[\.\-\+\ ]", "_", fixed)
+    fixed = re.sub('[.\-+ ]', '_', fixed)
 
     if not isinstance(fixed, unicode):
         fixed = unicode(fixed, 'utf-8', 'replace')
@@ -43,183 +57,180 @@ def prepareFailedName(release):
     return fixed
 
 
-def logFailed(release):
-    log_str = u""
+def add_failed(release):
+
     size = -1
-    provider = ""
-
-    release = prepareFailedName(release)
-
-    myDB = db.DBConnection('failed.db')
-    sql_results = myDB.select("SELECT * FROM history WHERE release=?", [release])
-
-    if len(sql_results) == 0:
-        logger.log(
-            u"Release not found in snatch history.", logger.WARNING)
-    elif len(sql_results) > 1:
-        logger.log(u"Multiple logged snatches found for release", logger.WARNING)
-        sizes = len(set(x["size"] for x in sql_results))
-        providers = len(set(x["provider"] for x in sql_results))
-        if sizes == 1:
-            logger.log(u"However, they're all the same size. Continuing with found size.", logger.WARNING)
-            size = sql_results[0]["size"]
+    provider = ''
+
+    release = prepare_failed_name(release)
+
+    sql_results = db_select('SELECT * FROM history t WHERE t.release=?', [release])
+
+    if not any(sql_results):
+        logger.log('Release not found in failed.db snatch history', logger.WARNING)
+
+    elif 1 < len(sql_results):
+        logger.log('Multiple logged snatches found for release in failed.db', logger.WARNING)
+        sizes = len(set(x['size'] for x in sql_results))
+        providers = len(set(x['provider'] for x in sql_results))
+
+        if 1 == sizes:
+            logger.log('However, they\'re all the same size. Continuing with found size', logger.WARNING)
+            size = sql_results[0]['size']
+
         else:
             logger.log(
-                u"They also vary in size. Deleting the logged snatches and recording this release with no size/provider",
+                'They also vary in size. Deleting logged snatches and recording this release with no size/provider',
                 logger.WARNING)
             for result in sql_results:
-                deleteLoggedSnatch(result["release"], result["size"], result["provider"])
+                remove_snatched(result['release'], result['size'], result['provider'])
 
-        if providers == 1:
-            logger.log(u"They're also from the same provider. Using it as well.")
-            provider = sql_results[0]["provider"]
+        if 1 == providers:
+            logger.log('They\'re also from the same provider. Using it as well')
+            provider = sql_results[0]['provider']
     else:
-        size = sql_results[0]["size"]
-        provider = sql_results[0]["provider"]
+        size = sql_results[0]['size']
+        provider = sql_results[0]['provider']
 
-    if not hasFailed(release, size, provider):
-        myDB = db.DBConnection('failed.db')
-        myDB.action("INSERT INTO failed (release, size, provider) VALUES (?, ?, ?)", [release, size, provider])
+    if not has_failed(release, size, provider):
+        db_action('INSERT INTO failed (`release`, `size`, `provider`) VALUES (?, ?, ?)', [release, size, provider])
 
-    deleteLoggedSnatch(release, size, provider)
+    remove_snatched(release, size, provider)
 
-    return log_str
 
+def add_snatched(search_result):
+    """
+    :param search_result: SearchResult object
+    """
 
-def logSuccess(release):
-    release = prepareFailedName(release)
+    log_date = datetime.datetime.today().strftime(dateFormat)
 
-    myDB = db.DBConnection('failed.db')
-    myDB.action("DELETE FROM history WHERE release=?", [release])
+    provider = 'unknown'
+    if None is not search_result.provider:
+        provider = search_result.provider.name
 
+    for episode in search_result.episodes:
+        db_action(
+            'INSERT INTO history (`date`, `size`, `release`, `provider`, `showid`, `season`, `episode`, `old_status`)'
+            'VALUES (?, ?, ?, ?, ?, ?, ?, ?)',
+            [log_date, search_result.size, prepare_failed_name(search_result.name), provider,
+             search_result.episodes[0].show.indexerid, episode.season, episode.episode, episode.status])
 
-def hasFailed(release, size, provider="%"):
-    """
-    Returns True if a release has previously failed.
 
-    If provider is given, return True only if the release is found
-    with that specific provider. Otherwise, return True if the release
-    is found with any provider.
-    """
+def set_episode_failed(ep_obj):
 
-    release = prepareFailedName(release)
+    try:
+        with ep_obj.lock:
+            quality = Quality.splitCompositeStatus(ep_obj.status)[1]
+            ep_obj.status = Quality.compositeStatus(FAILED, quality)
+            ep_obj.saveToDB()
 
-    myDB = db.DBConnection('failed.db')
-    sql_results = myDB.select(
-        "SELECT * FROM failed WHERE release=? AND size=? AND provider LIKE ?",
-        [release, size, provider])
+    except EpisodeNotFoundException as e:
+        logger.log('Unable to get episode, please set its status manually: %s' % ex(e), logger.WARNING)
 
-    return (len(sql_results) > 0)
 
+def remove_failed(release):
 
-def revertEpisode(epObj):
-    """Restore the episodes of a failed download to their original state"""
-    myDB = db.DBConnection('failed.db')
-    sql_results = myDB.select("SELECT * FROM history WHERE showid=? AND season=?",
-                              [epObj.show.indexerid, epObj.season])
+    db_action('DELETE FROM history WHERE %s=?' % '`release`', [prepare_failed_name(release)])
 
-    history_eps = dict([(res["episode"], res) for res in sql_results])
 
-    try:
-        logger.log(u"Reverting episode (%s, %s): %s" % (epObj.season, epObj.episode, epObj.name))
-        with epObj.lock:
-            if epObj.episode in history_eps:
-                logger.log(u"Found in history")
-                epObj.status = history_eps[epObj.episode]['old_status']
-            else:
-                logger.log(u"WARNING: Episode not found in history. Setting it back to WANTED",
-                           logger.WARNING)
-                epObj.status = WANTED
-                epObj.saveToDB()
+def remove_snatched(release, size, provider):
 
-    except EpisodeNotFoundException as e:
-        logger.log(u"Unable to create episode, please set its status manually: " + ex(e),
-                   logger.WARNING)
+    db_action('DELETE FROM history WHERE %s=? AND %s=? AND %s=?' % ('`release`', '`size`', '`provider`'),
+              [prepare_failed_name(release), size, provider])
 
 
-def markFailed(epObj):
-    log_str = u""
+def remove_old_history():
 
-    try:
-        with epObj.lock:
-            quality = Quality.splitCompositeStatus(epObj.status)[1]
-            epObj.status = Quality.compositeStatus(FAILED, quality)
-            epObj.saveToDB()
+    db_action('DELETE FROM history WHERE date < %s' %
+              str((datetime.datetime.today() - datetime.timedelta(days=30)).strftime(dateFormat)))
 
-    except EpisodeNotFoundException as e:
-        logger.log(u"Unable to get episode, please set its status manually: " + ex(e), logger.WARNING)
 
-    return log_str
+def has_failed(release, size, provider='%'):
+    """
+    Returns True if a release has previously failed.
 
+    If provider is given, return True only if the release is found
+    with that specific provider. Otherwise, return True if the release
+    is found with any provider.
+    """
+    return any(db_select('SELECT * FROM failed t WHERE t.release=? AND t.size=? AND t.provider LIKE ?',
+                         [prepare_failed_name(release), size, provider]))
 
-def logSnatch(searchResult):
-    logDate = datetime.datetime.today().strftime(dateFormat)
-    release = prepareFailedName(searchResult.name)
 
-    providerClass = searchResult.provider
-    if providerClass is not None:
-        provider = providerClass.name
-    else:
-        provider = "unknown"
+def revert_episode(ep_obj):
+    """Restore the episodes of a failed download to their original state"""
+    sql_results = db_select(
+        'SELECT * FROM history t WHERE t.showid=? AND t.season=?', [ep_obj.show.indexerid, ep_obj.season])
 
-    show_obj = searchResult.episodes[0].show
+    history_eps = {r['episode']: r for r in sql_results}
 
-    myDB = db.DBConnection('failed.db')
-    for episode in searchResult.episodes:
-        myDB.action(
-            "INSERT INTO history (date, size, release, provider, showid, season, episode, old_status)"
-            "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
-            [logDate, searchResult.size, release, provider, show_obj.indexerid, episode.season, episode.episode,
-             episode.status])
+    try:
+        logger.log('Reverting episode %sx%s: [%s]' % (ep_obj.season, ep_obj.episode, ep_obj.name))
+        with ep_obj.lock:
+            if ep_obj.episode in history_eps:
+                status_revert = history_eps[ep_obj.episode]['old_status']
+
+                status, quality = Quality.splitCompositeStatus(status_revert)
+                logger.log('Found in failed.db history with status: %s quality: %s' % (
+                    statusStrings[status], Quality.qualityStrings[quality]))
+            else:
+                status_revert = WANTED
 
+                logger.log('Episode not found in failed.db history. Setting it to WANTED', logger.WARNING)
 
-def deleteLoggedSnatch(release, size, provider):
-    release = prepareFailedName(release)
+            ep_obj.status = status_revert
+            ep_obj.saveToDB()
 
-    myDB = db.DBConnection('failed.db')
-    myDB.action("DELETE FROM history WHERE release=? AND size=? AND provider=?",
-                [release, size, provider])
+    except EpisodeNotFoundException as e:
+        logger.log('Unable to create episode, please set its status manually: %s' % ex(e), logger.WARNING)
 
 
-def trimHistory():
-    myDB = db.DBConnection('failed.db')
-    myDB.action("DELETE FROM history WHERE date < " + str(
-        (datetime.datetime.today() - datetime.timedelta(days=30)).strftime(dateFormat)))
+def find_old_status(ep_obj):
+    """
+    :param ep_obj:
+    :return: Old status if failed history item found
+    """
+    # Search for release in snatch history
+    results = db_select(
+        'SELECT t.old_status FROM history t WHERE t.showid=? AND t.season=? AND t.episode=? '
+        + 'ORDER BY t.date DESC LIMIT 1', [ep_obj.show.indexerid, ep_obj.season, ep_obj.episode])
+    if any(results):
+        return results[0]['old_status']
 
 
-def findRelease(epObj):
+def find_release(ep_obj):
     """
     Find releases in history by show ID and season.
     Return None for release if multiple found or no release found.
     """
-
-    release = None
-    provider = None
-
-
     # Clear old snatches for this release if any exist
-    myDB = db.DBConnection('failed.db')
-    myDB.action("DELETE FROM history WHERE showid=" + str(epObj.show.indexerid) + " AND season=" + str(
-        epObj.season) + " AND episode=" + str(
-        epObj.episode) + " AND date < (SELECT max(date) FROM history WHERE showid=" + str(
-        epObj.show.indexerid) + " AND season=" + str(epObj.season) + " AND episode=" + str(epObj.episode) + ")")
+    from_where = ' FROM history WHERE showid=%s AND season=%s AND episode=%s' % \
+                 (ep_obj.show.indexerid, ep_obj.season, ep_obj.episode)
+    db_action('DELETE %s AND date < (SELECT max(date) %s)' % (from_where, from_where))
 
     # Search for release in snatch history
-    results = myDB.select("SELECT release, provider, date FROM history WHERE showid=? AND season=? AND episode=?",
-                          [epObj.show.indexerid, epObj.season, epObj.episode])
+    results = db_select(
+        'SELECT t.release, t.provider, t.date FROM history t WHERE t.showid=? AND t.season=? AND t.episode=?',
+        [ep_obj.show.indexerid, ep_obj.season, ep_obj.episode])
 
-    for result in results:
-        release = str(result["release"])
-        provider = str(result["provider"])
-        date = result["date"]
+    if any(results):
+        r = results[0]
+        release = r['release']
+        provider = r['provider']
 
         # Clear any incomplete snatch records for this release if any exist
-        myDB.action("DELETE FROM history WHERE release=? AND date!=?", [release, date])
+        db_action('DELETE FROM history WHERE %s=? AND %s!=?' % ('`release`', '`date`'), [release, r['date']])
 
         # Found a previously failed release
-        logger.log(u"Failed release found for season (%s): (%s)" % (epObj.season, result["release"]), logger.DEBUG)
-        return (release, provider)
+        logger.log('Found failed.db history release %sx%s: [%s]' % (
+            ep_obj.season, ep_obj.episode, release), logger.DEBUG)
+    else:
+        release = None
+        provider = None
+
+        # Release not found
+        logger.log('No found failed.db history release for %sx%s: [%s]' % (
+            ep_obj.season, ep_obj.episode, ep_obj.show.name), logger.DEBUG)
 
-    # Release was not found
-    logger.log(u"No releases found for season (%s) of (%s)" % (epObj.season, epObj.show.indexerid), logger.DEBUG)
-    return (release, provider)
+    return release, provider

sickbeard/helpers.py

@@ -38,6 +38,7 @@ import subprocess
 import adba
 import requests
 import requests.exceptions
+from cfscrape import CloudflareScraper
 import sickbeard
 import subliminal
 
@@ -1106,7 +1107,7 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N
 
     # request session
     if None is session:
-        session = requests.session()
+        session = CloudflareScraper.create_scraper()
 
     if not kwargs.get('nocache'):
         cache_dir = sickbeard.CACHE_DIR or _getTempDir()
@@ -1118,6 +1119,8 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N
     req_headers = {'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'}
     if headers:
         req_headers.update(headers)
+    if hasattr(session, 'reserved') and 'headers' in session.reserved:
+        req_headers.update(session.reserved['headers'] or {})
     session.headers.update(req_headers)
 
     mute = []
@@ -1216,7 +1219,8 @@ def getURL(url, post_data=None, params=None, headers=None, timeout=30, session=N
 
     if json:
         try:
-            return resp.json()
+            data_json = resp.json()
+            return ({}, data_json)[isinstance(data_json, dict)]
         except (TypeError, Exception) as e:
             logger.log(u'JSON data issue from URL %s\r\nDetail... %s' % (url, e.message), logger.WARNING)
             return None
@@ -1231,12 +1235,14 @@ def _maybe_request_url(e, def_url=''):
 def download_file(url, filename, session=None):
     # create session
     if None is session:
-        session = requests.session()
+        session = CloudflareScraper.create_scraper()
     cache_dir = sickbeard.CACHE_DIR or _getTempDir()
     session = CacheControl(sess=session, cache=caches.FileCache(ek.ek(os.path.join, cache_dir, 'sessions')))
 
     # request session headers
     session.headers.update({'User-Agent': USER_AGENT, 'Accept-Encoding': 'gzip,deflate'})
+    if hasattr(session, 'reserved') and 'headers' in session.reserved:
+        session.headers.update(session.reserved['headers'] or {})
 
     # request session ssl verify
     session.verify = False

sickbeard/postProcessor.py

@@ -946,10 +946,10 @@ class PostProcessor(object):
 
         # Just want to keep this consistent for failed handling right now
         release_name = show_name_helpers.determineReleaseName(self.folder_path, self.nzb_name)
-        if None is not release_name:
-            failed_history.logSuccess(release_name)
-        else:
+        if None is release_name:
             self._log(u'No snatched release found in history', logger.WARNING)
+        elif sickbeard.USE_FAILED_DOWNLOADS:
+            failed_history.remove_failed(release_name)
 
         # find the destination folder
         try:

sickbeard/providers/__init__.py

@@ -31,7 +31,7 @@ from . import alpharatio, beyondhd, bithdtv, bitmetv, btn, btscene, dh, extrator
     iptorrents, limetorrents, morethan, ncore, pisexy, pretome, privatehd, ptf, \
     rarbg, revtt, scc, scenetime, shazbat, speedcd, \
     thepiratebay, torlock, torrentday, torrenting, torrentleech, \
-    torrentshack, torrentz2, transmithe_net, tvchaosuk, zooqle
+    torrentz2, transmithe_net, tvchaosuk, zooqle
 # anime
 from . import anizb, nyaatorrents, tokyotoshokan
 # custom
@@ -80,7 +80,6 @@ __all__ = ['omgwtfnzbs',
            'torrentday',
            'torrenting',
            'torrentleech',
-           'torrentshack',
            'torrentz2',
            'transmithe_net',
            'tvchaosuk',

sickbeard/providers/btn.py

@@ -21,7 +21,9 @@ import time
 
 from . import generic
 from sickbeard import helpers, logger, scene_exceptions, tvcache
+from sickbeard.bs4_parser import BS4Parser
 from sickbeard.helpers import tryInt
+from lib.unidecode import unidecode
 
 try:
     import json
@@ -35,24 +37,34 @@ class BTNProvider(generic.TorrentProvider):
     def __init__(self):
         generic.TorrentProvider.__init__(self, 'BTN')
 
-        self.url_base = 'https://broadcasthe.net'
+        self.url_base = 'https://broadcasthe.net/'
         self.url_api = 'https://api.btnapps.net'
 
+        self.urls = {'config_provider_home_uri': self.url_base, 'login': self.url_base + 'login.php',
+                     'search': self.url_base + 'torrents.php?searchstr=%s&action=basic&%s', 'get': self.url_base + '%s'}
+
         self.proper_search_terms = ['%.proper.%', '%.repack.%']
-        self.url = self.url_base
 
-        self.api_key, self.minseed, self.minleech = 3 * [None]
+        self.categories = {'Season': [2], 'Episode': [1]}
+        self.categories['Cache'] = self.categories['Season'] + self.categories['Episode']
+
+        self.url = self.urls['config_provider_home_uri']
+
+        self.api_key, self.username, self.password, self.auth_html, self.minseed, self.minleech = 6 * [None]
+        self.ua = self.session.headers['User-Agent']
         self.reject_m2ts = False
-        self.session.headers = {'Content-Type': 'application/json-rpc'}
         self.cache = BTNCache(self)
 
     def _authorised(self, **kwargs):
 
-        return self._check_auth()
+        if not self.api_key and not (self.username and self.password):
+            raise AuthException('Must set ApiKey or Username/Password for %s in config provider options' % self.name)
+        return True
 
     def _search_provider(self, search_params, age=0, **kwargs):
 
-        self._check_auth()
+        self._authorised()
+        self.auth_html = None
 
         results = []
 
@@ -66,6 +78,7 @@ class BTNProvider(generic.TorrentProvider):
                 else:
                     search_param and params.update(search_param)
                 age and params.update(dict(age='<=%i' % age))  # age in seconds
+                search_string = 'tvdb' in params and '%s %s' % (params.pop('series'), params['name']) or ''
 
                 json_rpc = (lambda param_dct, items_per_page=1000, offset=0:
                             '{"jsonrpc": "2.0", "id": "%s", "method": "getTorrents", "params": ["%s", %s, %s, %s]}' %
@@ -73,7 +86,13 @@ class BTNProvider(generic.TorrentProvider):
                              self.api_key, json.dumps(param_dct), items_per_page, offset))
 
                 try:
-                    response = helpers.getURL(self.url_api, post_data=json_rpc(params), session=self.session, json=True)
+                    response = None
+                    if self.api_key:
+                        self.session.headers['Content-Type'] = 'application/json-rpc'
+                        response = helpers.getURL(
+                            self.url_api, post_data=json_rpc(params), session=self.session, json=True)
+                    if not response:
+                        results = self.html(mode, search_string, results)
                     error_text = response['error']['message']
                     logger.log(
                         ('Call Limit' in error_text
@@ -138,6 +157,81 @@ class BTNProvider(generic.TorrentProvider):
                                      ('search_param: ' + str(search_param), self.name)['Cache' == mode])
 
                     results = self._sort_seeding(mode, results)
+                    break   # search first tvdb item only
+
+        return results
+
+    def _authorised_html(self):
+
+        if self.username and self.password:
+            return super(BTNProvider, self)._authorised(
+                post_params={'login': 'Log In!'}, logged_in=(lambda y='': 'casThe' in y[0:4096]))
+        raise AuthException('Password or Username for %s is empty in config provider options' % self.name)
+
+    def html(self, mode, search_string, results):
+
+        if 'Content-Type' in self.session.headers:
+            del (self.session.headers['Content-Type'])
+        setattr(self.session, 'reserved', {'headers': {
+            'Accept': 'text/html, application/xhtml+xml, */*', 'Accept-Language': 'en-GB',
+            'Cache-Control': 'no-cache', 'Referer': 'https://broadcasthe.net/login.php', 'User-Agent': self.ua}})
+        self.headers = None
+
+        if self.auth_html or self._authorised_html():
+            del (self.session.reserved['headers']['Referer'])
+            if 'Referer' in self.session.headers:
+                del (self.session.headers['Referer'])
+            self.auth_html = True
+
+            search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
+            search_url = self.urls['search'] % (search_string, self._categories_string(mode, 'filter_cat[%s]=1'))
+
+            html = helpers.getURL(search_url, session=self.session)
+            cnt = len(results)
+            try:
+                if not html or self._has_no_results(html):
+                    raise generic.HaltParseException
+
+                with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
+                    torrent_table = soup.find(id='torrent_table')
+                    torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
+
+                    if 2 > len(torrent_rows):
+                        raise generic.HaltParseException
+
+                    rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {
+                        'cats': '(?i)cat\[(?:%s)\]' % self._categories_string(mode, template='', delimiter='|'),
+                        'get': 'download'}.items())
+
+                    head = None
+                    for tr in torrent_rows[1:]:
+                        cells = tr.find_all('td')
+                        if 5 > len(cells):
+                            continue
+                        try:
+                            head = head if None is not head else self._header_row(tr)
+                            seeders, leechers, size = [tryInt(n, n) for n in [
+                                cells[head[x]].get_text().strip() for x in 'seed', 'leech', 'size']]
+                            if ((self.reject_m2ts and re.search(r'(?i)\[.*?m2?ts.*?\]', tr.get_text('', strip=True))) or
+                                    self._peers_fail(mode, seeders, leechers) or not tr.find('a', href=rc['cats'])):
+                                continue
+
+                            title = tr.select('td span[title]')[0].attrs.get('title').strip()
+                            download_url = self._link(tr.find('a', href=rc['get'])['href'])
+                        except (AttributeError, TypeError, ValueError, KeyError, IndexError):
+                            continue
+
+                        if title and download_url:
+                            results.append((title, download_url, seeders, self._bytesizer(size)))
+
+            except generic.HaltParseException:
+                pass
+            except (StandardError, Exception):
+                logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
+
+            self._log_search(mode, len(results) - cnt, search_url)
+
+            results = self._sort_seeding(mode, results)
 
         return results
 
@@ -187,17 +281,21 @@ class BTNProvider(generic.TorrentProvider):
 
         if 1 == ep_obj.show.indexer:
             base_params['tvdb'] = ep_obj.show.indexerid
+            base_params['series'] = ep_obj.show.name
             search_params.append(base_params)
         # elif 2 == ep_obj.show.indexer:
         #    current_params['tvrage'] = ep_obj.show.indexerid
         #    search_params.append(current_params)
-        else:
-            name_exceptions = list(
-                set([helpers.sanitizeSceneName(a) for a in
-                     scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
-            for name in name_exceptions:
-                series_param = {'series': name}
-                series_param.update(base_params)
+        # else:
+        name_exceptions = list(
+            set([helpers.sanitizeSceneName(a) for a in
+                 scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
+        dedupe = [ep_obj.show.name.replace(' ', '.')]
+        for name in name_exceptions:
+            if name.replace(' ', '.') not in dedupe:
+                dedupe += [name.replace(' ', '.')]
+                series_param = base_params.copy()
+                series_param['series'] = name
                 search_params.append(series_param)
 
         return [dict(Season=search_params)]
@@ -228,18 +326,23 @@ class BTNProvider(generic.TorrentProvider):
         # search
         if 1 == ep_obj.show.indexer:
             base_params['tvdb'] = ep_obj.show.indexerid
+            base_params['series'] = ep_obj.show.name
             search_params.append(base_params)
         # elif 2 == ep_obj.show.indexer:
         #    search_params['tvrage'] = ep_obj.show.indexerid
         #    to_return.append(search_params)
-        else:
+
+        # else:
             # add new query string for every exception
-            name_exceptions = list(
-                set([helpers.sanitizeSceneName(a) for a in
-                     scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
-            for name in name_exceptions:
-                series_param = {'series': name}
-                series_param.update(base_params)
+        name_exceptions = list(
+            set([helpers.sanitizeSceneName(a) for a in
+                 scene_exceptions.get_scene_exceptions(ep_obj.show.indexerid) + [ep_obj.show.name]]))
+        dedupe = [ep_obj.show.name.replace(' ', '.')]
+        for name in name_exceptions:
+            if name.replace(' ', '.') not in dedupe:
+                dedupe += [name.replace(' ', '.')]
+                series_param = base_params.copy()
+                series_param['series'] = name
                 search_params.append(series_param)
 
         return [dict(Episode=search_params)]

sickbeard/providers/generic.py

@@ -33,6 +33,7 @@ from base64 import b16encode, b32decode
 import sickbeard
 import requests
 import requests.cookies
+from cfscrape import CloudflareScraper
 from hachoir_parser import guessParser
 from hachoir_core.error import HachoirError
 from hachoir_core.stream import FileInputStream
@@ -74,7 +75,7 @@ class GenericProvider:
 
         self.cache = tvcache.TVCache(self)
 
-        self.session = requests.session()
+        self.session = CloudflareScraper.create_scraper()
 
         self.headers = {
             # Using USER_AGENT instead of Mozilla to keep same user agent along authentication and download phases,
@@ -208,7 +209,7 @@ class GenericProvider:
             cache_file = ek.ek(os.path.join, cache_dir, base_name)
 
             self.session.headers['Referer'] = url
-            if helpers.download_file(url, cache_file, session=self.session):
+            if getattr(result, 'cache_file', None) or helpers.download_file(url, cache_file, session=self.session):
 
                 if self._verify_download(cache_file):
                     logger.log(u'Downloaded %s result from %s' % (self.name, url))

sickbeard/providers/speedcd.py

@@ -67,7 +67,7 @@ class SpeedCDProvider(generic.TorrentProvider):
 
                 cnt = len(items[mode])
                 try:
-                    html = data_json.get('Fs')[0].get('Cn')[0].get('d')
+                    html = data_json.get('Fs', [{}])[0].get('Cn', [{}])[0].get('d')
                     if not html or self._has_no_results(html):
                         raise generic.HaltParseException
 

sickbeard/providers/torrentshack.py

@@ -1,121 +0,0 @@
-# coding=utf-8
-#
-# Author: SickGear
-#
-# This file is part of SickGear.
-#
-# SickGear is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# SickGear is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with SickGear.  If not, see <http://www.gnu.org/licenses/>.
-
-import re
-import traceback
-
-from . import generic
-from sickbeard import logger
-from sickbeard.bs4_parser import BS4Parser
-from sickbeard.helpers import tryInt
-from lib.unidecode import unidecode
-
-
-class TorrentShackProvider(generic.TorrentProvider):
-
-    def __init__(self):
-        generic.TorrentProvider.__init__(self, 'TorrentShack', cache_update_freq=20)
-
-        self.url_base = 'https://torrentshack.me/'
-        self.urls = {'config_provider_home_uri': self.url_base,
-                     'login_action': self.url_base + 'login.php',
-                     'search': self.url_base + 'torrents.php?searchstr=%s&%s&' + '&'.join(
-                         ['release_type=both', 'searchtags=', 'tags_type=0',
-                          'order_by=s3', 'order_way=desc', 'torrent_preset=all']),
-                     'get': self.url_base + '%s'}
-
-        self.categories = {'shows': [600, 620, 700, 981, 980], 'anime': [850]}
-
-        self.url = self.urls['config_provider_home_uri']
-
-        self.username, self.password, self.minseed, self.minleech = 4 * [None]
-
-    def _authorised(self, **kwargs):
-
-        return super(TorrentShackProvider, self)._authorised(logged_in=(lambda y=None: self.has_all_cookies('session')),
-                                                             post_params={'keeplogged': '1', 'form_tmpl': True})
-
-    def _search_provider(self, search_params, **kwargs):
-
-        results = []
-        if not self._authorised():
-            return results
-
-        items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []}
-
-        rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {
-            'info': 'view', 'get': 'download', 'title': 'view\s+torrent\s+', 'size': '\s{2,}.*'}.iteritems())
-        for mode in search_params.keys():
-            for search_string in search_params[mode]:
-                search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string
-                # fetch 15 results by default, and up to 100 if allowed in user profile
-                search_url = self.urls['search'] % (search_string, self._categories_string(mode, 'filter_cat[%s]=1'))
-
-                html = self.get_url(search_url)
-
-                cnt = len(items[mode])
-                try:
-                    if not html or self._has_no_results(html):
-                        raise generic.HaltParseException
-
-                    with BS4Parser(html, features=['html5lib', 'permissive']) as soup:
-                        torrent_table = soup.find('table', class_='torrent_table')
-                        torrent_rows = [] if not torrent_table else torrent_table.find_all('tr')
-
-                        if 2 > len(torrent_rows):
-                            raise generic.HaltParseException
-
-                        head = None
-                        for tr in torrent_rows[1:]:
-                            cells = tr.find_all('td')
-                            if 5 > len(cells):
-                                continue
-                            try:
-                                head = head if None is not head else self._header_row(tr)
-                                seeders, leechers, size = [tryInt(n, n) for n in [
-                                    cells[head[x]].get_text().strip() for x in 'seed', 'leech', 'size']]
-                                if self._peers_fail(mode, seeders, leechers):
-                                    continue
-
-                                size = rc['size'].sub('', size)
-                                info = tr.find('a', title=rc['info'])
-                                title = (rc['title'].sub('', info.attrs.get('title', '')) or info.get_text()).strip()
-                                download_url = self._link(tr.find('a', title=rc['get'])['href'])
-                            except (AttributeError, TypeError, ValueError, KeyError):
-                                continue
-
-                            if title and download_url:
-                                items[mode].append((title, download_url, seeders, self._bytesizer(size)))
-
-                except generic.HaltParseException:
-                    pass
-                except (StandardError, Exception):
-                    logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR)
-                self._log_search(mode, len(items[mode]) - cnt, search_url)
-
-            results = self._sort_seeding(mode, results + items[mode])
-
-        return results
-
-    def _episode_strings(self, ep_obj, **kwargs):
-
-        return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='.', **kwargs)
-
-
-provider = TorrentShackProvider()

sickbeard/providers/transmithe_net.py

@@ -77,7 +77,7 @@ class TransmithenetProvider(generic.TorrentProvider):
 
                 cnt = len(items[mode])
                 try:
-                    for item in data_json['response'].get('results', []):
+                    for item in data_json.get('response', {}).get('results', []):
                         if self.freeleech and not item.get('isFreeleech'):
                             continue
 
@@ -96,7 +96,7 @@ class TransmithenetProvider(generic.TorrentProvider):
                                 (maybe_res and [maybe_res[0]] or []) +
                                 [detail[0].strip(), detail[1], maybe_ext and maybe_ext[0].lower() or 'mkv']))
                         except (IndexError, KeyError):
-                            title = group_name
+                            title = self.regulate_title(item, group_name)
                         download_url = self.urls['get'] % (self.user_authkey, self.user_passkey, torrent_id)
 
                         if title and download_url:
@@ -110,6 +110,53 @@ class TransmithenetProvider(generic.TorrentProvider):
 
         return results
 
+    @staticmethod
+    def regulate_title(item, t_param):
+
+        if 'tags' not in item or not any(item['tags']):
+            return t_param
+
+        t = ['']
+        bl = '[*\[({]+\s*'
+        br = '\s*[})\]*]+'
+        title = re.sub('(.*?)((?i)%sproper%s)(.*)' % (bl, br), r'\1\3\2', item['groupName'])
+        for r in '\s+-\s+', '(?:19|20)\d\d(?:\-\d\d\-\d\d)?', 'S\d\d+(?:E\d\d+)?':
+            m = re.findall('(.*%s)(.*)' % r, title)
+            if any(m) and len(m[0][0]) > len(t[0]):
+                t = m[0]
+        t = (tuple(title), t)[any(t)]
+
+        tag_str = '_'.join(item['tags'])
+        tags = [re.findall(x, tag_str, flags=re.X) for x in
+                ('(?i)%sProper%s|\bProper\b$' % (bl, br),
+                 '(?i)\d{3,4}(?:[pi]|hd)',
+                 '''
+                 (?i)(hr.ws.pdtv|blu.?ray|hddvd|
+                 pdtv|hdtv|dsr|tvrip|web.?(?:dl|rip)|dvd.?rip|b[r|d]rip|mpeg-?2)
+                 ''', '''
+                 (?i)([hx].?26[45]|divx|xvid)
+                 ''', '''
+                 (?i)(avi|mkv|mp4|sub(?:b?ed|pack|s))
+                 ''')]
+
+        title = ('%s`%s' % (
+            re.sub('|'.join(['|'.join([re.escape(y) for y in x]) for x in tags if x]).strip('|'), '', t[-1]),
+            re.sub('(?i)(\d{3,4})hd', r'\1p', '`'.join(['`'.join(x) for x in tags[:-1]]).rstrip('`')) +
+            ('', '`hdtv')[not any(tags[2])] + ('', '`x264')[not any(tags[3])]))
+        for r in [('(?i)(?:\W(?:Series|Season))?\W(Repack)\W', r'`\1`'),
+                  ('(?i)%s(Proper)%s' % (bl, br), r'`\1`'), ('%s\s*%s' % (bl, br), '`')]:
+            title = re.sub(r[0], r[1], title)
+
+        grp = filter(lambda rn: '.release' in rn.lower(), item['tags'])
+        title = '%s%s-%s' % (('', t[0])[1 < len(t)], title,
+                             (any(grp) and grp[0] or 'nogrp').upper().replace('.RELEASE', ''))
+
+        for r in [('\s+[-]?\s+|\s+`|`\s+', '`'), ('`+', '.')]:
+            title = re.sub(r[0], r[1], title)
+
+        title += + any(tags[4]) and ('.%s' % tags[4][0]) or ''
+        return title
+
     def _season_strings(self, ep_obj, **kwargs):
 
         return generic.TorrentProvider._season_strings(self, ep_obj, scene=False)

sickbeard/providers/zooqle.py

@@ -32,7 +32,7 @@ class ZooqleProvider(generic.TorrentProvider):
 
         self.url_base = 'https://zooqle.com/'
         self.urls = {'config_provider_home_uri': self.url_base,
-                     'search': self.url_base + 'search?q=%s category:%s&s=dt&v=t&sd=d',
+                     'search': self.url_base + 'search?q=%s category:%s&s=ns&v=t&sd=d',
                      'get': self.url_base + 'download/%s.torrent'}
 
         self.categories = {'Season': ['TV'], 'Episode': ['TV'], 'anime': ['Anime']}

sickbeard/search.py

@@ -144,6 +144,9 @@ def snatch_episode(result, end_status=SNATCHED):
             # Snatches torrent with client
             client = clients.get_client_instance(sickbeard.TORRENT_METHOD)()
             dl_result = client.send_torrent(result)
+
+            if getattr(result, 'cache_file', None):
+                helpers.remove_file_failed(result.cache_file)
     else:
         logger.log(u'Unknown result type, unable to download it', logger.ERROR)
         dl_result = False
@@ -152,7 +155,7 @@ def snatch_episode(result, end_status=SNATCHED):
         return False
 
     if sickbeard.USE_FAILED_DOWNLOADS:
-        failed_history.logSnatch(result)
+        failed_history.add_snatched(result)
 
     ui.notifications.message(u'Episode snatched', result.name)
 
@@ -205,22 +208,22 @@ def pick_best_result(results, show, quality_list=None):
     best_result = None
     for cur_result in results:
 
-        logger.log(u'Quality is %s for %s' % (Quality.qualityStrings[cur_result.quality], cur_result.name))
+        logger.log(u'Quality is %s for [%s]' % (Quality.qualityStrings[cur_result.quality], cur_result.name))
 
         if show.is_anime and not show.release_groups.is_valid(cur_result):
             continue
 
         if quality_list and cur_result.quality not in quality_list:
-            logger.log(u'%s is an unwanted quality, rejecting it' % cur_result.name, logger.DEBUG)
+            logger.log(u'Rejecting unwanted quality [%s]' % cur_result.name, logger.DEBUG)
             continue
 
         if not pass_show_wordlist_checks(cur_result.name, show):
             continue
 
         cur_size = getattr(cur_result, 'size', None)
-        if sickbeard.USE_FAILED_DOWNLOADS and None is not cur_size and failed_history.hasFailed(
+        if sickbeard.USE_FAILED_DOWNLOADS and None is not cur_size and failed_history.has_failed(
                 cur_result.name, cur_size, cur_result.provider.name):
-            logger.log(u'%s has previously failed, rejecting it' % cur_result.name)
+            logger.log(u'Rejecting previously failed [%s]' % cur_result.name)
             continue
 
         if not best_result or best_result.quality < cur_result.quality != Quality.UNKNOWN:
@@ -233,11 +236,11 @@ def pick_best_result(results, show, quality_list=None):
             elif 'internal' in best_result.name.lower() and 'internal' not in cur_result.name.lower():
                 best_result = cur_result
             elif 'xvid' in best_result.name.lower() and 'x264' in cur_result.name.lower():
-                logger.log(u'Preferring %s (x264 over xvid)' % cur_result.name)
+                logger.log(u'Preferring (x264 over xvid) [%s]' % cur_result.name)
                 best_result = cur_result
 
     if best_result:
-        logger.log(u'Picked %s as the best' % best_result.name, logger.DEBUG)
+        logger.log(u'Picked as the best [%s]' % best_result.name, logger.DEBUG)
     else:
         logger.log(u'No result picked.', logger.DEBUG)
 
@@ -394,6 +397,8 @@ def wanted_episodes(show, from_date, make_dict=False, unaired=False):
                                     if cur_quality < i]
             # in case we don't want any quality for this episode, skip the episode
             if 0 == len(ep_obj.wantedQuality):
+                logger.log('Dropped episode, no wanted quality for %sx%s: [%s]' % (
+                    ep_obj.season, ep_obj.episode, ep_obj.show.name), logger.ERROR)
                 continue
             ep_obj.eps_aired_in_season = ep_count.get(helpers.tryInt(result['season']), 0)
             ep_obj.eps_aired_in_scene_season = ep_count_scene.get(
@@ -464,7 +469,7 @@ def search_for_needed_episodes(episodes):
     threading.currentThread().name = orig_thread_name
 
     if not len(providers):
-        logger.log('No NZB/Torrent sources enabled in Search Provider options to do recent searches', logger.WARNING)
+        logger.log('No NZB/Torrent sources enabled in Media Provider options to do recent searches', logger.WARNING)
     elif not search_done:
         logger.log('Failed recent search of %s enabled provider%s. More info in debug log.' % (
             len(providers), helpers.maybe_plural(len(providers))), logger.ERROR)
@@ -472,7 +477,7 @@ def search_for_needed_episodes(episodes):
     return found_results.values()
 
 
-def search_providers(show, episodes, manual_search=False, torrent_only=False, try_other_searches=False):
+def search_providers(show, episodes, manual_search=False, torrent_only=False, try_other_searches=False, old_status=None):
     found_results = {}
     final_results = []
 
@@ -480,6 +485,14 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr
 
     orig_thread_name = threading.currentThread().name
 
+    use_quality_list = None
+    if any([episodes]):
+        old_status = old_status or failed_history.find_old_status(episodes[0]) or episodes[0].status
+        if old_status:
+            status, quality = Quality.splitCompositeStatus(old_status)
+            use_quality_list = (status not in (
+                common.WANTED, common.FAILED, common.UNAIRED, common.SKIPPED, common.IGNORED, common.UNKNOWN))
+
     provider_list = [x for x in sickbeard.providers.sortedProviderList() if x.is_active() and x.enable_backlog and
                      (not torrent_only or x.providerType == GenericProvider.TORRENT)]
     for cur_provider in provider_list:
@@ -649,11 +662,11 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr
         if MULTI_EP_RESULT in found_results[provider_id]:
             for multi_result in found_results[provider_id][MULTI_EP_RESULT]:
 
-                logger.log(u'Checking usefulness of multi episode result %s' % multi_result.name, logger.DEBUG)
+                logger.log(u'Checking usefulness of multi episode result [%s]' % multi_result.name, logger.DEBUG)
 
-                if sickbeard.USE_FAILED_DOWNLOADS and failed_history.hasFailed(multi_result.name, multi_result.size,
-                                                                               multi_result.provider.name):
-                    logger.log(u'%s has previously failed, rejecting this multi episode result' % multi_result.name)
+                if sickbeard.USE_FAILED_DOWNLOADS and failed_history.has_failed(multi_result.name, multi_result.size,
+                                                                                multi_result.provider.name):
+                    logger.log(u'Rejecting previously failed multi episode result [%s]' % multi_result.name)
                     continue
 
                 # see how many of the eps that this result covers aren't covered by single results
@@ -708,6 +721,7 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr
 
         # of all the single ep results narrow it down to the best one for each episode
         final_results += set(multi_results.values())
+        quality_list = use_quality_list and (None, best_qualities)[any(best_qualities)] or None
         for cur_ep in found_results[provider_id]:
             if cur_ep in (MULTI_EP_RESULT, SEASON_RESULT):
                 continue
@@ -715,7 +729,7 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr
             if 0 == len(found_results[provider_id][cur_ep]):
                 continue
 
-            best_result = pick_best_result(found_results[provider_id][cur_ep], show)
+            best_result = pick_best_result(found_results[provider_id][cur_ep], show, quality_list)
 
             # if all results were rejected move on to the next episode
             if not best_result:
@@ -727,9 +741,18 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr
                     if 'blackhole' != sickbeard.TORRENT_METHOD:
                         best_result.content = None
                 else:
-                    td = best_result.provider.get_url(best_result.url)
-                    if not td:
+                    cache_file = ek.ek(os.path.join, sickbeard.CACHE_DIR or helpers._getTempDir(),
+                                       '%s.torrent' % (helpers.sanitizeFileName(best_result.name)))
+                    if not helpers.download_file(best_result.url, cache_file, session=best_result.provider.session):
                         continue
+
+                    try:
+                        with open(cache_file, 'rb') as fh:
+                            td = fh.read()
+                        setattr(best_result, 'cache_file', cache_file)
+                    except (StandardError, Exception):
+                        continue
+
                     if getattr(best_result.provider, 'chk_td', None):
                         name = None
                         try:
@@ -743,13 +766,13 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr
                                 if name:
                                     name = td[x: x + v]
                                     break
-                        except:
+                        except (StandardError, Exception):
                             continue
                         if name:
                             if not pass_show_wordlist_checks(name, show):
                                 continue
                             if not show_name_helpers.pass_wordlist_checks(name):
-                                logger.log(u'Ignored: %s (debug log has detail)' % name)
+                                logger.log('Ignored: %s (debug log has detail)' % name)
                                 continue
                             best_result.name = name
 
@@ -780,9 +803,11 @@ def search_providers(show, episodes, manual_search=False, torrent_only=False, tr
             break
 
     if not len(provider_list):
-        logger.log('No NZB/Torrent sources enabled in Search Provider options to do backlog searches', logger.WARNING)
+        logger.log('No NZB/Torrent sources enabled in Media Provider options to do backlog searches', logger.WARNING)
     elif not search_done:
         logger.log('Failed backlog search of %s enabled provider%s. More info in debug log.' % (
             len(provider_list), helpers.maybe_plural(len(provider_list))), logger.ERROR)
+    elif not any(final_results):
+        logger.log('No suitable candidates')
 
     return final_results

sickbeard/search_queue.py

@@ -248,7 +248,7 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
 
                             helpers.cpu_sleep()
 
-                except Exception:
+                except (StandardError, Exception):
                     logger.log(traceback.format_exc(), logger.DEBUG)
 
                 if None is self.success:
@@ -270,8 +270,9 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
         cur_time = datetime.datetime.now(network_timezones.sb_timezone)
 
         my_db = db.DBConnection()
-        sql_results = my_db.select('SELECT * FROM tv_episodes WHERE status = ? AND season > 0 AND airdate <= ? AND airdate > 1',
-                                   [common.UNAIRED, cur_date])
+        sql_results = my_db.select(
+            'SELECT * FROM tv_episodes WHERE status = ? AND season > 0 AND airdate <= ? AND airdate > 1',
+            [common.UNAIRED, cur_date])
 
         sql_l = []
         show = None
@@ -296,7 +297,7 @@ class RecentSearchQueueItem(generic_queue.QueueItem):
                 # filter out any episodes that haven't aired yet
                 if end_time > cur_time:
                     continue
-            except:
+            except (StandardError, Exception):
                 # if an error occurred assume the episode hasn't aired yet
                 continue
 
@@ -396,7 +397,7 @@ class ManualSearchQueueItem(generic_queue.QueueItem):
 
                 logger.log(u'Unable to find a download for: [%s]' % self.segment.prettyName())
 
-        except Exception:
+        except (StandardError, Exception):
             logger.log(traceback.format_exc(), logger.DEBUG)
 
         finally:
@@ -425,6 +426,7 @@ class BacklogQueueItem(generic_queue.QueueItem):
     def run(self):
         generic_queue.QueueItem.run(self)
 
+        is_error = False
         try:
             logger.log(u'Beginning backlog search for: [%s]' % self.show.name)
             search_result = search.search_providers(
@@ -440,10 +442,12 @@ class BacklogQueueItem(generic_queue.QueueItem):
                     helpers.cpu_sleep()
             else:
                 logger.log(u'No needed episodes found during backlog search for: [%s]' % self.show.name)
-        except Exception:
+        except (StandardError, Exception):
+            is_error = True
             logger.log(traceback.format_exc(), logger.DEBUG)
 
         finally:
+            logger.log('Completed backlog search %sfor: [%s]' % (('', 'with a debug error ')[is_error], self.show.name))
             self.finish()
 
 
@@ -462,21 +466,23 @@ class FailedQueueItem(generic_queue.QueueItem):
         self.started = True
 
         try:
-            for epObj in self.segment:
+            for ep_obj in self.segment:
 
-                logger.log(u'Marking episode as bad: [%s]' % epObj.prettyName())
+                logger.log(u'Marking episode as bad: [%s]' % ep_obj.prettyName())
 
-                failed_history.markFailed(epObj)
+                cur_status = ep_obj.status
 
-                (release, provider) = failed_history.findRelease(epObj)
+                failed_history.set_episode_failed(ep_obj)
+                (release, provider) = failed_history.find_release(ep_obj)
+                failed_history.revert_episode(ep_obj)
                 if release:
-                    failed_history.logFailed(release)
-                    history.logFailed(epObj, release, provider)
+                    failed_history.add_failed(release)
+                    history.logFailed(ep_obj, release, provider)
 
-                failed_history.revertEpisode(epObj)
-                logger.log(u'Beginning failed download search for: [%s]' % epObj.prettyName())
+                logger.log(u'Beginning failed download search for: [%s]' % ep_obj.prettyName())
 
-            search_result = search.search_providers(self.show, self.segment, True, try_other_searches=True)
+            search_result = search.search_providers(
+                self.show, self.segment, True, try_other_searches=True, old_status=cur_status)
 
             if search_result:
                 for result in search_result:
@@ -488,7 +494,7 @@ class FailedQueueItem(generic_queue.QueueItem):
             else:
                 pass
                 # logger.log(u'No valid episode found to retry for: [%s]' % self.segment.prettyName())
-        except Exception:
+        except (StandardError, Exception):
             logger.log(traceback.format_exc(), logger.DEBUG)
 
         finally:

sickbeard/show_updater.py

@@ -46,7 +46,7 @@ class ShowUpdater:
 
             # sure, why not?
             if sickbeard.USE_FAILED_DOWNLOADS:
-                failed_history.trimHistory()
+                failed_history.remove_old_history()
 
             # clear the data of unused providers
             sickbeard.helpers.clear_unused_providers()

sickbeard/webserve.py

@@ -63,12 +63,14 @@ from lib import adba
 from lib import subliminal
 from lib.dateutil import tz
 import lib.rarfile.rarfile as rarfile
+from unidecode import unidecode
 
 from lib.libtrakt import TraktAPI
 from lib.libtrakt.exceptions import TraktException, TraktAuthException
 from trakt_helpers import build_config, trakt_collection_remove_account
 from sickbeard.bs4_parser import BS4Parser
 from lib.tmdb_api import TMDB
+from lib.tvdb_api.tvdb_exceptions import tvdb_exception
 
 try:
     import json
@@ -2552,8 +2554,14 @@ class NewHomeAddShows(Home):
     def searchIndexersForShowName(self, search_term, lang='en', indexer=None):
         if not lang or lang == 'null':
             lang = 'en'
-
-        search_term = search_term.strip().encode('utf-8')
+        term = search_term.decode('utf-8').strip()
+        terms = []
+        try:
+            for t in term.encode('utf-8'), unidecode(term), term:
+                if t not in terms:
+                    terms += [t]
+        except (StandardError, Exception):
+            terms = [search_term.strip().encode('utf-8')]
 
         results = {}
         final_results = []
@@ -2592,14 +2600,32 @@ class NewHomeAddShows(Home):
                 else:
                     logger.log('Searching for shows using search term: %s from tv datasource %s' % (
                         search_term, sickbeard.indexerApi(indexer).name), logger.DEBUG)
-                    results.setdefault(indexer, []).extend(t[search_term])
+                    tvdb_ids = []
+                    for term in terms:
+                        try:
+                            for r in t[term]:
+                                tvdb_id = int(r['id'])
+                                if tvdb_id not in tvdb_ids:
+                                    tvdb_ids.append(tvdb_id)
+                                    results.setdefault(indexer, []).extend([r.copy()])
+                        except tvdb_exception:
+                            pass
             except Exception as e:
                 pass
 
             # Query trakt for tvdb ids
             try:
                 logger.log('Searching for show using search term: %s from tv datasource Trakt' % search_term, logger.DEBUG)
-                resp = self.getTrakt('/search/show?query=%s&extended=full' % search_term)
+                resp = []
+                for term in terms:
+                    result = self.getTrakt('/search/show?query=%s&extended=full' % term)
+                    resp += result
+                    match = False
+                    for r in result:
+                        if term == r.get('show', {}).get('title', ''):
+                            match = True
+                    if match:
+                        break
                 tvdb_ids = []
                 results_trakt = []
                 for item in resp: