Merge branch 'tv' of https://github.com/RuudBurger/CouchPotatoServer into tv_tvdb

couchpotato/api.py

@@ -1,14 +1,29 @@
 from couchpotato.core.helpers.request import getParams
+from functools import wraps
+from threading import Thread
+from tornado.gen import coroutine
 from tornado.web import RequestHandler, asynchronous
 import json
+import threading
+import tornado
 import urllib
 
 api = {}
+api_locks = {}
 api_nonblock = {}
 
 api_docs = {}
 api_docs_missing = []
 
+def run_async(func):
+    @wraps(func)
+    def async_func(*args, **kwargs):
+        func_hl = Thread(target = func, args = args, kwargs = kwargs)
+        func_hl.start()
+        return func_hl
+
+    return async_func
+
 # NonBlock API handler
 class NonBlockHandler(RequestHandler):
 
@@ -26,7 +41,7 @@ class NonBlockHandler(RequestHandler):
         if self.request.connection.stream.closed():
             return
 
-        self.finish(response)
+        self.write(response)
 
     def on_connection_close(self):
 
@@ -46,12 +61,15 @@ def addNonBlockApiView(route, func_tuple, docs = None, **kwargs):
 # Blocking API handler
 class ApiHandler(RequestHandler):
 
+    @coroutine
     def get(self, route, *args, **kwargs):
         route = route.strip('/')
         if not api.get(route):
             self.write('API call doesn\'t seem to exist')
             return
 
+        api_locks[route].acquire()
+
         kwargs = {}
         for x in self.request.arguments:
             kwargs[x] = urllib.unquote(self.get_argument(x))
@@ -63,8 +81,14 @@ class ApiHandler(RequestHandler):
         try: del kwargs['t']
         except: pass
 
+        # Add async callback handler
+        @run_async
+        def run_handler(callback):
+            result = api[route](**kwargs)
+            callback(result)
+        result = yield tornado.gen.Task(run_handler)
+
         # Check JSONP callback
-        result = api[route](**kwargs)
         jsonp_callback = self.get_argument('callback_func', default = None)
 
         if jsonp_callback:
@@ -74,10 +98,14 @@ class ApiHandler(RequestHandler):
         else:
             self.write(result)
 
+        api_locks[route].release()
+
 def addApiView(route, func, static = False, docs = None, **kwargs):
 
     if static: func(route)
-    else: api[route] = func
+    else:
+        api[route] = func
+        api_locks[route] = threading.Lock()
 
     if docs:
         api_docs[route[4:] if route[0:4] == 'api.' else route] = docs

couchpotato/core/downloaders/deluge/main.py

@@ -1,7 +1,7 @@
 from base64 import b64encode
-from couchpotato.core.helpers.variable import tryInt, tryFloat
 from couchpotato.core.downloaders.base import Downloader, StatusList
-from couchpotato.core.helpers.encoding import isInt
+from couchpotato.core.helpers.encoding import isInt, ss
+from couchpotato.core.helpers.variable import tryFloat
 from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
 from datetime import timedelta
@@ -31,7 +31,6 @@ class Deluge(Downloader):
         return self.drpc
 
     def download(self, data, movie, filedata = None):
-
         log.info('Sending "%s" (%s) to Deluge.', (data.get('name'), data.get('protocol')))
 
         if not self.connect():
@@ -72,7 +71,8 @@ class Deluge(Downloader):
         if data.get('protocol') == 'torrent_magnet':
             remote_torrent = self.drpc.add_torrent_magnet(data.get('url'), options)
         else:
-            remote_torrent = self.drpc.add_torrent_file(movie, b64encode(filedata), options)
+            filename = self.createFileName(data, filedata, movie)
+            remote_torrent = self.drpc.add_torrent_file(filename, b64encode(filedata), options)
 
         if not remote_torrent:
             log.error('Failed sending torrent to Deluge')
@@ -85,6 +85,10 @@ class Deluge(Downloader):
 
         log.debug('Checking Deluge download status.')
 
+        if not os.path.isdir(Env.setting('from', 'renamer')):
+            log.error('Renamer "from" folder doesn\'t to exist.')
+            return
+
         if not self.connect():
             return False
 
@@ -92,23 +96,24 @@ class Deluge(Downloader):
 
         queue = self.drpc.get_alltorrents()
 
-        if not (queue and queue.get('torrents')):
+        if not (queue):
             log.debug('Nothing in queue or error')
             return False
 
         for torrent_id in queue:
             item = queue[torrent_id]
-            log.debug('name=%s / id=%s / save_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / conf_ratio=%s/ is_seed=%s / is_finished=%s', (item['name'], item['hash'], item['save_path'], item['hash'], item['progress'], item['state'], item['eta'], item['ratio'], self.conf('ratio'), item['is_seed'], item['is_finished']))
+            log.debug('name=%s / id=%s / save_path=%s / move_completed_path=%s / hash=%s / progress=%s / state=%s / eta=%s / ratio=%s / stop_ratio=%s / is_seed=%s / is_finished=%s / paused=%s', (item['name'], item['hash'], item['save_path'], item['move_completed_path'], item['hash'], item['progress'], item['state'], item['eta'], item['ratio'], item['stop_ratio'], item['is_seed'], item['is_finished'], item['paused']))
-
-            if not os.path.isdir(Env.setting('from', 'renamer')):
-                log.error('Renamer "from" folder doesn\'t to exist.')
-                return
 
+            # Deluge has no easy way to work out if a torrent is stalled or failing.
+            #status = 'failed'
             status = 'busy'
-            # Deluge seems to set both is_seed and is_finished once everything has been downloaded.
+            if item['is_seed'] and tryFloat(item['ratio']) < tryFloat(item['stop_ratio']):
-            if item['is_seed'] or item['is_finished']:
+                # We have item['seeding_time'] to work out what the seeding time is, but we do not
+                # have access to the downloader seed_time, as with deluge we have no way to pass it
+                # when the torrent is added. So Deluge will only look at the ratio.
+                # See above comment in download().
                 status = 'seeding'
-            elif item['is_seed'] and item['is_finished'] and item['paused']:
+            elif item['is_seed'] and item['is_finished'] and item['paused'] and item['state'] == 'Paused':
                 status = 'completed'
 
             download_dir = item['save_path']
@@ -122,7 +127,7 @@ class Deluge(Downloader):
                 'original_status': item['state'],
                 'seed_ratio': item['ratio'],
                 'timeleft': str(timedelta(seconds = item['eta'])),
-                'folder': os.path.join(download_dir, item['name']),
+                'folder': ss(os.path.join(download_dir, item['name'])),
             })
 
         return statuses
@@ -169,22 +174,22 @@ class DelugeRPC(object):
             if options['label']:
                 self.client.label.set_torrent(torrent_id, options['label']).get()
         except Exception, err:
-            log.error('Failed to add torrent magnet: %s %s', err, traceback.format_exc())
+            log.error('Failed to add torrent magnet %s: %s %s', (torrent, err, traceback.format_exc()))
         finally:
             if self.client:
                 self.disconnect()
-            
+
         return torrent_id
 
-    def add_torrent_file(self, movie, torrent, options):
+    def add_torrent_file(self, filename, torrent, options):
         torrent_id = False
         try:
             self.connect()
-            torrent_id = self.client.core.add_torrent_file(movie, torrent, options).get()
+            torrent_id = self.client.core.add_torrent_file(filename, torrent, options).get()
             if options['label']:
                 self.client.label.set_torrent(torrent_id, options['label']).get()
         except Exception, err:
-            log.error('Failed to add torrent file: %s %s', err, traceback.format_exc())
+            log.error('Failed to add torrent file %s: %s %s', (filename, err, traceback.format_exc()))
         finally:
             if self.client:
                 self.disconnect()
@@ -197,7 +202,7 @@ class DelugeRPC(object):
             self.connect()
             ret = self.client.core.get_torrents_status({}, {}).get()
         except Exception, err:
-            log.error('Failed to get all torrents: %s %s', err, traceback.format_exc())
+            log.error('Failed to get all torrents: %s %s', (err, traceback.format_exc()))
         finally:
             if self.client:
                 self.disconnect()
@@ -208,7 +213,7 @@ class DelugeRPC(object):
             self.connect()
             self.client.core.pause_torrent(torrent_ids).get()
         except Exception, err:
-            log.error('Failed to pause torrent: %s %s', err, traceback.format_exc())
+            log.error('Failed to pause torrent: %s %s', (err, traceback.format_exc()))
         finally:
             if self.client:
                 self.disconnect()
@@ -218,7 +223,7 @@ class DelugeRPC(object):
             self.connect()
             self.client.core.resume_torrent(torrent_ids).get()
         except Exception, err:
-            log.error('Failed to resume torrent: %s %s', err, traceback.format_exc())
+            log.error('Failed to resume torrent: %s %s', (err, traceback.format_exc()))
         finally:
             if self.client:
                 self.disconnect()
@@ -229,7 +234,7 @@ class DelugeRPC(object):
             self.connect()
             ret = self.client.core.remove_torrent(torrent_id, remove_local_data).get()
         except Exception, err:
-            log.error('Failed to remove torrent: %s %s', err, traceback.format_exc())
+            log.error('Failed to remove torrent: %s %s', (err, traceback.format_exc()))
         finally:
             if self.client:
                 self.disconnect()

couchpotato/core/downloaders/nzbget/main.py

@@ -143,7 +143,7 @@ class NZBGet(Downloader):
                 'status': 'completed' if item['ParStatus'] == 'SUCCESS' and item['ScriptStatus'] == 'SUCCESS' else 'failed',
                 'original_status': item['ParStatus'] + ', ' + item['ScriptStatus'],
                 'timeleft': str(timedelta(seconds = 0)),
-                'folder': item['DestDir']
+                'folder': ss(item['DestDir'])
             })
 
         return statuses

couchpotato/core/downloaders/nzbvortex/main.py

@@ -57,7 +57,7 @@ class NZBVortex(Downloader):
                 'status': status,
                 'original_status': item['state'],
                 'timeleft':-1,
-                'folder': item['destinationPath'],
+                'folder': ss(item['destinationPath']),
             })
 
         return statuses

couchpotato/core/downloaders/rtorrent/main.py

@@ -2,6 +2,7 @@ from base64 import b16encode, b32decode
 from datetime import timedelta
 from hashlib import sha1
 import shutil
+from couchpotato.core.helpers.encoding import ss
 from rtorrent.err import MethodError
 
 from bencode import bencode, bdecode
@@ -157,9 +158,8 @@ class rTorrent(Downloader):
                     'status': status,
                     'seed_ratio': item.ratio,
                     'original_status': item.state,
-                    'timeleft': str(timedelta(seconds = float(item.left_bytes) / item.down_rate))
+                    'timeleft': str(timedelta(seconds = float(item.left_bytes) / item.down_rate)) if item.down_rate > 0 else -1,
-                    if item.down_rate > 0 else -1,
+                    'folder': ss(item.directory)
-                    'folder': item.directory
                 })
 
             return statuses

couchpotato/core/downloaders/sabnzbd/main.py

@@ -109,7 +109,7 @@ class Sabnzbd(Downloader):
                 'status': status,
                 'original_status': item['status'],
                 'timeleft': str(timedelta(seconds = 0)),
-                'folder': item['storage'],
+                'folder': ss(item['storage']),
             })
 
         return statuses

couchpotato/core/downloaders/transmission/main.py

@@ -1,6 +1,6 @@
 from base64 import b64encode
 from couchpotato.core.downloaders.base import Downloader, StatusList
-from couchpotato.core.helpers.encoding import isInt
+from couchpotato.core.helpers.encoding import isInt, ss
 from couchpotato.core.helpers.variable import tryInt, tryFloat
 from couchpotato.core.logger import CPLog
 from couchpotato.environment import Env
@@ -122,7 +122,7 @@ class Transmission(Downloader):
                 'original_status': item['status'],
                 'seed_ratio': item['uploadRatio'],
                 'timeleft': str(timedelta(seconds = item['eta'])),
-                'folder': os.path.join(item['downloadDir'], item['name']),
+                'folder': ss(os.path.join(item['downloadDir'], item['name'])),
             })
 
         return statuses

couchpotato/core/downloaders/utorrent/main.py

@@ -144,7 +144,7 @@ class uTorrent(Downloader):
                 'seed_ratio': float(item[7]) / 1000,
                 'original_status': item[1],
                 'timeleft': str(timedelta(seconds = item[10])),
-                'folder': item[26],
+                'folder': ss(item[26]),
             })
 
         return statuses

couchpotato/core/loader.py

@@ -16,10 +16,10 @@ class Loader(object):
         for filename in os.listdir(os.path.join(root, *base_path)):
             path = os.path.join(os.path.join(root, *base_path), filename)
             if os.path.isdir(path) and filename[:2] != '__':
-                if not u'__init__.py' in os.listdir(path):
+                if u'__init__.py' in os.listdir(path):
-                    return
+                    new_base_path = ''.join(s + '.' for s in base_path) + filename
-                new_base_path = ''.join(s + '.' for s in base_path) + filename
+                    self.paths[new_base_path.replace('.', '_')] = (priority, new_base_path, path)
-                self.paths[new_base_path.replace('.', '_')] = (priority, new_base_path, path)
+
                 if recursive:
                     self.addPath(root, base_path + [filename], priority, recursive = True)
 

couchpotato/core/media/movie/_base/main.py

@@ -23,7 +23,8 @@ class MovieBase(MovieTypeBase):
         'releases': {'status': {}, 'quality': {}, 'files':{}, 'info': {}},
         'library': {'titles': {}, 'files':{}},
         'files': {},
-        'status': {}
+        'status': {},
+        'category': {},
     }
 
     def __init__(self):
@@ -377,6 +378,7 @@ class MovieBase(MovieTypeBase):
         m = db.query(Media).filter_by(library_id = library.get('id')).first()
         added = True
         do_search = False
+        search_after = search_after and self.conf('search_on_add', section = 'moviesearcher')
         if not m:
             m = Media(
                 library_id = library.get('id'),

couchpotato/core/media/movie/_base/static/movie.actions.js

@@ -731,6 +731,7 @@ MA.Delete = new Class({
 		var self = this;
 		(e).preventDefault();
 
+		self.movie.removeView();
 		self.movie.slide('out');
 	},
 

couchpotato/core/media/movie/_base/static/movie.css

@@ -641,6 +641,12 @@
 					position: absolute;
 					z-index: 10;
 				}
+					@media only screen and (device-width: 768px) {
+						.trailer_container iframe {
+							margin-top: 25px;
+						}
+					}
+					
 					.trailer_container.hide {
 						height: 0 !important;
 					}

couchpotato/core/media/movie/library/movie/main.py

@@ -18,28 +18,28 @@ class MovieLibraryPlugin(LibraryBase):
     def __init__(self):
         addEvent('library.add.movie', self.add)
         addEvent('library.update.movie', self.update)
-        addEvent('library.update.movie_release_date', self.updateReleaseDate)
+        addEvent('library.update.movie.release_date', self.updateReleaseDate)
 
     def add(self, attrs = {}, update_after = True):
         # movies don't yet contain these, so lets make sure to set defaults
         type = attrs.get('type', 'movie')
         primary_provider = attrs.get('primary_provider', 'imdb')
-        
+
         db = get_session()
-        
+
         l = db.query(Library).filter_by(type = type, identifier = attrs.get('identifier')).first()
         if not l:
             status = fireEvent('status.get', 'needs_update', single = True)
             l = Library(
-                type = type, 
+                type = type,
-                primary_provider = primary_provider, 
+                primary_provider = primary_provider,
                 year = attrs.get('year'),
                 identifier = attrs.get('identifier'),
                 plot = toUnicode(attrs.get('plot')),
                 tagline = toUnicode(attrs.get('tagline')),
                 status_id = status.get('id'),
                 info = {},
-                parent = None, 
+                parent = None
             )
 
             title = LibraryTitle(

couchpotato/core/media/movie/searcher/__init__.py

@@ -33,6 +33,14 @@ config = [{
                     'description': 'Force run the searcher after (re)start.',
                 },
                 {
+                    'name': 'search_on_add',
+                    'label': 'Search after add',
+                    'advanced': True,
+                    'default': 1,
+                    'type': 'bool',
+                    'description': 'Disable this to only search for movies on cron.',
+                },
+                {
                     'name': 'cron_day',
                     'migrate_from': 'searcher',
                     'label': 'Day',

couchpotato/core/media/movie/searcher/main.py

@@ -93,7 +93,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
                     'profile': {'types': {'quality': {}}},
                     'releases': {'status': {}, 'quality': {}},
                     'library': {'titles': {}, 'files':{}},
-                    'files': {}
+                    'files': {},
                 })
 
                 try:
@@ -133,7 +133,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         db = get_session()
 
         pre_releases = fireEvent('quality.pre_releases', single = True)
-        release_dates = fireEvent('library.update.movie_release_date', identifier = movie['library']['identifier'], merge = True)
+        release_dates = fireEvent('library.update.movie.release_date', identifier = movie['library']['identifier'], merge = True)
         available_status, ignored_status, failed_status = fireEvent('status.get', ['available', 'ignored', 'failed'], single = True)
 
         found_releases = []
@@ -179,7 +179,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
 
                 download_preference = self.conf('preferred_method', section = 'searcher')
                 if download_preference != 'both':
-                    sorted_results = sorted(sorted_results, key = lambda k: k['type'][:3], reverse = (download_preference == 'torrent'))
+                    sorted_results = sorted(sorted_results, key = lambda k: k['protocol'][:3], reverse = (download_preference == 'torrent'))
 
                 # Check if movie isn't deleted while searching
                 if not db.query(Media).filter_by(id = movie.get('id')).first():
@@ -376,7 +376,7 @@ class MovieSearcher(SearcherBase, MovieTypeBase):
         else:
 
             # For movies before 1972
-            if dates.get('theater', 0) < 0 or dates.get('dvd', 0) < 0:
+            if not dates or dates.get('theater', 0) < 0 or dates.get('dvd', 0) < 0:
                 return True
 
             if is_pre_release:

couchpotato/core/notifications/core/main.py

@@ -19,7 +19,7 @@ log = CPLog(__name__)
 
 class CoreNotifier(Notification):
 
-    m_lock = threading.Lock()
+    m_lock = None
 
     def __init__(self):
         super(CoreNotifier, self).__init__()
@@ -57,6 +57,7 @@ class CoreNotifier(Notification):
 
         self.messages = []
         self.listeners = []
+        self.m_lock = threading.Lock()
 
     def clean(self):
 
@@ -116,7 +117,7 @@ class CoreNotifier(Notification):
         prop_name = 'messages.last_check'
         last_check = tryInt(Env.prop(prop_name, default = 0))
 
-        messages = fireEvent('cp.messages', last_check = last_check, single = True)
+        messages = fireEvent('cp.messages', last_check = last_check, single = True) or []
 
         for message in messages:
             if message.get('time') > last_check:
@@ -187,11 +188,14 @@ class CoreNotifier(Notification):
                     'result': messages,
                 })
 
+        self.m_lock.acquire()
         self.listeners.append((callback, last_id))
+        self.m_lock.release()
 
 
     def removeListener(self, callback):
 
+        self.m_lock.acquire()
         for list_tuple in self.listeners:
             try:
                 listener, last_id = list_tuple
@@ -199,6 +203,7 @@ class CoreNotifier(Notification):
                     self.listeners.remove(list_tuple)
             except:
                 log.debug('Failed removing listener: %s', traceback.format_exc())
+        self.m_lock.release()
 
     def cleanMessages(self):
 
@@ -222,7 +227,7 @@ class CoreNotifier(Notification):
         recent = []
         try:
             index = map(itemgetter('message_id'), self.messages).index(last_id)
-            recent = self.messages[index+1:]
+            recent = self.messages[index + 1:]
         except:
             pass
 

couchpotato/core/plugins/category/static/category.js

@@ -271,6 +271,11 @@ var Category = new Class({
 
 	del: function(){
 		var self = this;
+		
+		if(self.data.label == undefined){
+			self.el.destroy();
+			return;
+		}
 
 		var label = self.el.getElement('.category_label input').get('value');
 		var qObj = new Question('Are you sure you want to delete <strong>"'+label+'"</strong>?', '', [{

couchpotato/core/plugins/log/main.py

@@ -120,7 +120,7 @@ class Logging(Plugin):
             path = '%s%s' % (Env.get('log_path'), '.%s' % x if x > 0 else '')
 
             if not os.path.isfile(path):
-                break
+                continue
 
             try:
 

couchpotato/core/plugins/renamer/main.py

@@ -129,6 +129,7 @@ class Renamer(Plugin):
         download_info = self.extendDownloadInfo(download_info)
 
         # Unpack any archives
+        extr_files = None
         if self.conf('unrar'):
             folder, movie_folder, files, extr_files = self.extractFiles(folder = folder, movie_folder = movie_folder, files = files, \
                 cleanup = self.conf('cleanup') and not self.downloadIsTorrent(download_info))
@@ -187,7 +188,8 @@ class Renamer(Plugin):
                 fireEvent('renamer.before', group)
 
                 # Add extracted files to the before_rename list
-                group['before_rename'].extend(extr_files)
+                if extr_files:
+                    group['before_rename'].extend(extr_files)
 
                 # Remove weird chars from moviename
                 movie_name = re.sub(r"[\x00\/\\:\*\?\"<>\|]", '', movie_title)

couchpotato/core/plugins/scanner/main.py

@@ -120,13 +120,17 @@ class Scanner(Plugin):
                 files = []
                 for root, dirs, walk_files in os.walk(folder):
                     files.extend(os.path.join(root, filename) for filename in walk_files)
+
+                    # Break if CP wants to shut down
+                    if self.shuttingDown():
+                        break
+
             except:
                 log.error('Failed getting files from %s: %s', (folder, traceback.format_exc()))
         else:
             check_file_date = False
             files = [ss(x) for x in files]
 
-        db = get_session()
 
         for file_path in files:
 
@@ -339,6 +343,7 @@ class Scanner(Plugin):
             download_info = None
 
         # Determine file types
+        db = get_session()
         processed_movies = {}
         while True and not self.shuttingDown():
             try:
@@ -761,7 +766,7 @@ class Scanner(Plugin):
 
         # Year
         year = self.findYear(identifier)
-        if year:
+        if year and identifier[:4] != year:
             identifier = '%s %s' % (identifier.split(year)[0].strip(), year)
         else:
             identifier = identifier.split('::')[0]

couchpotato/core/plugins/suggestion/static/suggest.js

@@ -43,6 +43,8 @@ var SuggestList = new Class({
 	fill: function(json){
 
 		var self = this;
+		
+		if(!json) return;
 
 		Object.each(json.suggestions, function(movie){
 

couchpotato/core/plugins/userscript/main.py

@@ -81,8 +81,6 @@ class Userscript(Plugin):
 
     def getViaUrl(self, url = None, **kwargs):
 
-        print url
-
         params = {
             'url': url,
             'movie': fireEvent('userscript.get_movie_via_url', url = url, single = True)

couchpotato/core/providers/torrent/publichd/main.py

@@ -67,10 +67,15 @@ class PublicHD(TorrentMagnetProvider):
                 log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
 
     def getMoreInfo(self, item):
-        full_description = self.getCache('publichd.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
+
-        html = BeautifulSoup(full_description)
+        try:
-        nfo_pre = html.find('div', attrs = {'id':'torrmain'})
+            full_description = self.getCache('publichd.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)
-        description = toUnicode(nfo_pre.text) if nfo_pre else ''
+            html = BeautifulSoup(full_description)
+            nfo_pre = html.find('div', attrs = {'id':'torrmain'})
+            description = toUnicode(nfo_pre.text) if nfo_pre else ''
+        except:
+            log.error('Failed getting more info for %s', item['name'])
+            description = ''
 
         item['description'] = description
         return item

couchpotato/core/providers/torrent/torrentshack/main.py

@@ -27,7 +27,7 @@ class TorrentShack(TorrentProvider):
     ]
 
     http_time_between_calls = 1 #seconds
-    cat_backup_id = None
+    cat_backup_id = 400
 
     def _searchOnTitle(self, title, movie, quality, results):
 

couchpotato/runner.py

@@ -104,10 +104,14 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En
     for backup in backups:
         if total_backups > 3:
             if tryInt(os.path.basename(backup)) < time.time() - 259200:
-                for src_file in src_files:
+                for the_file in os.listdir(backup):
-                    b_file = toUnicode(os.path.join(backup, os.path.basename(src_file)))
+                    file_path = os.path.join(backup, the_file)
-                    if os.path.isfile(b_file):
+                    try:
-                        os.remove(b_file)
+                        if os.path.isfile(file_path):
+                            os.remove(file_path)
+                    except Exception, e:
+                        raise
+
                 os.rmdir(backup)
                 total_backups -= 1
 

libs/tornado/__init__.py

@@ -25,5 +25,5 @@ from __future__ import absolute_import, division, print_function, with_statement
 # is zero for an official release, positive for a development branch,
 # or negative for a release candidate or beta (after the base version
 # number has been incremented)
-version = "3.1b1"
+version = "3.2.dev2"
-version_info = (3, 1, 0, -98)
+version_info = (3, 2, 0, -99)

libs/tornado/auth.py

@@ -56,7 +56,7 @@ import hmac
 import time
 import uuid
 
-from tornado.concurrent import Future, chain_future, return_future
+from tornado.concurrent import TracebackFuture, chain_future, return_future
 from tornado import gen
 from tornado import httpclient
 from tornado import escape
@@ -99,7 +99,7 @@ def _auth_return_future(f):
 
     @functools.wraps(f)
     def wrapper(*args, **kwargs):
-        future = Future()
+        future = TracebackFuture()
         callback, args, kwargs = replacer.replace(future, args, kwargs)
         if callback is not None:
             future.add_done_callback(
@@ -306,10 +306,10 @@ class OAuthMixin(object):
         """Redirects the user to obtain OAuth authorization for this service.
 
         The ``callback_uri`` may be omitted if you have previously
-        registered a callback URI with the third-party service.  For some
+        registered a callback URI with the third-party service.  For
-        sevices (including Twitter and Friendfeed), you must use a
+        some sevices (including Friendfeed), you must use a
-        previously-registered callback URI and cannot specify a callback
+        previously-registered callback URI and cannot specify a
-        via this method.
+        callback via this method.
 
         This method sets a cookie called ``_oauth_request_token`` which is
         subsequently used (and cleared) in `get_authenticated_user` for
@@ -1158,7 +1158,7 @@ class FacebookMixin(object):
 class FacebookGraphMixin(OAuth2Mixin):
     """Facebook authentication using the new Graph API and OAuth2."""
     _OAUTH_ACCESS_TOKEN_URL = "https://graph.facebook.com/oauth/access_token?"
-    _OAUTH_AUTHORIZE_URL = "https://graph.facebook.com/oauth/authorize?"
+    _OAUTH_AUTHORIZE_URL = "https://www.facebook.com/dialog/oauth?"
     _OAUTH_NO_CALLBACKS = False
     _FACEBOOK_BASE_URL = "https://graph.facebook.com"
 

libs/tornado/escape.py

@@ -49,8 +49,9 @@ try:
 except NameError:
     unichr = chr
 
-_XHTML_ESCAPE_RE = re.compile('[&<>"]')
+_XHTML_ESCAPE_RE = re.compile('[&<>"\']')
-_XHTML_ESCAPE_DICT = {'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;'}
+_XHTML_ESCAPE_DICT = {'&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;',
+                      '\'': '&#39;'}
 
 
 def xhtml_escape(value):

libs/tornado/httpclient.py

@@ -33,7 +33,7 @@ import functools
 import time
 import weakref
 
-from tornado.concurrent import Future
+from tornado.concurrent import TracebackFuture
 from tornado.escape import utf8
 from tornado import httputil, stack_context
 from tornado.ioloop import IOLoop
@@ -144,9 +144,16 @@ class AsyncHTTPClient(Configurable):
 
     def close(self):
         """Destroys this HTTP client, freeing any file descriptors used.
-        Not needed in normal use, but may be helpful in unittests that
+
-        create and destroy http clients.  No other methods may be called
+        This method is **not needed in normal use** due to the way
-        on the `AsyncHTTPClient` after ``close()``.
+        that `AsyncHTTPClient` objects are transparently reused.
+        ``close()`` is generally only necessary when either the
+        `.IOLoop` is also being closed, or the ``force_instance=True``
+        argument was used when creating the `AsyncHTTPClient`.
+
+        No other methods may be called on the `AsyncHTTPClient` after
+        ``close()``.
+
         """
         if self._async_clients().get(self.io_loop) is self:
             del self._async_clients()[self.io_loop]
@@ -174,7 +181,7 @@ class AsyncHTTPClient(Configurable):
         # where normal dicts get converted to HTTPHeaders objects.
         request.headers = httputil.HTTPHeaders(request.headers)
         request = _RequestProxy(request, self.defaults)
-        future = Future()
+        future = TracebackFuture()
         if callback is not None:
             callback = stack_context.wrap(callback)
 

libs/tornado/ioloop.py

@@ -59,6 +59,9 @@ except ImportError:
 from tornado.platform.auto import set_close_exec, Waker
 
 
+_POLL_TIMEOUT = 3600.0
+
+
 class TimeoutError(Exception):
     pass
 
@@ -356,7 +359,7 @@ class IOLoop(Configurable):
                 if isinstance(result, Future):
                     future_cell[0] = result
                 else:
-                    future_cell[0] = Future()
+                    future_cell[0] = TracebackFuture()
                     future_cell[0].set_result(result)
             self.add_future(future_cell[0], lambda future: self.stop())
         self.add_callback(run)
@@ -596,7 +599,7 @@ class PollIOLoop(IOLoop):
                 pass
 
         while True:
-            poll_timeout = 3600.0
+            poll_timeout = _POLL_TIMEOUT
 
             # Prevent IO event starvation by delaying new callbacks
             # to the next iteration of the event loop.
@@ -605,6 +608,9 @@ class PollIOLoop(IOLoop):
                 self._callbacks = []
             for callback in callbacks:
                 self._run_callback(callback)
+            # Closures may be holding on to a lot of memory, so allow
+            # them to be freed before we go into our poll wait.
+            callbacks = callback = None
 
             if self._timeouts:
                 now = self.time()
@@ -616,6 +622,7 @@ class PollIOLoop(IOLoop):
                     elif self._timeouts[0].deadline <= now:
                         timeout = heapq.heappop(self._timeouts)
                         self._run_callback(timeout.callback)
+                        del timeout
                     else:
                         seconds = self._timeouts[0].deadline - now
                         poll_timeout = min(seconds, poll_timeout)
@@ -669,17 +676,16 @@ class PollIOLoop(IOLoop):
             while self._events:
                 fd, events = self._events.popitem()
                 try:
-                    self._handlers[fd](fd, events)
+                    if self._handlers.has_key(fd):
+                        self._handlers[fd](fd, events)
                 except (OSError, IOError) as e:
                     if e.args[0] == errno.EPIPE:
                         # Happens when the client closes the connection
                         pass
                     else:
-                        app_log.error("Exception in I/O handler for fd %s",
+                        self.handle_callback_exception(self._handlers.get(fd))
-                                      fd, exc_info=True)
                 except Exception:
-                    app_log.error("Exception in I/O handler for fd %s",
+                    self.handle_callback_exception(self._handlers.get(fd))
-                                  fd, exc_info=True)
         # reset the stopped flag so another start/stop pair can be issued
         self._stopped = False
         if self._blocking_signal_threshold is not None:
@@ -717,14 +723,14 @@ class PollIOLoop(IOLoop):
             list_empty = not self._callbacks
             self._callbacks.append(functools.partial(
                 stack_context.wrap(callback), *args, **kwargs))
-        if list_empty and thread.get_ident() != self._thread_ident:
+            if list_empty and thread.get_ident() != self._thread_ident:
-            # If we're in the IOLoop's thread, we know it's not currently
+                # If we're in the IOLoop's thread, we know it's not currently
-            # polling.  If we're not, and we added the first callback to an
+                # polling.  If we're not, and we added the first callback to an
-            # empty list, we may need to wake it up (it may wake up on its
+                # empty list, we may need to wake it up (it may wake up on its
-            # own, but an occasional extra wake is harmless).  Waking
+                # own, but an occasional extra wake is harmless).  Waking
-            # up a polling IOLoop is relatively expensive, so we try to
+                # up a polling IOLoop is relatively expensive, so we try to
-            # avoid it when we can.
+                # avoid it when we can.
-            self._waker.wake()
+                self._waker.wake()
 
     def add_callback_from_signal(self, callback, *args, **kwargs):
         with stack_context.NullContext():
@@ -813,7 +819,7 @@ class PeriodicCallback(object):
         try:
             self.callback()
         except Exception:
-            app_log.error("Error in periodic callback", exc_info=True)
+            self.io_loop.handle_callback_exception(self.callback)
         self._schedule_next()
 
     def _schedule_next(self):

libs/tornado/iostream.py

@@ -46,6 +46,14 @@ try:
 except ImportError:
     _set_nonblocking = None
 
+# These errnos indicate that a non-blocking operation must be retried
+# at a later time.  On most platforms they're the same value, but on
+# some they differ.
+_ERRNO_WOULDBLOCK = (errno.EWOULDBLOCK, errno.EAGAIN)
+
+# These errnos indicate that a connection has been abruptly terminated.
+# They should be caught and handled less noisily than other errors.
+_ERRNO_CONNRESET = (errno.ECONNRESET, errno.ECONNABORTED, errno.EPIPE)
 
 class StreamClosedError(IOError):
     """Exception raised by `IOStream` methods when the stream is closed.
@@ -257,15 +265,19 @@ class BaseIOStream(object):
         self._maybe_run_close_callback()
 
     def _maybe_run_close_callback(self):
-        if (self.closed() and self._close_callback and
+        # If there are pending callbacks, don't run the close callback
-                self._pending_callbacks == 0):
+        # until they're done (see _maybe_add_error_handler)
-            # if there are pending callbacks, don't run the close callback
+        if self.closed() and self._pending_callbacks == 0:
-            # until they're done (see _maybe_add_error_handler)
+            if self._close_callback is not None:
-            cb = self._close_callback
+                cb = self._close_callback
-            self._close_callback = None
+                self._close_callback = None
-            self._run_callback(cb)
+                self._run_callback(cb)
             # Delete any unfinished callbacks to break up reference cycles.
             self._read_callback = self._write_callback = None
+            # Clear the buffers so they can be cleared immediately even
+            # if the IOStream object is kept alive by a reference cycle.
+            # TODO: Clear the read buffer too; it currently breaks some tests.
+            self._write_buffer = None
 
     def reading(self):
         """Returns true if we are currently reading from the stream."""
@@ -447,7 +459,7 @@ class BaseIOStream(object):
             chunk = self.read_from_fd()
         except (socket.error, IOError, OSError) as e:
             # ssl.SSLError is a subclass of socket.error
-            if e.args[0] == errno.ECONNRESET:
+            if e.args[0] in _ERRNO_CONNRESET:
                 # Treat ECONNRESET as a connection close rather than
                 # an error to minimize log spam  (the exception will
                 # be available on self.error for apps that care).
@@ -550,12 +562,12 @@ class BaseIOStream(object):
                 self._write_buffer_frozen = False
                 _merge_prefix(self._write_buffer, num_bytes)
                 self._write_buffer.popleft()
-            except socket.error as e:
+            except (socket.error, IOError, OSError) as e:
-                if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+                if e.args[0] in _ERRNO_WOULDBLOCK:
                     self._write_buffer_frozen = True
                     break
                 else:
-                    if e.args[0] not in (errno.EPIPE, errno.ECONNRESET):
+                    if e.args[0] not in _ERRNO_CONNRESET:
                         # Broken pipe errors are usually caused by connection
                         # reset, and its better to not log EPIPE errors to
                         # minimize log spam
@@ -682,7 +694,7 @@ class IOStream(BaseIOStream):
         try:
             chunk = self.socket.recv(self.read_chunk_size)
         except socket.error as e:
-            if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+            if e.args[0] in _ERRNO_WOULDBLOCK:
                 return None
             else:
                 raise
@@ -725,7 +737,8 @@ class IOStream(BaseIOStream):
             # returned immediately when attempting to connect to
             # localhost, so handle them the same way as an error
             # reported later in _handle_connect.
-            if e.args[0] not in (errno.EINPROGRESS, errno.EWOULDBLOCK):
+            if (e.args[0] != errno.EINPROGRESS and
+                e.args[0] not in _ERRNO_WOULDBLOCK):
                 gen_log.warning("Connect error on fd %d: %s",
                                 self.socket.fileno(), e)
                 self.close(exc_info=True)
@@ -789,6 +802,17 @@ class SSLIOStream(IOStream):
         self._ssl_connect_callback = None
         self._server_hostname = None
 
+        # If the socket is already connected, attempt to start the handshake.
+        try:
+            self.socket.getpeername()
+        except socket.error:
+            pass
+        else:
+            # Indirectly start the handshake, which will run on the next
+            # IOLoop iteration and then the real IO state will be set in
+            # _handle_events.
+            self._add_io_state(self.io_loop.WRITE)
+
     def reading(self):
         return self._handshake_reading or super(SSLIOStream, self).reading()
 
@@ -821,7 +845,7 @@ class SSLIOStream(IOStream):
                 return self.close(exc_info=True)
             raise
         except socket.error as err:
-            if err.args[0] in (errno.ECONNABORTED, errno.ECONNRESET):
+            if err.args[0] in _ERRNO_CONNRESET:
                 return self.close(exc_info=True)
         except AttributeError:
             # On Linux, if the connection was reset before the call to
@@ -917,7 +941,7 @@ class SSLIOStream(IOStream):
             else:
                 raise
         except socket.error as e:
-            if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+            if e.args[0] in _ERRNO_WOULDBLOCK:
                 return None
             else:
                 raise
@@ -953,7 +977,7 @@ class PipeIOStream(BaseIOStream):
         try:
             chunk = os.read(self.fd, self.read_chunk_size)
         except (IOError, OSError) as e:
-            if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
+            if e.args[0] in _ERRNO_WOULDBLOCK:
                 return None
             elif e.args[0] == errno.EBADF:
                 # If the writing half of a pipe is closed, select will

libs/tornado/netutil.py

@@ -159,6 +159,10 @@ def is_valid_ip(ip):
 
     Supports IPv4 and IPv6.
     """
+    if not ip or '\x00' in ip:
+        # getaddrinfo resolves empty strings to localhost, and truncates
+        # on zero bytes.
+        return False
     try:
         res = socket.getaddrinfo(ip, 0, socket.AF_UNSPEC,
                                  socket.SOCK_STREAM,

libs/tornado/process.py

@@ -190,23 +190,34 @@ class Subprocess(object):
 
     def __init__(self, *args, **kwargs):
         self.io_loop = kwargs.pop('io_loop', None) or ioloop.IOLoop.current()
+        # All FDs we create should be closed on error; those in to_close
+        # should be closed in the parent process on success.
+        pipe_fds = []
         to_close = []
         if kwargs.get('stdin') is Subprocess.STREAM:
             in_r, in_w = _pipe_cloexec()
             kwargs['stdin'] = in_r
+            pipe_fds.extend((in_r, in_w))
             to_close.append(in_r)
             self.stdin = PipeIOStream(in_w, io_loop=self.io_loop)
         if kwargs.get('stdout') is Subprocess.STREAM:
             out_r, out_w = _pipe_cloexec()
             kwargs['stdout'] = out_w
+            pipe_fds.extend((out_r, out_w))
             to_close.append(out_w)
             self.stdout = PipeIOStream(out_r, io_loop=self.io_loop)
         if kwargs.get('stderr') is Subprocess.STREAM:
             err_r, err_w = _pipe_cloexec()
             kwargs['stderr'] = err_w
+            pipe_fds.extend((err_r, err_w))
             to_close.append(err_w)
             self.stderr = PipeIOStream(err_r, io_loop=self.io_loop)
-        self.proc = subprocess.Popen(*args, **kwargs)
+        try:
+            self.proc = subprocess.Popen(*args, **kwargs)
+        except:
+            for fd in pipe_fds:
+                os.close(fd)
+            raise
         for fd in to_close:
             os.close(fd)
         for attr in ['stdin', 'stdout', 'stderr', 'pid']:

libs/tornado/template.py

@@ -169,6 +169,10 @@ with ``{# ... #}``.
 
         {% module Template("foo.html", arg=42) %}
 
+    ``UIModules`` are a feature of the `tornado.web.RequestHandler`
+    class (and specifically its ``render`` method) and will not work
+    when the template system is used on its own in other contexts.
+
 ``{% raw *expr* %}``
     Outputs the result of the given expression without autoescaping.
 

libs/tornado/web.py

@@ -437,15 +437,25 @@ class RequestHandler(object):
             morsel[k] = v
 
     def clear_cookie(self, name, path="/", domain=None):
-        """Deletes the cookie with the given name."""
+        """Deletes the cookie with the given name.
+
+        Due to limitations of the cookie protocol, you must pass the same
+        path and domain to clear a cookie as were used when that cookie
+        was set (but there is no way to find out on the server side
+        which values were used for a given cookie).
+        """
         expires = datetime.datetime.utcnow() - datetime.timedelta(days=365)
         self.set_cookie(name, value="", path=path, expires=expires,
                         domain=domain)
 
-    def clear_all_cookies(self):
+    def clear_all_cookies(self, path="/", domain=None):
-        """Deletes all the cookies the user sent with this request."""
+        """Deletes all the cookies the user sent with this request.
+
+        See `clear_cookie` for more information on the path and domain
+        parameters.
+        """
         for name in self.request.cookies:
-            self.clear_cookie(name)
+            self.clear_cookie(name, path=path, domain=domain)
 
     def set_secure_cookie(self, name, value, expires_days=30, **kwargs):
         """Signs and timestamps a cookie so it cannot be forged.
@@ -751,10 +761,10 @@ class RequestHandler(object):
 
         if hasattr(self.request, "connection"):
             # Now that the request is finished, clear the callback we
-            # set on the IOStream (which would otherwise prevent the
+            # set on the HTTPConnection (which would otherwise prevent the
             # garbage collection of the RequestHandler when there
             # are keepalive connections)
-            self.request.connection.stream.set_close_callback(None)
+            self.request.connection.set_close_callback(None)
 
         if not self.application._wsgi:
             self.flush(include_footers=True)
@@ -1142,7 +1152,7 @@ class RequestHandler(object):
             elif isinstance(result, Future):
                 if result.done():
                     if result.result() is not None:
-                        raise ValueError('Expected None, got %r' % result)
+                        raise ValueError('Expected None, got %r' % result.result())
                     callback()
                 else:
                     # Delayed import of IOLoop because it's not available
@@ -1827,6 +1837,10 @@ class StaticFileHandler(RequestHandler):
                 return
             if start is not None and start < 0:
                 start += size
+            if end is not None and end > size:
+                # Clients sometimes blindly use a large range to limit their
+                # download size; cap the endpoint at the actual file size.
+                end = size
             # Note: only return HTTP 206 if less than the entire range has been
             # requested. Not only is this semantically correct, but Chrome
             # refuses to play audio if it gets an HTTP 206 in response to
@@ -2305,9 +2319,12 @@ class UIModule(object):
         self.handler = handler
         self.request = handler.request
         self.ui = handler.ui
-        self.current_user = handler.current_user
         self.locale = handler.locale
 
+    @property
+    def current_user(self):
+        return self.handler.current_user
+
     def render(self, *args, **kwargs):
         """Overridden in subclasses to return this module's output."""
         raise NotImplementedError()

libs/tornado/websocket.py

@@ -31,7 +31,7 @@ import time
 import tornado.escape
 import tornado.web
 
-from tornado.concurrent import Future
+from tornado.concurrent import TracebackFuture
 from tornado.escape import utf8, native_str
 from tornado import httpclient
 from tornado.ioloop import IOLoop
@@ -51,6 +51,10 @@ class WebSocketError(Exception):
     pass
 
 
+class WebSocketClosedError(WebSocketError):
+    pass
+
+
 class WebSocketHandler(tornado.web.RequestHandler):
     """Subclass this class to create a basic WebSocket handler.
 
@@ -160,6 +164,8 @@ class WebSocketHandler(tornado.web.RequestHandler):
         message will be sent as utf8; in binary mode any byte string
         is allowed.
         """
+        if self.ws_connection is None:
+            raise WebSocketClosedError()
         if isinstance(message, dict):
             message = tornado.escape.json_encode(message)
         self.ws_connection.write_message(message, binary=binary)
@@ -195,6 +201,8 @@ class WebSocketHandler(tornado.web.RequestHandler):
 
     def ping(self, data):
         """Send ping frame to the remote end."""
+        if self.ws_connection is None:
+            raise WebSocketClosedError()
         self.ws_connection.write_ping(data)
 
     def on_pong(self, data):
@@ -210,8 +218,9 @@ class WebSocketHandler(tornado.web.RequestHandler):
 
         Once the close handshake is successful the socket will be closed.
         """
-        self.ws_connection.close()
+        if self.ws_connection:
-        self.ws_connection = None
+            self.ws_connection.close()
+            self.ws_connection = None
 
     def allow_draft76(self):
         """Override to enable support for the older "draft76" protocol.
@@ -764,7 +773,7 @@ class WebSocketProtocol13(WebSocketProtocol):
 class WebSocketClientConnection(simple_httpclient._HTTPConnection):
     """WebSocket client connection."""
     def __init__(self, io_loop, request):
-        self.connect_future = Future()
+        self.connect_future = TracebackFuture()
         self.read_future = None
         self.read_queue = collections.deque()
         self.key = base64.b64encode(os.urandom(16))
@@ -825,7 +834,7 @@ class WebSocketClientConnection(simple_httpclient._HTTPConnection):
         ready.
         """
         assert self.read_future is None
-        future = Future()
+        future = TracebackFuture()
         if self.read_queue:
             future.set_result(self.read_queue.popleft())
         else:

libs/tornado/wsgi.py

@@ -242,10 +242,12 @@ class WSGIContainer(object):
             return response.append
         app_response = self.wsgi_application(
             WSGIContainer.environ(request), start_response)
-        response.extend(app_response)
+        try:
-        body = b"".join(response)
+            response.extend(app_response)
-        if hasattr(app_response, "close"):
+            body = b"".join(response)
-            app_response.close()
+        finally:
+            if hasattr(app_response, "close"):
+                app_response.close()
         if not data:
             raise Exception("WSGI app did not call start_response")