From 9631be1ee4ddf3b61de0d23d8594af6a9c331ebe Mon Sep 17 00:00:00 2001 From: Ruud Date: Sat, 15 Mar 2014 11:47:49 +0100 Subject: [PATCH] Move tv branch to nosql --- couchpotato/core/media/show/__init__.py | 6 + couchpotato/core/media/show/_base/__init__.py | 4 + couchpotato/core/media/show/_base/main.py | 240 +++ couchpotato/core/media/show/_base/static/search.js | 232 +++ couchpotato/core/media/show/library/__init__.py | 0 couchpotato/core/media/show/library/episode.py | 269 ++++ couchpotato/core/media/show/library/season.py | 244 +++ couchpotato/core/media/show/library/show.py | 232 +++ couchpotato/core/media/show/matcher.py | 130 ++ couchpotato/core/media/show/providers/__init__.py | 0 couchpotato/core/media/show/providers/base.py | 13 + .../core/media/show/providers/info/__init__.py | 0 .../core/media/show/providers/info/thetvdb.py | 495 ++++++ couchpotato/core/media/show/providers/info/xem.py | 194 +++ .../core/media/show/providers/nzb/__init__.py | 0 .../core/media/show/providers/nzb/binsearch.py | 51 + .../core/media/show/providers/nzb/newznab.py | 49 + .../core/media/show/providers/nzb/nzbclub.py | 52 + .../core/media/show/providers/nzb/nzbindex.py | 51 + .../core/media/show/providers/torrent/__init__.py | 0 .../core/media/show/providers/torrent/bithdtv.py | 36 + .../core/media/show/providers/torrent/bitsoup.py | 41 + .../media/show/providers/torrent/iptorrents.py | 37 + .../core/media/show/providers/torrent/publichd.py | 27 + .../media/show/providers/torrent/sceneaccess.py | 60 + .../media/show/providers/torrent/thepiratebay.py | 46 + .../media/show/providers/torrent/torrentday.py | 34 + .../media/show/providers/torrent/torrentleech.py | 42 + .../media/show/providers/torrent/torrentpotato.py | 38 + .../media/show/providers/torrent/torrentshack.py | 52 + couchpotato/core/media/show/searcher.py | 190 +++ libs/qcond/__init__.py | 42 + libs/qcond/compat.py | 23 + libs/qcond/helpers.py | 84 + libs/qcond/transformers/__init__.py | 0 libs/qcond/transformers/base.py | 21 + libs/qcond/transformers/merge.py | 241 +++ libs/qcond/transformers/slice.py | 280 ++++ libs/qcond/transformers/strip_common.py | 26 + libs/tvdb_api/.gitignore | 4 + libs/tvdb_api/.travis.yml | 9 + libs/tvdb_api/MANIFEST.in | 4 + libs/tvdb_api/Rakefile | 103 ++ libs/tvdb_api/UNLICENSE | 26 + libs/tvdb_api/__init__.py | 0 libs/tvdb_api/readme.md | 109 ++ libs/tvdb_api/setup.py | 35 + libs/tvdb_api/tests/gprof2dot.py | 1638 ++++++++++++++++++++ libs/tvdb_api/tests/runtests.py | 28 + libs/tvdb_api/tests/test_tvdb_api.py | 526 +++++++ libs/tvdb_api/tvdb_api.py | 874 +++++++++++ libs/tvdb_api/tvdb_cache.py | 251 +++ libs/tvdb_api/tvdb_exceptions.py | 52 + libs/tvdb_api/tvdb_ui.py | 153 ++ 54 files changed, 7394 insertions(+) create mode 100644 couchpotato/core/media/show/__init__.py create mode 100644 couchpotato/core/media/show/_base/__init__.py create mode 100644 couchpotato/core/media/show/_base/main.py create mode 100644 couchpotato/core/media/show/_base/static/search.js create mode 100644 couchpotato/core/media/show/library/__init__.py create mode 100644 couchpotato/core/media/show/library/episode.py create mode 100644 couchpotato/core/media/show/library/season.py create mode 100644 couchpotato/core/media/show/library/show.py create mode 100644 couchpotato/core/media/show/matcher.py create mode 100644 couchpotato/core/media/show/providers/__init__.py create mode 100644 couchpotato/core/media/show/providers/base.py create mode 100644 couchpotato/core/media/show/providers/info/__init__.py create mode 100644 couchpotato/core/media/show/providers/info/thetvdb.py create mode 100644 couchpotato/core/media/show/providers/info/xem.py create mode 100644 couchpotato/core/media/show/providers/nzb/__init__.py create mode 100644 couchpotato/core/media/show/providers/nzb/binsearch.py create mode 100644 couchpotato/core/media/show/providers/nzb/newznab.py create mode 100644 couchpotato/core/media/show/providers/nzb/nzbclub.py create mode 100644 couchpotato/core/media/show/providers/nzb/nzbindex.py create mode 100644 couchpotato/core/media/show/providers/torrent/__init__.py create mode 100644 couchpotato/core/media/show/providers/torrent/bithdtv.py create mode 100644 couchpotato/core/media/show/providers/torrent/bitsoup.py create mode 100644 couchpotato/core/media/show/providers/torrent/iptorrents.py create mode 100644 couchpotato/core/media/show/providers/torrent/publichd.py create mode 100644 couchpotato/core/media/show/providers/torrent/sceneaccess.py create mode 100644 couchpotato/core/media/show/providers/torrent/thepiratebay.py create mode 100644 couchpotato/core/media/show/providers/torrent/torrentday.py create mode 100644 couchpotato/core/media/show/providers/torrent/torrentleech.py create mode 100644 couchpotato/core/media/show/providers/torrent/torrentpotato.py create mode 100644 couchpotato/core/media/show/providers/torrent/torrentshack.py create mode 100644 couchpotato/core/media/show/searcher.py create mode 100644 libs/qcond/__init__.py create mode 100644 libs/qcond/compat.py create mode 100644 libs/qcond/helpers.py create mode 100644 libs/qcond/transformers/__init__.py create mode 100644 libs/qcond/transformers/base.py create mode 100644 libs/qcond/transformers/merge.py create mode 100644 libs/qcond/transformers/slice.py create mode 100644 libs/qcond/transformers/strip_common.py create mode 100644 libs/tvdb_api/.gitignore create mode 100644 libs/tvdb_api/.travis.yml create mode 100644 libs/tvdb_api/MANIFEST.in create mode 100644 libs/tvdb_api/Rakefile create mode 100644 libs/tvdb_api/UNLICENSE create mode 100644 libs/tvdb_api/__init__.py create mode 100644 libs/tvdb_api/readme.md create mode 100644 libs/tvdb_api/setup.py create mode 100644 libs/tvdb_api/tests/gprof2dot.py create mode 100644 libs/tvdb_api/tests/runtests.py create mode 100644 libs/tvdb_api/tests/test_tvdb_api.py create mode 100644 libs/tvdb_api/tvdb_api.py create mode 100644 libs/tvdb_api/tvdb_cache.py create mode 100644 libs/tvdb_api/tvdb_exceptions.py create mode 100644 libs/tvdb_api/tvdb_ui.py diff --git a/couchpotato/core/media/show/__init__.py b/couchpotato/core/media/show/__init__.py new file mode 100644 index 0000000..89af436 --- /dev/null +++ b/couchpotato/core/media/show/__init__.py @@ -0,0 +1,6 @@ +from couchpotato.core.media import MediaBase + + +class ShowTypeBase(MediaBase): + + _type = 'show' diff --git a/couchpotato/core/media/show/_base/__init__.py b/couchpotato/core/media/show/_base/__init__.py new file mode 100644 index 0000000..e1fb58b --- /dev/null +++ b/couchpotato/core/media/show/_base/__init__.py @@ -0,0 +1,4 @@ +from .main import ShowBase + +def autoload(): + return ShowBase() diff --git a/couchpotato/core/media/show/_base/main.py b/couchpotato/core/media/show/_base/main.py new file mode 100644 index 0000000..5207929 --- /dev/null +++ b/couchpotato/core/media/show/_base/main.py @@ -0,0 +1,240 @@ +import time + +from couchpotato import get_session +from couchpotato.api import addApiView +from couchpotato.core.event import fireEvent, fireEventAsync, addEvent +from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.logger import CPLog +from couchpotato.core.media import MediaBase + + +log = CPLog(__name__) + + +class ShowBase(MediaBase): + + _type = 'show' + + def __init__(self): + super(ShowBase, self).__init__() + + addApiView('show.add', self.addView, docs = { + 'desc': 'Add new movie to the wanted list', + 'params': { + 'identifier': {'desc': 'IMDB id of the movie your want to add.'}, + 'profile_id': {'desc': 'ID of quality profile you want the add the movie in. If empty will use the default profile.'}, + 'title': {'desc': 'Movie title to use for searches. Has to be one of the titles returned by movie.search.'}, + } + }) + + addEvent('show.add', self.add) + + def addView(self, **kwargs): + add_dict = self.add(params = kwargs) + + return { + 'success': True if add_dict else False, + 'show': add_dict, + } + + def add(self, params = {}, force_readd = True, search_after = True, update_library = False, status_id = None): + """ + params + {'category_id': u'-1', + 'identifier': u'tt1519931', + 'profile_id': u'12', + 'thetvdb_id': u'158661', + 'title': u'Haven'} + """ + log.debug("show.add") + + # Add show parent to db first; need to update library so maps will be in place (if any) + parent = self.addToDatabase(params = params, update_library = True, type = 'show') + + # TODO: add by airdate + + # Add by Season/Episode numbers + self.addBySeasonEpisode(parent, + params = params, + force_readd = force_readd, + search_after = search_after, + update_library = update_library, + status_id = status_id + ) + + def addBySeasonEpisode(self, parent, params = {}, force_readd = True, search_after = True, update_library = False, status_id = None): + identifier = params.get('id') + # 'tvdb' will always be the master for our purpose. All mapped data can be mapped + # to another source for downloading, but it will always be remapped back to tvdb numbering + # when renamed so media can be used in media players that use tvdb for info provider + # + # This currently means the episode must actually exist in tvdb in order to be found but + # the numbering can be different + + #master = 'tvdb' + #destination = 'scene' + #destination = 'anidb' + #destination = 'rage' + #destination = 'trakt' + # TODO: auto mode. if anime exists use it. if scene exists use it else use tvdb + + # XXX: We should abort adding show, etc if either tvdb or xem is down or we will have incorrent mappings + # I think if tvdb gets error we wont have anydata anyway, but we must make sure XEM returns!!!! + + # Only the master should return results here; all other info providers should just return False + # since we are just interested in the structure at this point. + seasons = fireEvent('season.info', merge = True, identifier = identifier) + if seasons is not None: + for season in seasons: + # Make sure we are only dealing with 'tvdb' responses at this point + if season.get('primary_provider', None) != 'thetvdb': + continue + season_id = season.get('id', None) + if season_id is None: continue + + season_params = {'season_identifier': season_id} + # Calling all info providers; merge your info now for individual season + single_season = fireEvent('season.info', merge = True, identifier = identifier, params = season_params) + single_season['category_id'] = params.get('category_id') + single_season['profile_id'] = params.get('profile_id') + single_season['title'] = single_season.get('original_title', None) + single_season['identifier'] = season_id + single_season['parent_identifier'] = identifier + log.info("Adding Season %s" % season_id) + s = self.addToDatabase(params = single_season, type = "season") + + episode_params = {'season_identifier': season_id} + episodes = fireEvent('episode.info', merge = True, identifier = identifier, params = episode_params) + if episodes is not None: + for episode in episodes: + # Make sure we are only dealing with 'tvdb' responses at this point + if episode.get('primary_provider', None) != 'thetvdb': + continue + episode_id = episode.get('id', None) + if episode_id is None: continue + try: + episode_number = int(episode.get('episodenumber', None)) + except (ValueError, TypeError): + continue + try: + absolute_number = int(episode.get('absolute_number', None)) + except (ValueError, TypeError): + absolute_number = None + + episode_params = {'season_identifier': season_id, + 'episode_identifier': episode_id, + 'episode': episode_number} + if absolute_number: + episode_params['absolute'] = absolute_number + # Calling all info providers; merge your info now for individual episode + single_episode = fireEvent('episode.info', merge = True, identifier = identifier, params = episode_params) + single_episode['category_id'] = params.get('category_id') + single_episode['profile_id'] = params.get('profile_id') + single_episode['title'] = single_episode.get('original_title', None) + single_episode['identifier'] = episode_id + single_episode['parent_identifier'] = single_season['identifier'] + log.info("Adding [%sx%s] %s - %s" % (season_id, + episode_number, + params['title'], + single_episode.get('original_title', ''))) + e = self.addToDatabase(params = single_episode, type = "episode") + + # Start searching now that all the media has been added + if search_after: + onComplete = self.createOnComplete(parent['id']) + onComplete() + + return parent + + def addToDatabase(self, params = {}, type = "show", force_readd = True, search_after = False, update_library = False, status_id = None): + log.debug("show.addToDatabase") + + if not params.get('identifier'): + msg = 'Can\'t add show without imdb identifier.' + log.error(msg) + fireEvent('notify.frontend', type = 'show.is_tvshow', message = msg) + return False + #else: + #try: + #is_show = fireEvent('movie.is_show', identifier = params.get('identifier'), single = True) + #if not is_show: + #msg = 'Can\'t add show, seems to be a TV show.' + #log.error(msg) + #fireEvent('notify.frontend', type = 'show.is_tvshow', message = msg) + #return False + #except: + #pass + + library = fireEvent('library.add.%s' % type, single = True, attrs = params, update_after = update_library) + if not library: + return False + + # Status + status_active, snatched_status, ignored_status, done_status, downloaded_status = \ + fireEvent('status.get', ['active', 'snatched', 'ignored', 'done', 'downloaded'], single = True) + + default_profile = fireEvent('profile.default', single = True) + cat_id = params.get('category_id', None) + + db = get_session() + m = db.query(Media).filter_by(library_id = library.get('id')).first() + added = True + do_search = False + if not m: + m = Media( + type = type, + library_id = library.get('id'), + profile_id = params.get('profile_id', default_profile.get('id')), + status_id = status_id if status_id else status_active.get('id'), + category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None, + ) + db.add(m) + db.commit() + + onComplete = None + if search_after: + onComplete = self.createOnComplete(m.id) + + fireEventAsync('library.update.%s' % type, params.get('identifier'), default_title = params.get('title', ''), on_complete = onComplete) + search_after = False + elif force_readd: + + # Clean snatched history + for release in m.releases: + if release.status_id in [downloaded_status.get('id'), snatched_status.get('id'), done_status.get('id')]: + if params.get('ignore_previous', False): + release.status_id = ignored_status.get('id') + else: + fireEvent('release.delete', release.id, single = True) + + m.profile_id = params.get('profile_id', default_profile.get('id')) + m.category_id = tryInt(cat_id) if cat_id is not None and tryInt(cat_id) > 0 else None + else: + log.debug('Show already exists, not updating: %s', params) + added = False + + if force_readd: + m.status_id = status_id if status_id else status_active.get('id') + m.last_edit = int(time.time()) + do_search = True + + db.commit() + + # Remove releases + available_status = fireEvent('status.get', 'available', single = True) + for rel in m.releases: + if rel.status_id is available_status.get('id'): + db.delete(rel) + db.commit() + + show_dict = m.to_dict(self.default_dict) + + if do_search and search_after: + onComplete = self.createOnComplete(m.id) + onComplete() + + if added: + fireEvent('notify.frontend', type = 'show.added', data = show_dict, message = 'Successfully added "%s" to your wanted list.' % params.get('title', '')) + + db.expire_all() + return show_dict diff --git a/couchpotato/core/media/show/_base/static/search.js b/couchpotato/core/media/show/_base/static/search.js new file mode 100644 index 0000000..f01d866 --- /dev/null +++ b/couchpotato/core/media/show/_base/static/search.js @@ -0,0 +1,232 @@ +Block.Search.ShowItem = new Class({ + + Implements: [Options, Events], + + initialize: function(info, options){ + var self = this; + self.setOptions(options); + + self.info = info; + self.alternative_titles = []; + + self.create(); + }, + + create: function(){ + var self = this, + info = self.info; + + self.el = new Element('div.media_result', { + 'id': info.id + }).adopt( + self.thumbnail = info.images && info.images.poster.length > 0 ? new Element('img.thumbnail', { + 'src': info.images.poster[0], + 'height': null, + 'width': null + }) : null, + self.options_el = new Element('div.options.inlay'), + self.data_container = new Element('div.data', { + 'events': { + 'click': self.showOptions.bind(self) + } + }).adopt( + self.info_container = new Element('div.info').adopt( + new Element('h2').adopt( + self.title = new Element('span.title', { + 'text': info.titles && info.titles.length > 0 ? info.titles[0] : 'Unknown' + }), + self.year = info.year ? new Element('span.year', { + 'text': info.year + }) : null + ) + ) + ) + ) + + if(info.titles) + info.titles.each(function(title){ + self.alternativeTitle({ + 'title': title + }); + }) + }, + + alternativeTitle: function(alternative){ + var self = this; + + self.alternative_titles.include(alternative); + }, + + getTitle: function(){ + var self = this; + try { + return self.info.original_title ? self.info.original_title : self.info.titles[0]; + } + catch(e){ + return 'Unknown'; + } + }, + + get: function(key){ + return this.info[key] + }, + + showOptions: function(){ + var self = this; + + self.createOptions(); + + self.data_container.addClass('open'); + self.el.addEvent('outerClick', self.closeOptions.bind(self)) + + }, + + closeOptions: function(){ + var self = this; + + self.data_container.removeClass('open'); + self.el.removeEvents('outerClick') + }, + + add: function(e){ + var self = this; + + if(e) + (e).preventDefault(); + + self.loadingMask(); + + Api.request('show.add', { + 'data': { + 'identifier': self.info.id, + 'id': self.info.id, + 'type': self.info.type, + 'primary_provider': self.info.primary_provider, + 'title': self.title_select.get('value'), + 'profile_id': self.profile_select.get('value'), + 'category_id': self.category_select.get('value') + }, + 'onComplete': function(json){ + self.options_el.empty(); + self.options_el.adopt( + new Element('div.message', { + 'text': json.added ? 'Show successfully added.' : 'Show didn\'t add properly. Check logs' + }) + ); + self.mask.fade('out'); + + self.fireEvent('added'); + }, + 'onFailure': function(){ + self.options_el.empty(); + self.options_el.adopt( + new Element('div.message', { + 'text': 'Something went wrong, check the logs for more info.' + }) + ); + self.mask.fade('out'); + } + }); + }, + + createOptions: function(){ + var self = this, + info = self.info; + + if(!self.options_el.hasClass('set')){ + + if(self.info.in_library){ + var in_library = []; + self.info.in_library.releases.each(function(release){ + in_library.include(release.quality.label) + }); + } + + self.options_el.grab( + new Element('div', { + 'class': self.info.in_wanted && self.info.in_wanted.profile_id || in_library ? 'in_library_wanted' : '' + }).adopt( + self.info.in_wanted && self.info.in_wanted.profile_id ? new Element('span.in_wanted', { + 'text': 'Already in wanted list: ' + Quality.getProfile(self.info.in_wanted.profile_id).get('label') + }) : (in_library ? new Element('span.in_library', { + 'text': 'Already in library: ' + in_library.join(', ') + }) : null), + self.title_select = new Element('select', { + 'name': 'title' + }), + self.profile_select = new Element('select', { + 'name': 'profile' + }), + self.category_select = new Element('select', { + 'name': 'category' + }).grab( + new Element('option', {'value': -1, 'text': 'None'}) + ), + self.add_button = new Element('a.button', { + 'text': 'Add', + 'events': { + 'click': self.add.bind(self) + } + }) + ) + ); + + Array.each(self.alternative_titles, function(alt){ + new Element('option', { + 'text': alt.title + }).inject(self.title_select) + }) + + + // Fill categories + var categories = CategoryList.getAll(); + + if(categories.length == 0) + self.category_select.hide(); + else { + self.category_select.show(); + categories.each(function(category){ + new Element('option', { + 'value': category.data.id, + 'text': category.data.label + }).inject(self.category_select); + }); + } + + // Fill profiles + var profiles = Quality.getActiveProfiles(); + if(profiles.length == 1) + self.profile_select.hide(); + + profiles.each(function(profile){ + new Element('option', { + 'value': profile.id ? profile.id : profile.data.id, + 'text': profile.label ? profile.label : profile.data.label + }).inject(self.profile_select) + }); + + self.options_el.addClass('set'); + + if(categories.length == 0 && self.title_select.getElements('option').length == 1 && profiles.length == 1 && + !(self.info.in_wanted && self.info.in_wanted.profile_id || in_library)) + self.add(); + + } + + }, + + loadingMask: function(){ + var self = this; + + self.mask = new Element('div.mask').inject(self.el).fade('hide') + + createSpinner(self.mask) + self.mask.fade('in') + + }, + + toElement: function(){ + return this.el + } + +}); diff --git a/couchpotato/core/media/show/library/__init__.py b/couchpotato/core/media/show/library/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/couchpotato/core/media/show/library/episode.py b/couchpotato/core/media/show/library/episode.py new file mode 100644 index 0000000..e92216a --- /dev/null +++ b/couchpotato/core/media/show/library/episode.py @@ -0,0 +1,269 @@ +from string import ascii_letters +import time +import traceback + +from couchpotato import get_session +from couchpotato.core.event import addEvent, fireEventAsync, fireEvent +from couchpotato.core.helpers.encoding import toUnicode, simplifyString +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.library.base import LibraryBase +from couchpotato.core.helpers.variable import tryInt + + +log = CPLog(__name__) + +autload = 'EpisodeLibraryPlugin' + + +class EpisodeLibraryPlugin(LibraryBase): + + default_dict = {'titles': {}, 'files':{}} + + def __init__(self): + addEvent('library.query', self.query) + addEvent('library.identifier', self.identifier) + addEvent('library.add.episode', self.add) + addEvent('library.update.episode', self.update) + addEvent('library.update.episode_release_date', self.updateReleaseDate) + + def query(self, library, first = True, condense = True, include_identifier = True, **kwargs): + if library is list or library.get('type') != 'episode': + return + + # Get the titles of the season + if not library.get('related_libraries', {}).get('season', []): + log.warning('Invalid library, unable to determine title.') + return + + titles = fireEvent( + 'library.query', + library['related_libraries']['season'][0], + first=False, + include_identifier=include_identifier, + condense=condense, + + single=True + ) + + identifier = fireEvent('library.identifier', library, single = True) + + # Add episode identifier to titles + if include_identifier and identifier.get('episode'): + titles = [title + ('E%02d' % identifier['episode']) for title in titles] + + + if first: + return titles[0] if titles else None + + return titles + + + def identifier(self, library): + if library.get('type') != 'episode': + return + + identifier = { + 'season': None, + 'episode': None + } + + scene_map = library['info'].get('map_episode', {}).get('scene') + + if scene_map: + # Use scene mappings if they are available + identifier['season'] = scene_map.get('season') + identifier['episode'] = scene_map.get('episode') + else: + # Fallback to normal season/episode numbers + identifier['season'] = library.get('season_number') + identifier['episode'] = library.get('episode_number') + + + # Cast identifiers to integers + # TODO this will need changing to support identifiers with trailing 'a', 'b' characters + identifier['season'] = tryInt(identifier['season'], None) + identifier['episode'] = tryInt(identifier['episode'], None) + + return identifier + + def add(self, attrs = {}, update_after = True): + type = attrs.get('type', 'episode') + primary_provider = attrs.get('primary_provider', 'thetvdb') + + db = get_session() + parent_identifier = attrs.get('parent_identifier', None) + + parent = None + if parent_identifier: + parent = db.query(SeasonLibrary).filter_by(primary_provider = primary_provider, identifier = attrs.get('parent_identifier')).first() + + l = db.query(EpisodeLibrary).filter_by(type = type, identifier = attrs.get('identifier')).first() + if not l: + status = fireEvent('status.get', 'needs_update', single = True) + l = EpisodeLibrary( + type = type, + primary_provider = primary_provider, + year = attrs.get('year'), + identifier = attrs.get('identifier'), + plot = toUnicode(attrs.get('plot')), + tagline = toUnicode(attrs.get('tagline')), + status_id = status.get('id'), + info = {}, + parent = parent, + season_number = tryInt(attrs.get('seasonnumber', None)), + episode_number = tryInt(attrs.get('episodenumber', None)), + absolute_number = tryInt(attrs.get('absolute_number', None)) + ) + + title = LibraryTitle( + title = toUnicode(attrs.get('title')), + simple_title = self.simplifyTitle(attrs.get('title')), + ) + + l.titles.append(title) + + db.add(l) + db.commit() + + # Update library info + if update_after is not False: + handle = fireEventAsync if update_after is 'async' else fireEvent + handle('library.update.episode', identifier = l.identifier, default_title = toUnicode(attrs.get('title', ''))) + + library_dict = l.to_dict(self.default_dict) + + db.expire_all() + return library_dict + + def update(self, identifier, default_title = '', force = False): + + if self.shuttingDown(): + return + + db = get_session() + library = db.query(EpisodeLibrary).filter_by(identifier = identifier).first() + done_status = fireEvent('status.get', 'done', single = True) + + if library: + library_dict = library.to_dict(self.default_dict) + + do_update = True + + parent_identifier = None + if library.parent is not None: + parent_identifier = library.parent.identifier + + if library.status_id == done_status.get('id') and not force: + do_update = False + + episode_params = {'season_identifier': parent_identifier, + 'episode_identifier': identifier, + 'episode': library.episode_number, + 'absolute': library.absolute_number,} + info = fireEvent('episode.info', merge = True, params = episode_params) + + # Don't need those here + try: del info['in_wanted'] + except: pass + try: del info['in_library'] + except: pass + + if not info or len(info) == 0: + log.error('Could not update, no movie info to work with: %s', identifier) + return False + + # Main info + if do_update: + library.plot = toUnicode(info.get('plot', '')) + library.tagline = toUnicode(info.get('tagline', '')) + library.year = info.get('year', 0) + library.status_id = done_status.get('id') + library.season_number = tryInt(info.get('seasonnumber', None)) + library.episode_number = tryInt(info.get('episodenumber', None)) + library.absolute_number = tryInt(info.get('absolute_number', None)) + try: + library.last_updated = int(info.get('lastupdated')) + except: + library.last_updated = int(time.time()) + library.info.update(info) + db.commit() + + # Titles + [db.delete(title) for title in library.titles] + db.commit() + + titles = info.get('titles', []) + log.debug('Adding titles: %s', titles) + counter = 0 + for title in titles: + if not title: + continue + title = toUnicode(title) + t = LibraryTitle( + title = title, + simple_title = self.simplifyTitle(title), + default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == u'' and toUnicode(titles[0]) == title) + ) + library.titles.append(t) + counter += 1 + + db.commit() + + # Files + images = info.get('images', []) + for image_type in ['poster']: + for image in images.get(image_type, []): + if not isinstance(image, (str, unicode)): + continue + + file_path = fireEvent('file.download', url = image, single = True) + if file_path: + file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True) + try: + file_obj = db.query(File).filter_by(id = file_obj.get('id')).one() + library.files.append(file_obj) + db.commit() + + break + except: + log.debug('Failed to attach to library: %s', traceback.format_exc()) + + library_dict = library.to_dict(self.default_dict) + db.expire_all() + return library_dict + + def updateReleaseDate(self, identifier): + '''XXX: Not sure what this is for yet in relation to an episode''' + pass + #db = get_session() + #library = db.query(EpisodeLibrary).filter_by(identifier = identifier).first() + + #if not library.info: + #library_dict = self.update(identifier, force = True) + #dates = library_dict.get('info', {}).get('release_date') + #else: + #dates = library.info.get('release_date') + + #if dates and dates.get('expires', 0) < time.time() or not dates: + #dates = fireEvent('movie.release_date', identifier = identifier, merge = True) + #library.info.update({'release_date': dates }) + #db.commit() + + #db.expire_all() + #return dates + + + #TODO: Add to base class + def simplifyTitle(self, title): + + title = toUnicode(title) + + nr_prefix = '' if title[0] in ascii_letters else '#' + title = simplifyString(title) + + for prefix in ['the ']: + if prefix == title[:len(prefix)]: + title = title[len(prefix):] + break + + return nr_prefix + title diff --git a/couchpotato/core/media/show/library/season.py b/couchpotato/core/media/show/library/season.py new file mode 100644 index 0000000..925063f --- /dev/null +++ b/couchpotato/core/media/show/library/season.py @@ -0,0 +1,244 @@ +from string import ascii_letters +import traceback + +from couchpotato import get_session +from couchpotato.core.event import addEvent, fireEventAsync, fireEvent +from couchpotato.core.helpers.encoding import toUnicode, simplifyString +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.library.base import LibraryBase +from couchpotato.core.helpers.variable import tryInt + + +log = CPLog(__name__) + +autload = 'SeasonLibraryPlugin' + + +class SeasonLibraryPlugin(LibraryBase): + + default_dict = {'titles': {}, 'files':{}} + + def __init__(self): + addEvent('library.query', self.query) + addEvent('library.identifier', self.identifier) + addEvent('library.add.season', self.add) + addEvent('library.update.season', self.update) + addEvent('library.update.season_release_date', self.updateReleaseDate) + + def query(self, library, first = True, condense = True, include_identifier = True, **kwargs): + if library is list or library.get('type') != 'season': + return + + # Get the titles of the show + if not library.get('related_libraries', {}).get('show', []): + log.warning('Invalid library, unable to determine title.') + return + + titles = fireEvent( + 'library.query', + library['related_libraries']['show'][0], + first=False, + condense=condense, + + single=True + ) + + # Add season map_names if they exist + if 'map_names' in library['info']: + season_names = library['info']['map_names'].get(str(library['season_number']), {}) + + # Add titles from all locations + # TODO only add name maps from a specific location + for location, names in season_names.items(): + titles += [name for name in names if name and name not in titles] + + + identifier = fireEvent('library.identifier', library, single = True) + + # Add season identifier to titles + if include_identifier and identifier.get('season') is not None: + titles = [title + (' S%02d' % identifier['season']) for title in titles] + + + if first: + return titles[0] if titles else None + + return titles + + def identifier(self, library): + if library.get('type') != 'season': + return + + return { + 'season': tryInt(library['season_number'], None) + } + + def add(self, attrs = {}, update_after = True): + type = attrs.get('type', 'season') + primary_provider = attrs.get('primary_provider', 'thetvdb') + + db = get_session() + parent_identifier = attrs.get('parent_identifier', None) + + parent = None + if parent_identifier: + parent = db.query(ShowLibrary).filter_by(primary_provider = primary_provider, identifier = attrs.get('parent_identifier')).first() + + l = db.query(SeasonLibrary).filter_by(type = type, identifier = attrs.get('identifier')).first() + if not l: + status = fireEvent('status.get', 'needs_update', single = True) + l = SeasonLibrary( + type = type, + primary_provider = primary_provider, + year = attrs.get('year'), + identifier = attrs.get('identifier'), + plot = toUnicode(attrs.get('plot')), + tagline = toUnicode(attrs.get('tagline')), + status_id = status.get('id'), + info = {}, + parent = parent, + ) + + title = LibraryTitle( + title = toUnicode(attrs.get('title')), + simple_title = self.simplifyTitle(attrs.get('title')), + ) + + l.titles.append(title) + + db.add(l) + db.commit() + + # Update library info + if update_after is not False: + handle = fireEventAsync if update_after is 'async' else fireEvent + handle('library.update.season', identifier = l.identifier, default_title = toUnicode(attrs.get('title', ''))) + + library_dict = l.to_dict(self.default_dict) + db.expire_all() + return library_dict + + def update(self, identifier, default_title = '', force = False): + + if self.shuttingDown(): + return + + db = get_session() + library = db.query(SeasonLibrary).filter_by(identifier = identifier).first() + done_status = fireEvent('status.get', 'done', single = True) + + if library: + library_dict = library.to_dict(self.default_dict) + + do_update = True + + parent_identifier = None + if library.parent is not None: + parent_identifier = library.parent.identifier + + if library.status_id == done_status.get('id') and not force: + do_update = False + + season_params = {'season_identifier': identifier} + info = fireEvent('season.info', merge = True, identifier = parent_identifier, params = season_params) + + # Don't need those here + try: del info['in_wanted'] + except: pass + try: del info['in_library'] + except: pass + + if not info or len(info) == 0: + log.error('Could not update, no movie info to work with: %s', identifier) + return False + + # Main info + if do_update: + library.plot = toUnicode(info.get('plot', '')) + library.tagline = toUnicode(info.get('tagline', '')) + library.year = info.get('year', 0) + library.status_id = done_status.get('id') + library.season_number = tryInt(info.get('seasonnumber', None)) + library.info.update(info) + db.commit() + + # Titles + [db.delete(title) for title in library.titles] + db.commit() + + titles = info.get('titles', []) + log.debug('Adding titles: %s', titles) + counter = 0 + for title in titles: + if not title: + continue + title = toUnicode(title) + t = LibraryTitle( + title = title, + simple_title = self.simplifyTitle(title), + # XXX: default was None; so added a quick hack since we don't really need titiles for seasons anyway + #default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == u'' and toUnicode(titles[0]) == title) + default = True, + ) + library.titles.append(t) + counter += 1 + + db.commit() + + # Files + images = info.get('images', []) + for image_type in ['poster']: + for image in images.get(image_type, []): + if not isinstance(image, (str, unicode)): + continue + + file_path = fireEvent('file.download', url = image, single = True) + if file_path: + file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True) + try: + file_obj = db.query(File).filter_by(id = file_obj.get('id')).one() + library.files.append(file_obj) + db.commit() + break + except: + log.debug('Failed to attach to library: %s', traceback.format_exc()) + + library_dict = library.to_dict(self.default_dict) + db.expire_all() + return library_dict + + def updateReleaseDate(self, identifier): + '''XXX: Not sure what this is for yet in relation to a tvshow''' + pass + #db = get_session() + #library = db.query(SeasonLibrary).filter_by(identifier = identifier).first() + + #if not library.info: + #library_dict = self.update(identifier, force = True) + #dates = library_dict.get('info', {}).get('release_date') + #else: + #dates = library.info.get('release_date') + + #if dates and dates.get('expires', 0) < time.time() or not dates: + #dates = fireEvent('movie.release_date', identifier = identifier, merge = True) + #library.info.update({'release_date': dates }) + #db.commit() + + #db.expire_all() + #return dates + + + #TODO: Add to base class + def simplifyTitle(self, title): + + title = toUnicode(title) + + nr_prefix = '' if title[0] in ascii_letters else '#' + title = simplifyString(title) + + for prefix in ['the ']: + if prefix == title[:len(prefix)]: + title = title[len(prefix):] + break + + return nr_prefix + title diff --git a/couchpotato/core/media/show/library/show.py b/couchpotato/core/media/show/library/show.py new file mode 100644 index 0000000..d1e2464 --- /dev/null +++ b/couchpotato/core/media/show/library/show.py @@ -0,0 +1,232 @@ +from string import ascii_letters +import time +import traceback + +from couchpotato import get_session +from couchpotato.core.event import addEvent, fireEventAsync, fireEvent +from couchpotato.core.helpers.encoding import toUnicode, simplifyString +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.library.base import LibraryBase +from qcond.helpers import simplify +from qcond import QueryCondenser + + +log = CPLog(__name__) + +autload = 'SeasonLibraryPlugin' + + +class ShowLibraryPlugin(LibraryBase): + + default_dict = {'titles': {}, 'files':{}} + + def __init__(self): + self.query_condenser = QueryCondenser() + + addEvent('library.query', self.query) + addEvent('library.add.show', self.add) + addEvent('library.update.show', self.update) + addEvent('library.update.show_release_date', self.updateReleaseDate) + + def query(self, library, first = True, condense = True, **kwargs): + if library is list or library.get('type') != 'show': + return + + titles = [title['title'] for title in library['titles']] + + if condense: + # Use QueryCondenser to build a list of optimal search titles + condensed_titles = self.query_condenser.distinct(titles) + + if condensed_titles: + # Use condensed titles if we got a valid result + titles = condensed_titles + else: + # Fallback to simplifying titles + titles = [simplify(title) for title in titles] + + if first: + return titles[0] if titles else None + + return titles + + def add(self, attrs = {}, update_after = True): + type = attrs.get('type', 'show') + primary_provider = attrs.get('primary_provider', 'thetvdb') + + db = get_session() + + l = db.query(ShowLibrary).filter_by(type = type, identifier = attrs.get('identifier')).first() + if not l: + status = fireEvent('status.get', 'needs_update', single = True) + l = ShowLibrary( + type = type, + primary_provider = primary_provider, + year = attrs.get('year'), + identifier = attrs.get('identifier'), + plot = toUnicode(attrs.get('plot')), + tagline = toUnicode(attrs.get('tagline')), + status_id = status.get('id'), + info = {}, + parent = None, + ) + + title = LibraryTitle( + title = toUnicode(attrs.get('title')), + simple_title = self.simplifyTitle(attrs.get('title')), + ) + + l.titles.append(title) + + db.add(l) + db.commit() + + # Update library info + if update_after is not False: + handle = fireEventAsync if update_after is 'async' else fireEvent + handle('library.update.show', identifier = l.identifier, default_title = toUnicode(attrs.get('title', ''))) + + library_dict = l.to_dict(self.default_dict) + db.expire_all() + return library_dict + + def update(self, identifier, default_title = '', force = False): + + if self.shuttingDown(): + return + + db = get_session() + library = db.query(ShowLibrary).filter_by(identifier = identifier).first() + done_status = fireEvent('status.get', 'done', single = True) + + if library: + library_dict = library.to_dict(self.default_dict) + + do_update = True + + info = fireEvent('show.info', merge = True, identifier = identifier) + + # Don't need those here + try: del info['in_wanted'] + except: pass + try: del info['in_library'] + except: pass + + if not info or len(info) == 0: + log.error('Could not update, no show info to work with: %s', identifier) + return False + + # Main info + if do_update: + library.plot = toUnicode(info.get('plot', '')) + library.tagline = toUnicode(info.get('tagline', '')) + library.year = info.get('year', 0) + library.status_id = done_status.get('id') + library.show_status = toUnicode(info.get('status', '').lower()) + library.airs_time = info.get('airs_time', None) + + # Bits + days_of_week_map = { + u'Monday': 1, + u'Tuesday': 2, + u'Wednesday': 4, + u'Thursday': 8, + u'Friday': 16, + u'Saturday': 32, + u'Sunday': 64, + u'Daily': 127, + } + try: + library.airs_dayofweek = days_of_week_map.get(info.get('airs_dayofweek')) + except: + library.airs_dayofweek = 0 + + try: + library.last_updated = int(info.get('lastupdated')) + except: + library.last_updated = int(time.time()) + + library.info.update(info) + + db.commit() + + # Titles + [db.delete(title) for title in library.titles] + db.commit() + + titles = info.get('titles', []) + log.debug('Adding titles: %s', titles) + counter = 0 + for title in titles: + if not title: + continue + title = toUnicode(title) + t = LibraryTitle( + title = title, + simple_title = self.simplifyTitle(title), + default = (len(default_title) == 0 and counter == 0) or len(titles) == 1 or title.lower() == toUnicode(default_title.lower()) or (toUnicode(default_title) == u'' and toUnicode(titles[0]) == title) + ) + library.titles.append(t) + counter += 1 + + db.commit() + + # Files + images = info.get('images', []) + for image_type in ['poster']: + for image in images.get(image_type, []): + if not isinstance(image, (str, unicode)): + continue + + file_path = fireEvent('file.download', url = image, single = True) + if file_path: + file_obj = fireEvent('file.add', path = file_path, type_tuple = ('image', image_type), single = True) + try: + file_obj = db.query(File).filter_by(id = file_obj.get('id')).one() + library.files.append(file_obj) + db.commit() + + break + except: + log.debug('Failed to attach to library: %s', traceback.format_exc()) + + library_dict = library.to_dict(self.default_dict) + + db.expire_all() + return library_dict + + def updateReleaseDate(self, identifier): + '''XXX: Not sure what this is for yet in relation to a show''' + pass + #db = get_session() + #library = db.query(ShowLibrary).filter_by(identifier = identifier).first() + + #if not library.info: + #library_dict = self.update(identifier, force = True) + #dates = library_dict.get('info', {}).get('release_date') + #else: + #dates = library.info.get('release_date') + + #if dates and dates.get('expires', 0) < time.time() or not dates: + #dates = fireEvent('movie.release_date', identifier = identifier, merge = True) + #library.info.update({'release_date': dates }) + #db.commit() + + #db.expire_all() + #return dates + + + #TODO: Add to base class + def simplifyTitle(self, title): + + title = toUnicode(title) + + nr_prefix = '' if title[0] in ascii_letters else '#' + title = simplifyString(title) + + for prefix in ['the ']: + if prefix == title[:len(prefix)]: + title = title[len(prefix):] + break + + return nr_prefix + title diff --git a/couchpotato/core/media/show/matcher.py b/couchpotato/core/media/show/matcher.py new file mode 100644 index 0000000..05d7873 --- /dev/null +++ b/couchpotato/core/media/show/matcher.py @@ -0,0 +1,130 @@ +from couchpotato import CPLog +from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.media._base.matcher.base import MatcherBase +from couchpotato.core.media._base.providers.base import MultiProvider + +log = CPLog(__name__) + +autoload = 'ShowMatcher' + + +class ShowMatcher(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Base(MatcherBase): + + # TODO come back to this later, think this could be handled better, this is starting to get out of hand.... + quality_map = { + 'bluray_1080p': {'resolution': ['1080p'], 'source': ['bluray']}, + 'bluray_720p': {'resolution': ['720p'], 'source': ['bluray']}, + + 'bdrip_1080p': {'resolution': ['1080p'], 'source': ['BDRip']}, + 'bdrip_720p': {'resolution': ['720p'], 'source': ['BDRip']}, + + 'brrip_1080p': {'resolution': ['1080p'], 'source': ['BRRip']}, + 'brrip_720p': {'resolution': ['720p'], 'source': ['BRRip']}, + + 'webdl_1080p': {'resolution': ['1080p'], 'source': ['webdl', ['web', 'dl']]}, + 'webdl_720p': {'resolution': ['720p'], 'source': ['webdl', ['web', 'dl']]}, + 'webdl_480p': {'resolution': ['480p'], 'source': ['webdl', ['web', 'dl']]}, + + 'hdtv_720p': {'resolution': ['720p'], 'source': ['hdtv']}, + 'hdtv_sd': {'resolution': ['480p', None], 'source': ['hdtv']}, + } + + def __init__(self): + super(Base, self).__init__() + + addEvent('%s.matcher.correct_identifier' % self.type, self.correctIdentifier) + + def correct(self, chain, release, media, quality): + log.info("Checking if '%s' is valid", release['name']) + log.info2('Release parsed as: %s', chain.info) + + if not fireEvent('matcher.correct_quality', chain, quality, self.quality_map, single = True): + log.info('Wrong: %s, quality does not match', release['name']) + return False + + if not fireEvent('%s.matcher.correct_identifier' % self.type, chain, media): + log.info('Wrong: %s, identifier does not match', release['name']) + return False + + if not fireEvent('matcher.correct_title', chain, media): + log.info("Wrong: '%s', undetermined naming.", (' '.join(chain.info['show_name']))) + return False + + return True + + def correctIdentifier(self, chain, media): + raise NotImplementedError() + + def getChainIdentifier(self, chain): + if 'identifier' not in chain.info: + return None + + identifier = self.flattenInfo(chain.info['identifier']) + + # Try cast values to integers + for key, value in identifier.items(): + if isinstance(value, list): + if len(value) <= 1: + value = value[0] + else: + log.warning('Wrong: identifier contains multiple season or episode values, unsupported') + return None + + identifier[key] = tryInt(value, value) + + return identifier + + +class Episode(Base): + type = 'episode' + + def correctIdentifier(self, chain, media): + identifier = self.getChainIdentifier(chain) + if not identifier: + log.info2('Wrong: release identifier is not valid (unsupported or missing identifier)') + return False + + # TODO - Parse episode ranges from identifier to determine if they are multi-part episodes + if any([x in identifier for x in ['episode_from', 'episode_to']]): + log.info2('Wrong: releases with identifier ranges are not supported yet') + return False + + required = fireEvent('library.identifier', media['library'], single = True) + + # TODO - Support air by date episodes + # TODO - Support episode parts + + if identifier != required: + log.info2('Wrong: required identifier (%s) does not match release identifier (%s)', (required, identifier)) + return False + + return True + +class Season(Base): + type = 'season' + + def correctIdentifier(self, chain, media): + identifier = self.getChainIdentifier(chain) + if not identifier: + log.info2('Wrong: release identifier is not valid (unsupported or missing identifier)') + return False + + # TODO - Parse episode ranges from identifier to determine if they are season packs + if any([x in identifier for x in ['episode_from', 'episode_to']]): + log.info2('Wrong: releases with identifier ranges are not supported yet') + return False + + required = fireEvent('library.identifier', media['library'], single = True) + + if identifier != required: + log.info2('Wrong: required identifier (%s) does not match release identifier (%s)', (required, identifier)) + return False + + return True diff --git a/couchpotato/core/media/show/providers/__init__.py b/couchpotato/core/media/show/providers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/couchpotato/core/media/show/providers/base.py b/couchpotato/core/media/show/providers/base.py new file mode 100644 index 0000000..8ad4a7a --- /dev/null +++ b/couchpotato/core/media/show/providers/base.py @@ -0,0 +1,13 @@ +from couchpotato.core.media._base.providers.info.base import BaseInfoProvider + + +class ShowProvider(BaseInfoProvider): + type = 'show' + + +class SeasonProvider(BaseInfoProvider): + type = 'season' + + +class EpisodeProvider(BaseInfoProvider): + type = 'episode' diff --git a/couchpotato/core/media/show/providers/info/__init__.py b/couchpotato/core/media/show/providers/info/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/couchpotato/core/media/show/providers/info/thetvdb.py b/couchpotato/core/media/show/providers/info/thetvdb.py new file mode 100644 index 0000000..66d5f6d --- /dev/null +++ b/couchpotato/core/media/show/providers/info/thetvdb.py @@ -0,0 +1,495 @@ +from datetime import datetime +import traceback +import os + +from couchpotato.core.event import addEvent +from couchpotato.core.helpers.encoding import simplifyString, toUnicode +from couchpotato.core.logger import CPLog +from couchpotato.core.media.show.providers.base import ShowProvider +from couchpotato.environment import Env +from tvdb_api import tvdb_exceptions +import tvdb_api + + +log = CPLog(__name__) + +autoload = 'TheTVDb' + + +class TheTVDb(ShowProvider): + + # TODO: Consider grabbing zips to put less strain on tvdb + # TODO: Unicode stuff (check) + # TODO: Notigy frontend on error (tvdb down at monent) + # TODO: Expose apikey in setting so it can be changed by user + + def __init__(self): + addEvent('info.search', self.search, priority = 1) + addEvent('show.search', self.search, priority = 1) + addEvent('show.info', self.getShowInfo, priority = 1) + addEvent('season.info', self.getSeasonInfo, priority = 1) + addEvent('episode.info', self.getEpisodeInfo, priority = 1) + + self.tvdb_api_parms = { + 'apikey': self.conf('api_key'), + 'banners': True, + 'language': 'en', + 'cache': os.path.join(Env.get('cache_dir'), 'thetvdb_api'), + } + self._setup() + + def _setup(self): + self.tvdb = tvdb_api.Tvdb(**self.tvdb_api_parms) + self.valid_languages = self.tvdb.config['valid_languages'] + + def search(self, q, limit = 12, language = 'en'): + ''' Find show by name + show = { 'id': 74713, + 'language': 'en', + 'lid': 7, + 'seriesid': '74713', + 'seriesname': u'Breaking Bad',} + ''' + + if self.isDisabled(): + return False + + if language != self.tvdb_api_parms['language'] and language in self.valid_languages: + self.tvdb_api_parms['language'] = language + self._setup() + + search_string = simplifyString(q) + cache_key = 'thetvdb.cache.%s.%s' % (search_string, limit) + results = self.getCache(cache_key) + + if not results: + log.debug('Searching for show: %s', q) + raw = None + try: + raw = self.tvdb.search(search_string) + except (tvdb_exceptions.tvdb_error, IOError), e: + log.error('Failed searching TheTVDB for "%s": %s', (search_string, traceback.format_exc())) + return False + + results = [] + if raw: + try: + nr = 0 + for show_info in raw: + show = self.tvdb[int(show_info['id'])] + results.append(self._parseShow(show)) + nr += 1 + if nr == limit: + break + log.info('Found: %s', [result['titles'][0] + ' (' + str(result.get('year', 0)) + ')' for result in results]) + self.setCache(cache_key, results) + return results + except (tvdb_exceptions.tvdb_error, IOError), e: + log.error('Failed parsing TheTVDB for "%s": %s', (show, traceback.format_exc())) + return False + return results + + def getShow(self, identifier = None): + show = None + try: + log.debug('Getting show: %s', identifier) + show = self.tvdb[int(identifier)] + except (tvdb_exceptions.tvdb_error, IOError), e: + log.error('Failed to getShowInfo for show id "%s": %s', (identifier, traceback.format_exc())) + return None + + return show + + def getShowInfo(self, identifier = None): + if not identifier: + return None + + cache_key = 'thetvdb.cache.%s' % identifier + log.debug('Getting showInfo: %s', cache_key) + result = self.getCache(cache_key) or {} + if result: + return result + + show = self.getShow(identifier = identifier) + if show: + result = self._parseShow(show) + self.setCache(cache_key, result) + + return result + + def getSeasonInfo(self, identifier = None, params = {}): + """Either return a list of all seasons or a single season by number. + identifier is the show 'id' + """ + if not identifier: + return False + + season_identifier = params.get('season_identifier', None) + + # season_identifier must contain the 'show id : season number' since there is no tvdb id + # for season and we need a reference to both the show id and season number + if season_identifier: + try: season_identifier = int(season_identifier.split(':')[1]) + except: return False + + cache_key = 'thetvdb.cache.%s.%s' % (identifier, season_identifier) + log.debug('Getting SeasonInfo: %s', cache_key) + result = self.getCache(cache_key) or {} + if result: + return result + + try: + show = self.tvdb[int(identifier)] + except (tvdb_exceptions.tvdb_error, IOError), e: + log.error('Failed parsing TheTVDB SeasonInfo for "%s" id "%s": %s', (show, identifier, traceback.format_exc())) + return False + + result = [] + for number, season in show.items(): + if season_identifier is not None and number == season_identifier: + result = self._parseSeason(show, (number, season)) + self.setCache(cache_key, result) + return result + else: + result.append(self._parseSeason(show, (number, season))) + + self.setCache(cache_key, result) + return result + + def getEpisodeInfo(self, identifier = None, params = {}): + """Either return a list of all episodes or a single episode. + If episode_identifer contains an episode number to search for + """ + season_identifier = params.get('season_identifier', None) + episode_identifier = params.get('episode_identifier', None) + + if not identifier and season_identifier is None: + return False + + # season_identifier must contain the 'show id : season number' since there is no tvdb id + # for season and we need a reference to both the show id and season number + if season_identifier: + try: + identifier, season_identifier = season_identifier.split(':') + season_identifier = int(season_identifier) + except: return None + + cache_key = 'thetvdb.cache.%s.%s.%s' % (identifier, episode_identifier, season_identifier) + log.debug('Getting EpisodeInfo: %s', cache_key) + result = self.getCache(cache_key) or {} + if result: + return result + + try: + show = self.tvdb[int(identifier)] + except (tvdb_exceptions.tvdb_error, IOError), e: + log.error('Failed parsing TheTVDB EpisodeInfo for "%s" id "%s": %s', (show, identifier, traceback.format_exc())) + return False + + result = [] + for number, season in show.items(): + if season_identifier is not None and number != season_identifier: + continue + + for episode in season.values(): + if episode_identifier is not None and episode['id'] == toUnicode(episode_identifier): + result = self._parseEpisode(show, episode) + self.setCache(cache_key, result) + return result + else: + result.append(self._parseEpisode(show, episode)) + + self.setCache(cache_key, result) + return result + + def _parseShow(self, show): + """ + 'actors': u'|Bryan Cranston|Aaron Paul|Dean Norris|RJ Mitte|Betsy Brandt|Anna Gunn|Laura Fraser|Jesse Plemons|Christopher Cousins|Steven Michael Quezada|Jonathan Banks|Giancarlo Esposito|Bob Odenkirk|', + 'added': None, + 'addedby': None, + 'airs_dayofweek': u'Sunday', + 'airs_time': u'9:00 PM', + 'banner': u'http://thetvdb.com/banners/graphical/81189-g13.jpg', + 'contentrating': u'TV-MA', + 'fanart': u'http://thetvdb.com/banners/fanart/original/81189-28.jpg', + 'firstaired': u'2008-01-20', + 'genre': u'|Crime|Drama|Suspense|', + 'id': u'81189', + 'imdb_id': u'tt0903747', + 'language': u'en', + 'lastupdated': u'1376620212', + 'network': u'AMC', + 'networkid': None, + 'overview': u"Walter White, a struggling high school chemistry teacher is diagnosed with advanced lung cancer. He turns to a life of crime, producing and selling methamphetamine accompanied by a former student, Jesse Pinkman with the aim of securing his family's financial future before he dies.", + 'poster': u'http://thetvdb.com/banners/posters/81189-22.jpg', + 'rating': u'9.3', + 'ratingcount': u'473', + 'runtime': u'60', + 'seriesid': u'74713', + 'seriesname': u'Breaking Bad', + 'status': u'Continuing', + 'zap2it_id': u'SH01009396' + """ + + # + # NOTE: show object only allows direct access via + # show['id'], not show.get('id') + # + + # TODO: Make sure we have a valid show id, not '' or None + #if len (show['id']) is 0: + # return None + + ## Images + poster = show['poster'] or None + backdrop = show['fanart'] or None + + genres = [] if show['genre'] is None else show['genre'].strip('|').split('|') + if show['firstaired'] is not None: + try: year = datetime.strptime(show['firstaired'], '%Y-%m-%d').year + except: year = None + else: + year = None + + try: + id = int(show['id']) + except: + id = None + + show_data = { + 'id': id, + 'type': 'show', + 'primary_provider': 'thetvdb', + 'titles': [show['seriesname'] or u'', ], + 'original_title': show['seriesname'] or u'', + 'images': { + 'poster': [poster] if poster else [], + 'backdrop': [backdrop] if backdrop else [], + 'poster_original': [], + 'backdrop_original': [], + }, + 'year': year, + 'genres': genres, + 'imdb': show['imdb_id'] or None, + 'zap2it_id': show['zap2it_id'] or None, + 'seriesid': show['seriesid'] or None, + 'network': show['network'] or None, + 'networkid': show['networkid'] or None, + 'airs_dayofweek': show['airs_dayofweek'] or None, + 'airs_time': show['airs_time'] or None, + 'firstaired': show['firstaired'] or None, + 'released': show['firstaired'] or None, + 'runtime': show['runtime'] or None, + 'contentrating': show['contentrating'] or None, + 'rating': show['rating'] or None, + 'ratingcount': show['ratingcount'] or None, + 'actors': show['actors'] or None, + 'lastupdated': show['lastupdated'] or None, + 'status': show['status'] or None, + 'language': show['language'] or None, + } + + show_data = dict((k, v) for k, v in show_data.iteritems() if v) + + # Add alternative titles + try: + raw = self.tvdb.search(show['seriesname']) + if raw: + for show_info in raw: + if show_info['id'] == show_data['id'] and show_info.get('aliasnames', None): + for alt_name in show_info['aliasnames'].split('|'): + show_data['titles'].append(toUnicode(alt_name)) + except (tvdb_exceptions.tvdb_error, IOError), e: + log.error('Failed searching TheTVDB for "%s": %s', (show['seriesname'], traceback.format_exc())) + + return show_data + + def _parseSeason(self, show, season_tuple): + """ + contains no data + """ + + number, season = season_tuple + title = toUnicode('%s - Season %s' % (show['seriesname'] or u'', str(number))) + poster = [] + try: + for id, data in show.data['_banners']['season']['season'].items(): + if data.get('season', None) == str(number) and data['bannertype'] == 'season' and data['bannertype2'] == 'season': + poster.append(data.get('_bannerpath')) + break # Only really need one + except: + pass + + try: + id = (show['id'] + ':' + str(number)) + except: + id = None + + # XXX: work on title; added defualt_title to fix an error + season_data = { + 'id': id, + 'type': 'season', + 'primary_provider': 'thetvdb', + 'titles': [title, ], + 'original_title': title, + 'via_thetvdb': True, + 'parent_identifier': show['id'] or None, + 'seasonnumber': str(number), + 'images': { + 'poster': poster, + 'backdrop': [], + 'poster_original': [], + 'backdrop_original': [], + }, + 'year': None, + 'genres': None, + 'imdb': None, + } + + season_data = dict((k, v) for k, v in season_data.iteritems() if v) + return season_data + + def _parseEpisode(self, show, episode): + """ + ('episodenumber', u'1'), + ('thumb_added', None), + ('rating', u'7.7'), + ('overview', + u'Experienced waitress Max Black meets her new co-worker, former rich-girl Caroline Channing, and puts her skills to the test at an old but re-emerging Brooklyn diner. Despite her initial distaste for Caroline, Max eventually softens and the two team up for a new business venture.'), + ('dvd_episodenumber', None), + ('dvd_discid', None), + ('combined_episodenumber', u'1'), + ('epimgflag', u'7'), + ('id', u'4099506'), + ('seasonid', u'465948'), + ('thumb_height', u'225'), + ('tms_export', u'1374789754'), + ('seasonnumber', u'1'), + ('writer', u'|Michael Patrick King|Whitney Cummings|'), + ('lastupdated', u'1371420338'), + ('filename', u'http://thetvdb.com/banners/episodes/248741/4099506.jpg'), + ('absolute_number', u'1'), + ('ratingcount', u'102'), + ('combined_season', u'1'), + ('thumb_width', u'400'), + ('imdb_id', u'tt1980319'), + ('director', u'James Burrows'), + ('dvd_chapter', None), + ('dvd_season', None), + ('gueststars', + u'|Brooke Lyons|Noah Mills|Shoshana Bush|Cale Hartmann|Adam Korson|Alex Enriquez|Matt Cook|Bill Parks|Eugene Shaw|Sergey Brusilovsky|Greg Lewis|Cocoa Brown|Nick Jameson|'), + ('seriesid', u'248741'), + ('language', u'en'), + ('productioncode', u'296793'), + ('firstaired', u'2011-09-19'), + ('episodename', u'Pilot')] + """ + + poster = episode.get('filename', []) + backdrop = [] + genres = [] + plot = "%s - %sx%s - %s" % (show['seriesname'] or u'', + episode.get('seasonnumber', u'?'), + episode.get('episodenumber', u'?'), + episode.get('overview', u'')) + if episode.get('firstaired', None) is not None: + try: year = datetime.strptime(episode['firstaired'], '%Y-%m-%d').year + except: year = None + else: + year = None + + try: + id = int(episode['id']) + except: + id = None + + episode_data = { + 'id': id, + 'type': 'episode', + 'primary_provider': 'thetvdb', + 'via_thetvdb': True, + 'thetvdb_id': id, + 'titles': [episode.get('episodename', u''), ], + 'original_title': episode.get('episodename', u'') , + 'images': { + 'poster': [poster] if poster else [], + 'backdrop': [backdrop] if backdrop else [], + 'poster_original': [], + 'backdrop_original': [], + }, + 'imdb': episode.get('imdb_id', None), + 'runtime': None, + 'released': episode.get('firstaired', None), + 'year': year, + 'plot': plot, + 'genres': genres, + 'parent_identifier': show['id'] or None, + 'seasonnumber': episode.get('seasonnumber', None), + 'episodenumber': episode.get('episodenumber', None), + 'combined_episodenumber': episode.get('combined_episodenumber', None), + 'absolute_number': episode.get('absolute_number', None), + 'combined_season': episode.get('combined_season', None), + 'productioncode': episode.get('productioncode', None), + 'seriesid': episode.get('seriesid', None), + 'seasonid': episode.get('seasonid', None), + 'firstaired': episode.get('firstaired', None), + 'thumb_added': episode.get('thumb_added', None), + 'thumb_height': episode.get('thumb_height', None), + 'thumb_width': episode.get('thumb_width', None), + 'rating': episode.get('rating', None), + 'ratingcount': episode.get('ratingcount', None), + 'epimgflag': episode.get('epimgflag', None), + 'dvd_episodenumber': episode.get('dvd_episodenumber', None), + 'dvd_discid': episode.get('dvd_discid', None), + 'dvd_chapter': episode.get('dvd_chapter', None), + 'dvd_season': episode.get('dvd_season', None), + 'tms_export': episode.get('tms_export', None), + 'writer': episode.get('writer', None), + 'director': episode.get('director', None), + 'gueststars': episode.get('gueststars', None), + 'lastupdated': episode.get('lastupdated', None), + 'language': episode.get('language', None), + } + + episode_data = dict((k, v) for k, v in episode_data.iteritems() if v) + return episode_data + + #def getImage(self, show, type = 'poster', size = 'cover'): + #"""""" + ## XXX: Need to implement size + #image_url = '' + + #for res, res_data in show['_banners'].get(type, {}).items(): + #for bid, banner_info in res_data.items(): + #image_url = banner_info.get('_bannerpath', '') + #break + + #return image_url + + def isDisabled(self): + if self.conf('api_key') == '': + log.error('No API key provided.') + True + else: + False + + +config = [{ + 'name': 'thetvdb', + 'groups': [ + { + 'tab': 'providers', + 'name': 'tmdb', + 'label': 'TheTVDB', + 'hidden': True, + 'description': 'Used for all calls to TheTVDB.', + 'options': [ + { + 'name': 'api_key', + 'default': '7966C02F860586D2', + 'label': 'Api Key', + }, + ], + }, + ], +}] diff --git a/couchpotato/core/media/show/providers/info/xem.py b/couchpotato/core/media/show/providers/info/xem.py new file mode 100644 index 0000000..ec7d343 --- /dev/null +++ b/couchpotato/core/media/show/providers/info/xem.py @@ -0,0 +1,194 @@ +from couchpotato.core.event import addEvent +from couchpotato.core.logger import CPLog +from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode +from couchpotato.core.media.show.providers.base import ShowProvider + +log = CPLog(__name__) + + +class Xem(ShowProvider): + ''' + Mapping Information + =================== + + Single + ------ + You will need the id / identifier of the show e.g. tvdb-id for American Dad! is 73141 + the origin is the name of the site/entity the episode, season (and/or absolute) numbers are based on + + http://thexem.de/map/single?id=&origin=&episode=&season=&absolute= + + episode, season and absolute are all optional but it wont work if you don't provide either episode and season OR absolute in + addition you can provide destination as the name of the wished destination, if not provided it will output all available + + When a destination has two or more addresses another entry will be added as _ ... for now the second address gets the index "2" + (the first index is omitted) and so on + + http://thexem.de/map/single?id=7529&origin=anidb&season=1&episode=2&destination=trakt + { + "result":"success", + "data":{ + "trakt": {"season":1,"episode":3,"absolute":3}, + "trakt_2":{"season":1,"episode":4,"absolute":4} + }, + "message":"single mapping for 7529 on anidb." + } + + All + --- + Basically same as "single" just a little easier + The origin address is added into the output too!! + + http://thexem.de/map/all?id=7529&origin=anidb + + All Names + --------- + Get all names xem has to offer + non optional params: origin(an entity string like 'tvdb') + optional params: season, language + - season: a season number or a list like: 1,3,5 or a compare operator like ne,gt,ge,lt,le,eq and a season number. default would + return all + - language: a language string like 'us' or 'jp' default is all + - defaultNames: 1(yes) or 0(no) should the default names be added to the list ? default is 0(no) + + http://thexem.de/map/allNames?origin=tvdb&season=le1 + + { + "result": "success", + "data": { + "248812": ["Dont Trust the Bitch in Apartment 23", "Don't Trust the Bitch in Apartment 23"], + "257571": ["Nazo no Kanojo X"], + "257875": ["Lupin III - Mine Fujiko to Iu Onna", "Lupin III Fujiko to Iu Onna", "Lupin the Third - Mine Fujiko to Iu Onna"] + }, + "message": "" + } + ''' + + def __init__(self): + addEvent('show.info', self.getShowInfo, priority = 5) + addEvent('episode.info', self.getEpisodeInfo, priority = 5) + + self.config = {} + self.config['base_url'] = "http://thexem.de" + self.config['url_single'] = u"%(base_url)s/map/single?" % self.config + self.config['url_all'] = u"%(base_url)s/map/all?" % self.config + self.config['url_names'] = u"%(base_url)s/map/names?" % self.config + self.config['url_all_names'] = u"%(base_url)s/map/allNames?" % self.config + + # TODO: Also get show aliases (store as titles) + def getShowInfo(self, identifier = None): + if self.isDisabled(): + return {} + + cache_key = 'xem.cache.%s' % identifier + log.debug('Getting showInfo: %s', cache_key) + result = self.getCache(cache_key) or {} + if result: + return result + + # Create season/episode and absolute mappings + url = self.config['url_all'] + "id=%s&origin=tvdb" % tryUrlencode(identifier) + response = self.getJsonData(url) + if response: + if response.get('result') == 'success': + data = response.get('data', None) + result = self._parse(data) + + # Create name alias mappings + url = self.config['url_names'] + "id=%s&origin=tvdb" % tryUrlencode(identifier) + response = self.getJsonData(url) + if response: + if response.get('result') == 'success': + data = response.get('data', None) + result.update({'map_names': data}) + + self.setCache(cache_key, result) + return result + + def getEpisodeInfo(self, identifier = None, params = {}): + episode = params.get('episode', None) + if episode is None: + return False + + season_identifier = params.get('season_identifier', None) + if season_identifier is None: + return False + + episode_identifier = params.get('episode_identifier', None) + absolute = params.get('absolute', None) + + # season_identifier must contain the 'show id : season number' since there is no tvdb id + # for season and we need a reference to both the show id and season number + if season_identifier: + try: + identifier, season_identifier = season_identifier.split(':') + season = int(season_identifier) + except: return False + + result = self.getShowInfo(identifier) + map = {} + if result: + map_episode = result.get('map_episode', {}).get(season, {}).get(episode, {}) + if map_episode: + map.update({'map_episode': map_episode}) + + if absolute: + map_absolute = result.get('map_absolute', {}).get(absolute, {}) + if map_absolute: + map.update({'map_absolute': map_absolute}) + + map_names = result.get('map_names', {}).get(toUnicode(season), {}) + if map_names: + map.update({'map_names': map_names}) + + return map + + + def _parse(self, data, master = 'tvdb'): + '''parses xem map and returns a custom formatted dict map + + To retreive map for scene: + if 'scene' in map['map_episode'][1][1]: + print map['map_episode'][1][1]['scene']['season'] + ''' + if not isinstance(data, list): + return {} + + map = {'map_episode': {}, 'map_absolute': {}} + for maps in data: + origin = maps.pop(master, None) + if origin is None: + continue # No master origin to map to + map.get('map_episode').setdefault(origin['season'], {}).setdefault(origin['episode'], maps.copy()) + map.get('map_absolute').setdefault(origin['absolute'], maps.copy()) + + return map + + def isDisabled(self): + if __name__ == '__main__': + return False + if self.conf('enabled'): + return False + else: + return True + + +config = [{ + 'name': 'xem', + 'groups': [ + { + 'tab': 'providers', + 'name': 'xem', + 'label': 'TheXem', + 'hidden': True, + 'description': 'Used for all calls to TheXem.', + 'options': [ + { + 'name': 'enabled', + 'default': True, + 'label': 'Enabled', + }, + ], + }, + ], +}] diff --git a/couchpotato/core/media/show/providers/nzb/__init__.py b/couchpotato/core/media/show/providers/nzb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/couchpotato/core/media/show/providers/nzb/binsearch.py b/couchpotato/core/media/show/providers/nzb/binsearch.py new file mode 100644 index 0000000..0492424 --- /dev/null +++ b/couchpotato/core/media/show/providers/nzb/binsearch.py @@ -0,0 +1,51 @@ +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.event import fireEvent +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media._base.providers.nzb.binsearch import Base +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.environment import Env + +log = CPLog(__name__) + +autoload = 'BinSearch' + + +class BinSearch(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + def buildUrl(self, media, quality): + query = tryUrlencode({ + 'q': fireEvent('library.query', media, single = True), + 'm': 'n', + 'max': 400, + 'adv_age': Env.setting('retention', 'nzb'), + 'adv_sort': 'date', + 'adv_col': 'on', + 'adv_nfo': 'on', + 'minsize': quality.get('size_min'), + 'maxsize': quality.get('size_max'), + }) + return query + + +class Episode(EpisodeProvider, Base): + + def buildUrl(self, media, quality): + query = tryUrlencode({ + 'q': fireEvent('library.query', media, single = True), + 'm': 'n', + 'max': 400, + 'adv_age': Env.setting('retention', 'nzb'), + 'adv_sort': 'date', + 'adv_col': 'on', + 'adv_nfo': 'on', + 'minsize': quality.get('size_min'), + 'maxsize': quality.get('size_max'), + }) + return query diff --git a/couchpotato/core/media/show/providers/nzb/newznab.py b/couchpotato/core/media/show/providers/nzb/newznab.py new file mode 100644 index 0000000..0c9d49a --- /dev/null +++ b/couchpotato/core/media/show/providers/nzb/newznab.py @@ -0,0 +1,49 @@ +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.event import fireEvent +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media._base.providers.nzb.newznab import Base +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider + +log = CPLog(__name__) + +autoload = 'Newznab' + + +class Newznab(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + def buildUrl(self, media, api_key): + search_title = fireEvent('library.query', media, include_identifier = False, single = True) + identifier = fireEvent('library.identifier', media, single = True) + + query = tryUrlencode({ + 't': 'tvsearch', + 'q': search_title, + 'season': identifier['season'], + 'apikey': api_key, + 'extended': 1 + }) + return query + + +class Episode(EpisodeProvider, Base): + + def buildUrl(self, media, api_key): + search_title = fireEvent('library.query', media, include_identifier = False, single = True) + identifier = fireEvent('library.identifier', media, single = True) + + query = tryUrlencode({ + 't': 'tvsearch', + 'q': search_title, + 'season': identifier['season'], + 'ep': identifier['episode'], + 'apikey': api_key, + 'extended': 1 + }) + return query diff --git a/couchpotato/core/media/show/providers/nzb/nzbclub.py b/couchpotato/core/media/show/providers/nzb/nzbclub.py new file mode 100644 index 0000000..d3fb696 --- /dev/null +++ b/couchpotato/core/media/show/providers/nzb/nzbclub.py @@ -0,0 +1,52 @@ +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.event import fireEvent +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.nzb.nzbclub import Base + +log = CPLog(__name__) + +autoload = 'NZBClub' + + +class NZBClub(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + def buildUrl(self, media): + + q = tryUrlencode({ + 'q': fireEvent('library.query', media, single = True), + }) + + query = tryUrlencode({ + 'ig': 1, + 'rpp': 200, + 'st': 5, + 'sp': 1, + 'ns': 1, + }) + return '%s&%s' % (q, query) + + +class Episode(EpisodeProvider, Base): + + def buildUrl(self, media): + + q = tryUrlencode({ + 'q': fireEvent('library.query', media, single = True), + }) + + query = tryUrlencode({ + 'ig': 1, + 'rpp': 200, + 'st': 5, + 'sp': 1, + 'ns': 1, + }) + return '%s&%s' % (q, query) diff --git a/couchpotato/core/media/show/providers/nzb/nzbindex.py b/couchpotato/core/media/show/providers/nzb/nzbindex.py new file mode 100644 index 0000000..7900df4 --- /dev/null +++ b/couchpotato/core/media/show/providers/nzb/nzbindex.py @@ -0,0 +1,51 @@ +from couchpotato import Env +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.event import fireEvent +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.nzb.nzbindex import Base + +log = CPLog(__name__) + +autoload = 'NzbIndex' + + +class NzbIndex(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + def buildUrl(self, media, quality): + query = tryUrlencode({ + 'q': fireEvent('library.query', media, single = True), + 'age': Env.setting('retention', 'nzb'), + 'sort': 'agedesc', + 'minsize': quality.get('size_min'), + 'maxsize': quality.get('size_max'), + 'rating': 1, + 'max': 250, + 'more': 1, + 'complete': 1, + }) + return query + + +class Episode(EpisodeProvider, Base): + + def buildUrl(self, media, quality): + query = tryUrlencode({ + 'q': fireEvent('library.query', media, single = True), + 'age': Env.setting('retention', 'nzb'), + 'sort': 'agedesc', + 'minsize': quality.get('size_min'), + 'maxsize': quality.get('size_max'), + 'rating': 1, + 'max': 250, + 'more': 1, + 'complete': 1, + }) + return query diff --git a/couchpotato/core/media/show/providers/torrent/__init__.py b/couchpotato/core/media/show/providers/torrent/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/couchpotato/core/media/show/providers/torrent/bithdtv.py b/couchpotato/core/media/show/providers/torrent/bithdtv.py new file mode 100644 index 0000000..6f8488d --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/bithdtv.py @@ -0,0 +1,36 @@ +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.event import fireEvent +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.bithdtv import Base + +log = CPLog(__name__) + +autoload = 'BitHDTV' + + +class BiTHDTV(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + def buildUrl(self, media): + query = tryUrlencode({ + 'search': fireEvent('library.query', media, single = True), + 'cat': 12 # Season cat + }) + return query + + +class Episode(EpisodeProvider, Base): + + def buildUrl(self, media): + query = tryUrlencode({ + 'search': fireEvent('library.query', media, single = True), + 'cat': 10 # Episode cat + }) + return query diff --git a/couchpotato/core/media/show/providers/torrent/bitsoup.py b/couchpotato/core/media/show/providers/torrent/bitsoup.py new file mode 100644 index 0000000..fcaeffd --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/bitsoup.py @@ -0,0 +1,41 @@ +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.event import fireEvent +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.bitsoup import Base + +log = CPLog(__name__) + +autoload = 'Bitsoup' + + +class Bitsoup(MultiProvider): + + def getTypes(self): + return [Movie, Season, Episode] + + +class Season(SeasonProvider, Base): + # For season bundles, bitsoup currently only has one category + def buildUrl(self, media, quality): + query = tryUrlencode({ + 'search': fireEvent('library.query', media, single = True), + 'cat': 45 # TV-Packs Category + }) + return query + + +class Episode(EpisodeProvider, Base): + cat_ids = [ + ([42], ['hdtv_720p', 'webdl_720p', 'webdl_1080p', 'bdrip_1080p', 'bdrip_720p', 'brrip_1080p', 'brrip_720p']), + ([49], ['hdtv_sd', 'webdl_480p']) + ] + cat_backup_id = 0 + + def buildUrl(self, media, quality): + query = tryUrlencode({ + 'search': fireEvent('library.query', media, single = True), + 'cat': self.getCatId(quality['identifier'])[0], + }) + return query diff --git a/couchpotato/core/media/show/providers/torrent/iptorrents.py b/couchpotato/core/media/show/providers/torrent/iptorrents.py new file mode 100644 index 0000000..8183675 --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/iptorrents.py @@ -0,0 +1,37 @@ +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.iptorrents import Base + +log = CPLog(__name__) + +autoload = 'IPTorrents' + + +class IPTorrents(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + # TODO come back to this later, a better quality system needs to be created + cat_ids = [ + ([65], [ + 'bluray_1080p', 'bluray_720p', + 'bdrip_1080p', 'bdrip_720p', + 'brrip_1080p', 'brrip_720p', + 'webdl_1080p', 'webdl_720p', 'webdl_480p', + 'hdtv_720p', 'hdtv_sd' + ]), + ] + + +class Episode(EpisodeProvider, Base): + + # TODO come back to this later, a better quality system needs to be created + cat_ids = [ + ([5], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ([4, 78, 79], ['hdtv_sd']) + ] diff --git a/couchpotato/core/media/show/providers/torrent/publichd.py b/couchpotato/core/media/show/providers/torrent/publichd.py new file mode 100644 index 0000000..5865070 --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/publichd.py @@ -0,0 +1,27 @@ +from couchpotato.core.logger import CPLog +from couchpotato.core.event import fireEvent +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.publichd import Base + +log = CPLog(__name__) + +autoload = 'PublicHD' + + +class PublicHD(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + def buildUrl(self, media): + return fireEvent('library.query', media, single = True) + + +class Episode(EpisodeProvider, Base): + + def buildUrl(self, media): + return fireEvent('library.query', media, single = True) diff --git a/couchpotato/core/media/show/providers/torrent/sceneaccess.py b/couchpotato/core/media/show/providers/torrent/sceneaccess.py new file mode 100644 index 0000000..2fa266d --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/sceneaccess.py @@ -0,0 +1,60 @@ +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.event import fireEvent +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.sceneaccess import Base + + +log = CPLog(__name__) + +autoload = 'SceneAccess' + + +class SceneAccess(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + cat_ids = [ + ([26], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ] + + def buildUrl(self, media, quality): + url = self.urls['archive'] % ( + self.getCatId(quality['identifier'])[0], + self.getCatId(quality['identifier'])[0] + ) + + arguments = tryUrlencode({ + 'search': fireEvent('library.query', media, single = True), + 'method': 3, + }) + query = "%s&%s" % (url, arguments) + + return query + + +class Episode(EpisodeProvider, Base): + + cat_ids = [ + ([27], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ([17, 11], ['hdtv_sd']) + ] + + def buildUrl(self, media, quality): + url = self.urls['search'] % ( + self.getCatId(quality['identifier'])[0], + self.getCatId(quality['identifier'])[0] + ) + + arguments = tryUrlencode({ + 'search': fireEvent('library.query', media, single = True), + 'method': 3, + }) + query = "%s&%s" % (url, arguments) + + return query diff --git a/couchpotato/core/media/show/providers/torrent/thepiratebay.py b/couchpotato/core/media/show/providers/torrent/thepiratebay.py new file mode 100644 index 0000000..b648fe0 --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/thepiratebay.py @@ -0,0 +1,46 @@ +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.event import fireEvent +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.thepiratebay import Base + +log = CPLog(__name__) + +autoload = 'ThePirateBay' + + +class ThePirateBay(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + cat_ids = [ + ([208], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ([205], ['hdtv_sd']) + ] + + def buildUrl(self, media, page, cats): + return ( + tryUrlencode('"%s"' % fireEvent('library.query', media, single = True)), + page, + ','.join(str(x) for x in cats) + ) + + +class Episode(EpisodeProvider, Base): + + cat_ids = [ + ([208], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ([205], ['hdtv_sd']) + ] + + def buildUrl(self, media, page, cats): + return ( + tryUrlencode('"%s"' % fireEvent('library.query', media, single = True)), + page, + ','.join(str(x) for x in cats) + ) diff --git a/couchpotato/core/media/show/providers/torrent/torrentday.py b/couchpotato/core/media/show/providers/torrent/torrentday.py new file mode 100644 index 0000000..68b4c01 --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/torrentday.py @@ -0,0 +1,34 @@ +from couchpotato.core.logger import CPLog +from couchpotato.core.event import fireEvent +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.torrentday import Base + +log = CPLog(__name__) + +autoload = 'TorrentDay' + + +class TorrentDay(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + cat_ids = [ + ([14], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ] + def buildUrl(self, media): + return fireEvent('library.query', media, single = True) + + +class Episode(EpisodeProvider, Base): + cat_ids = [ + ([7], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ([2], [24], [26], ['hdtv_sd']) + ] + def buildUrl(self, media): + return fireEvent('library.query', media, single = True) + diff --git a/couchpotato/core/media/show/providers/torrent/torrentleech.py b/couchpotato/core/media/show/providers/torrent/torrentleech.py new file mode 100644 index 0000000..312ec19 --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/torrentleech.py @@ -0,0 +1,42 @@ +from couchpotato import fireEvent +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.torrentleech import Base + +log = CPLog(__name__) + +autoload = 'TorrentLeech' + + +class TorrentLeech(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + cat_ids = [ + ([27], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ] + + def buildUrl(self, media, quality): + return ( + tryUrlencode(fireEvent('library.query', media, single = True)), + self.getCatId(quality['identifier'])[0] + ) + +class Episode(EpisodeProvider, Base): + + cat_ids = [ + ([32], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']), + ([26], ['hdtv_sd']) + ] + + def buildUrl(self, media, quality): + return ( + tryUrlencode(fireEvent('library.query', media, single = True)), + self.getCatId(quality['identifier'])[0] + ) diff --git a/couchpotato/core/media/show/providers/torrent/torrentpotato.py b/couchpotato/core/media/show/providers/torrent/torrentpotato.py new file mode 100644 index 0000000..35028a2 --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/torrentpotato.py @@ -0,0 +1,38 @@ +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.event import fireEvent +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.torrentpotato import Base + +log = CPLog(__name__) + +autoload = 'TorrentPotato' + + +class TorrentPotato(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + + def buildUrl(self, media, host): + arguments = tryUrlencode({ + 'user': host['name'], + 'passkey': host['pass_key'], + 'search': fireEvent('library.query', media, single = True) + }) + return '%s?%s' % (host['host'], arguments) + + +class Episode(EpisodeProvider, Base): + + def buildUrl(self, media, host): + arguments = tryUrlencode({ + 'user': host['name'], + 'passkey': host['pass_key'], + 'search': fireEvent('library.query', media, single = True) + }) + return '%s?%s' % (host['host'], arguments) diff --git a/couchpotato/core/media/show/providers/torrent/torrentshack.py b/couchpotato/core/media/show/providers/torrent/torrentshack.py new file mode 100644 index 0000000..8d11759 --- /dev/null +++ b/couchpotato/core/media/show/providers/torrent/torrentshack.py @@ -0,0 +1,52 @@ +from couchpotato.core.event import fireEvent +from couchpotato.core.helpers.encoding import tryUrlencode +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.providers.base import MultiProvider +from couchpotato.core.media.show.providers.base import SeasonProvider, EpisodeProvider +from couchpotato.core.media._base.providers.torrent.torrentshack import Base + +log = CPLog(__name__) + +autoload = 'TorrentShack' + + +class TorrentShack(MultiProvider): + + def getTypes(self): + return [Season, Episode] + + +class Season(SeasonProvider, Base): + # TorrentShack tv season search categories + # TV-SD Pack - 980 + # TV-HD Pack - 981 + # Full Blu-ray - 970 + cat_ids = [ + ([980], ['hdtv_sd']), + ([981], ['hdtv_720p', 'webdl_720p', 'webdl_1080p', 'bdrip_1080p', 'bdrip_720p', 'brrip_1080p', 'brrip_720p']), + ([970], ['bluray_1080p', 'bluray_720p']), + ] + cat_backup_id = 980 + + def buildUrl(self, media, quality): + query = (tryUrlencode(fireEvent('library.query', media, single = True)), + self.getCatId(quality['identifier'])[0], + self.getSceneOnly()) + return query + +class Episode(EpisodeProvider, Base): + # TorrentShack tv episode search categories + # TV/x264-HD - 600 + # TV/x264-SD - 620 + # TV/DVDrip - 700 + cat_ids = [ + ([600], ['hdtv_720p', 'webdl_720p', 'webdl_1080p', 'bdrip_1080p', 'bdrip_720p', 'brrip_1080p', 'brrip_720p']), + ([620], ['hdtv_sd']) + ] + cat_backup_id = 620 + + def buildUrl(self, media, quality): + query = (tryUrlencode(fireEvent('library.query', media, single = True)), + self.getCatId(quality['identifier'])[0], + self.getSceneOnly()) + return query diff --git a/couchpotato/core/media/show/searcher.py b/couchpotato/core/media/show/searcher.py new file mode 100644 index 0000000..bd10a5e --- /dev/null +++ b/couchpotato/core/media/show/searcher.py @@ -0,0 +1,190 @@ +from couchpotato import Env, get_session +from couchpotato.core.event import addEvent, fireEvent +from couchpotato.core.helpers.variable import getTitle, toIterable +from couchpotato.core.logger import CPLog +from couchpotato.core.media._base.searcher.base import SearcherBase +from couchpotato.core.media._base.searcher.main import SearchSetupError +from couchpotato.core.media.show import ShowTypeBase +from couchpotato.core.media.show._base import ShowBase +from qcond import QueryCondenser + +log = CPLog(__name__) + +autoload = 'ShowSearcher' + + +class ShowSearcher(SearcherBase, ShowTypeBase): + + type = ['show', 'season', 'episode'] + + in_progress = False + + def __init__(self): + super(ShowSearcher, self).__init__() + + self.query_condenser = QueryCondenser() + + for type in toIterable(self.type): + addEvent('%s.searcher.single' % type, self.single) + + addEvent('searcher.correct_release', self.correctRelease) + + def single(self, media, search_protocols = None, manual = False): + show, season, episode = self.getLibraries(media['library']) + + db = get_session() + + if media['type'] == 'show': + for library in season: + # TODO ideally we shouldn't need to fetch the media for each season library here + m = db.query(Media).filter_by(library_id = library['library_id']).first() + + fireEvent('season.searcher.single', m.to_dict(ShowBase.search_dict)) + + return + + # Find out search type + try: + if not search_protocols: + search_protocols = fireEvent('searcher.protocols', single = True) + except SearchSetupError: + return + + done_status, available_status, ignored_status, failed_status = fireEvent('status.get', ['done', 'available', 'ignored', 'failed'], single = True) + + if not media['profile'] or media['status_id'] == done_status.get('id'): + log.debug('Episode doesn\'t have a profile or already done, assuming in manage tab.') + return + + #pre_releases = fireEvent('quality.pre_releases', single = True) + + found_releases = [] + too_early_to_search = [] + + default_title = fireEvent('library.query', media['library'], condense = False, single=True) + if not default_title: + log.error('No proper info found for episode, removing it from library to cause it from having more issues.') + #fireEvent('episode.delete', episode['id'], single = True) + return + + if not show or not season: + log.error('Unable to find show or season library in database, missing required data for searching') + return + + fireEvent('notify.frontend', type = 'show.searcher.started.%s' % media['id'], data = True, message = 'Searching for "%s"' % default_title) + + ret = False + has_better_quality = None + + for quality_type in media['profile']['types']: + # TODO check air date? + #if not self.conf('always_search') and not self.couldBeReleased(quality_type['quality']['identifier'] in pre_releases, release_dates, movie['library']['year']): + # too_early_to_search.append(quality_type['quality']['identifier']) + # continue + + has_better_quality = 0 + + # See if better quality is available + for release in media['releases']: + if release['quality']['order'] <= quality_type['quality']['order'] and release['status_id'] not in [available_status.get('id'), ignored_status.get('id'), failed_status.get('id')]: + has_better_quality += 1 + + # Don't search for quality lower then already available. + if has_better_quality is 0: + + log.info('Search for %s S%02d%s in %s', ( + getTitle(show), + season['season_number'], + "E%02d" % episode['episode_number'] if episode and len(episode) == 1 else "", + quality_type['quality']['label']) + ) + quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True) + + results = fireEvent('searcher.search', search_protocols, media, quality, single = True) + if len(results) == 0: + log.debug('Nothing found for %s in %s', (default_title, quality_type['quality']['label'])) + + # Check if movie isn't deleted while searching + if not db.query(Media).filter_by(id = media.get('id')).first(): + break + + # Add them to this movie releases list + found_releases += fireEvent('release.create_from_search', results, media, quality_type, single = True) + + # Try find a valid result and download it + if fireEvent('release.try_download_result', results, media, quality_type, manual, single = True): + ret = True + + # Remove releases that aren't found anymore + for release in media.get('releases', []): + if release.get('status_id') == available_status.get('id') and release.get('identifier') not in found_releases: + fireEvent('release.delete', release.get('id'), single = True) + else: + log.info('Better quality (%s) already available or snatched for %s', (quality_type['quality']['label'], default_title)) + fireEvent('media.restatus', media['id']) + break + + # Break if CP wants to shut down + if self.shuttingDown() or ret: + break + + if len(too_early_to_search) > 0: + log.info2('Too early to search for %s, %s', (too_early_to_search, default_title)) + elif media['type'] == 'season' and not ret and has_better_quality is 0: + # If nothing was found, start searching for episodes individually + log.info('No season pack found, starting individual episode search') + + for library in episode: + # TODO ideally we shouldn't need to fetch the media for each episode library here + m = db.query(Media).filter_by(library_id = library['library_id']).first() + + fireEvent('episode.searcher.single', m.to_dict(ShowBase.search_dict)) + + + fireEvent('notify.frontend', type = 'show.searcher.ended.%s' % media['id'], data = True) + + return ret + + def correctRelease(self, release = None, media = None, quality = None, **kwargs): + + if media.get('type') not in ['season', 'episode']: return + + retention = Env.setting('retention', section = 'nzb') + + if release.get('seeders') is None and 0 < retention < release.get('age', 0): + log.info2('Wrong: Outside retention, age is %s, needs %s or lower: %s', (release['age'], retention, release['name'])) + return False + + # Check for required and ignored words + if not fireEvent('searcher.correct_words', release['name'], media, single = True): + return False + + # TODO Matching is quite costly, maybe we should be caching release matches somehow? (also look at caper optimizations) + match = fireEvent('matcher.match', release, media, quality, single = True) + if match: + return match.weight + + return False + + def getLibraries(self, library): + if 'related_libraries' not in library: + log.warning("'related_libraries' missing from media library, unable to continue searching") + return None, None, None + + libraries = library['related_libraries'] + + # Show always collapses as there can never be any multiples + show = libraries.get('show', []) + show = show[0] if len(show) else None + + # Season collapses if the subject is a season or episode + season = libraries.get('season', []) + if library['type'] in ['season', 'episode']: + season = season[0] if len(season) else None + + # Episode collapses if the subject is a episode + episode = libraries.get('episode', []) + if library['type'] == 'episode': + episode = episode[0] if len(episode) else None + + return show, season, episode diff --git a/libs/qcond/__init__.py b/libs/qcond/__init__.py new file mode 100644 index 0000000..be64e7b --- /dev/null +++ b/libs/qcond/__init__.py @@ -0,0 +1,42 @@ +# Copyright 2013 Dean Gardiner +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from qcond.transformers.merge import MergeTransformer +from qcond.transformers.slice import SliceTransformer +from qcond.transformers.strip_common import StripCommonTransformer + + +__version_info__ = ('0', '1', '0') +__version_branch__ = 'master' + +__version__ = "%s%s" % ( + '.'.join(__version_info__), + '-' + __version_branch__ if __version_branch__ else '' +) + + +class QueryCondenser(object): + def __init__(self): + self.transformers = [ + MergeTransformer(), + SliceTransformer(), + StripCommonTransformer() + ] + + def distinct(self, titles): + for transformer in self.transformers: + titles = transformer.run(titles) + + return titles diff --git a/libs/qcond/compat.py b/libs/qcond/compat.py new file mode 100644 index 0000000..f3f0925 --- /dev/null +++ b/libs/qcond/compat.py @@ -0,0 +1,23 @@ +# Copyright 2013 Dean Gardiner +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import sys + +PY3 = sys.version_info[0] == 3 + +if PY3: + xrange = range +else: + xrange = xrange diff --git a/libs/qcond/helpers.py b/libs/qcond/helpers.py new file mode 100644 index 0000000..a341b6e --- /dev/null +++ b/libs/qcond/helpers.py @@ -0,0 +1,84 @@ +# Copyright 2013 Dean Gardiner +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from difflib import SequenceMatcher +import re +import sys +from logr import Logr +from qcond.compat import xrange + + +PY3 = sys.version_info[0] == 3 + + +def simplify(s): + s = s.lower() + s = re.sub(r"(\w)'(\w)", r"\1\2", s) + return s + + +def strip(s): + return re.sub(r"^(\W*)(.*?)(\W*)$", r"\2", s) + + +def create_matcher(a, b, swap_longest = True, case_sensitive = False): + # Ensure longest string is a + if swap_longest and len(b) > len(a): + a_ = a + a = b + b = a_ + + if not case_sensitive: + a = a.upper() + b = b.upper() + + return SequenceMatcher(None, a, b) + + +def first(function_or_none, sequence): + if PY3: + for item in filter(function_or_none, sequence): + return item + else: + result = filter(function_or_none, sequence) + if len(result): + return result[0] + + return None + +def sorted_append(sequence, item, func): + if not len(sequence): + sequence.insert(0, item) + return + + x = 0 + for x in xrange(len(sequence)): + if func(sequence[x]): + sequence.insert(x, item) + return + + sequence.append(item) + +def itemsMatch(L1, L2): + return len(L1) == len(L2) and sorted(L1) == sorted(L2) + +def distinct(sequence): + result = [] + + for item in sequence: + if item not in result: + result.append(item) + + return result \ No newline at end of file diff --git a/libs/qcond/transformers/__init__.py b/libs/qcond/transformers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libs/qcond/transformers/base.py b/libs/qcond/transformers/base.py new file mode 100644 index 0000000..7054729 --- /dev/null +++ b/libs/qcond/transformers/base.py @@ -0,0 +1,21 @@ +# Copyright 2013 Dean Gardiner +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +class Transformer(object): + def __init__(self): + pass + + def run(self, titles): + raise NotImplementedError() diff --git a/libs/qcond/transformers/merge.py b/libs/qcond/transformers/merge.py new file mode 100644 index 0000000..f98dea3 --- /dev/null +++ b/libs/qcond/transformers/merge.py @@ -0,0 +1,241 @@ +# Copyright 2013 Dean Gardiner +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from operator import itemgetter +from logr import Logr +from qcond.helpers import simplify, strip, first, sorted_append, distinct +from qcond.transformers.base import Transformer +from qcond.compat import xrange + + +class MergeTransformer(Transformer): + def __init__(self): + super(MergeTransformer, self).__init__() + + def run(self, titles): + titles = distinct([simplify(title) for title in titles]) + + Logr.info(str(titles)) + + Logr.debug("------------------------------------------------------------") + + root, tails = self.parse(titles) + + Logr.debug("--------------------------PARSE-----------------------------") + + for node in root: + print_tree(node) + + Logr.debug("--------------------------MERGE-----------------------------") + + self.merge(root) + + Logr.debug("--------------------------FINAL-----------------------------") + + for node in root: + print_tree(node) + + Logr.debug("--------------------------RESULT-----------------------------") + + scores = {} + results = [] + + for tail in tails: + score, value, original_value = tail.full_value() + + if value in scores: + scores[value] += score + else: + results.append((value, original_value)) + scores[value] = score + + Logr.debug("%s %s %s", score, value, original_value) + + sorted_results = sorted(results, key=lambda item: (scores[item[0]], item[1]), reverse = True) + + return [result[0] for result in sorted_results] + + def parse(self, titles): + root = [] + tails = [] + + for title in titles: + Logr.debug(title) + + cur = None + words = title.split(' ') + + for wx in xrange(len(words)): + word = strip(words[wx]) + + if cur is None: + cur = find_node(root, word) + + if cur is None: + cur = DNode(word, None, num_children=len(words) - wx, original_value=title) + root.append(cur) + else: + parent = cur + parent.weight += 1 + + cur = find_node(parent.right, word) + + if cur is None: + Logr.debug("%s %d", word, len(words) - wx) + cur = DNode(word, parent, num_children=len(words) - wx) + sorted_append(parent.right, cur, lambda a: a.num_children < cur.num_children) + else: + cur.weight += 1 + + tails.append(cur) + + return root, tails + + def merge(self, root): + for x in range(len(root)): + Logr.debug(root[x]) + root[x].right = self._merge(root[x].right) + Logr.debug('=================================================================') + + return root + + def get_nodes_right(self, value): + if type(value) is not list: + value = [value] + + nodes = [] + + for node in value: + nodes.append(node) + + for child in self.get_nodes_right(node.right): + nodes.append(child) + + return nodes + + def destroy_nodes_right(self, value): + nodes = self.get_nodes_right(value) + + for node in nodes: + node.value = None + node.dead = True + + def _merge(self, nodes, depth = 0): + Logr.debug(str('\t' * depth) + str(nodes)) + + if not len(nodes): + return [] + + top = nodes[0] + + # Merge into top + for x in range(len(nodes)): + # Merge extra results into top + if x > 0: + top.value = None + top.weight += nodes[x].weight + self.destroy_nodes_right(top.right) + + if len(nodes[x].right): + top.join_right(nodes[x].right) + + Logr.debug("= %s joined %s", nodes[x], top) + + nodes[x].dead = True + + nodes = [n for n in nodes if not n.dead] + + # Traverse further + for node in nodes: + if len(node.right): + node.right = self._merge(node.right, depth + 1) + + return nodes + + +def print_tree(node, depth = 0): + Logr.debug(str('\t' * depth) + str(node)) + + if len(node.right): + for child in node.right: + print_tree(child, depth + 1) + else: + Logr.debug(node.full_value()[1]) + + +def find_node(node_list, value): + # Try find adjacent node match + for node in node_list: + if node.value == value: + return node + + return None + + +class DNode(object): + def __init__(self, value, parent, right=None, weight=1, num_children=None, original_value=None): + self.value = value + + self.parent = parent + + if right is None: + right = [] + self.right = right + + self.weight = weight + + self.original_value = original_value + self.num_children = num_children + + self.dead = False + + def join_right(self, nodes): + for node in nodes: + duplicate = first(lambda x: x.value == node.value, self.right) + + if duplicate: + duplicate.weight += node.weight + duplicate.join_right(node.right) + else: + node.parent = self + self.right.append(node) + + def full_value(self): + words = [] + total_score = 0 + + cur = self + root = None + + while cur is not None: + if cur.value and not cur.dead: + words.insert(0, cur.value) + total_score += cur.weight + + if cur.parent is None: + root = cur + cur = cur.parent + + return float(total_score) / len(words), ' '.join(words), root.original_value if root else None + + def __repr__(self): + return '<%s value:"%s", weight: %s, num_children: %s%s%s>' % ( + 'DNode', + self.value, + self.weight, + self.num_children, + (', original_value: %s' % self.original_value) if self.original_value else '', + ' REMOVING' if self.dead else '' + ) diff --git a/libs/qcond/transformers/slice.py b/libs/qcond/transformers/slice.py new file mode 100644 index 0000000..864f673 --- /dev/null +++ b/libs/qcond/transformers/slice.py @@ -0,0 +1,280 @@ +# Copyright 2013 Dean Gardiner +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from logr import Logr +from qcond.helpers import create_matcher +from qcond.transformers.base import Transformer + + +class SliceTransformer(Transformer): + def __init__(self): + super(SliceTransformer, self).__init__() + + def run(self, titles): + nodes = [] + + # Create a node for each title + for title in titles: + nodes.append(SimNode(title)) + + # Calculate similarities between nodes + for node in nodes: + calculate_sim_links(node, [n for n in nodes if n != node]) + + kill_nodes_above(nodes, 0.90) + + Logr.debug('---------------------------------------------------------------------') + + print_link_tree(nodes) + Logr.debug('%s %s', len(nodes), [n.value for n in nodes]) + + Logr.debug('---------------------------------------------------------------------') + + kill_trailing_nodes(nodes) + + Logr.debug('---------------------------------------------------------------------') + + # Sort remaining nodes by 'num_merges' + nodes = sorted(nodes, key=lambda n: n.num_merges, reverse=True) + + print_link_tree(nodes) + + Logr.debug('---------------------------------------------------------------------') + + Logr.debug('%s %s', len(nodes), [n.value for n in nodes]) + + return [n.value for n in nodes] + + +class SimLink(object): + def __init__(self, similarity, opcodes, stats): + self.similarity = similarity + self.opcodes = opcodes + self.stats = stats + + +class SimNode(object): + def __init__(self, value): + self.value = value + + self.dead = False + self.num_merges = 0 + + self.links = {} # {: } + + +def kill_nodes(nodes, killed_nodes): + # Remove killed nodes from root list + for node in killed_nodes: + if node in nodes: + nodes.remove(node) + + # Remove killed nodes from links + for killed_node in killed_nodes: + for node in nodes: + if killed_node in node.links: + node.links.pop(killed_node) + + +def kill_nodes_above(nodes, above_sim): + killed_nodes = [] + + for node in nodes: + if node.dead: + continue + + Logr.debug(node.value) + + for link_node, link in node.links.items(): + if link_node.dead: + continue + + Logr.debug('\t%0.2f -- %s', link.similarity, link_node.value) + + if link.similarity >= above_sim: + if len(link_node.value) > len(node.value): + Logr.debug('\t\tvery similar, killed this node') + link_node.dead = True + node.num_merges += 1 + killed_nodes.append(link_node) + else: + Logr.debug('\t\tvery similar, killed owner') + node.dead = True + link_node.num_merges += 1 + killed_nodes.append(node) + + kill_nodes(nodes, killed_nodes) + + +def print_link_tree(nodes): + for node in nodes: + Logr.debug(node.value) + Logr.debug('\tnum_merges: %s', node.num_merges) + + if len(node.links): + Logr.debug('\t========== LINKS ==========') + for link_node, link in node.links.items(): + Logr.debug('\t%0.2f -- %s', link.similarity, link_node.value) + + Logr.debug('\t---------------------------') + + +def kill_trailing_nodes(nodes): + killed_nodes = [] + + for node in nodes: + if node.dead: + continue + + Logr.debug(node.value) + + for link_node, link in node.links.items(): + if link_node.dead: + continue + + is_valid = link.stats.get('valid', False) + + has_deletions = False + has_insertions = False + has_replacements = False + + for opcode in link.opcodes: + if opcode[0] == 'delete': + has_deletions = True + if opcode[0] == 'insert': + has_insertions = True + if opcode[0] == 'replace': + has_replacements = True + + equal_perc = link.stats.get('equal', 0) / float(len(node.value)) + insert_perc = link.stats.get('insert', 0) / float(len(node.value)) + + Logr.debug('\t({0:<24}) [{1:02d}:{2:02d} = {3:02d} {4:3.0f}% {5:3.0f}%] -- {6:<45}'.format( + 'd:%s, i:%s, r:%s' % (has_deletions, has_insertions, has_replacements), + len(node.value), len(link_node.value), link.stats.get('equal', 0), + equal_perc * 100, insert_perc * 100, + '"{0}"'.format(link_node.value) + )) + + Logr.debug('\t\t%s', link.stats) + + kill = all([ + is_valid, + equal_perc >= 0.5, + insert_perc < 2, + has_insertions, + not has_deletions, + not has_replacements + ]) + + if kill: + Logr.debug('\t\tkilled this node') + + link_node.dead = True + node.num_merges += 1 + killed_nodes.append(link_node) + + kill_nodes(nodes, killed_nodes) + +stats_print_format = "\t{0:<8} ({1:2d}:{2:2d}) ({3:2d}:{4:2d})" + + +def get_index_values(iterable, a, b): + return ( + iterable[a] if a else None, + iterable[b] if b else None + ) + + +def get_indices(iterable, a, b): + return ( + a if 0 < a < len(iterable) else None, + b if 0 < b < len(iterable) else None + ) + + +def get_opcode_stats(for_node, node, opcodes): + stats = {} + + for tag, i1, i2, j1, j2 in opcodes: + Logr.debug(stats_print_format.format( + tag, i1, i2, j1, j2 + )) + + if tag in ['insert', 'delete']: + ax = None, None + bx = None, None + + if tag == 'insert': + ax = get_indices(for_node.value, i1 - 1, i1) + bx = get_indices(node.value, j1, j2 - 1) + + if tag == 'delete': + ax = get_indices(for_node.value, j1 - 1, j1) + bx = get_indices(node.value, i1, i2 - 1) + + av = get_index_values(for_node.value, *ax) + bv = get_index_values(node.value, *bx) + + Logr.debug( + '\t\t%s %s [%s><%s] <---> %s %s [%s><%s]', + ax, av, av[0], av[1], + bx, bv, bv[0], bv[1] + ) + + head_valid = av[0] in [None, ' '] or bv[0] in [None, ' '] + tail_valid = av[1] in [None, ' '] or bv[1] in [None, ' '] + valid = head_valid and tail_valid + + if 'valid' not in stats or (stats['valid'] and not valid): + stats['valid'] = valid + + Logr.debug('\t\t' + ('VALID' if valid else 'INVALID')) + + if tag not in stats: + stats[tag] = 0 + + stats[tag] += (i2 - i1) or (j2 - j1) + + return stats + + +def calculate_sim_links(for_node, other_nodes): + for node in other_nodes: + if node in for_node.links: + continue + + Logr.debug('calculating similarity between "%s" and "%s"', for_node.value, node.value) + + # Get similarity + similarity_matcher = create_matcher(for_node.value, node.value) + similarity = similarity_matcher.quick_ratio() + + # Get for_node -> node opcodes + a_opcodes_matcher = create_matcher(for_node.value, node.value, swap_longest = False) + a_opcodes = a_opcodes_matcher.get_opcodes() + a_stats = get_opcode_stats(for_node, node, a_opcodes) + + Logr.debug('-' * 100) + + # Get node -> for_node opcodes + b_opcodes_matcher = create_matcher(node.value, for_node.value, swap_longest = False) + b_opcodes = b_opcodes_matcher.get_opcodes() + b_stats = get_opcode_stats(for_node, node, b_opcodes) + + for_node.links[node] = SimLink(similarity, a_opcodes, a_stats) + node.links[for_node] = SimLink(similarity, b_opcodes, b_stats) + + #raw_input('Press ENTER to continue') diff --git a/libs/qcond/transformers/strip_common.py b/libs/qcond/transformers/strip_common.py new file mode 100644 index 0000000..47b8401 --- /dev/null +++ b/libs/qcond/transformers/strip_common.py @@ -0,0 +1,26 @@ +# Copyright 2013 Dean Gardiner +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from qcond.transformers.base import Transformer + + +COMMON_WORDS = [ + 'the' +] + + +class StripCommonTransformer(Transformer): + def run(self, titles): + return [title for title in titles if title.lower() not in COMMON_WORDS] diff --git a/libs/tvdb_api/.gitignore b/libs/tvdb_api/.gitignore new file mode 100644 index 0000000..e42c383 --- /dev/null +++ b/libs/tvdb_api/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +*.pyc +*.egg-info/* +dist/*.tar.gz diff --git a/libs/tvdb_api/.travis.yml b/libs/tvdb_api/.travis.yml new file mode 100644 index 0000000..36f5e51 --- /dev/null +++ b/libs/tvdb_api/.travis.yml @@ -0,0 +1,9 @@ +language: python +python: + - 2.5 + - 2.6 + - 2.7 + +install: pip install nose + +script: nosetests diff --git a/libs/tvdb_api/MANIFEST.in b/libs/tvdb_api/MANIFEST.in new file mode 100644 index 0000000..bd227aa --- /dev/null +++ b/libs/tvdb_api/MANIFEST.in @@ -0,0 +1,4 @@ +include UNLICENSE +include readme.md +include tests/*.py +include Rakefile diff --git a/libs/tvdb_api/Rakefile b/libs/tvdb_api/Rakefile new file mode 100644 index 0000000..561deb7 --- /dev/null +++ b/libs/tvdb_api/Rakefile @@ -0,0 +1,103 @@ +require 'fileutils' + +task :default => [:clean] + +task :clean do + [".", "tests"].each do |cd| + puts "Cleaning directory #{cd}" + Dir.new(cd).each do |t| + if t =~ /.*\.pyc$/ + puts "Removing #{File.join(cd, t)}" + File.delete(File.join(cd, t)) + end + end + end +end + +desc "Upversion files" +task :upversion do + puts "Upversioning" + + Dir.glob("*.py").each do |filename| + f = File.new(filename, File::RDWR) + contents = f.read() + + contents.gsub!(/__version__ = ".+?"/){|m| + cur_version = m.scan(/\d+\.\d+/)[0].to_f + new_version = cur_version + 0.1 + + puts "Current version: #{cur_version}" + puts "New version: #{new_version}" + + new_line = "__version__ = \"#{new_version}\"" + + puts "Old line: #{m}" + puts "New line: #{new_line}" + + m = new_line + } + + puts contents[0] + + f.truncate(0) # empty the existing file + f.seek(0) + f.write(contents.to_s) # write modified file + f.close() + end +end + +desc "Upload current version to PyPi" +task :topypi => :test do + cur_file = File.open("tvdb_api.py").read() + tvdb_api_version = cur_file.scan(/__version__ = "(.*)"/) + tvdb_api_version = tvdb_api_version[0][0].to_f + + puts "Build sdist and send tvdb_api v#{tvdb_api_version} to PyPi?" + if $stdin.gets.chomp == "y" + puts "Sending source-dist (sdist) to PyPi" + + if system("python setup.py sdist register upload") + puts "tvdb_api uploaded!" + end + + else + puts "Cancelled" + end +end + +desc "Profile by running unittests" +task :profile do + cd "tests" + puts "Profiling.." + `python -m cProfile -o prof_runtest.prof runtests.py` + puts "Converting prof to dot" + `python gprof2dot.py -o prof_runtest.dot -f pstats prof_runtest.prof` + puts "Generating graph" + `~/Applications/dev/graphviz.app/Contents/macOS/dot -Tpng -o profile.png prof_runtest.dot -Gbgcolor=black` + puts "Cleanup" + rm "prof_runtest.dot" + rm "prof_runtest.prof" +end + +task :test do + puts "Nosetest'ing" + if not system("nosetests -v --with-doctest") + raise "Test failed!" + end + + puts "Doctesting *.py (excluding setup.py)" + Dir.glob("*.py").select{|e| ! e.match(/setup.py/)}.each do |filename| + if filename =~ /^setup\.py/ + skip + end + puts "Doctesting #{filename}" + if not system("python", "-m", "doctest", filename) + raise "Failed doctest" + end + end + + puts "Doctesting readme.md" + if not system("python", "-m", "doctest", "readme.md") + raise "Doctest" + end +end diff --git a/libs/tvdb_api/UNLICENSE b/libs/tvdb_api/UNLICENSE new file mode 100644 index 0000000..c4205d4 --- /dev/null +++ b/libs/tvdb_api/UNLICENSE @@ -0,0 +1,26 @@ +Copyright 2011-2012 Ben Dickson (dbr) + +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to diff --git a/libs/tvdb_api/__init__.py b/libs/tvdb_api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/libs/tvdb_api/readme.md b/libs/tvdb_api/readme.md new file mode 100644 index 0000000..a34726e --- /dev/null +++ b/libs/tvdb_api/readme.md @@ -0,0 +1,109 @@ +# `tvdb_api` + +`tvdb_api` is an easy to use interface to [thetvdb.com][tvdb] + +`tvnamer` has moved to a separate repository: [github.com/dbr/tvnamer][tvnamer] - it is a utility which uses `tvdb_api` to rename files from `some.show.s01e03.blah.abc.avi` to `Some Show - [01x03] - The Episode Name.avi` (which works by getting the episode name from `tvdb_api`) + +[![Build Status](https://secure.travis-ci.org/dbr/tvdb_api.png?branch=master)](http://travis-ci.org/dbr/tvdb_api) + +## To install + +You can easily install `tvdb_api` via `easy_install` + + easy_install tvdb_api + +You may need to use sudo, depending on your setup: + + sudo easy_install tvdb_api + +The [`tvnamer`][tvnamer] command-line tool can also be installed via `easy_install`, this installs `tvdb_api` as a dependancy: + + easy_install tvnamer + + +## Basic usage + + import tvdb_api + t = tvdb_api.Tvdb() + episode = t['My Name Is Earl'][1][3] # get season 1, episode 3 of show + print episode['episodename'] # Print episode name + +## Advanced usage + +Most of the documentation is in docstrings. The examples are tested (using doctest) so will always be up to date and working. + +The docstring for `Tvdb.__init__` lists all initialisation arguments, including support for non-English searches, custom "Select Series" interfaces and enabling the retrieval of banners and extended actor information. You can also override the default API key using `apikey`, recommended if you're using `tvdb_api` in a larger script or application + +### Exceptions + +There are several exceptions you may catch, these can be imported from `tvdb_api`: + +- `tvdb_error` - this is raised when there is an error communicating with [thetvdb.com][tvdb] (a network error most commonly) +- `tvdb_userabort` - raised when a user aborts the Select Series dialog (by `ctrl+c`, or entering `q`) +- `tvdb_shownotfound` - raised when `t['show name']` cannot find anything +- `tvdb_seasonnotfound` - raised when the requested series (`t['show name][99]`) does not exist +- `tvdb_episodenotfound` - raised when the requested episode (`t['show name][1][99]`) does not exist. +- `tvdb_attributenotfound` - raised when the requested attribute is not found (`t['show name']['an attribute']`, `t['show name'][1]['an attribute']`, or ``t['show name'][1][1]['an attribute']``) + +### Series data + +All data exposed by [thetvdb.com][tvdb] is accessible via the `Show` class. A Show is retrieved by doing.. + + >>> import tvdb_api + >>> t = tvdb_api.Tvdb() + >>> show = t['scrubs'] + >>> type(show) + + +For example, to find out what network Scrubs is aired: + + >>> t['scrubs']['network'] + u'ABC' + +The data is stored in an attribute named `data`, within the Show instance: + + >>> t['scrubs'].data.keys() + ['networkid', 'rating', 'airs_dayofweek', 'contentrating', 'seriesname', 'id', 'airs_time', 'network', 'fanart', 'lastupdated', 'actors', 'ratingcount', 'status', 'added', 'poster', 'imdb_id', 'genre', 'banner', 'seriesid', 'language', 'zap2it_id', 'addedby', 'firstaired', 'runtime', 'overview'] + +Although each element is also accessible via `t['scrubs']` for ease-of-use: + + >>> t['scrubs']['rating'] + u'9.0' + +This is the recommended way of retrieving "one-off" data (for example, if you are only interested in "seriesname"). If you wish to iterate over all data, or check if a particular show has a specific piece of data, use the `data` attribute, + + >>> 'rating' in t['scrubs'].data + True + +### Banners and actors + +Since banners and actors are separate XML files, retrieving them by default is undesirable. If you wish to retrieve banners (and other fanart), use the `banners` Tvdb initialisation argument: + + >>> from tvdb_api import Tvdb + >>> t = Tvdb(banners = True) + +Then access the data using a `Show`'s `_banner` key: + + >>> t['scrubs']['_banners'].keys() + ['fanart', 'poster', 'series', 'season'] + +The banner data structure will be improved in future versions. + +Extended actor data is accessible similarly: + + >>> t = Tvdb(actors = True) + >>> actors = t['scrubs']['_actors'] + >>> actors[0] + + >>> actors[0].keys() + ['sortorder', 'image', 'role', 'id', 'name'] + >>> actors[0]['role'] + u'Dr. John Michael "J.D." Dorian' + +Remember a simple list of actors is accessible via the default Show data: + + >>> t['scrubs']['actors'] + u'|Zach Braff|Donald Faison|Sarah Chalke|Christa Miller|Aloma Wright|Robert Maschio|Sam Lloyd|Neil Flynn|Ken Jenkins|Judy Reyes|John C. McGinley|Travis Schuldt|Johnny Kastl|Heather Graham|Michael Mosley|Kerry Bish\xe9|Dave Franco|Eliza Coupe|' + +[tvdb]: http://thetvdb.com +[tvnamer]: http://github.com/dbr/tvnamer diff --git a/libs/tvdb_api/setup.py b/libs/tvdb_api/setup.py new file mode 100644 index 0000000..18eb7d1 --- /dev/null +++ b/libs/tvdb_api/setup.py @@ -0,0 +1,35 @@ +from setuptools import setup +setup( +name = 'tvdb_api', +version='1.8.2', + +author='dbr/Ben', +description='Interface to thetvdb.com', +url='http://github.com/dbr/tvdb_api/tree/master', +license='unlicense', + +long_description="""\ +An easy to use API interface to TheTVDB.com +Basic usage is: + +>>> import tvdb_api +>>> t = tvdb_api.Tvdb() +>>> ep = t['My Name Is Earl'][1][22] +>>> ep + +>>> ep['episodename'] +u'Stole a Badge' +""", + +py_modules = ['tvdb_api', 'tvdb_ui', 'tvdb_exceptions', 'tvdb_cache'], + +classifiers=[ + "Intended Audience :: Developers", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Topic :: Multimedia", + "Topic :: Utilities", + "Topic :: Software Development :: Libraries :: Python Modules", +] +) diff --git a/libs/tvdb_api/tests/gprof2dot.py b/libs/tvdb_api/tests/gprof2dot.py new file mode 100644 index 0000000..4503ec7 --- /dev/null +++ b/libs/tvdb_api/tests/gprof2dot.py @@ -0,0 +1,1638 @@ +#!/usr/bin/env python +# +# Copyright 2008 Jose Fonseca +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see . +# + +"""Generate a dot graph from the output of several profilers.""" + +__author__ = "Jose Fonseca" + +__version__ = "1.0" + + +import sys +import math +import os.path +import re +import textwrap +import optparse + + +try: + # Debugging helper module + import debug +except ImportError: + pass + + +def percentage(p): + return "%.02f%%" % (p*100.0,) + +def add(a, b): + return a + b + +def equal(a, b): + if a == b: + return a + else: + return None + +def fail(a, b): + assert False + + +def ratio(numerator, denominator): + numerator = float(numerator) + denominator = float(denominator) + assert 0.0 <= numerator + assert numerator <= denominator + try: + return numerator/denominator + except ZeroDivisionError: + # 0/0 is undefined, but 1.0 yields more useful results + return 1.0 + + +class UndefinedEvent(Exception): + """Raised when attempting to get an event which is undefined.""" + + def __init__(self, event): + Exception.__init__(self) + self.event = event + + def __str__(self): + return 'unspecified event %s' % self.event.name + + +class Event(object): + """Describe a kind of event, and its basic operations.""" + + def __init__(self, name, null, aggregator, formatter = str): + self.name = name + self._null = null + self._aggregator = aggregator + self._formatter = formatter + + def __eq__(self, other): + return self is other + + def __hash__(self): + return id(self) + + def null(self): + return self._null + + def aggregate(self, val1, val2): + """Aggregate two event values.""" + assert val1 is not None + assert val2 is not None + return self._aggregator(val1, val2) + + def format(self, val): + """Format an event value.""" + assert val is not None + return self._formatter(val) + + +MODULE = Event("Module", None, equal) +PROCESS = Event("Process", None, equal) + +CALLS = Event("Calls", 0, add) +SAMPLES = Event("Samples", 0, add) + +TIME = Event("Time", 0.0, add, lambda x: '(' + str(x) + ')') +TIME_RATIO = Event("Time ratio", 0.0, add, lambda x: '(' + percentage(x) + ')') +TOTAL_TIME = Event("Total time", 0.0, fail) +TOTAL_TIME_RATIO = Event("Total time ratio", 0.0, fail, percentage) + +CALL_RATIO = Event("Call ratio", 0.0, add, percentage) + +PRUNE_RATIO = Event("Prune ratio", 0.0, add, percentage) + + +class Object(object): + """Base class for all objects in profile which can store events.""" + + def __init__(self, events=None): + if events is None: + self.events = {} + else: + self.events = events + + def __hash__(self): + return id(self) + + def __eq__(self, other): + return self is other + + def __contains__(self, event): + return event in self.events + + def __getitem__(self, event): + try: + return self.events[event] + except KeyError: + raise UndefinedEvent(event) + + def __setitem__(self, event, value): + if value is None: + if event in self.events: + del self.events[event] + else: + self.events[event] = value + + +class Call(Object): + """A call between functions. + + There should be at most one call object for every pair of functions. + """ + + def __init__(self, callee_id): + Object.__init__(self) + self.callee_id = callee_id + + +class Function(Object): + """A function.""" + + def __init__(self, id, name): + Object.__init__(self) + self.id = id + self.name = name + self.calls = {} + self.cycle = None + + def add_call(self, call): + if call.callee_id in self.calls: + sys.stderr.write('warning: overwriting call from function %s to %s\n' % (str(self.id), str(call.callee_id))) + self.calls[call.callee_id] = call + + # TODO: write utility functions + + def __repr__(self): + return self.name + + +class Cycle(Object): + """A cycle made from recursive function calls.""" + + def __init__(self): + Object.__init__(self) + # XXX: Do cycles need an id? + self.functions = set() + + def add_function(self, function): + assert function not in self.functions + self.functions.add(function) + # XXX: Aggregate events? + if function.cycle is not None: + for other in function.cycle.functions: + if function not in self.functions: + self.add_function(other) + function.cycle = self + + +class Profile(Object): + """The whole profile.""" + + def __init__(self): + Object.__init__(self) + self.functions = {} + self.cycles = [] + + def add_function(self, function): + if function.id in self.functions: + sys.stderr.write('warning: overwriting function %s (id %s)\n' % (function.name, str(function.id))) + self.functions[function.id] = function + + def add_cycle(self, cycle): + self.cycles.append(cycle) + + def validate(self): + """Validate the edges.""" + + for function in self.functions.itervalues(): + for callee_id in function.calls.keys(): + assert function.calls[callee_id].callee_id == callee_id + if callee_id not in self.functions: + sys.stderr.write('warning: call to undefined function %s from function %s\n' % (str(callee_id), function.name)) + del function.calls[callee_id] + + def find_cycles(self): + """Find cycles using Tarjan's strongly connected components algorithm.""" + + # Apply the Tarjan's algorithm successively until all functions are visited + visited = set() + for function in self.functions.itervalues(): + if function not in visited: + self._tarjan(function, 0, [], {}, {}, visited) + cycles = [] + for function in self.functions.itervalues(): + if function.cycle is not None and function.cycle not in cycles: + cycles.append(function.cycle) + self.cycles = cycles + if 0: + for cycle in cycles: + sys.stderr.write("Cycle:\n") + for member in cycle.functions: + sys.stderr.write("\t%s\n" % member.name) + + def _tarjan(self, function, order, stack, orders, lowlinks, visited): + """Tarjan's strongly connected components algorithm. + + See also: + - http://en.wikipedia.org/wiki/Tarjan's_strongly_connected_components_algorithm + """ + + visited.add(function) + orders[function] = order + lowlinks[function] = order + order += 1 + pos = len(stack) + stack.append(function) + for call in function.calls.itervalues(): + callee = self.functions[call.callee_id] + # TODO: use a set to optimize lookup + if callee not in orders: + order = self._tarjan(callee, order, stack, orders, lowlinks, visited) + lowlinks[function] = min(lowlinks[function], lowlinks[callee]) + elif callee in stack: + lowlinks[function] = min(lowlinks[function], orders[callee]) + if lowlinks[function] == orders[function]: + # Strongly connected component found + members = stack[pos:] + del stack[pos:] + if len(members) > 1: + cycle = Cycle() + for member in members: + cycle.add_function(member) + return order + + def call_ratios(self, event): + # Aggregate for incoming calls + cycle_totals = {} + for cycle in self.cycles: + cycle_totals[cycle] = 0.0 + function_totals = {} + for function in self.functions.itervalues(): + function_totals[function] = 0.0 + for function in self.functions.itervalues(): + for call in function.calls.itervalues(): + if call.callee_id != function.id: + callee = self.functions[call.callee_id] + function_totals[callee] += call[event] + if callee.cycle is not None and callee.cycle is not function.cycle: + cycle_totals[callee.cycle] += call[event] + + # Compute the ratios + for function in self.functions.itervalues(): + for call in function.calls.itervalues(): + assert CALL_RATIO not in call + if call.callee_id != function.id: + callee = self.functions[call.callee_id] + if callee.cycle is not None and callee.cycle is not function.cycle: + total = cycle_totals[callee.cycle] + else: + total = function_totals[callee] + call[CALL_RATIO] = ratio(call[event], total) + + def integrate(self, outevent, inevent): + """Propagate function time ratio allong the function calls. + + Must be called after finding the cycles. + + See also: + - http://citeseer.ist.psu.edu/graham82gprof.html + """ + + # Sanity checking + assert outevent not in self + for function in self.functions.itervalues(): + assert outevent not in function + assert inevent in function + for call in function.calls.itervalues(): + assert outevent not in call + if call.callee_id != function.id: + assert CALL_RATIO in call + + # Aggregate the input for each cycle + for cycle in self.cycles: + total = inevent.null() + for function in self.functions.itervalues(): + total = inevent.aggregate(total, function[inevent]) + self[inevent] = total + + # Integrate along the edges + total = inevent.null() + for function in self.functions.itervalues(): + total = inevent.aggregate(total, function[inevent]) + self._integrate_function(function, outevent, inevent) + self[outevent] = total + + def _integrate_function(self, function, outevent, inevent): + if function.cycle is not None: + return self._integrate_cycle(function.cycle, outevent, inevent) + else: + if outevent not in function: + total = function[inevent] + for call in function.calls.itervalues(): + if call.callee_id != function.id: + total += self._integrate_call(call, outevent, inevent) + function[outevent] = total + return function[outevent] + + def _integrate_call(self, call, outevent, inevent): + assert outevent not in call + assert CALL_RATIO in call + callee = self.functions[call.callee_id] + subtotal = call[CALL_RATIO]*self._integrate_function(callee, outevent, inevent) + call[outevent] = subtotal + return subtotal + + def _integrate_cycle(self, cycle, outevent, inevent): + if outevent not in cycle: + + total = inevent.null() + for member in cycle.functions: + subtotal = member[inevent] + for call in member.calls.itervalues(): + callee = self.functions[call.callee_id] + if callee.cycle is not cycle: + subtotal += self._integrate_call(call, outevent, inevent) + total += subtotal + cycle[outevent] = total + + callees = {} + for function in self.functions.itervalues(): + if function.cycle is not cycle: + for call in function.calls.itervalues(): + callee = self.functions[call.callee_id] + if callee.cycle is cycle: + try: + callees[callee] += call[CALL_RATIO] + except KeyError: + callees[callee] = call[CALL_RATIO] + + for callee, call_ratio in callees.iteritems(): + ranks = {} + call_ratios = {} + partials = {} + self._rank_cycle_function(cycle, callee, 0, ranks) + self._call_ratios_cycle(cycle, callee, ranks, call_ratios, set()) + partial = self._integrate_cycle_function(cycle, callee, call_ratio, partials, ranks, call_ratios, outevent, inevent) + assert partial == max(partials.values()) + assert not total or abs(1.0 - partial/(call_ratio*total)) <= 0.001 + + return cycle[outevent] + + def _rank_cycle_function(self, cycle, function, rank, ranks): + if function not in ranks or ranks[function] > rank: + ranks[function] = rank + for call in function.calls.itervalues(): + if call.callee_id != function.id: + callee = self.functions[call.callee_id] + if callee.cycle is cycle: + self._rank_cycle_function(cycle, callee, rank + 1, ranks) + + def _call_ratios_cycle(self, cycle, function, ranks, call_ratios, visited): + if function not in visited: + visited.add(function) + for call in function.calls.itervalues(): + if call.callee_id != function.id: + callee = self.functions[call.callee_id] + if callee.cycle is cycle: + if ranks[callee] > ranks[function]: + call_ratios[callee] = call_ratios.get(callee, 0.0) + call[CALL_RATIO] + self._call_ratios_cycle(cycle, callee, ranks, call_ratios, visited) + + def _integrate_cycle_function(self, cycle, function, partial_ratio, partials, ranks, call_ratios, outevent, inevent): + if function not in partials: + partial = partial_ratio*function[inevent] + for call in function.calls.itervalues(): + if call.callee_id != function.id: + callee = self.functions[call.callee_id] + if callee.cycle is not cycle: + assert outevent in call + partial += partial_ratio*call[outevent] + else: + if ranks[callee] > ranks[function]: + callee_partial = self._integrate_cycle_function(cycle, callee, partial_ratio, partials, ranks, call_ratios, outevent, inevent) + call_ratio = ratio(call[CALL_RATIO], call_ratios[callee]) + call_partial = call_ratio*callee_partial + try: + call[outevent] += call_partial + except UndefinedEvent: + call[outevent] = call_partial + partial += call_partial + partials[function] = partial + try: + function[outevent] += partial + except UndefinedEvent: + function[outevent] = partial + return partials[function] + + def aggregate(self, event): + """Aggregate an event for the whole profile.""" + + total = event.null() + for function in self.functions.itervalues(): + try: + total = event.aggregate(total, function[event]) + except UndefinedEvent: + return + self[event] = total + + def ratio(self, outevent, inevent): + assert outevent not in self + assert inevent in self + for function in self.functions.itervalues(): + assert outevent not in function + assert inevent in function + function[outevent] = ratio(function[inevent], self[inevent]) + for call in function.calls.itervalues(): + assert outevent not in call + if inevent in call: + call[outevent] = ratio(call[inevent], self[inevent]) + self[outevent] = 1.0 + + def prune(self, node_thres, edge_thres): + """Prune the profile""" + + # compute the prune ratios + for function in self.functions.itervalues(): + try: + function[PRUNE_RATIO] = function[TOTAL_TIME_RATIO] + except UndefinedEvent: + pass + + for call in function.calls.itervalues(): + callee = self.functions[call.callee_id] + + if TOTAL_TIME_RATIO in call: + # handle exact cases first + call[PRUNE_RATIO] = call[TOTAL_TIME_RATIO] + else: + try: + # make a safe estimate + call[PRUNE_RATIO] = min(function[TOTAL_TIME_RATIO], callee[TOTAL_TIME_RATIO]) + except UndefinedEvent: + pass + + # prune the nodes + for function_id in self.functions.keys(): + function = self.functions[function_id] + try: + if function[PRUNE_RATIO] < node_thres: + del self.functions[function_id] + except UndefinedEvent: + pass + + # prune the egdes + for function in self.functions.itervalues(): + for callee_id in function.calls.keys(): + call = function.calls[callee_id] + try: + if callee_id not in self.functions or call[PRUNE_RATIO] < edge_thres: + del function.calls[callee_id] + except UndefinedEvent: + pass + + def dump(self): + for function in self.functions.itervalues(): + sys.stderr.write('Function %s:\n' % (function.name,)) + self._dump_events(function.events) + for call in function.calls.itervalues(): + callee = self.functions[call.callee_id] + sys.stderr.write(' Call %s:\n' % (callee.name,)) + self._dump_events(call.events) + + def _dump_events(self, events): + for event, value in events.iteritems(): + sys.stderr.write(' %s: %s\n' % (event.name, event.format(value))) + + +class Struct: + """Masquerade a dictionary with a structure-like behavior.""" + + def __init__(self, attrs = None): + if attrs is None: + attrs = {} + self.__dict__['_attrs'] = attrs + + def __getattr__(self, name): + try: + return self._attrs[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name, value): + self._attrs[name] = value + + def __str__(self): + return str(self._attrs) + + def __repr__(self): + return repr(self._attrs) + + +class ParseError(Exception): + """Raised when parsing to signal mismatches.""" + + def __init__(self, msg, line): + self.msg = msg + # TODO: store more source line information + self.line = line + + def __str__(self): + return '%s: %r' % (self.msg, self.line) + + +class Parser: + """Parser interface.""" + + def __init__(self): + pass + + def parse(self): + raise NotImplementedError + + +class LineParser(Parser): + """Base class for parsers that read line-based formats.""" + + def __init__(self, file): + Parser.__init__(self) + self._file = file + self.__line = None + self.__eof = False + + def readline(self): + line = self._file.readline() + if not line: + self.__line = '' + self.__eof = True + self.__line = line.rstrip('\r\n') + + def lookahead(self): + assert self.__line is not None + return self.__line + + def consume(self): + assert self.__line is not None + line = self.__line + self.readline() + return line + + def eof(self): + assert self.__line is not None + return self.__eof + + +class GprofParser(Parser): + """Parser for GNU gprof output. + + See also: + - Chapter "Interpreting gprof's Output" from the GNU gprof manual + http://sourceware.org/binutils/docs-2.18/gprof/Call-Graph.html#Call-Graph + - File "cg_print.c" from the GNU gprof source code + http://sourceware.org/cgi-bin/cvsweb.cgi/~checkout~/src/gprof/cg_print.c?rev=1.12&cvsroot=src + """ + + def __init__(self, fp): + Parser.__init__(self) + self.fp = fp + self.functions = {} + self.cycles = {} + + def readline(self): + line = self.fp.readline() + if not line: + sys.stderr.write('error: unexpected end of file\n') + sys.exit(1) + line = line.rstrip('\r\n') + return line + + _int_re = re.compile(r'^\d+$') + _float_re = re.compile(r'^\d+\.\d+$') + + def translate(self, mo): + """Extract a structure from a match object, while translating the types in the process.""" + attrs = {} + groupdict = mo.groupdict() + for name, value in groupdict.iteritems(): + if value is None: + value = None + elif self._int_re.match(value): + value = int(value) + elif self._float_re.match(value): + value = float(value) + attrs[name] = (value) + return Struct(attrs) + + _cg_header_re = re.compile( + # original gprof header + r'^\s+called/total\s+parents\s*$|' + + r'^index\s+%time\s+self\s+descendents\s+called\+self\s+name\s+index\s*$|' + + r'^\s+called/total\s+children\s*$|' + + # GNU gprof header + r'^index\s+%\s+time\s+self\s+children\s+called\s+name\s*$' + ) + + _cg_ignore_re = re.compile( + # spontaneous + r'^\s+\s*$|' + # internal calls (such as "mcount") + r'^.*\((\d+)\)$' + ) + + _cg_primary_re = re.compile( + r'^\[(?P\d+)\]' + + r'\s+(?P\d+\.\d+)' + + r'\s+(?P\d+\.\d+)' + + r'\s+(?P\d+\.\d+)' + + r'\s+(?:(?P\d+)(?:\+(?P\d+))?)?' + + r'\s+(?P\S.*?)' + + r'(?:\s+\d+)>)?' + + r'\s\[(\d+)\]$' + ) + + _cg_parent_re = re.compile( + r'^\s+(?P\d+\.\d+)?' + + r'\s+(?P\d+\.\d+)?' + + r'\s+(?P\d+)(?:/(?P\d+))?' + + r'\s+(?P\S.*?)' + + r'(?:\s+\d+)>)?' + + r'\s\[(?P\d+)\]$' + ) + + _cg_child_re = _cg_parent_re + + _cg_cycle_header_re = re.compile( + r'^\[(?P\d+)\]' + + r'\s+(?P\d+\.\d+)' + + r'\s+(?P\d+\.\d+)' + + r'\s+(?P\d+\.\d+)' + + r'\s+(?:(?P\d+)(?:\+(?P\d+))?)?' + + r'\s+\d+)\sas\sa\swhole>' + + r'\s\[(\d+)\]$' + ) + + _cg_cycle_member_re = re.compile( + r'^\s+(?P\d+\.\d+)?' + + r'\s+(?P\d+\.\d+)?' + + r'\s+(?P\d+)(?:\+(?P\d+))?' + + r'\s+(?P\S.*?)' + + r'(?:\s+\d+)>)?' + + r'\s\[(?P\d+)\]$' + ) + + _cg_sep_re = re.compile(r'^--+$') + + def parse_function_entry(self, lines): + parents = [] + children = [] + + while True: + if not lines: + sys.stderr.write('warning: unexpected end of entry\n') + line = lines.pop(0) + if line.startswith('['): + break + + # read function parent line + mo = self._cg_parent_re.match(line) + if not mo: + if self._cg_ignore_re.match(line): + continue + sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) + else: + parent = self.translate(mo) + parents.append(parent) + + # read primary line + mo = self._cg_primary_re.match(line) + if not mo: + sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) + return + else: + function = self.translate(mo) + + while lines: + line = lines.pop(0) + + # read function subroutine line + mo = self._cg_child_re.match(line) + if not mo: + if self._cg_ignore_re.match(line): + continue + sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) + else: + child = self.translate(mo) + children.append(child) + + function.parents = parents + function.children = children + + self.functions[function.index] = function + + def parse_cycle_entry(self, lines): + + # read cycle header line + line = lines[0] + mo = self._cg_cycle_header_re.match(line) + if not mo: + sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) + return + cycle = self.translate(mo) + + # read cycle member lines + cycle.functions = [] + for line in lines[1:]: + mo = self._cg_cycle_member_re.match(line) + if not mo: + sys.stderr.write('warning: unrecognized call graph entry: %r\n' % line) + continue + call = self.translate(mo) + cycle.functions.append(call) + + self.cycles[cycle.cycle] = cycle + + def parse_cg_entry(self, lines): + if lines[0].startswith("["): + self.parse_cycle_entry(lines) + else: + self.parse_function_entry(lines) + + def parse_cg(self): + """Parse the call graph.""" + + # skip call graph header + while not self._cg_header_re.match(self.readline()): + pass + line = self.readline() + while self._cg_header_re.match(line): + line = self.readline() + + # process call graph entries + entry_lines = [] + while line != '\014': # form feed + if line and not line.isspace(): + if self._cg_sep_re.match(line): + self.parse_cg_entry(entry_lines) + entry_lines = [] + else: + entry_lines.append(line) + line = self.readline() + + def parse(self): + self.parse_cg() + self.fp.close() + + profile = Profile() + profile[TIME] = 0.0 + + cycles = {} + for index in self.cycles.iterkeys(): + cycles[index] = Cycle() + + for entry in self.functions.itervalues(): + # populate the function + function = Function(entry.index, entry.name) + function[TIME] = entry.self + if entry.called is not None: + function[CALLS] = entry.called + if entry.called_self is not None: + call = Call(entry.index) + call[CALLS] = entry.called_self + function[CALLS] += entry.called_self + + # populate the function calls + for child in entry.children: + call = Call(child.index) + + assert child.called is not None + call[CALLS] = child.called + + if child.index not in self.functions: + # NOTE: functions that were never called but were discovered by gprof's + # static call graph analysis dont have a call graph entry so we need + # to add them here + missing = Function(child.index, child.name) + function[TIME] = 0.0 + function[CALLS] = 0 + profile.add_function(missing) + + function.add_call(call) + + profile.add_function(function) + + if entry.cycle is not None: + cycles[entry.cycle].add_function(function) + + profile[TIME] = profile[TIME] + function[TIME] + + for cycle in cycles.itervalues(): + profile.add_cycle(cycle) + + # Compute derived events + profile.validate() + profile.ratio(TIME_RATIO, TIME) + profile.call_ratios(CALLS) + profile.integrate(TOTAL_TIME, TIME) + profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME) + + return profile + + +class OprofileParser(LineParser): + """Parser for oprofile callgraph output. + + See also: + - http://oprofile.sourceforge.net/doc/opreport.html#opreport-callgraph + """ + + _fields_re = { + 'samples': r'(?P\d+)', + '%': r'(?P\S+)', + 'linenr info': r'(?P\(no location information\)|\S+:\d+)', + 'image name': r'(?P\S+(?:\s\(tgid:[^)]*\))?)', + 'app name': r'(?P\S+)', + 'symbol name': r'(?P\(no symbols\)|.+?)', + } + + def __init__(self, infile): + LineParser.__init__(self, infile) + self.entries = {} + self.entry_re = None + + def add_entry(self, callers, function, callees): + try: + entry = self.entries[function.id] + except KeyError: + self.entries[function.id] = (callers, function, callees) + else: + callers_total, function_total, callees_total = entry + self.update_subentries_dict(callers_total, callers) + function_total.samples += function.samples + self.update_subentries_dict(callees_total, callees) + + def update_subentries_dict(self, totals, partials): + for partial in partials.itervalues(): + try: + total = totals[partial.id] + except KeyError: + totals[partial.id] = partial + else: + total.samples += partial.samples + + def parse(self): + # read lookahead + self.readline() + + self.parse_header() + while self.lookahead(): + self.parse_entry() + + profile = Profile() + + reverse_call_samples = {} + + # populate the profile + profile[SAMPLES] = 0 + for _callers, _function, _callees in self.entries.itervalues(): + function = Function(_function.id, _function.name) + function[SAMPLES] = _function.samples + profile.add_function(function) + profile[SAMPLES] += _function.samples + + if _function.application: + function[PROCESS] = os.path.basename(_function.application) + if _function.image: + function[MODULE] = os.path.basename(_function.image) + + total_callee_samples = 0 + for _callee in _callees.itervalues(): + total_callee_samples += _callee.samples + + for _callee in _callees.itervalues(): + if not _callee.self: + call = Call(_callee.id) + call[SAMPLES] = _callee.samples + function.add_call(call) + + # compute derived data + profile.validate() + profile.find_cycles() + profile.ratio(TIME_RATIO, SAMPLES) + profile.call_ratios(SAMPLES) + profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO) + + return profile + + def parse_header(self): + while not self.match_header(): + self.consume() + line = self.lookahead() + fields = re.split(r'\s\s+', line) + entry_re = r'^\s*' + r'\s+'.join([self._fields_re[field] for field in fields]) + r'(?P\s+\[self\])?$' + self.entry_re = re.compile(entry_re) + self.skip_separator() + + def parse_entry(self): + callers = self.parse_subentries() + if self.match_primary(): + function = self.parse_subentry() + if function is not None: + callees = self.parse_subentries() + self.add_entry(callers, function, callees) + self.skip_separator() + + def parse_subentries(self): + subentries = {} + while self.match_secondary(): + subentry = self.parse_subentry() + subentries[subentry.id] = subentry + return subentries + + def parse_subentry(self): + entry = Struct() + line = self.consume() + mo = self.entry_re.match(line) + if not mo: + raise ParseError('failed to parse', line) + fields = mo.groupdict() + entry.samples = int(fields.get('samples', 0)) + entry.percentage = float(fields.get('percentage', 0.0)) + if 'source' in fields and fields['source'] != '(no location information)': + source = fields['source'] + filename, lineno = source.split(':') + entry.filename = filename + entry.lineno = int(lineno) + else: + source = '' + entry.filename = None + entry.lineno = None + entry.image = fields.get('image', '') + entry.application = fields.get('application', '') + if 'symbol' in fields and fields['symbol'] != '(no symbols)': + entry.symbol = fields['symbol'] + else: + entry.symbol = '' + if entry.symbol.startswith('"') and entry.symbol.endswith('"'): + entry.symbol = entry.symbol[1:-1] + entry.id = ':'.join((entry.application, entry.image, source, entry.symbol)) + entry.self = fields.get('self', None) != None + if entry.self: + entry.id += ':self' + if entry.symbol: + entry.name = entry.symbol + else: + entry.name = entry.image + return entry + + def skip_separator(self): + while not self.match_separator(): + self.consume() + self.consume() + + def match_header(self): + line = self.lookahead() + return line.startswith('samples') + + def match_separator(self): + line = self.lookahead() + return line == '-'*len(line) + + def match_primary(self): + line = self.lookahead() + return not line[:1].isspace() + + def match_secondary(self): + line = self.lookahead() + return line[:1].isspace() + + +class SharkParser(LineParser): + """Parser for MacOSX Shark output. + + Author: tom@dbservice.com + """ + + def __init__(self, infile): + LineParser.__init__(self, infile) + self.stack = [] + self.entries = {} + + def add_entry(self, function): + try: + entry = self.entries[function.id] + except KeyError: + self.entries[function.id] = (function, { }) + else: + function_total, callees_total = entry + function_total.samples += function.samples + + def add_callee(self, function, callee): + func, callees = self.entries[function.id] + try: + entry = callees[callee.id] + except KeyError: + callees[callee.id] = callee + else: + entry.samples += callee.samples + + def parse(self): + self.readline() + self.readline() + self.readline() + self.readline() + + match = re.compile(r'(?P[|+ ]*)(?P\d+), (?P[^,]+), (?P.*)') + + while self.lookahead(): + line = self.consume() + mo = match.match(line) + if not mo: + raise ParseError('failed to parse', line) + + fields = mo.groupdict() + prefix = len(fields.get('prefix', 0)) / 2 - 1 + + symbol = str(fields.get('symbol', 0)) + image = str(fields.get('image', 0)) + + entry = Struct() + entry.id = ':'.join([symbol, image]) + entry.samples = int(fields.get('samples', 0)) + + entry.name = symbol + entry.image = image + + # adjust the callstack + if prefix < len(self.stack): + del self.stack[prefix:] + + if prefix == len(self.stack): + self.stack.append(entry) + + # if the callstack has had an entry, it's this functions caller + if prefix > 0: + self.add_callee(self.stack[prefix - 1], entry) + + self.add_entry(entry) + + profile = Profile() + profile[SAMPLES] = 0 + for _function, _callees in self.entries.itervalues(): + function = Function(_function.id, _function.name) + function[SAMPLES] = _function.samples + profile.add_function(function) + profile[SAMPLES] += _function.samples + + if _function.image: + function[MODULE] = os.path.basename(_function.image) + + for _callee in _callees.itervalues(): + call = Call(_callee.id) + call[SAMPLES] = _callee.samples + function.add_call(call) + + # compute derived data + profile.validate() + profile.find_cycles() + profile.ratio(TIME_RATIO, SAMPLES) + profile.call_ratios(SAMPLES) + profile.integrate(TOTAL_TIME_RATIO, TIME_RATIO) + + return profile + + +class PstatsParser: + """Parser python profiling statistics saved with te pstats module.""" + + def __init__(self, *filename): + import pstats + self.stats = pstats.Stats(*filename) + self.profile = Profile() + self.function_ids = {} + + def get_function_name(self, (filename, line, name)): + module = os.path.splitext(filename)[0] + module = os.path.basename(module) + return "%s:%d:%s" % (module, line, name) + + def get_function(self, key): + try: + id = self.function_ids[key] + except KeyError: + id = len(self.function_ids) + name = self.get_function_name(key) + function = Function(id, name) + self.profile.functions[id] = function + self.function_ids[key] = id + else: + function = self.profile.functions[id] + return function + + def parse(self): + self.profile[TIME] = 0.0 + self.profile[TOTAL_TIME] = self.stats.total_tt + for fn, (cc, nc, tt, ct, callers) in self.stats.stats.iteritems(): + callee = self.get_function(fn) + callee[CALLS] = nc + callee[TOTAL_TIME] = ct + callee[TIME] = tt + self.profile[TIME] += tt + self.profile[TOTAL_TIME] = max(self.profile[TOTAL_TIME], ct) + for fn, value in callers.iteritems(): + caller = self.get_function(fn) + call = Call(callee.id) + if isinstance(value, tuple): + for i in xrange(0, len(value), 4): + nc, cc, tt, ct = value[i:i+4] + if CALLS in call: + call[CALLS] += cc + else: + call[CALLS] = cc + + if TOTAL_TIME in call: + call[TOTAL_TIME] += ct + else: + call[TOTAL_TIME] = ct + + else: + call[CALLS] = value + call[TOTAL_TIME] = ratio(value, nc)*ct + + caller.add_call(call) + #self.stats.print_stats() + #self.stats.print_callees() + + # Compute derived events + self.profile.validate() + self.profile.ratio(TIME_RATIO, TIME) + self.profile.ratio(TOTAL_TIME_RATIO, TOTAL_TIME) + + return self.profile + + +class Theme: + + def __init__(self, + bgcolor = (0.0, 0.0, 1.0), + mincolor = (0.0, 0.0, 0.0), + maxcolor = (0.0, 0.0, 1.0), + fontname = "Arial", + minfontsize = 10.0, + maxfontsize = 10.0, + minpenwidth = 0.5, + maxpenwidth = 4.0, + gamma = 2.2): + self.bgcolor = bgcolor + self.mincolor = mincolor + self.maxcolor = maxcolor + self.fontname = fontname + self.minfontsize = minfontsize + self.maxfontsize = maxfontsize + self.minpenwidth = minpenwidth + self.maxpenwidth = maxpenwidth + self.gamma = gamma + + def graph_bgcolor(self): + return self.hsl_to_rgb(*self.bgcolor) + + def graph_fontname(self): + return self.fontname + + def graph_fontsize(self): + return self.minfontsize + + def node_bgcolor(self, weight): + return self.color(weight) + + def node_fgcolor(self, weight): + return self.graph_bgcolor() + + def node_fontsize(self, weight): + return self.fontsize(weight) + + def edge_color(self, weight): + return self.color(weight) + + def edge_fontsize(self, weight): + return self.fontsize(weight) + + def edge_penwidth(self, weight): + return max(weight*self.maxpenwidth, self.minpenwidth) + + def edge_arrowsize(self, weight): + return 0.5 * math.sqrt(self.edge_penwidth(weight)) + + def fontsize(self, weight): + return max(weight**2 * self.maxfontsize, self.minfontsize) + + def color(self, weight): + weight = min(max(weight, 0.0), 1.0) + + hmin, smin, lmin = self.mincolor + hmax, smax, lmax = self.maxcolor + + h = hmin + weight*(hmax - hmin) + s = smin + weight*(smax - smin) + l = lmin + weight*(lmax - lmin) + + return self.hsl_to_rgb(h, s, l) + + def hsl_to_rgb(self, h, s, l): + """Convert a color from HSL color-model to RGB. + + See also: + - http://www.w3.org/TR/css3-color/#hsl-color + """ + + h = h % 1.0 + s = min(max(s, 0.0), 1.0) + l = min(max(l, 0.0), 1.0) + + if l <= 0.5: + m2 = l*(s + 1.0) + else: + m2 = l + s - l*s + m1 = l*2.0 - m2 + r = self._hue_to_rgb(m1, m2, h + 1.0/3.0) + g = self._hue_to_rgb(m1, m2, h) + b = self._hue_to_rgb(m1, m2, h - 1.0/3.0) + + # Apply gamma correction + r **= self.gamma + g **= self.gamma + b **= self.gamma + + return (r, g, b) + + def _hue_to_rgb(self, m1, m2, h): + if h < 0.0: + h += 1.0 + elif h > 1.0: + h -= 1.0 + if h*6 < 1.0: + return m1 + (m2 - m1)*h*6.0 + elif h*2 < 1.0: + return m2 + elif h*3 < 2.0: + return m1 + (m2 - m1)*(2.0/3.0 - h)*6.0 + else: + return m1 + + +TEMPERATURE_COLORMAP = Theme( + mincolor = (2.0/3.0, 0.80, 0.25), # dark blue + maxcolor = (0.0, 1.0, 0.5), # satured red + gamma = 1.0 +) + +PINK_COLORMAP = Theme( + mincolor = (0.0, 1.0, 0.90), # pink + maxcolor = (0.0, 1.0, 0.5), # satured red +) + +GRAY_COLORMAP = Theme( + mincolor = (0.0, 0.0, 0.85), # light gray + maxcolor = (0.0, 0.0, 0.0), # black +) + +BW_COLORMAP = Theme( + minfontsize = 8.0, + maxfontsize = 24.0, + mincolor = (0.0, 0.0, 0.0), # black + maxcolor = (0.0, 0.0, 0.0), # black + minpenwidth = 0.1, + maxpenwidth = 8.0, +) + + +class DotWriter: + """Writer for the DOT language. + + See also: + - "The DOT Language" specification + http://www.graphviz.org/doc/info/lang.html + """ + + def __init__(self, fp): + self.fp = fp + + def graph(self, profile, theme): + self.begin_graph() + + fontname = theme.graph_fontname() + + self.attr('graph', fontname=fontname, ranksep=0.25, nodesep=0.125) + self.attr('node', fontname=fontname, shape="box", style="filled,rounded", fontcolor="white", width=0, height=0) + self.attr('edge', fontname=fontname) + + for function in profile.functions.itervalues(): + labels = [] + for event in PROCESS, MODULE: + if event in function.events: + label = event.format(function[event]) + labels.append(label) + labels.append(function.name) + for event in TOTAL_TIME_RATIO, TIME_RATIO, CALLS: + if event in function.events: + label = event.format(function[event]) + labels.append(label) + + try: + weight = function[PRUNE_RATIO] + except UndefinedEvent: + weight = 0.0 + + label = '\n'.join(labels) + self.node(function.id, + label = label, + color = self.color(theme.node_bgcolor(weight)), + fontcolor = self.color(theme.node_fgcolor(weight)), + fontsize = "%.2f" % theme.node_fontsize(weight), + ) + + for call in function.calls.itervalues(): + callee = profile.functions[call.callee_id] + + labels = [] + for event in TOTAL_TIME_RATIO, CALLS: + if event in call.events: + label = event.format(call[event]) + labels.append(label) + + try: + weight = call[PRUNE_RATIO] + except UndefinedEvent: + try: + weight = callee[PRUNE_RATIO] + except UndefinedEvent: + weight = 0.0 + + label = '\n'.join(labels) + + self.edge(function.id, call.callee_id, + label = label, + color = self.color(theme.edge_color(weight)), + fontcolor = self.color(theme.edge_color(weight)), + fontsize = "%.2f" % theme.edge_fontsize(weight), + penwidth = "%.2f" % theme.edge_penwidth(weight), + labeldistance = "%.2f" % theme.edge_penwidth(weight), + arrowsize = "%.2f" % theme.edge_arrowsize(weight), + ) + + self.end_graph() + + def begin_graph(self): + self.write('digraph {\n') + + def end_graph(self): + self.write('}\n') + + def attr(self, what, **attrs): + self.write("\t") + self.write(what) + self.attr_list(attrs) + self.write(";\n") + + def node(self, node, **attrs): + self.write("\t") + self.id(node) + self.attr_list(attrs) + self.write(";\n") + + def edge(self, src, dst, **attrs): + self.write("\t") + self.id(src) + self.write(" -> ") + self.id(dst) + self.attr_list(attrs) + self.write(";\n") + + def attr_list(self, attrs): + if not attrs: + return + self.write(' [') + first = True + for name, value in attrs.iteritems(): + if first: + first = False + else: + self.write(", ") + self.id(name) + self.write('=') + self.id(value) + self.write(']') + + def id(self, id): + if isinstance(id, (int, float)): + s = str(id) + elif isinstance(id, str): + if id.isalnum(): + s = id + else: + s = self.escape(id) + else: + raise TypeError + self.write(s) + + def color(self, (r, g, b)): + + def float2int(f): + if f <= 0.0: + return 0 + if f >= 1.0: + return 255 + return int(255.0*f + 0.5) + + return "#" + "".join(["%02x" % float2int(c) for c in (r, g, b)]) + + def escape(self, s): + s = s.encode('utf-8') + s = s.replace('\\', r'\\') + s = s.replace('\n', r'\n') + s = s.replace('\t', r'\t') + s = s.replace('"', r'\"') + return '"' + s + '"' + + def write(self, s): + self.fp.write(s) + + +class Main: + """Main program.""" + + themes = { + "color": TEMPERATURE_COLORMAP, + "pink": PINK_COLORMAP, + "gray": GRAY_COLORMAP, + "bw": BW_COLORMAP, + } + + def main(self): + """Main program.""" + + parser = optparse.OptionParser( + usage="\n\t%prog [options] [file] ...", + version="%%prog %s" % __version__) + parser.add_option( + '-o', '--output', metavar='FILE', + type="string", dest="output", + help="output filename [stdout]") + parser.add_option( + '-n', '--node-thres', metavar='PERCENTAGE', + type="float", dest="node_thres", default=0.5, + help="eliminate nodes below this threshold [default: %default]") + parser.add_option( + '-e', '--edge-thres', metavar='PERCENTAGE', + type="float", dest="edge_thres", default=0.1, + help="eliminate edges below this threshold [default: %default]") + parser.add_option( + '-f', '--format', + type="choice", choices=('prof', 'oprofile', 'pstats', 'shark'), + dest="format", default="prof", + help="profile format: prof, oprofile, or pstats [default: %default]") + parser.add_option( + '-c', '--colormap', + type="choice", choices=('color', 'pink', 'gray', 'bw'), + dest="theme", default="color", + help="color map: color, pink, gray, or bw [default: %default]") + parser.add_option( + '-s', '--strip', + action="store_true", + dest="strip", default=False, + help="strip function parameters, template parameters, and const modifiers from demangled C++ function names") + parser.add_option( + '-w', '--wrap', + action="store_true", + dest="wrap", default=False, + help="wrap function names") + (self.options, self.args) = parser.parse_args(sys.argv[1:]) + + if len(self.args) > 1 and self.options.format != 'pstats': + parser.error('incorrect number of arguments') + + try: + self.theme = self.themes[self.options.theme] + except KeyError: + parser.error('invalid colormap \'%s\'' % self.options.theme) + + if self.options.format == 'prof': + if not self.args: + fp = sys.stdin + else: + fp = open(self.args[0], 'rt') + parser = GprofParser(fp) + elif self.options.format == 'oprofile': + if not self.args: + fp = sys.stdin + else: + fp = open(self.args[0], 'rt') + parser = OprofileParser(fp) + elif self.options.format == 'pstats': + if not self.args: + parser.error('at least a file must be specified for pstats input') + parser = PstatsParser(*self.args) + elif self.options.format == 'shark': + if not self.args: + fp = sys.stdin + else: + fp = open(self.args[0], 'rt') + parser = SharkParser(fp) + else: + parser.error('invalid format \'%s\'' % self.options.format) + + self.profile = parser.parse() + + if self.options.output is None: + self.output = sys.stdout + else: + self.output = open(self.options.output, 'wt') + + self.write_graph() + + _parenthesis_re = re.compile(r'\([^()]*\)') + _angles_re = re.compile(r'<[^<>]*>') + _const_re = re.compile(r'\s+const$') + + def strip_function_name(self, name): + """Remove extraneous information from C++ demangled function names.""" + + # Strip function parameters from name by recursively removing paired parenthesis + while True: + name, n = self._parenthesis_re.subn('', name) + if not n: + break + + # Strip const qualifier + name = self._const_re.sub('', name) + + # Strip template parameters from name by recursively removing paired angles + while True: + name, n = self._angles_re.subn('', name) + if not n: + break + + return name + + def wrap_function_name(self, name): + """Split the function name on multiple lines.""" + + if len(name) > 32: + ratio = 2.0/3.0 + height = max(int(len(name)/(1.0 - ratio) + 0.5), 1) + width = max(len(name)/height, 32) + # TODO: break lines in symbols + name = textwrap.fill(name, width, break_long_words=False) + + # Take away spaces + name = name.replace(", ", ",") + name = name.replace("> >", ">>") + name = name.replace("> >", ">>") # catch consecutive + + return name + + def compress_function_name(self, name): + """Compress function name according to the user preferences.""" + + if self.options.strip: + name = self.strip_function_name(name) + + if self.options.wrap: + name = self.wrap_function_name(name) + + # TODO: merge functions with same resulting name + + return name + + def write_graph(self): + dot = DotWriter(self.output) + profile = self.profile + profile.prune(self.options.node_thres/100.0, self.options.edge_thres/100.0) + + for function in profile.functions.itervalues(): + function.name = self.compress_function_name(function.name) + + dot.graph(profile, self.theme) + + +if __name__ == '__main__': + Main().main() diff --git a/libs/tvdb_api/tests/runtests.py b/libs/tvdb_api/tests/runtests.py new file mode 100644 index 0000000..ebb73d9 --- /dev/null +++ b/libs/tvdb_api/tests/runtests.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +#encoding:utf-8 +#author:dbr/Ben +#project:tvdb_api +#repository:http://github.com/dbr/tvdb_api +#license:unlicense (http://unlicense.org/) + +import sys +import unittest + +import test_tvdb_api + +def main(): + suite = unittest.TestSuite([ + unittest.TestLoader().loadTestsFromModule(test_tvdb_api) + ]) + + runner = unittest.TextTestRunner(verbosity=2) + result = runner.run(suite) + if result.wasSuccessful(): + return 0 + else: + return 1 + +if __name__ == '__main__': + sys.exit( + int(main()) + ) diff --git a/libs/tvdb_api/tests/test_tvdb_api.py b/libs/tvdb_api/tests/test_tvdb_api.py new file mode 100644 index 0000000..0947461 --- /dev/null +++ b/libs/tvdb_api/tests/test_tvdb_api.py @@ -0,0 +1,526 @@ +#!/usr/bin/env python +#encoding:utf-8 +#author:dbr/Ben +#project:tvdb_api +#repository:http://github.com/dbr/tvdb_api +#license:unlicense (http://unlicense.org/) + +"""Unittests for tvdb_api +""" + +import os +import sys +import datetime +import unittest + +# Force parent directory onto path +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import tvdb_api +import tvdb_ui +from tvdb_api import (tvdb_shownotfound, tvdb_seasonnotfound, +tvdb_episodenotfound, tvdb_attributenotfound) + +class test_tvdb_basic(unittest.TestCase): + # Used to store the cached instance of Tvdb() + t = None + + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, banners = False) + + def test_different_case(self): + """Checks the auto-correction of show names is working. + It should correct the weirdly capitalised 'sCruBs' to 'Scrubs' + """ + self.assertEquals(self.t['scrubs'][1][4]['episodename'], 'My Old Lady') + self.assertEquals(self.t['sCruBs']['seriesname'], 'Scrubs') + + def test_spaces(self): + """Checks shownames with spaces + """ + self.assertEquals(self.t['My Name Is Earl']['seriesname'], 'My Name Is Earl') + self.assertEquals(self.t['My Name Is Earl'][1][4]['episodename'], 'Faked His Own Death') + + def test_numeric(self): + """Checks numeric show names + """ + self.assertEquals(self.t['24'][2][20]['episodename'], 'Day 2: 3:00 A.M.-4:00 A.M.') + self.assertEquals(self.t['24']['seriesname'], '24') + + def test_show_iter(self): + """Iterating over a show returns each seasons + """ + self.assertEquals( + len( + [season for season in self.t['Life on Mars']] + ), + 2 + ) + + def test_season_iter(self): + """Iterating over a show returns episodes + """ + self.assertEquals( + len( + [episode for episode in self.t['Life on Mars'][1]] + ), + 8 + ) + + def test_get_episode_overview(self): + """Checks episode overview is retrieved correctly. + """ + self.assertEquals( + self.t['Battlestar Galactica (2003)'][1][6]['overview'].startswith( + 'When a new copy of Doral, a Cylon who had been previously'), + True + ) + + def test_get_parent(self): + """Check accessing series from episode instance + """ + show = self.t['Battlestar Galactica (2003)'] + season = show[1] + episode = show[1][1] + + self.assertEquals( + season.show, + show + ) + + self.assertEquals( + episode.season, + season + ) + + self.assertEquals( + episode.season.show, + show + ) + + +class test_tvdb_errors(unittest.TestCase): + # Used to store the cached instance of Tvdb() + t = None + + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, banners = False) + + def test_seasonnotfound(self): + """Checks exception is thrown when season doesn't exist. + """ + self.assertRaises(tvdb_seasonnotfound, lambda:self.t['CNNNN'][10][1]) + + def test_shownotfound(self): + """Checks exception is thrown when episode doesn't exist. + """ + self.assertRaises(tvdb_shownotfound, lambda:self.t['the fake show thingy']) + + def test_episodenotfound(self): + """Checks exception is raised for non-existent episode + """ + self.assertRaises(tvdb_episodenotfound, lambda:self.t['Scrubs'][1][30]) + + def test_attributenamenotfound(self): + """Checks exception is thrown for if an attribute isn't found. + """ + self.assertRaises(tvdb_attributenotfound, lambda:self.t['CNNNN'][1][6]['afakeattributething']) + self.assertRaises(tvdb_attributenotfound, lambda:self.t['CNNNN']['afakeattributething']) + +class test_tvdb_search(unittest.TestCase): + # Used to store the cached instance of Tvdb() + t = None + + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, banners = False) + + def test_search_len(self): + """There should be only one result matching + """ + self.assertEquals(len(self.t['My Name Is Earl'].search('Faked His Own Death')), 1) + + def test_search_checkname(self): + """Checks you can get the episode name of a search result + """ + self.assertEquals(self.t['Scrubs'].search('my first')[0]['episodename'], 'My First Day') + self.assertEquals(self.t['My Name Is Earl'].search('Faked His Own Death')[0]['episodename'], 'Faked His Own Death') + + def test_search_multiresults(self): + """Checks search can return multiple results + """ + self.assertEquals(len(self.t['Scrubs'].search('my first')) >= 3, True) + + def test_search_no_params_error(self): + """Checks not supplying search info raises TypeError""" + self.assertRaises( + TypeError, + lambda: self.t['Scrubs'].search() + ) + + def test_search_season(self): + """Checks the searching of a single season""" + self.assertEquals( + len(self.t['Scrubs'][1].search("First")), + 3 + ) + + def test_search_show(self): + """Checks the searching of an entire show""" + self.assertEquals( + len(self.t['CNNNN'].search('CNNNN', key='episodename')), + 3 + ) + + def test_aired_on(self): + """Tests airedOn show method""" + sr = self.t['Scrubs'].airedOn(datetime.date(2001, 10, 2)) + self.assertEquals(len(sr), 1) + self.assertEquals(sr[0]['episodename'], u'My First Day') + +class test_tvdb_data(unittest.TestCase): + # Used to store the cached instance of Tvdb() + t = None + + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, banners = False) + + def test_episode_data(self): + """Check the firstaired value is retrieved + """ + self.assertEquals( + self.t['lost']['firstaired'], + '2004-09-22' + ) + +class test_tvdb_misc(unittest.TestCase): + # Used to store the cached instance of Tvdb() + t = None + + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, banners = False) + + def test_repr_show(self): + """Check repr() of Season + """ + self.assertEquals( + repr(self.t['CNNNN']), + "" + ) + def test_repr_season(self): + """Check repr() of Season + """ + self.assertEquals( + repr(self.t['CNNNN'][1]), + "" + ) + def test_repr_episode(self): + """Check repr() of Episode + """ + self.assertEquals( + repr(self.t['CNNNN'][1][1]), + "" + ) + def test_have_all_languages(self): + """Check valid_languages is up-to-date (compared to languages.xml) + """ + et = self.t._getetsrc( + "http://thetvdb.com/api/%s/languages.xml" % ( + self.t.config['apikey'] + ) + ) + languages = [x.find("abbreviation").text for x in et.findall("Language")] + + self.assertEquals( + sorted(languages), + sorted(self.t.config['valid_languages']) + ) + +class test_tvdb_languages(unittest.TestCase): + def test_episode_name_french(self): + """Check episode data is in French (language="fr") + """ + t = tvdb_api.Tvdb(cache = True, language = "fr") + self.assertEquals( + t['scrubs'][1][1]['episodename'], + "Mon premier jour" + ) + self.assertTrue( + t['scrubs']['overview'].startswith( + u"J.D. est un jeune m\xe9decin qui d\xe9bute" + ) + ) + + def test_episode_name_spanish(self): + """Check episode data is in Spanish (language="es") + """ + t = tvdb_api.Tvdb(cache = True, language = "es") + self.assertEquals( + t['scrubs'][1][1]['episodename'], + "Mi Primer Dia" + ) + self.assertTrue( + t['scrubs']['overview'].startswith( + u'Scrubs es una divertida comedia' + ) + ) + + def test_multilanguage_selection(self): + """Check selected language is used + """ + class SelectEnglishUI(tvdb_ui.BaseUI): + def selectSeries(self, allSeries): + return [x for x in allSeries if x['language'] == "en"][0] + + class SelectItalianUI(tvdb_ui.BaseUI): + def selectSeries(self, allSeries): + return [x for x in allSeries if x['language'] == "it"][0] + + t_en = tvdb_api.Tvdb( + cache=True, + custom_ui = SelectEnglishUI, + language = "en") + t_it = tvdb_api.Tvdb( + cache=True, + custom_ui = SelectItalianUI, + language = "it") + + self.assertEquals( + t_en['dexter'][1][2]['episodename'], "Crocodile" + ) + self.assertEquals( + t_it['dexter'][1][2]['episodename'], "Lacrime di coccodrillo" + ) + + +class test_tvdb_unicode(unittest.TestCase): + def test_search_in_chinese(self): + """Check searching for show with language=zh returns Chinese seriesname + """ + t = tvdb_api.Tvdb(cache = True, language = "zh") + show = t[u'T\xecnh Ng\u01b0\u1eddi Hi\u1ec7n \u0110\u1ea1i'] + self.assertEquals( + type(show), + tvdb_api.Show + ) + + self.assertEquals( + show['seriesname'], + u'T\xecnh Ng\u01b0\u1eddi Hi\u1ec7n \u0110\u1ea1i' + ) + + def test_search_in_all_languages(self): + """Check search_all_languages returns Chinese show, with language=en + """ + t = tvdb_api.Tvdb(cache = True, search_all_languages = True, language="en") + show = t[u'T\xecnh Ng\u01b0\u1eddi Hi\u1ec7n \u0110\u1ea1i'] + self.assertEquals( + type(show), + tvdb_api.Show + ) + + self.assertEquals( + show['seriesname'], + u'Virtues Of Harmony II' + ) + +class test_tvdb_banners(unittest.TestCase): + # Used to store the cached instance of Tvdb() + t = None + + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, banners = True) + + def test_have_banners(self): + """Check banners at least one banner is found + """ + self.assertEquals( + len(self.t['scrubs']['_banners']) > 0, + True + ) + + def test_banner_url(self): + """Checks banner URLs start with http:// + """ + for banner_type, banner_data in self.t['scrubs']['_banners'].items(): + for res, res_data in banner_data.items(): + for bid, banner_info in res_data.items(): + self.assertEquals( + banner_info['_bannerpath'].startswith("http://"), + True + ) + + def test_episode_image(self): + """Checks episode 'filename' image is fully qualified URL + """ + self.assertEquals( + self.t['scrubs'][1][1]['filename'].startswith("http://"), + True + ) + + def test_show_artwork(self): + """Checks various image URLs within season data are fully qualified + """ + for key in ['banner', 'fanart', 'poster']: + self.assertEquals( + self.t['scrubs'][key].startswith("http://"), + True + ) + +class test_tvdb_actors(unittest.TestCase): + t = None + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, actors = True) + + def test_actors_is_correct_datatype(self): + """Check show/_actors key exists and is correct type""" + self.assertTrue( + isinstance( + self.t['scrubs']['_actors'], + tvdb_api.Actors + ) + ) + + def test_actors_has_actor(self): + """Check show has at least one Actor + """ + self.assertTrue( + isinstance( + self.t['scrubs']['_actors'][0], + tvdb_api.Actor + ) + ) + + def test_actor_has_name(self): + """Check first actor has a name""" + self.assertEquals( + self.t['scrubs']['_actors'][0]['name'], + "Zach Braff" + ) + + def test_actor_image_corrected(self): + """Check image URL is fully qualified + """ + for actor in self.t['scrubs']['_actors']: + if actor['image'] is not None: + # Actor's image can be None, it displays as the placeholder + # image on thetvdb.com + self.assertTrue( + actor['image'].startswith("http://") + ) + +class test_tvdb_doctest(unittest.TestCase): + # Used to store the cached instance of Tvdb() + t = None + + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, banners = False) + + def test_doctest(self): + """Check docstring examples works""" + import doctest + doctest.testmod(tvdb_api) + + +class test_tvdb_custom_caching(unittest.TestCase): + def test_true_false_string(self): + """Tests setting cache to True/False/string + + Basic tests, only checking for errors + """ + + tvdb_api.Tvdb(cache = True) + tvdb_api.Tvdb(cache = False) + tvdb_api.Tvdb(cache = "/tmp") + + def test_invalid_cache_option(self): + """Tests setting cache to invalid value + """ + + try: + tvdb_api.Tvdb(cache = 2.3) + except ValueError: + pass + else: + self.fail("Expected ValueError from setting cache to float") + + def test_custom_urlopener(self): + class UsedCustomOpener(Exception): + pass + + import urllib2 + class TestOpener(urllib2.BaseHandler): + def default_open(self, request): + print request.get_method() + raise UsedCustomOpener("Something") + + custom_opener = urllib2.build_opener(TestOpener()) + t = tvdb_api.Tvdb(cache = custom_opener) + try: + t['scrubs'] + except UsedCustomOpener: + pass + else: + self.fail("Did not use custom opener") + +class test_tvdb_by_id(unittest.TestCase): + t = None + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, actors = True) + + def test_actors_is_correct_datatype(self): + """Check show/_actors key exists and is correct type""" + self.assertEquals( + self.t[76156]['seriesname'], + 'Scrubs' + ) + + +class test_tvdb_zip(unittest.TestCase): + # Used to store the cached instance of Tvdb() + t = None + + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, useZip = True) + + def test_get_series_from_zip(self): + """ + """ + self.assertEquals(self.t['scrubs'][1][4]['episodename'], 'My Old Lady') + self.assertEquals(self.t['sCruBs']['seriesname'], 'Scrubs') + + def test_spaces_from_zip(self): + """Checks shownames with spaces + """ + self.assertEquals(self.t['My Name Is Earl']['seriesname'], 'My Name Is Earl') + self.assertEquals(self.t['My Name Is Earl'][1][4]['episodename'], 'Faked His Own Death') + + +class test_tvdb_show_search(unittest.TestCase): + # Used to store the cached instance of Tvdb() + t = None + + def setUp(self): + if self.t is None: + self.__class__.t = tvdb_api.Tvdb(cache = True, useZip = True) + + def test_search(self): + """Test Tvdb.search method + """ + results = self.t.search("my name is earl") + all_ids = [x['seriesid'] for x in results] + self.assertTrue('75397' in all_ids) + + +if __name__ == '__main__': + runner = unittest.TextTestRunner(verbosity = 2) + unittest.main(testRunner = runner) diff --git a/libs/tvdb_api/tvdb_api.py b/libs/tvdb_api/tvdb_api.py new file mode 100644 index 0000000..4bfe78a --- /dev/null +++ b/libs/tvdb_api/tvdb_api.py @@ -0,0 +1,874 @@ +#!/usr/bin/env python +#encoding:utf-8 +#author:dbr/Ben +#project:tvdb_api +#repository:http://github.com/dbr/tvdb_api +#license:unlicense (http://unlicense.org/) + +"""Simple-to-use Python interface to The TVDB's API (thetvdb.com) + +Example usage: + +>>> from tvdb_api import Tvdb +>>> t = Tvdb() +>>> t['Lost'][4][11]['episodename'] +u'Cabin Fever' +""" +__author__ = "dbr/Ben" +__version__ = "1.8.2" + +import os +import time +import urllib +import urllib2 +import getpass +import StringIO +import tempfile +import warnings +import logging +import datetime +import zipfile + +try: + import xml.etree.cElementTree as ElementTree +except ImportError: + import xml.etree.ElementTree as ElementTree + +try: + import gzip +except ImportError: + gzip = None + + +from tvdb_cache import CacheHandler + +from tvdb_ui import BaseUI, ConsoleUI +from tvdb_exceptions import (tvdb_error, tvdb_userabort, tvdb_shownotfound, + tvdb_seasonnotfound, tvdb_episodenotfound, tvdb_attributenotfound) + +lastTimeout = None + +def log(): + return logging.getLogger("tvdb_api") + + +class ShowContainer(dict): + """Simple dict that holds a series of Show instances + """ + + def __init__(self): + self._stack = [] + self._lastgc = time.time() + + def __setitem__(self, key, value): + self._stack.append(key) + + #keep only the 100th latest results + if time.time() - self._lastgc > 20: + tbd = self._stack[:-100] + i = 0 + for o in tbd: + del self[o] + del self._stack[i] + i += 1 + + _lastgc = time.time() + del tbd + + super(ShowContainer, self).__setitem__(key, value) + + +class Show(dict): + """Holds a dict of seasons, and show data. + """ + def __init__(self): + dict.__init__(self) + self.data = {} + + def __repr__(self): + return "" % ( + self.data.get(u'seriesname', 'instance'), + len(self) + ) + + def __getitem__(self, key): + if key in self: + # Key is an episode, return it + return dict.__getitem__(self, key) + + if key in self.data: + # Non-numeric request is for show-data + return dict.__getitem__(self.data, key) + + # Data wasn't found, raise appropriate error + if isinstance(key, int) or key.isdigit(): + # Episode number x was not found + raise tvdb_seasonnotfound("Could not find season %s" % (repr(key))) + else: + # If it's not numeric, it must be an attribute name, which + # doesn't exist, so attribute error. + raise tvdb_attributenotfound("Cannot find attribute %s" % (repr(key))) + + def airedOn(self, date): + ret = self.search(str(date), 'firstaired') + if len(ret) == 0: + raise tvdb_episodenotfound("Could not find any episodes that aired on %s" % date) + return ret + + def search(self, term = None, key = None): + """ + Search all episodes in show. Can search all data, or a specific key (for + example, episodename) + + Always returns an array (can be empty). First index contains the first + match, and so on. + + Each array index is an Episode() instance, so doing + search_results[0]['episodename'] will retrieve the episode name of the + first match. + + Search terms are converted to lower case (unicode) strings. + + # Examples + + These examples assume t is an instance of Tvdb(): + + >>> t = Tvdb() + >>> + + To search for all episodes of Scrubs with a bit of data + containing "my first day": + + >>> t['Scrubs'].search("my first day") + [] + >>> + + Search for "My Name Is Earl" episode named "Faked His Own Death": + + >>> t['My Name Is Earl'].search('Faked His Own Death', key = 'episodename') + [] + >>> + + To search Scrubs for all episodes with "mentor" in the episode name: + + >>> t['scrubs'].search('mentor', key = 'episodename') + [, ] + >>> + + # Using search results + + >>> results = t['Scrubs'].search("my first") + >>> print results[0]['episodename'] + My First Day + >>> for x in results: print x['episodename'] + My First Day + My First Step + My First Kill + >>> + """ + results = [] + for cur_season in self.values(): + searchresult = cur_season.search(term = term, key = key) + if len(searchresult) != 0: + results.extend(searchresult) + + return results + + +class Season(dict): + def __init__(self, show = None): + """The show attribute points to the parent show + """ + self.show = show + + def __repr__(self): + return "" % ( + len(self.keys()) + ) + + def __getitem__(self, episode_number): + if episode_number not in self: + raise tvdb_episodenotfound("Could not find episode %s" % (repr(episode_number))) + else: + return dict.__getitem__(self, episode_number) + + def search(self, term = None, key = None): + """Search all episodes in season, returns a list of matching Episode + instances. + + >>> t = Tvdb() + >>> t['scrubs'][1].search('first day') + [] + >>> + + See Show.search documentation for further information on search + """ + results = [] + for ep in self.values(): + searchresult = ep.search(term = term, key = key) + if searchresult is not None: + results.append( + searchresult + ) + return results + + +class Episode(dict): + def __init__(self, season = None): + """The season attribute points to the parent season + """ + self.season = season + + def __repr__(self): + seasno = int(self.get(u'seasonnumber', 0)) + epno = int(self.get(u'episodenumber', 0)) + epname = self.get(u'episodename') + if epname is not None: + return "" % (seasno, epno, epname) + else: + return "" % (seasno, epno) + + def __getitem__(self, key): + try: + return dict.__getitem__(self, key) + except KeyError: + raise tvdb_attributenotfound("Cannot find attribute %s" % (repr(key))) + + def search(self, term = None, key = None): + """Search episode data for term, if it matches, return the Episode (self). + The key parameter can be used to limit the search to a specific element, + for example, episodename. + + This primarily for use use by Show.search and Season.search. See + Show.search for further information on search + + Simple example: + + >>> e = Episode() + >>> e['episodename'] = "An Example" + >>> e.search("examp") + + >>> + + Limiting by key: + + >>> e.search("examp", key = "episodename") + + >>> + """ + if term == None: + raise TypeError("must supply string to search for (contents)") + + term = unicode(term).lower() + for cur_key, cur_value in self.items(): + cur_key, cur_value = unicode(cur_key).lower(), unicode(cur_value).lower() + if key is not None and cur_key != key: + # Do not search this key + continue + if cur_value.find( unicode(term).lower() ) > -1: + return self + + +class Actors(list): + """Holds all Actor instances for a show + """ + pass + + +class Actor(dict): + """Represents a single actor. Should contain.. + + id, + image, + name, + role, + sortorder + """ + def __repr__(self): + return "" % (self.get("name")) + + +class Tvdb: + """Create easy-to-use interface to name of season/episode name + >>> t = Tvdb() + >>> t['Scrubs'][1][24]['episodename'] + u'My Last Day' + """ + def __init__(self, + interactive = False, + select_first = False, + debug = False, + cache = True, + banners = False, + actors = False, + custom_ui = None, + language = None, + search_all_languages = False, + apikey = None, + forceConnect=False, + useZip=False): + + """interactive (True/False): + When True, uses built-in console UI is used to select the correct show. + When False, the first search result is used. + + select_first (True/False): + Automatically selects the first series search result (rather + than showing the user a list of more than one series). + Is overridden by interactive = False, or specifying a custom_ui + + debug (True/False) DEPRECATED: + Replaced with proper use of logging module. To show debug messages: + + >>> import logging + >>> logging.basicConfig(level = logging.DEBUG) + + cache (True/False/str/unicode/urllib2 opener): + Retrieved XML are persisted to to disc. If true, stores in + tvdb_api folder under your systems TEMP_DIR, if set to + str/unicode instance it will use this as the cache + location. If False, disables caching. Can also be passed + an arbitrary Python object, which is used as a urllib2 + opener, which should be created by urllib2.build_opener + + banners (True/False): + Retrieves the banners for a show. These are accessed + via the _banners key of a Show(), for example: + + >>> Tvdb(banners=True)['scrubs']['_banners'].keys() + ['fanart', 'poster', 'series', 'season'] + + actors (True/False): + Retrieves a list of the actors for a show. These are accessed + via the _actors key of a Show(), for example: + + >>> t = Tvdb(actors=True) + >>> t['scrubs']['_actors'][0]['name'] + u'Zach Braff' + + custom_ui (tvdb_ui.BaseUI subclass): + A callable subclass of tvdb_ui.BaseUI (overrides interactive option) + + language (2 character language abbreviation): + The language of the returned data. Is also the language search + uses. Default is "en" (English). For full list, run.. + + >>> Tvdb().config['valid_languages'] #doctest: +ELLIPSIS + ['da', 'fi', 'nl', ...] + + search_all_languages (True/False): + By default, Tvdb will only search in the language specified using + the language option. When this is True, it will search for the + show in and language + + apikey (str/unicode): + Override the default thetvdb.com API key. By default it will use + tvdb_api's own key (fine for small scripts), but you can use your + own key if desired - this is recommended if you are embedding + tvdb_api in a larger application) + See http://thetvdb.com/?tab=apiregister to get your own key + + forceConnect (bool): + If true it will always try to connect to theTVDB.com even if we + recently timed out. By default it will wait one minute before + trying again, and any requests within that one minute window will + return an exception immediately. + + useZip (bool): + Download the zip archive where possibale, instead of the xml. + This is only used when all episodes are pulled. + And only the main language xml is used, the actor and banner xml are lost. + """ + + global lastTimeout + + # if we're given a lastTimeout that is less than 1 min just give up + if not forceConnect and lastTimeout != None and datetime.datetime.now() - lastTimeout < datetime.timedelta(minutes=1): + raise tvdb_error("We recently timed out, so giving up early this time") + + self.shows = ShowContainer() # Holds all Show classes + self.corrections = {} # Holds show-name to show_id mapping + + self.config = {} + + if apikey is not None: + self.config['apikey'] = apikey + else: + self.config['apikey'] = "0629B785CE550C8D" # tvdb_api's API key + + self.config['debug_enabled'] = debug # show debugging messages + + self.config['custom_ui'] = custom_ui + + self.config['interactive'] = interactive # prompt for correct series? + + self.config['select_first'] = select_first + + self.config['search_all_languages'] = search_all_languages + + self.config['useZip'] = useZip + + + if cache is True: + self.config['cache_enabled'] = True + self.config['cache_location'] = self._getTempDir() + self.urlopener = urllib2.build_opener( + CacheHandler(self.config['cache_location']) + ) + + elif cache is False: + self.config['cache_enabled'] = False + self.urlopener = urllib2.build_opener() # default opener with no caching + + elif isinstance(cache, basestring): + self.config['cache_enabled'] = True + self.config['cache_location'] = cache + self.urlopener = urllib2.build_opener( + CacheHandler(self.config['cache_location']) + ) + + elif isinstance(cache, urllib2.OpenerDirector): + # If passed something from urllib2.build_opener, use that + log().debug("Using %r as urlopener" % cache) + self.config['cache_enabled'] = True + self.urlopener = cache + + else: + raise ValueError("Invalid value for Cache %r (type was %s)" % (cache, type(cache))) + + self.config['banners_enabled'] = banners + self.config['actors_enabled'] = actors + + if self.config['debug_enabled']: + warnings.warn("The debug argument to tvdb_api.__init__ will be removed in the next version. " + "To enable debug messages, use the following code before importing: " + "import logging; logging.basicConfig(level=logging.DEBUG)") + logging.basicConfig(level=logging.DEBUG) + + + # List of language from http://thetvdb.com/api/0629B785CE550C8D/languages.xml + # Hard-coded here as it is realtively static, and saves another HTTP request, as + # recommended on http://thetvdb.com/wiki/index.php/API:languages.xml + self.config['valid_languages'] = [ + "da", "fi", "nl", "de", "it", "es", "fr","pl", "hu","el","tr", + "ru","he","ja","pt","zh","cs","sl", "hr","ko","en","sv","no" + ] + + # thetvdb.com should be based around numeric language codes, + # but to link to a series like http://thetvdb.com/?tab=series&id=79349&lid=16 + # requires the language ID, thus this mapping is required (mainly + # for usage in tvdb_ui - internally tvdb_api will use the language abbreviations) + self.config['langabbv_to_id'] = {'el': 20, 'en': 7, 'zh': 27, + 'it': 15, 'cs': 28, 'es': 16, 'ru': 22, 'nl': 13, 'pt': 26, 'no': 9, + 'tr': 21, 'pl': 18, 'fr': 17, 'hr': 31, 'de': 14, 'da': 10, 'fi': 11, + 'hu': 19, 'ja': 25, 'he': 24, 'ko': 32, 'sv': 8, 'sl': 30} + + if language is None: + self.config['language'] = 'en' + else: + if language not in self.config['valid_languages']: + raise ValueError("Invalid language %s, options are: %s" % ( + language, self.config['valid_languages'] + )) + else: + self.config['language'] = language + + # The following url_ configs are based of the + # http://thetvdb.com/wiki/index.php/Programmers_API + self.config['base_url'] = "http://thetvdb.com" + + if self.config['search_all_languages']: + self.config['url_getSeries'] = u"%(base_url)s/api/GetSeries.php?seriesname=%%s&language=all" % self.config + else: + self.config['url_getSeries'] = u"%(base_url)s/api/GetSeries.php?seriesname=%%s&language=%(language)s" % self.config + + self.config['url_epInfo'] = u"%(base_url)s/api/%(apikey)s/series/%%s/all/%%s.xml" % self.config + self.config['url_epInfo_zip'] = u"%(base_url)s/api/%(apikey)s/series/%%s/all/%%s.zip" % self.config + + self.config['url_seriesInfo'] = u"%(base_url)s/api/%(apikey)s/series/%%s/%%s.xml" % self.config + self.config['url_actorsInfo'] = u"%(base_url)s/api/%(apikey)s/series/%%s/actors.xml" % self.config + + self.config['url_seriesBanner'] = u"%(base_url)s/api/%(apikey)s/series/%%s/banners.xml" % self.config + self.config['url_artworkPrefix'] = u"%(base_url)s/banners/%%s" % self.config + + def _getTempDir(self): + """Returns the [system temp dir]/tvdb_api-u501 (or + tvdb_api-myuser) + """ + if hasattr(os, 'getuid'): + uid = "u%d" % (os.getuid()) + else: + # For Windows + try: + uid = getpass.getuser() + except ImportError: + return os.path.join(tempfile.gettempdir(), "tvdb_api") + + return os.path.join(tempfile.gettempdir(), "tvdb_api-%s" % (uid)) + + def _loadUrl(self, url, recache = False, language=None): + global lastTimeout + try: + log().debug("Retrieving URL %s" % url) + resp = self.urlopener.open(url) + if 'x-local-cache' in resp.headers: + log().debug("URL %s was cached in %s" % ( + url, + resp.headers['x-local-cache']) + ) + if recache: + log().debug("Attempting to recache %s" % url) + resp.recache() + except (IOError, urllib2.URLError), errormsg: + if not str(errormsg).startswith('HTTP Error'): + lastTimeout = datetime.datetime.now() + raise tvdb_error("Could not connect to server: %s" % (errormsg)) + + + # handle gzipped content, + # http://dbr.lighthouseapp.com/projects/13342/tickets/72-gzipped-data-patch + if 'gzip' in resp.headers.get("Content-Encoding", ''): + if gzip: + stream = StringIO.StringIO(resp.read()) + gz = gzip.GzipFile(fileobj=stream) + return gz.read() + + raise tvdb_error("Received gzip data from thetvdb.com, but could not correctly handle it") + + if 'application/zip' in resp.headers.get("Content-Type", ''): + try: + # TODO: The zip contains actors.xml and banners.xml, which are currently ignored [GH-20] + log().debug("We recived a zip file unpacking now ...") + zipdata = StringIO.StringIO() + zipdata.write(resp.read()) + myzipfile = zipfile.ZipFile(zipdata) + return myzipfile.read('%s.xml' % language) + except zipfile.BadZipfile: + if 'x-local-cache' in resp.headers: + resp.delete_cache() + raise tvdb_error("Bad zip file received from thetvdb.com, could not read it") + + return resp.read() + + def _getetsrc(self, url, language=None): + """Loads a URL using caching, returns an ElementTree of the source + """ + src = self._loadUrl(url, language=language) + try: + # TVDB doesn't sanitize \r (CR) from user input in some fields, + # remove it to avoid errors. Change from SickBeard, from will14m + return ElementTree.fromstring(src.rstrip("\r")) + except SyntaxError: + src = self._loadUrl(url, recache=True, language=language) + try: + return ElementTree.fromstring(src.rstrip("\r")) + except SyntaxError, exceptionmsg: + errormsg = "There was an error with the XML retrieved from thetvdb.com:\n%s" % ( + exceptionmsg + ) + + if self.config['cache_enabled']: + errormsg += "\nFirst try emptying the cache folder at..\n%s" % ( + self.config['cache_location'] + ) + + errormsg += "\nIf this does not resolve the issue, please try again later. If the error persists, report a bug on" + errormsg += "\nhttp://dbr.lighthouseapp.com/projects/13342-tvdb_api/overview\n" + raise tvdb_error(errormsg) + + def _setItem(self, sid, seas, ep, attrib, value): + """Creates a new episode, creating Show(), Season() and + Episode()s as required. Called by _getShowData to populate show + + Since the nice-to-use tvdb[1][24]['name] interface + makes it impossible to do tvdb[1][24]['name] = "name" + and still be capable of checking if an episode exists + so we can raise tvdb_shownotfound, we have a slightly + less pretty method of setting items.. but since the API + is supposed to be read-only, this is the best way to + do it! + The problem is that calling tvdb[1][24]['episodename'] = "name" + calls __getitem__ on tvdb[1], there is no way to check if + tvdb.__dict__ should have a key "1" before we auto-create it + """ + if sid not in self.shows: + self.shows[sid] = Show() + if seas not in self.shows[sid]: + self.shows[sid][seas] = Season(show = self.shows[sid]) + if ep not in self.shows[sid][seas]: + self.shows[sid][seas][ep] = Episode(season = self.shows[sid][seas]) + self.shows[sid][seas][ep][attrib] = value + + def _setShowData(self, sid, key, value): + """Sets self.shows[sid] to a new Show instance, or sets the data + """ + if sid not in self.shows: + self.shows[sid] = Show() + self.shows[sid].data[key] = value + + def _cleanData(self, data): + """Cleans up strings returned by TheTVDB.com + + Issues corrected: + - Replaces & with & + - Trailing whitespace + """ + data = data.replace(u"&", u"&") + data = data.strip() + return data + + def search(self, series): + """This searches TheTVDB.com for the series name + and returns the result list + """ + series = urllib.quote(series.encode("utf-8")) + log().debug("Searching for show %s" % series) + seriesEt = self._getetsrc(self.config['url_getSeries'] % (series)) + allSeries = [] + for series in seriesEt: + result = dict((k.tag.lower(), k.text) for k in series.getchildren()) + result['id'] = int(result['id']) + result['lid'] = self.config['langabbv_to_id'][result['language']] + log().debug('Found series %(seriesname)s' % result) + allSeries.append(result) + + return allSeries + + def _getSeries(self, series): + """This searches TheTVDB.com for the series name, + If a custom_ui UI is configured, it uses this to select the correct + series. If not, and interactive == True, ConsoleUI is used, if not + BaseUI is used to select the first result. + """ + allSeries = self.search(series) + + if len(allSeries) == 0: + log().debug('Series result returned zero') + raise tvdb_shownotfound("Show-name search returned zero results (cannot find show on TVDB)") + + if self.config['custom_ui'] is not None: + log().debug("Using custom UI %s" % (repr(self.config['custom_ui']))) + ui = self.config['custom_ui'](config = self.config) + else: + if not self.config['interactive']: + log().debug('Auto-selecting first search result using BaseUI') + ui = BaseUI(config = self.config) + else: + log().debug('Interactively selecting show using ConsoleUI') + ui = ConsoleUI(config = self.config) + + return ui.selectSeries(allSeries) + + def _parseBanners(self, sid): + """Parses banners XML, from + http://thetvdb.com/api/[APIKEY]/series/[SERIES ID]/banners.xml + + Banners are retrieved using t['show name]['_banners'], for example: + + >>> t = Tvdb(banners = True) + >>> t['scrubs']['_banners'].keys() + ['fanart', 'poster', 'series', 'season'] + >>> t['scrubs']['_banners']['poster']['680x1000']['35308']['_bannerpath'] + u'http://thetvdb.com/banners/posters/76156-2.jpg' + >>> + + Any key starting with an underscore has been processed (not the raw + data from the XML) + + This interface will be improved in future versions. + """ + log().debug('Getting season banners for %s' % (sid)) + bannersEt = self._getetsrc( self.config['url_seriesBanner'] % (sid) ) + banners = {} + for cur_banner in bannersEt.findall('Banner'): + bid = cur_banner.find('id').text + btype = cur_banner.find('BannerType') + btype2 = cur_banner.find('BannerType2') + if btype is None or btype2 is None: + continue + btype, btype2 = btype.text, btype2.text + if not btype in banners: + banners[btype] = {} + if not btype2 in banners[btype]: + banners[btype][btype2] = {} + if not bid in banners[btype][btype2]: + banners[btype][btype2][bid] = {} + + for cur_element in cur_banner.getchildren(): + tag = cur_element.tag.lower() + value = cur_element.text + if tag is None or value is None: + continue + tag, value = tag.lower(), value.lower() + banners[btype][btype2][bid][tag] = value + + for k, v in banners[btype][btype2][bid].items(): + if k.endswith("path"): + new_key = "_%s" % (k) + log().debug("Transforming %s to %s" % (k, new_key)) + new_url = self.config['url_artworkPrefix'] % (v) + banners[btype][btype2][bid][new_key] = new_url + + self._setShowData(sid, "_banners", banners) + + def _parseActors(self, sid): + """Parsers actors XML, from + http://thetvdb.com/api/[APIKEY]/series/[SERIES ID]/actors.xml + + Actors are retrieved using t['show name]['_actors'], for example: + + >>> t = Tvdb(actors = True) + >>> actors = t['scrubs']['_actors'] + >>> type(actors) + + >>> type(actors[0]) + + >>> actors[0] + + >>> sorted(actors[0].keys()) + ['id', 'image', 'name', 'role', 'sortorder'] + >>> actors[0]['name'] + u'Zach Braff' + >>> actors[0]['image'] + u'http://thetvdb.com/banners/actors/43640.jpg' + + Any key starting with an underscore has been processed (not the raw + data from the XML) + """ + log().debug("Getting actors for %s" % (sid)) + actorsEt = self._getetsrc(self.config['url_actorsInfo'] % (sid)) + + cur_actors = Actors() + for curActorItem in actorsEt.findall("Actor"): + curActor = Actor() + for curInfo in curActorItem: + tag = curInfo.tag.lower() + value = curInfo.text + if value is not None: + if tag == "image": + value = self.config['url_artworkPrefix'] % (value) + else: + value = self._cleanData(value) + curActor[tag] = value + cur_actors.append(curActor) + self._setShowData(sid, '_actors', cur_actors) + + def _getShowData(self, sid, language): + """Takes a series ID, gets the epInfo URL and parses the TVDB + XML file into the shows dict in layout: + shows[series_id][season_number][episode_number] + """ + + if self.config['language'] is None: + log().debug('Config language is none, using show language') + if language is None: + raise tvdb_error("config['language'] was None, this should not happen") + getShowInLanguage = language + else: + log().debug( + 'Configured language %s override show language of %s' % ( + self.config['language'], + language + ) + ) + getShowInLanguage = self.config['language'] + + # Parse show information + log().debug('Getting all series data for %s' % (sid)) + seriesInfoEt = self._getetsrc( + self.config['url_seriesInfo'] % (sid, getShowInLanguage) + ) + for curInfo in seriesInfoEt.findall("Series")[0]: + tag = curInfo.tag.lower() + value = curInfo.text + + if value is not None: + if tag in ['banner', 'fanart', 'poster']: + value = self.config['url_artworkPrefix'] % (value) + else: + value = self._cleanData(value) + + self._setShowData(sid, tag, value) + + # Parse banners + if self.config['banners_enabled']: + self._parseBanners(sid) + + # Parse actors + if self.config['actors_enabled']: + self._parseActors(sid) + + # Parse episode data + log().debug('Getting all episodes of %s' % (sid)) + + if self.config['useZip']: + url = self.config['url_epInfo_zip'] % (sid, language) + else: + url = self.config['url_epInfo'] % (sid, language) + + epsEt = self._getetsrc( url, language=language) + + for cur_ep in epsEt.findall("Episode"): + seas_no = int(cur_ep.find('SeasonNumber').text) + ep_no = int(cur_ep.find('EpisodeNumber').text) + for cur_item in cur_ep.getchildren(): + tag = cur_item.tag.lower() + value = cur_item.text + if value is not None: + if tag == 'filename': + value = self.config['url_artworkPrefix'] % (value) + else: + value = self._cleanData(value) + self._setItem(sid, seas_no, ep_no, tag, value) + + def _nameToSid(self, name): + """Takes show name, returns the correct series ID (if the show has + already been grabbed), or grabs all episodes and returns + the correct SID. + """ + if name in self.corrections: + log().debug('Correcting %s to %s' % (name, self.corrections[name]) ) + sid = self.corrections[name] + else: + log().debug('Getting show %s' % (name)) + selected_series = self._getSeries( name ) + sname, sid = selected_series['seriesname'], selected_series['id'] + log().debug('Got %(seriesname)s, id %(id)s' % selected_series) + + self.corrections[name] = sid + self._getShowData(selected_series['id'], selected_series['language']) + + return sid + + def __getitem__(self, key): + """Handles tvdb_instance['seriesname'] calls. + The dict index should be the show id + """ + if isinstance(key, (int, long)): + # Item is integer, treat as show id + if key not in self.shows: + self._getShowData(key, self.config['language']) + return self.shows[key] + + key = key.lower() # make key lower case + sid = self._nameToSid(key) + log().debug('Got series id %s' % (sid)) + return self.shows[sid] + + def __repr__(self): + return str(self.shows) + + +def main(): + """Simple example of using tvdb_api - it just + grabs an episode name interactively. + """ + import logging + logging.basicConfig(level=logging.DEBUG) + + tvdb_instance = Tvdb(interactive=True, cache=False) + print tvdb_instance['Lost']['seriesname'] + print tvdb_instance['Lost'][1][4]['episodename'] + +if __name__ == '__main__': + main() diff --git a/libs/tvdb_api/tvdb_cache.py b/libs/tvdb_api/tvdb_cache.py new file mode 100644 index 0000000..d77c545 --- /dev/null +++ b/libs/tvdb_api/tvdb_cache.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python +#encoding:utf-8 +#author:dbr/Ben +#project:tvdb_api +#repository:http://github.com/dbr/tvdb_api +#license:unlicense (http://unlicense.org/) + +""" +urllib2 caching handler +Modified from http://code.activestate.com/recipes/491261/ +""" +from __future__ import with_statement + +__author__ = "dbr/Ben" +__version__ = "1.8.2" + +import os +import time +import errno +import httplib +import urllib2 +import StringIO +from hashlib import md5 +from threading import RLock + +cache_lock = RLock() + +def locked_function(origfunc): + """Decorator to execute function under lock""" + def wrapped(*args, **kwargs): + cache_lock.acquire() + try: + return origfunc(*args, **kwargs) + finally: + cache_lock.release() + return wrapped + +def calculate_cache_path(cache_location, url): + """Checks if [cache_location]/[hash_of_url].headers and .body exist + """ + thumb = md5(url).hexdigest() + header = os.path.join(cache_location, thumb + ".headers") + body = os.path.join(cache_location, thumb + ".body") + return header, body + +def check_cache_time(path, max_age): + """Checks if a file has been created/modified in the [last max_age] seconds. + False means the file is too old (or doesn't exist), True means it is + up-to-date and valid""" + if not os.path.isfile(path): + return False + cache_modified_time = os.stat(path).st_mtime + time_now = time.time() + if cache_modified_time < time_now - max_age: + # Cache is old + return False + else: + return True + +@locked_function +def exists_in_cache(cache_location, url, max_age): + """Returns if header AND body cache file exist (and are up-to-date)""" + hpath, bpath = calculate_cache_path(cache_location, url) + if os.path.exists(hpath) and os.path.exists(bpath): + return( + check_cache_time(hpath, max_age) + and check_cache_time(bpath, max_age) + ) + else: + # File does not exist + return False + +@locked_function +def store_in_cache(cache_location, url, response): + """Tries to store response in cache.""" + hpath, bpath = calculate_cache_path(cache_location, url) + try: + outf = open(hpath, "wb") + headers = str(response.info()) + outf.write(headers) + outf.close() + + outf = open(bpath, "wb") + outf.write(response.read()) + outf.close() + except IOError: + return True + else: + return False + +@locked_function +def delete_from_cache(cache_location, url): + """Deletes a response in cache.""" + hpath, bpath = calculate_cache_path(cache_location, url) + try: + if os.path.exists(hpath): + os.remove(hpath) + if os.path.exists(bpath): + os.remove(bpath) + except IOError: + return True + else: + return False + +class CacheHandler(urllib2.BaseHandler): + """Stores responses in a persistant on-disk cache. + + If a subsequent GET request is made for the same URL, the stored + response is returned, saving time, resources and bandwidth + """ + @locked_function + def __init__(self, cache_location, max_age = 21600): + """The location of the cache directory""" + self.max_age = max_age + self.cache_location = cache_location + if not os.path.exists(self.cache_location): + try: + os.mkdir(self.cache_location) + except OSError, e: + if e.errno == errno.EEXIST and os.path.isdir(self.cache_location): + # File exists, and it's a directory, + # another process beat us to creating this dir, that's OK. + pass + else: + # Our target dir is already a file, or different error, + # relay the error! + raise + + def default_open(self, request): + """Handles GET requests, if the response is cached it returns it + """ + if request.get_method() is not "GET": + return None # let the next handler try to handle the request + + if exists_in_cache( + self.cache_location, request.get_full_url(), self.max_age + ): + return CachedResponse( + self.cache_location, + request.get_full_url(), + set_cache_header = True + ) + else: + return None + + def http_response(self, request, response): + """Gets a HTTP response, if it was a GET request and the status code + starts with 2 (200 OK etc) it caches it and returns a CachedResponse + """ + if (request.get_method() == "GET" + and str(response.code).startswith("2") + ): + if 'x-local-cache' not in response.info(): + # Response is not cached + set_cache_header = store_in_cache( + self.cache_location, + request.get_full_url(), + response + ) + else: + set_cache_header = True + + return CachedResponse( + self.cache_location, + request.get_full_url(), + set_cache_header = set_cache_header + ) + else: + return response + +class CachedResponse(StringIO.StringIO): + """An urllib2.response-like object for cached responses. + + To determine if a response is cached or coming directly from + the network, check the x-local-cache header rather than the object type. + """ + + @locked_function + def __init__(self, cache_location, url, set_cache_header=True): + self.cache_location = cache_location + hpath, bpath = calculate_cache_path(cache_location, url) + + StringIO.StringIO.__init__(self, file(bpath, "rb").read()) + + self.url = url + self.code = 200 + self.msg = "OK" + headerbuf = file(hpath, "rb").read() + if set_cache_header: + headerbuf += "x-local-cache: %s\r\n" % (bpath) + self.headers = httplib.HTTPMessage(StringIO.StringIO(headerbuf)) + + def info(self): + """Returns headers + """ + return self.headers + + def geturl(self): + """Returns original URL + """ + return self.url + + @locked_function + def recache(self): + new_request = urllib2.urlopen(self.url) + set_cache_header = store_in_cache( + self.cache_location, + new_request.url, + new_request + ) + CachedResponse.__init__(self, self.cache_location, self.url, True) + + @locked_function + def delete_cache(self): + delete_from_cache( + self.cache_location, + self.url + ) + + +if __name__ == "__main__": + def main(): + """Quick test/example of CacheHandler""" + opener = urllib2.build_opener(CacheHandler("/tmp/")) + response = opener.open("http://google.com") + print response.headers + print "Response:", response.read() + + response.recache() + print response.headers + print "After recache:", response.read() + + # Test usage in threads + from threading import Thread + class CacheThreadTest(Thread): + lastdata = None + def run(self): + req = opener.open("http://google.com") + newdata = req.read() + if self.lastdata is None: + self.lastdata = newdata + assert self.lastdata == newdata, "Data was not consistent, uhoh" + req.recache() + threads = [CacheThreadTest() for x in range(50)] + print "Starting threads" + [t.start() for t in threads] + print "..done" + print "Joining threads" + [t.join() for t in threads] + print "..done" + main() diff --git a/libs/tvdb_api/tvdb_exceptions.py b/libs/tvdb_api/tvdb_exceptions.py new file mode 100644 index 0000000..cacbb93 --- /dev/null +++ b/libs/tvdb_api/tvdb_exceptions.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +#encoding:utf-8 +#author:dbr/Ben +#project:tvdb_api +#repository:http://github.com/dbr/tvdb_api +#license:unlicense (http://unlicense.org/) + +"""Custom exceptions used or raised by tvdb_api +""" + +__author__ = "dbr/Ben" +__version__ = "1.8.2" + +__all__ = ["tvdb_error", "tvdb_userabort", "tvdb_shownotfound", +"tvdb_seasonnotfound", "tvdb_episodenotfound", "tvdb_attributenotfound"] + +class tvdb_exception(Exception): + """Any exception generated by tvdb_api + """ + pass + +class tvdb_error(tvdb_exception): + """An error with thetvdb.com (Cannot connect, for example) + """ + pass + +class tvdb_userabort(tvdb_exception): + """User aborted the interactive selection (via + the q command, ^c etc) + """ + pass + +class tvdb_shownotfound(tvdb_exception): + """Show cannot be found on thetvdb.com (non-existant show) + """ + pass + +class tvdb_seasonnotfound(tvdb_exception): + """Season cannot be found on thetvdb.com + """ + pass + +class tvdb_episodenotfound(tvdb_exception): + """Episode cannot be found on thetvdb.com + """ + pass + +class tvdb_attributenotfound(tvdb_exception): + """Raised if an episode does not have the requested + attribute (such as a episode name) + """ + pass diff --git a/libs/tvdb_api/tvdb_ui.py b/libs/tvdb_api/tvdb_ui.py new file mode 100644 index 0000000..a4b6e95 --- /dev/null +++ b/libs/tvdb_api/tvdb_ui.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +#encoding:utf-8 +#author:dbr/Ben +#project:tvdb_api +#repository:http://github.com/dbr/tvdb_api +#license:unlicense (http://unlicense.org/) + +"""Contains included user interfaces for Tvdb show selection. + +A UI is a callback. A class, it's __init__ function takes two arguments: + +- config, which is the Tvdb config dict, setup in tvdb_api.py +- log, which is Tvdb's logger instance (which uses the logging module). You can +call log.info() log.warning() etc + +It must have a method "selectSeries", this is passed a list of dicts, each dict +contains the the keys "name" (human readable show name), and "sid" (the shows +ID as on thetvdb.com). For example: + +[{'name': u'Lost', 'sid': u'73739'}, + {'name': u'Lost Universe', 'sid': u'73181'}] + +The "selectSeries" method must return the appropriate dict, or it can raise +tvdb_userabort (if the selection is aborted), tvdb_shownotfound (if the show +cannot be found). + +A simple example callback, which returns a random series: + +>>> import random +>>> from tvdb_ui import BaseUI +>>> class RandomUI(BaseUI): +... def selectSeries(self, allSeries): +... import random +... return random.choice(allSeries) + +Then to use it.. + +>>> from tvdb_api import Tvdb +>>> t = Tvdb(custom_ui = RandomUI) +>>> random_matching_series = t['Lost'] +>>> type(random_matching_series) + +""" + +__author__ = "dbr/Ben" +__version__ = "1.8.2" + +import logging +import warnings + +from tvdb_exceptions import tvdb_userabort + +def log(): + return logging.getLogger(__name__) + +class BaseUI: + """Default non-interactive UI, which auto-selects first results + """ + def __init__(self, config, log = None): + self.config = config + if log is not None: + warnings.warn("the UI's log parameter is deprecated, instead use\n" + "use import logging; logging.getLogger('ui').info('blah')\n" + "The self.log attribute will be removed in the next version") + self.log = logging.getLogger(__name__) + + def selectSeries(self, allSeries): + return allSeries[0] + + +class ConsoleUI(BaseUI): + """Interactively allows the user to select a show from a console based UI + """ + + def _displaySeries(self, allSeries, limit = 6): + """Helper function, lists series with corresponding ID + """ + if limit is not None: + toshow = allSeries[:limit] + else: + toshow = allSeries + + print "TVDB Search Results:" + for i, cshow in enumerate(toshow): + i_show = i + 1 # Start at more human readable number 1 (not 0) + log().debug('Showing allSeries[%s], series %s)' % (i_show, allSeries[i]['seriesname'])) + if i == 0: + extra = " (default)" + else: + extra = "" + + print "%s -> %s [%s] # http://thetvdb.com/?tab=series&id=%s&lid=%s%s" % ( + i_show, + cshow['seriesname'].encode("UTF-8", "ignore"), + cshow['language'].encode("UTF-8", "ignore"), + str(cshow['id']), + cshow['lid'], + extra + ) + + def selectSeries(self, allSeries): + self._displaySeries(allSeries) + + if len(allSeries) == 1: + # Single result, return it! + print "Automatically selecting only result" + return allSeries[0] + + if self.config['select_first'] is True: + print "Automatically returning first search result" + return allSeries[0] + + while True: # return breaks this loop + try: + print "Enter choice (first number, return for default, 'all', ? for help):" + ans = raw_input() + except KeyboardInterrupt: + raise tvdb_userabort("User aborted (^c keyboard interupt)") + except EOFError: + raise tvdb_userabort("User aborted (EOF received)") + + log().debug('Got choice of: %s' % (ans)) + try: + selected_id = int(ans) - 1 # The human entered 1 as first result, not zero + except ValueError: # Input was not number + if len(ans.strip()) == 0: + # Default option + log().debug('Default option, returning first series') + return allSeries[0] + if ans == "q": + log().debug('Got quit command (q)') + raise tvdb_userabort("User aborted ('q' quit command)") + elif ans == "?": + print "## Help" + print "# Enter the number that corresponds to the correct show." + print "# a - display all results" + print "# all - display all results" + print "# ? - this help" + print "# q - abort tvnamer" + print "# Press return with no input to select first result" + elif ans.lower() in ["a", "all"]: + self._displaySeries(allSeries, limit = None) + else: + log().debug('Unknown keypress %s' % (ans)) + else: + log().debug('Trying to return ID: %d' % (selected_id)) + try: + return allSeries[selected_id] + except IndexError: + log().debug('Invalid show number entered!') + print "Invalid number (%s) selected!" + self._displaySeries(allSeries) +