Merge pull request #2549 from fuzeman/tv_searcher

[TV] Searcher cleanup and matcher updates

couchpotato/core/media/__init__.py

@@ -1,5 +1,6 @@
 from couchpotato import get_session
 from couchpotato.core.event import addEvent, fireEventAsync, fireEvent
+from couchpotato.core.helpers.variable import mergeDicts
 from couchpotato.core.plugins.base import Plugin
 from couchpotato.core.settings.model import Media
 
@@ -17,6 +18,13 @@ class MediaBase(Plugin):
         'category': {},
     }
 
+    search_dict = mergeDicts(default_dict, {
+        'library': {
+            'related_libraries': {},
+            'root_library': {}
+        },
+    })
+
     def initType(self):
         addEvent('media.types', self.getType)
 
@@ -28,7 +36,7 @@ class MediaBase(Plugin):
         def onComplete():
             db = get_session()
             media = db.query(Media).filter_by(id = id).first()
-            fireEventAsync('%s.searcher.single' % media.type, media.to_dict(self.default_dict), on_complete = self.createNotifyFront(id))
+            fireEventAsync('%s.searcher.single' % media.type, media.to_dict(self.search_dict), on_complete = self.createNotifyFront(id))
             db.expire_all()
 
         return onComplete

couchpotato/core/media/show/library/episode/main.py

@@ -17,10 +17,39 @@ class EpisodeLibraryPlugin(LibraryBase):
     default_dict = {'titles': {}, 'files':{}}
 
     def __init__(self):
+        addEvent('library.identifier', self.identifier)
         addEvent('library.add.episode', self.add)
         addEvent('library.update.episode', self.update)
         addEvent('library.update.episode_release_date', self.updateReleaseDate)
 
+    def identifier(self, library):
+        if library.get('type') != 'episode':
+            return
+
+        identifier = {
+            'season': None,
+            'episode': None
+        }
+
+        scene_map = library['info'].get('map_episode', {}).get('scene')
+
+        if scene_map:
+            # Use scene mappings if they are available
+            identifier['season'] = scene_map.get('season')
+            identifier['episode'] = scene_map.get('episode')
+        else:
+            # Fallback to normal season/episode numbers
+            identifier['season'] = library.get('season_number')
+            identifier['episode'] = library.get('episode_number')
+
+
+        # Cast identifiers to integers
+        # TODO this will need changing to support identifiers with trailing 'a', 'b' characters
+        identifier['season'] = tryInt(identifier['season'], None)
+        identifier['episode'] = tryInt(identifier['episode'], None)
+
+        return identifier
+
     def add(self, attrs = {}, update_after = True):
         type = attrs.get('type', 'episode')
         primary_provider = attrs.get('primary_provider', 'thetvdb')

couchpotato/core/media/show/library/season/main.py

@@ -17,10 +17,22 @@ class SeasonLibraryPlugin(LibraryBase):
     default_dict = {'titles': {}, 'files':{}}
 
     def __init__(self):
+        addEvent('library.identifier', self.identifier)
         addEvent('library.add.season', self.add)
         addEvent('library.update.season', self.update)
         addEvent('library.update.season_release_date', self.updateReleaseDate)
 
+    def identifier(self, library):
+        if library.get('type') != 'season':
+            return
+
+        season_num = tryInt(library['season_number'], None)
+
+        return {
+            'season': season_num,
+            'episode': None
+        }
+
     def add(self, attrs = {}, update_after = True):
         type = attrs.get('type', 'season')
         primary_provider = attrs.get('primary_provider', 'thetvdb')

couchpotato/core/media/show/searcher/main.py

@@ -1,10 +1,10 @@
-from couchpotato import get_session, Env
+from couchpotato import Env, get_session
 from couchpotato.core.event import addEvent, fireEvent
-from couchpotato.core.helpers.variable import getTitle, tryInt
+from couchpotato.core.helpers.variable import getTitle, tryInt, toIterable
 from couchpotato.core.logger import CPLog
 from couchpotato.core.media._base.searcher.main import SearchSetupError
 from couchpotato.core.plugins.base import Plugin
-from couchpotato.core.settings.model import Media, Library
+from couchpotato.core.settings.model import Media
 from qcond import QueryCondenser
 from qcond.helpers import simplify
 
@@ -13,6 +13,8 @@ log = CPLog(__name__)
 
 class ShowSearcher(Plugin):
 
+    type = ['show', 'season', 'episode']
+
     in_progress = False
 
     # TODO come back to this later, think this could be handled better
@@ -29,16 +31,17 @@ class ShowSearcher(Plugin):
 
         self.query_condenser = QueryCondenser()
 
-        addEvent('show.searcher.single', self.single)
+        for type in toIterable(self.type):
+            addEvent('%s.searcher.single' % type, self.single)
+
         addEvent('searcher.get_search_title', self.getSearchTitle)
 
         addEvent('searcher.correct_match', self.correctMatch)
         addEvent('searcher.correct_release', self.correctRelease)
 
-        addEvent('searcher.get_media_identifier', self.getMediaIdentifier)
-        addEvent('searcher.get_media_root', self.getMediaRoot)
-
     def single(self, media, search_protocols = None, manual = False):
+        show, season, episode = self.getLibraries(media['library'])
+
         if media['type'] == 'show':
             # TODO handle show searches (scan all seasons)
             return
@@ -69,8 +72,7 @@ class ShowSearcher(Plugin):
             #fireEvent('episode.delete', episode['id'], single = True)
             return
 
-        show, season, episode = self.getMedia(media)
+        if not show or not season:
-        if show is None or season is None:
             log.error('Unable to find show or season library in database, missing required data for searching')
             return
 
@@ -93,7 +95,7 @@ class ShowSearcher(Plugin):
             # Don't search for quality lower then already available.
             if has_better_quality is 0:
 
-                log.info('Search for %s S%02d%s in %s', (getTitle(show), season.season_number, "E%02d" % episode.episode_number if episode else "", quality_type['quality']['label']))
+                log.info('Search for %s S%02d%s in %s', (getTitle(show), season['season_number'], "E%02d" % episode['episode_number'] if episode else "", quality_type['quality']['label']))
                 quality = fireEvent('quality.single', identifier = quality_type['quality']['identifier'], single = True)
 
                 results = fireEvent('searcher.search', search_protocols, media, quality, single = True)
@@ -135,15 +137,16 @@ class ShowSearcher(Plugin):
         if media['type'] not in ['show', 'season', 'episode']:
             return
 
-        show, season, episode = self.getMedia(media)
+        show, season, episode = self.getLibraries(media['library'])
-        if show is None:
+
+        if not show:
             return None
 
         titles = []
 
         # Add season map_names if they exist
-        if season is not None and 'map_names' in show.info:
+        if season is not None and 'map_names' in show['info']:
-            season_names = show.info['map_names'].get(str(season.season_number), {})
+            season_names = show['info']['map_names'].get(str(season['season_number']), {})
 
             # Add titles from all locations
             # TODO only add name maps from a specific location
@@ -151,7 +154,7 @@ class ShowSearcher(Plugin):
                 titles += [name for name in names if name not in titles]
 
         # Add show titles
-        titles += [title.title for title in show.titles if title.title not in titles]
+        titles += [title['title'] for title in show['titles'] if title['title'] not in titles]
 
         # Use QueryCondenser to build a list of optimal search titles
         condensed_titles = self.query_condenser.distinct(titles)
@@ -170,9 +173,9 @@ class ShowSearcher(Plugin):
             return None
 
         # Add the identifier to search title
-        # TODO supporting other identifier formats
+        identifier = fireEvent('library.identifier', media['library'], single = True)
-        identifier = fireEvent('searcher.get_media_identifier', media['library'], single = True)
 
+        # TODO this needs to support other identifier formats
         if identifier['season']:
             title += ' S%02d' % identifier['season']
 
@@ -195,11 +198,7 @@ class ShowSearcher(Plugin):
         if not fireEvent('searcher.correct_words', release['name'], media, single = True):
             return False
 
-        show, season, episode = self.getMedia(media)
+        # TODO Matching is quite costly, maybe we should be caching release matches somehow? (also look at caper optimizations)
-        if show is None or season is None:
-            log.error('Unable to find show or season library in database, missing required data for searching')
-            return
-
         match = fireEvent('matcher.best', release, media, quality, single = True)
         if match:
             return match.weight
@@ -224,68 +223,24 @@ class ShowSearcher(Plugin):
 
         return True
 
-    # TODO move this somewhere else
+    def getLibraries(self, library):
-    def getMediaIdentifier(self, media_library):
+        if 'related_libraries' not in library:
-        if media_library['type'] not in ['show', 'season', 'episode']:
+            log.warning("'related_libraries' missing from media library, unable to continue searching")
-            return None
+            return None, None, None
-
-        identifier = {
-            'season': None,
-            'episode': None
-        }
 
-        if media_library['type'] == 'episode':
+        libraries = library['related_libraries']
-            map_episode = media_library['info'].get('map_episode')
 
-            if map_episode and 'scene' in map_episode:
+        # Get libraries and return lists only if there is multiple items
-                identifier['season'] = map_episode['scene'].get('season')
+        show = libraries.get('show', [])
-                identifier['episode'] = map_episode['scene'].get('episode')
+        if len(show) <= 1:
-            else:
+            show = show[0] if len(show) else None
-                # TODO xem mapping?
-                identifier['season'] = media_library.get('season_number')
-                identifier['episode'] = media_library.get('episode_number')
 
-        if media_library['type'] == 'season':
+        season = libraries.get('season', [])
-            identifier['season'] = media_library.get('season_number')
+        if len(season) <= 1:
+            season = season[0] if len(season) else None
 
-        # Try cast identifier values to integers
+        episode = libraries.get('episode', [])
-        identifier['season'] = tryInt(identifier['season'], None)
+        if len(episode) <= 1:
-        identifier['episode'] = tryInt(identifier['episode'], None)
+            episode = episode[0] if len(episode) else None
-
-        return identifier
-
-    # TODO move this somewhere else
-    def getMediaRoot(self, media):
-        if media['type'] not in ['show', 'season', 'episode']:
-            return None
-
-        show, season, episode = self.getMedia(media)
-        if show is None or season is None:
-            log.error('Unable to find show or season library in database, missing required data for searching')
-            return
-
-        return show.to_dict()
-
-    # TODO move this somewhere else
-    def getMedia(self, media):
-        db = get_session()
-
-        media_library = db.query(Library).filter_by(id = media['library_id']).first()
-
-        show = None
-        season = None
-        episode = None
-
-        if media['type'] == 'episode':
-            show = media_library.parent.parent
-            season = media_library.parent
-            episode = media_library
-
-        if media['type'] == 'season':
-            show = media_library.parent
-            season = media_library
-
-        if media['type'] == 'show':
-            show = media_library
 
         return show, season, episode

couchpotato/core/plugins/matcher/main.py

@@ -33,7 +33,7 @@ class Matcher(Plugin):
             if fireEvent('searcher.correct_match', chain, release, media, quality, single = True):
                 return chain
 
-        return None
+        return False
 
     def chainMatch(self, chain, group, tags):
         found_tags = []
@@ -50,7 +50,7 @@ class Matcher(Plugin):
         return set([key for key, value in tags.items() if None not in value]) == set(found_tags)
 
     def correctIdentifier(self, chain, media):
-        required_id = fireEvent('searcher.get_media_identifier', media['library'], single = True)
+        required_id = fireEvent('library.identifier', media['library'], single = True)
 
         if 'identifier' not in chain.info:
             return False
@@ -73,7 +73,7 @@ class Matcher(Plugin):
         return True
 
     def correctTitle(self, chain, media):
-        root_library = fireEvent('searcher.get_media_root', media['library'], single = True)
+        root_library = media['library']['root_library']
 
         if 'show_name' not in chain.info or not len(chain.info['show_name']):
             log.info('Wrong: missing show name in parsed result')

couchpotato/core/providers/base.py

@@ -105,7 +105,6 @@ class YarrProvider(Provider):
     type = 'movie'
 
     cat_ids = {}
-    cat_ids_structure = None
     cat_backup_id = None
 
     sizeGb = ['gb', 'gib']
@@ -250,33 +249,9 @@ class YarrProvider(Provider):
 
         return 0
 
-    def _discoverCatIdStructure(self):
+    def getCatId(self, identifier):
-        # Discover cat_ids structure (single or groups)
-        for group_name, group_cat_ids in self.cat_ids:
-            if len(group_cat_ids) > 0:
-                if type(group_cat_ids[0]) is tuple:
-                    self.cat_ids_structure = 'group'
-                if type(group_cat_ids[0]) is str:
-                    self.cat_ids_structure = 'single'
 
-    def getCatId(self, identifier, group = None):
+        for ids, qualities in self.cat_ids:
-
-        cat_ids = self.cat_ids
-
-        if not self.cat_ids_structure:
-            self._discoverCatIdStructure()
-
-        # If cat_ids is in a 'groups' structure, locate the media group
-        if self.cat_ids_structure == 'group':
-            if not group:
-                raise ValueError("group is required on group cat_ids structure")
-
-            for group_type, group_cat_ids in cat_ids:
-                if group in toIterable(group_type):
-                    cat_ids = group_cat_ids
-
-        for cats in cat_ids:
-            ids, qualities = cats
             if identifier in qualities:
                 return ids
 

couchpotato/core/providers/info/base.py

@@ -6,4 +6,12 @@ class MovieProvider(Provider):
 
 
 class ShowProvider(Provider):
-    type = ['season', 'episode']
+    type = 'show'
+
+
+class SeasonProvider(Provider):
+    type = 'season'
+
+
+class EpisodeProvider(Provider):
+    type = 'episode'

couchpotato/core/providers/torrent/iptorrents/main.py

@@ -3,7 +3,7 @@ from couchpotato.core.helpers.encoding import tryUrlencode
 from couchpotato.core.helpers.variable import tryInt
 from couchpotato.core.logger import CPLog
 from couchpotato.core.providers.base import MultiProvider
-from couchpotato.core.providers.info.base import MovieProvider, ShowProvider
+from couchpotato.core.providers.info.base import MovieProvider, SeasonProvider, EpisodeProvider
 from couchpotato.core.providers.torrent.base import TorrentProvider
 import traceback
 
@@ -13,7 +13,7 @@ log = CPLog(__name__)
 class IPTorrents(MultiProvider):
 
     def getTypes(self):
-        return [Movie, Show]
+        return [Movie, Season, Episode]
 
 
 class Base(TorrentProvider):
@@ -29,13 +29,16 @@ class Base(TorrentProvider):
     http_time_between_calls = 1 #seconds
     cat_backup_id = None
 
-    def _buildUrl(self, query, quality_identifier, cat_ids_group = None):
+    def buildUrl(self, title, media, quality):
+        return self._buildUrl(title.replace(':', ''), quality['identifier'])
+
+    def _buildUrl(self, query, quality_identifier):
 
-        cat_ids = self.getCatId(quality_identifier, cat_ids_group)
+        cat_ids = self.getCatId(quality_identifier)
 
-        if not cat_ids or not len(cat_ids):
+        if not cat_ids:
-            log.warning('Unable to find category for quality %s', quality_identifier)
+            log.warning('Unable to find category ids for identifier "%s"', quality_identifier)
-            return
+            return None
 
         return self.urls['search'] % ("&".join(("l%d=" % x) for x in cat_ids), tryUrlencode(query).replace('%', '%%'))
 
@@ -133,20 +136,16 @@ class Movie(MovieProvider, Base):
         return self._buildUrl(query, quality['identifier'])
 
 
-class Show(ShowProvider, Base):
+class Season(SeasonProvider, Base):
 
     cat_ids = [
-        ('season', [
+        ([65], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']),
-            ([65], ['hdtv_sd', 'hdtv_720p', 'webdl_720p', 'webdl_1080p']),
-        ]),
-        ('episode', [
-            ([5], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']),
-            ([4, 78, 79], ['hdtv_sd'])
-        ])
     ]
 
-    def buildUrl(self, title, media, quality):
-        if media['type'] not in ['season', 'episode']:
-            return
 
-        return self._buildUrl(title.replace(':', ''), quality['identifier'], media['type'])
+class Episode(EpisodeProvider, Base):
+
+    cat_ids = [
+        ([5], ['hdtv_720p', 'webdl_720p', 'webdl_1080p']),
+        ([4, 78, 79], ['hdtv_sd'])
+    ]

couchpotato/core/settings/model.py

@@ -90,6 +90,7 @@ class Media(Entity):
     files = ManyToMany('File', cascade = 'all, delete-orphan', single_parent = True)
 
 
+
 class Library(Entity):
     """"""
     using_options(inheritance = 'multi')
@@ -112,6 +113,79 @@ class Library(Entity):
     parent = ManyToOne('Library')
     children = OneToMany('Library')
 
+    def getRelated(self, include_parents = True, include_self = True, include_children = True, merge=False):
+        libraries = []
+
+        if include_parents and self.parent is not None:
+            libraries += self.parent.getRelated(include_children = False)
+
+        if include_self:
+            libraries += [(self.type, self)]
+
+        if include_children:
+            for child in self.children:
+                libraries += child.getRelated(include_parents = False)
+
+        # Return plain results if we aren't merging the results
+        if not merge:
+            return libraries
+
+        # Merge the results into a dict ({type: [<library>,...]})
+        root_key = None
+        results = {}
+
+        for key, library in libraries:
+            if root_key is None:
+                root_key = key
+
+            if key not in results:
+                results[key] = []
+
+            results[key].append(library)
+
+        return root_key, results
+
+    def to_dict(self, deep = None, exclude = None):
+        if not exclude: exclude = []
+        if not deep: deep = {}
+
+        include_related = False
+        include_root = False
+
+        if any(x in deep for x in ['related_libraries', 'root_library']):
+            deep = deep.copy()
+
+            include_related = deep.pop('related_libraries', None) is not None
+            include_root = deep.pop('root_library', None) is not None
+
+        orig_dict = super(Library, self).to_dict(deep = deep, exclude = exclude)
+
+        # Include related libraries (parents and children)
+        if include_related:
+            # Fetch child and parent libraries and determine root type
+            root_key, related_libraries = self.getRelated(include_self = False, merge=True)
+
+            # Serialize libraries
+            related_libraries = dict([
+                (key, [library.to_dict(deep, exclude) for library in libraries])
+                for (key, libraries) in related_libraries.items()
+            ])
+
+            # Add a reference to the current library dict into related_libraries
+            if orig_dict['type'] not in related_libraries:
+                related_libraries[orig_dict['type']] = []
+
+            related_libraries[orig_dict['type']].append(orig_dict)
+
+            # Update the dict for this library
+            orig_dict['related_libraries'] = related_libraries
+
+            if include_root:
+                root_library = related_libraries.get(root_key)
+                orig_dict['root_library'] = root_library[0] if len(root_library) else None
+
+        return orig_dict
+
 
 class ShowLibrary(Library, DictMixin):
     using_options(inheritance = 'multi')

libs/caper/__init__.py

@@ -19,7 +19,7 @@ from caper.parsers.anime import AnimeParser
 from caper.parsers.scene import SceneParser
 
 
-__version_info__ = ('0', '2', '2')
+__version_info__ = ('0', '2', '3')
 __version_branch__ = 'master'
 
 __version__ = "%s%s" % (
@@ -43,10 +43,10 @@ CL_END = 1
 
 
 class Caper(object):
-    def __init__(self):
+    def __init__(self, debug=False):
         self.parsers = {
-            'scene': SceneParser(),
+            'scene': SceneParser(debug),
-            'anime': AnimeParser()
+            'anime': AnimeParser(debug)
         }
 
     def _closure_split(self, name):

libs/caper/parsers/anime.py

@@ -53,8 +53,8 @@ PATTERN_GROUPS = [
 
 
 class AnimeParser(Parser):
-    def __init__(self):
+    def __init__(self, debug=False):
-        super(AnimeParser, self).__init__(PATTERN_GROUPS)
+        super(AnimeParser, self).__init__(PATTERN_GROUPS, debug)
 
     def capture_group(self, fragment):
         match = REGEX_GROUP.match(fragment.value)

libs/caper/parsers/base.py

@@ -18,7 +18,9 @@ from caper.result import CaperResult, CaperClosureNode
 
 
 class Parser(object):
-    def __init__(self, pattern_groups):
+    def __init__(self, pattern_groups, debug=False):
+        self.debug = debug
+
         self.matcher = FragmentMatcher(pattern_groups)
 
         self.closures = None

libs/caper/parsers/scene.py

@@ -98,8 +98,8 @@ PATTERN_GROUPS = [
 
 
 class SceneParser(Parser):
-    def __init__(self):
+    def __init__(self, debug=False):
-        super(SceneParser, self).__init__(PATTERN_GROUPS)
+        super(SceneParser, self).__init__(PATTERN_GROUPS, debug)
 
     def capture_group(self, fragment):
         if fragment.left_sep == '-' and not fragment.right:
@@ -133,6 +133,9 @@ class SceneParser(Parser):
         return self.result
 
     def print_tree(self, heads):
+        if not self.debug:
+            return
+
         for head in heads:
             head = head if type(head) is list else [head]
 

libs/logr/__init__.py

@@ -32,8 +32,11 @@ class Logr(object):
     loggers = {}
     handler = None
 
+    trace_origin = False
+    name = "Logr"
+
     @staticmethod
-    def configure(level=logging.WARNING, handler=None, formatter=None):
+    def configure(level=logging.WARNING, handler=None, formatter=None, trace_origin=False, name="Logr"):
         """Configure Logr
 
         @param handler: Logger message handler
@@ -52,6 +55,9 @@ class Logr(object):
         handler.setLevel(level)
         Logr.handler = handler
 
+        Logr.trace_origin = trace_origin
+        Logr.name = name
+
     @staticmethod
     def configure_check():
         if Logr.handler is None:
@@ -65,7 +71,29 @@ class Logr(object):
             return "<unknown>"
 
     @staticmethod
+    def get_frame_class(frame):
+        if len(frame.f_code.co_varnames) <= 0:
+            return None
+
+        farg = frame.f_code.co_varnames[0]
+
+        if farg not in frame.f_locals:
+            return None
+
+        if farg == 'self':
+            return frame.f_locals[farg].__class__
+
+        if farg == 'cls':
+            return frame.f_locals[farg]
+
+        return None
+
+
+    @staticmethod
     def get_logger_name():
+        if not Logr.trace_origin:
+            return Logr.name
+
         stack = inspect.stack()
 
         for x in xrange_six(len(stack)):
@@ -73,20 +101,16 @@ class Logr(object):
             name = None
 
             # Try find name of function defined inside a class
-            if len(frame.f_code.co_varnames) > 0:
+            frame_class = Logr.get_frame_class(frame)
-                self_argument = frame.f_code.co_varnames[0]
-
-                if self_argument == 'self' and self_argument in frame.f_locals:
-                    instance = frame.f_locals[self_argument]
 
-                    class_ = instance.__class__
+            if frame_class:
-                    class_name = class_.__name__
+                class_name = frame_class.__name__
-                    module_name = class_.__module__
+                module_name = frame_class.__module__
 
-                    if module_name != '__main__':
+                if module_name != '__main__':
-                        name = module_name + '.' + class_name
+                    name = module_name + '.' + class_name
-                    else:
+                else:
-                        name = class_name
+                    name = class_name
 
             # Try find name of function defined outside of a class
             if name is None: