SickRage now stores the XEM scene data to the main DB under tv_episodes for faster lookups and better linkages with epiosde objects, insures we have scene data on hand when we want it!

Adding functions to lookup scene season and ep info via scene absolute numbers.

We now try and create a show object from parsed release info before we consider it a valid parsed result to insure we even have the show in our show list, also used to validate anime releases.

Misc bug fixes here and there.

lib/adba/aniDBAbstracter.py

@@ -103,7 +103,7 @@ class aniDBabstractObject(object):
 
 
 class Anime(aniDBabstractObject):
-    def __init__(self, aniDB, name=None, aid=None, tvdbid=None, tvrageid=None, paramsA=None, autoCorrectName=False, load=False):
+    def __init__(self, aniDB, name=None, aid=None, tvdbid=None, paramsA=None, autoCorrectName=False, load=False):
 
         self.maper = AniDBMaper()
         self.tvDBMap = TvDBMap()

lib/adba/aniDBcommands.py

@@ -19,8 +19,10 @@ from threading import Lock
 from aniDBresponses import *
 from aniDBerrors import *
 
+
 class Command:
     queue = {None: None}
+
     def __init__(self, command, **parameters):
         self.command = command
         self.parameters = parameters
@@ -74,27 +76,33 @@ class Command:
     def cache(self, interface, database):
         pass
 
+
 #first run
 class AuthCommand(Command):
     def __init__(self, username, password, protover, client, clientver, nat=None, comp=None, enc=None, mtu=None):
-		parameters={'user':username,'pass':password,'protover':protover,'client':client,'clientver':clientver,'nat':nat,'comp':comp,'enc':enc,'mtu':mtu}
+        parameters = {'user': username, 'pass': password, 'protover': protover, 'client': client,
+                      'clientver': clientver, 'nat': nat, 'comp': comp, 'enc': enc, 'mtu': mtu}
         Command.__init__(self, 'AUTH', **parameters)
 
+
 class LogoutCommand(Command):
     def __init__(self):
         Command.__init__(self, 'LOGOUT')
 
+
 #third run (at the same time as second)
 class PushCommand(Command):
     def __init__(self, notify, msg, buddy=None):
         parameters = {'notify': notify, 'msg': msg, 'buddy': buddy}
         Command.__init__(self, 'PUSH', **parameters)
 
+
 class PushAckCommand(Command):
     def __init__(self, nid):
         parameters = {'nid': nid}
         Command.__init__(self, 'PUSHACK', **parameters)
 
+
 class NotifyAddCommand(Command):
     def __init__(self, aid=None, gid=None, type=None, priority=None):
         if not (aid or gid) or (aid and gid):
@@ -102,25 +110,30 @@ class NotifyAddCommand(Command):
         parameters = {'aid': aid, "gid": gid, "type": type, "priority": priority}
         Command.__init__(self, 'NOTIFICATIONADD', **parameters)
 
+
 class NotifyCommand(Command):
     def __init__(self, buddy=None):
         parameters = {'buddy': buddy}
         Command.__init__(self, 'NOTIFY', **parameters)
 
+
 class NotifyListCommand(Command):
     def __init__(self):
         Command.__init__(self, 'NOTIFYLIST')
 
+
 class NotifyGetCommand(Command):
     def __init__(self, type, id):
         parameters = {'type': type, 'id': id}
         Command.__init__(self, 'NOTIFYGET', **parameters)
 
+
 class NotifyAckCommand(Command):
     def __init__(self, type, id):
         parameters = {'type': type, 'id': id}
         Command.__init__(self, 'NOTIFYACK', **parameters)
 
+
 class BuddyAddCommand(Command):
     def __init__(self, uid=None, uname=None):
         if not (uid or uname) or (uid and uname):
@@ -128,31 +141,37 @@ class BuddyAddCommand(Command):
         parameters = {'uid': uid, 'uname': uname.lower()}
         Command.__init__(self, 'BUDDYADD', **parameters)
 
+
 class BuddyDelCommand(Command):
     def __init__(self, uid):
         parameters = {'uid': uid}
         Command.__init__(self, 'BUDDYDEL', **parameters)
 
+
 class BuddyAcceptCommand(Command):
     def __init__(self, uid):
         parameters = {'uid': uid}
         Command.__init__(self, 'BUDDYACCEPT', **parameters)
 
+
 class BuddyDenyCommand(Command):
     def __init__(self, uid):
         parameters = {'uid': uid}
         Command.__init__(self, 'BUDDYDENY', **parameters)
 
+
 class BuddyListCommand(Command):
     def __init__(self, startat):
         parameters = {'startat': startat}
         Command.__init__(self, 'BUDDYLIST', **parameters)
 
+
 class BuddyStateCommand(Command):
     def __init__(self, startat):
         parameters = {'startat': startat}
         Command.__init__(self, 'BUDDYSTATE', **parameters)
 
+
 #first run
 class AnimeCommand(Command):
     def __init__(self, aid=None, aname=None, amask=None):
@@ -161,6 +180,7 @@ class AnimeCommand(Command):
         parameters = {'aid': aid, 'aname': aname, 'amask': amask}
         Command.__init__(self, 'ANIME', **parameters)
 
+
 class EpisodeCommand(Command):
     def __init__(self, eid=None, aid=None, aname=None, epno=None):
         if not (eid or ((aname or aid) and epno)) or (aname and aid) or (eid and (aname or aid or epno)):
@@ -168,13 +188,21 @@ class EpisodeCommand(Command):
         parameters = {'eid': eid, 'aid': aid, 'aname': aname, 'epno': epno}
         Command.__init__(self, 'EPISODE', **parameters)
 
+
 class FileCommand(Command):
-	def __init__(self,fid=None,size=None,ed2k=None,aid=None,aname=None,gid=None,gname=None,epno=None,fmask=None,amask=None):
-		if not (fid or (size and ed2k) or ((aid or aname) and (gid or gname) and epno)) or (fid and (size or ed2k or aid or aname or gid or gname or epno)) or ((size and ed2k) and (fid or aid or aname or gid or gname or epno)) or (((aid or aname) and (gid or gname) and epno) and (fid or size or ed2k)) or (aid and aname) or (gid and gname):
+    def __init__(self, fid=None, size=None, ed2k=None, aid=None, aname=None, gid=None, gname=None, epno=None,
+                 fmask=None, amask=None):
+        if not (fid or (size and ed2k) or ((aid or aname) and (gid or gname) and epno)) or (
+            fid and (size or ed2k or aid or aname or gid or gname or epno)) or (
+            (size and ed2k) and (fid or aid or aname or gid or gname or epno)) or (
+            ((aid or aname) and (gid or gname) and epno) and (fid or size or ed2k)) or (aid and aname) or (
+            gid and gname):
             raise AniDBIncorrectParameterError, "You must provide <fid XOR size+ed2k XOR a(id|name)+g(id|name)+epno> for FILE command"
-		parameters={'fid':fid,'size':size,'ed2k':ed2k,'aid':aid,'aname':aname,'gid':gid,'gname':gname,'epno':epno,'fmask':fmask,'amask':amask}
+        parameters = {'fid': fid, 'size': size, 'ed2k': ed2k, 'aid': aid, 'aname': aname, 'gid': gid, 'gname': gname,
+                      'epno': epno, 'fmask': fmask, 'amask': amask}
         Command.__init__(self, 'FILE', **parameters)
 
+
 class GroupCommand(Command):
     def __init__(self, gid=None, gname=None):
         if not (gid or gname) or (gid and gname):
@@ -182,6 +210,7 @@ class GroupCommand(Command):
         parameters = {'gid': gid, 'gname': gname}
         Command.__init__(self, 'GROUP', **parameters)
 
+
 class GroupstatusCommand(Command):
     def __init__(self, aid=None, status=None):
         if not aid:
@@ -189,6 +218,7 @@ class GroupstatusCommand(Command):
         parameters = {'aid': aid, 'status': status}
         Command.__init__(self, 'GROUPSTATUS', **parameters)
 
+
 class ProducerCommand(Command):
     def __init__(self, pid=None, pname=None):
         if not (pid or pname) or (pid and pname):
@@ -233,11 +263,17 @@ class ProducerCommand(Command):
 
             db.insert('ptb', names, valueholders, *values)
 
+
 class MyListCommand(Command):
     def __init__(self, lid=None, fid=None, size=None, ed2k=None, aid=None, aname=None, gid=None, gname=None, epno=None):
-		if not (lid or fid or (size and ed2k) or (aid or aname)) or (lid and (fid or size or ed2k or aid or aname or gid or gname or epno)) or (fid and (lid or size or ed2k or aid or aname or gid or gname or epno)) or ((size and ed2k) and (lid or fid or aid or aname or gid or gname or epno)) or ((aid or aname) and (lid or fid or size or ed2k)) or (aid and aname) or (gid and gname):
+        if not (lid or fid or (size and ed2k) or (aid or aname)) or (
+            lid and (fid or size or ed2k or aid or aname or gid or gname or epno)) or (
+            fid and (lid or size or ed2k or aid or aname or gid or gname or epno)) or (
+            (size and ed2k) and (lid or fid or aid or aname or gid or gname or epno)) or (
+            (aid or aname) and (lid or fid or size or ed2k)) or (aid and aname) or (gid and gname):
             raise AniDBIncorrectParameterError, "You must provide <lid XOR fid XOR size+ed2k XOR a(id|name)+g(id|name)+epno> for MYLIST command"
-		parameters={'lid':lid,'fid':fid,'size':size,'ed2k':ed2k,'aid':aid,'aname':aname,'gid':gid,'gname':gname,'epno':epno}
+        parameters = {'lid': lid, 'fid': fid, 'size': size, 'ed2k': ed2k, 'aid': aid, 'aname': aname, 'gid': gid,
+                      'gname': gname, 'epno': epno}
         Command.__init__(self, 'MYLIST', **parameters)
 
     def cached(self, intr, db):
@@ -320,24 +356,39 @@ class MyListCommand(Command):
 
             db.insert('ltb', names, valueholders, *values)
 
+
 class MyListAddCommand(Command):
-	def __init__(self,lid=None,fid=None,size=None,ed2k=None,aid=None,aname=None,gid=None,gname=None,epno=None,edit=None,state=None,viewed=None,source=None,storage=None,other=None):
-		if not (lid or fid or (size and ed2k) or ((aid or aname) and (gid or gname))) or (lid and (fid or size or ed2k or aid or aname or gid or gname or epno)) or (fid and (lid or size or ed2k or aid or aname or gid or gname or epno)) or ((size and ed2k) and (lid or fid or aid or aname or gid or gname or epno)) or (((aid or aname) and (gid or gname)) and (lid or fid or size or ed2k)) or (aid and aname) or (gid and gname) or (lid and not edit):
+    def __init__(self, lid=None, fid=None, size=None, ed2k=None, aid=None, aname=None, gid=None, gname=None, epno=None,
+                 edit=None, state=None, viewed=None, source=None, storage=None, other=None):
+        if not (lid or fid or (size and ed2k) or ((aid or aname) and (gid or gname))) or (
+            lid and (fid or size or ed2k or aid or aname or gid or gname or epno)) or (
+            fid and (lid or size or ed2k or aid or aname or gid or gname or epno)) or (
+            (size and ed2k) and (lid or fid or aid or aname or gid or gname or epno)) or (
+            ((aid or aname) and (gid or gname)) and (lid or fid or size or ed2k)) or (aid and aname) or (
+            gid and gname) or (lid and not edit):
             raise AniDBIncorrectParameterError, "You must provide <lid XOR fid XOR size+ed2k XOR a(id|name)+g(id|name)+epno> for MYLISTADD command"
-		parameters={'lid':lid,'fid':fid,'size':size,'ed2k':ed2k,'aid':aid,'aname':aname,'gid':gid,'gname':gname,'epno':epno,'edit':edit,'state':state,'viewed':viewed,'source':source,'storage':storage,'other':other}
+        parameters = {'lid': lid, 'fid': fid, 'size': size, 'ed2k': ed2k, 'aid': aid, 'aname': aname, 'gid': gid,
+                      'gname': gname, 'epno': epno, 'edit': edit, 'state': state, 'viewed': viewed, 'source': source,
+                      'storage': storage, 'other': other}
         Command.__init__(self, 'MYLISTADD', **parameters)
 
+
 class MyListDelCommand(Command):
     def __init__(self, lid=None, fid=None, aid=None, aname=None, gid=None, gname=None, epno=None):
-		if not (lid or fid or ((aid or aname) and (gid or gname) and epno)) or (lid and (fid or aid or aname or gid or gname or epno)) or (fid and (lid or aid or aname or gid or gname or epno)) or (((aid or aname) and (gid or gname) and epno) and (lid or fid)) or (aid and aname) or (gid and gname):
+        if not (lid or fid or ((aid or aname) and (gid or gname) and epno)) or (
+            lid and (fid or aid or aname or gid or gname or epno)) or (
+            fid and (lid or aid or aname or gid or gname or epno)) or (
+            ((aid or aname) and (gid or gname) and epno) and (lid or fid)) or (aid and aname) or (gid and gname):
             raise AniDBIncorrectParameterError, "You must provide <lid+edit=1 XOR fid XOR a(id|name)+g(id|name)+epno> for MYLISTDEL command"
         parameters = {'lid': lid, 'fid': fid, 'aid': aid, 'aname': aname, 'gid': gid, 'gname': gname, 'epno': epno}
         Command.__init__(self, 'MYLISTDEL', **parameters)
 
+
 class MyListStatsCommand(Command):
     def __init__(self):
         Command.__init__(self, 'MYLISTSTATS')
 
+
 class VoteCommand(Command):
     def __init__(self, type, id=None, name=None, value=None, epno=None):
         if not (id or name) or (id and name):
@@ -345,15 +396,18 @@ class VoteCommand(Command):
         parameters = {'type': type, 'id': id, 'name': name, 'value': value, 'epno': epno}
         Command.__init__(self, 'VOTE', **parameters)
 
+
 class RandomAnimeCommand(Command):
     def __init__(self, type):
         parameters = {'type': type}
         Command.__init__(self, 'RANDOMANIME', **parameters)
 
+
 class PingCommand(Command):
     def __init__(self):
         Command.__init__(self, 'PING')
 
+
 #second run
 class EncryptCommand(Command):
     def __init__(self, user, apipassword, type):
@@ -361,11 +415,13 @@ class EncryptCommand(Command):
         parameters = {'user': user.lower(), 'type': type}
         Command.__init__(self, 'ENCRYPT', **parameters)
 
+
 class EncodingCommand(Command):
     def __init__(self, name):
         parameters = {'name': type}
         Command.__init__(self, 'ENCODING', **parameters)
 
+
 class SendMsgCommand(Command):
     def __init__(self, to, title, body):
         if len(title) > 50 or len(body) > 900:
@@ -373,15 +429,18 @@ class SendMsgCommand(Command):
         parameters = {'to': to.lower(), 'title': title, 'body': body}
         Command.__init__(self, 'SENDMSG', **parameters)
 
+
 class UserCommand(Command):
     def __init__(self, user):
         parameters = {'user': user}
         Command.__init__(self, 'USER', **parameters)
 
+
 class UptimeCommand(Command):
     def __init__(self):
         Command.__init__(self, 'UPTIME')
 
+
 class VersionCommand(Command):
     def __init__(self):
         Command.__init__(self, 'VERSION')

lib/adba/aniDBmaper.py

@@ -22,11 +22,11 @@ class AniDBMaper:
     
     def getAnimeBitsA(self,amask):
         map = self.getAnimeMapA()
-        return self._getBitChain(map,amask);
+        return self._getBitChain(map,amask)
     
     def getAnimeCodesA(self,aBitChain):
         amap = self.getAnimeMapA()
-        return self._getCodes(amap,aBitChain);
+        return self._getCodes(amap,aBitChain)
 
         
     def getFileBitsF(self,fmask):

sickbeard/databases/cache_db.py

@@ -46,7 +46,6 @@ class AddSceneExceptions(InitialSchema):
         self.connection.action(
             "CREATE TABLE scene_exceptions (exception_id INTEGER PRIMARY KEY, indexer_id INTEGER KEY, show_name TEXT)")
 
-
 class AddSceneNameCache(AddSceneExceptions):
     def test(self):
         return self.hasTable("scene_names")
@@ -62,38 +61,13 @@ class AddNetworkTimezones(AddSceneNameCache):
     def execute(self):
         self.connection.action("CREATE TABLE network_timezones (network_name TEXT PRIMARY KEY, timezone TEXT)")
 
-
-class AddXemNumbering(AddNetworkTimezones):
-    def test(self):
-        return self.hasTable("xem_numbering")
-
-    def execute(self):
-        self.connection.action(
-            "CREATE TABLE xem_numbering (indexer TEXT, indexer_id INTEGER, season INTEGER, episode INTEGER, absolute_number INTEGER, scene_season INTEGER, scene_episode INTEGER, scene_absolute_number INTEGER, PRIMARY KEY (indexer_id, scene_season, scene_episode))")
-
-class AddXemRefresh(AddXemNumbering):
-    def test(self):
-        return self.hasTable("xem_refresh")
-
-    def execute(self):
-        self.connection.action(
-            "CREATE TABLE xem_refresh (indexer TEXT, indexer_id INTEGER PRIMARY KEY, last_refreshed INTEGER)")
-
-class AddLastSearch(AddXemRefresh):
+class AddLastSearch(AddNetworkTimezones):
     def test(self):
         return self.hasTable("lastSearch")
 
     def execute(self):
         self.connection.action("CREATE TABLE lastSearch (provider TEXT, time NUMERIC)")
 
-class AddAbsoluteNumbering(AddLastSearch):
-    def test(self):
-        return self.hasColumn("xem_numbering", "absolute_number")
-
-    def execute(self):
-        self.addColumn("xem_numbering", "absolute_number", "NUMERIC", "0")
-        self.addColumn("xem_numbering", "scene_absolute_number", "NUMERIC", "0")
-
 class AddSceneExceptionsSeasons(AddSceneNameCache):
     def test(self):
         return self.hasColumn("scene_exceptions", "season")

sickbeard/databases/mainDB.py

@@ -27,7 +27,7 @@ from sickbeard import encodingKludge as ek
 from sickbeard.name_parser.parser import NameParser, InvalidNameException
 
 MIN_DB_VERSION = 9  # oldest db version we support migrating from
-MAX_DB_VERSION = 36
+MAX_DB_VERSION = 37
 
 class MainSanityCheck(db.DBSanityCheck):
     def check(self):
@@ -861,3 +861,16 @@ class AddSceneAbsoluteNumbering(AddAnimeBlacklistWhitelist):
         self.addColumn("tv_episodes", "scene_absolute_number", "NUMERIC", "0")
 
         self.incDBVersion()
+
+class AddXemRefresh(AddSceneAbsoluteNumbering):
+    def test(self):
+        return self.checkDBVersion() >= 37
+
+    def execute(self):
+        backupDatabase(37)
+
+        logger.log(u"Creating table xem_refresh")
+        self.connection.action(
+            "CREATE TABLE xem_refresh (indexer TEXT, indexer_id INTEGER PRIMARY KEY, last_refreshed INTEGER)")
+
+        self.incDBVersion()

sickbeard/metadata/generic.py

@@ -784,9 +784,9 @@ class GenericMetadata():
         if image_url is None:
             for show_name in set(allPossibleShowNames(show_obj)):
                 if image_type in ('poster', 'poster_thumb'):
-                    image_url = self._retrieve_show_images_from_tmdb(show_name, poster=True)
+                    image_url = self._retrieve_show_images_from_tmdb(show_obj, poster=True)
                 elif image_type == 'fanart':
-                    image_url = self._retrieve_show_images_from_tmdb(show_name, backdrop=True)
+                    image_url = self._retrieve_show_images_from_tmdb(show_obj, backdrop=True)
 
                 if image_url:
                     break
@@ -964,11 +964,11 @@ class GenericMetadata():
 
         return (indexer_id, name, indexer)
 
-    def _retrieve_show_images_from_tmdb(self, name, id=None, backdrop=False, poster=False):
-        tmdb = TMDB(sickbeard.TMDB_API_KEY)
-        result = None
+    def _retrieve_show_images_from_tmdb(self, show, backdrop=False, poster=False):
+        tmdb_id = None
 
         # get TMDB configuration info
+        tmdb = TMDB(sickbeard.TMDB_API_KEY)
         config = tmdb.Configuration()
         response = config.info()
         base_url = response['images']['base_url']
@@ -980,27 +980,15 @@ class GenericMetadata():
         max_size = max(sizes, key=size_str_to_int)
 
         try:
-            if id is None:
-                search = tmdb.Search()
-                response = search.collection({'query': name})
-                id = response['results'][0]['id']
-
-            result = tmdb.Collections(id)
-        except:
-            try:
-                if id is None:
             search = tmdb.Search()
-                    response = search.tv({'query': name})
-                    id = response['results'][0]['id']
-
-                result = tmdb.TV(id)
-            except:
-                pass
-
-        if result is None:
-            return None
+            for result in search.collection({'query': show.name}) + search.tv({'query': show.name}):
+                tmdb_id = result['id']
+                external_ids = tmdb.TV(tmdb_id).external_ids()
+                if show.indexerid in [external_ids['tvdb_id'], external_ids['tvrage_id']]:
+                    break
 
-        images = result.images()
+            if tmdb_id:
+                images = tmdb.Collections(tmdb_id).images()
                 if len(images) > 0:
                     # get backdrop urls
                     if backdrop:
@@ -1013,5 +1001,7 @@ class GenericMetadata():
                         rel_path = images['posters'][0]['file_path']
                         url = "{0}{1}{2}".format(base_url, max_size, rel_path)
                         return url
+        except:
+            pass
 
-        return None
+        logger.log(u"Could not find any posters or background for " + show.name, logger.DEBUG)

sickbeard/name_parser/parser.py

@@ -116,7 +116,10 @@ class NameParser(object):
                         self.compiled_regexes[regex_type].update({cur_pattern_name: cur_regex})
 
     def _parse_string(self, name):
-        for cur_regex_type, cur_regexes in self.compiled_regexes.items() if name else []:
+        if not name:
+            return
+
+        for cur_regex_type, cur_regexes in self.compiled_regexes.items():
             for cur_regex_name, cur_regex in cur_regexes.items():
                 match = cur_regex.match(name)
 
@@ -133,6 +136,15 @@ class NameParser(object):
                     if result.series_name:
                         result.series_name = self.clean_series_name(result.series_name)
 
+                        cur_show = helpers.get_show_by_name(result.series_name, useIndexer=self.useIndexers)
+                        if self.show and cur_show:
+                            if self.show.indexerid != cur_show.indexerid:
+                                logger.log(
+                                    u"I expected an episode of the show " + self.show.name + " but the parser thinks its the show " + cur_show.name + ". I will continue thinking its " + self.show.name,
+                                    logger.WARNING)
+                            else:
+                                result.show = self.show
+
                 if 'season_num' in named_groups:
                     tmp_season = int(match.group('season_num'))
                     if cur_regex_name == 'bare' and tmp_season in (19, 20):
@@ -195,20 +207,11 @@ class NameParser(object):
                 if 'release_group' in named_groups:
                     result.release_group = match.group('release_group')
 
-                # determin show object for correct regex matching
-                if not self.show:
-                    show = helpers.get_show_by_name(result.series_name, useIndexer=self.useIndexers)
-                else:
-                    show = self.show
-
-                if show and show.is_anime and cur_regex_type in ['anime', 'normal']:
-                    result.show = show
+                if result.show and result.show.is_anime and cur_regex_type in ['anime', 'normal']:
                     return result
-                elif show and show.is_sports and cur_regex_type == 'sports':
-                    result.show = show
+                elif result.show and result.show.is_sports and cur_regex_type == 'sports':
                     return result
                 elif cur_regex_type == 'normal':
-                    result.show = show if show else None
                     return result
 
         return None
@@ -241,25 +244,38 @@ class NameParser(object):
                 obj = unicode(obj, encoding)
         return obj
 
-    def _convert_number(self, number):
+    def _convert_number(self, org_number):
+        """
+         Convert org_number into an integer
+         org_number: integer or representation of a number: string or unicode
+         Try force converting to int first, on error try converting from Roman numerals
+         returns integer or 0
+         """
 
         try:
-            return int(number)
+            # try forcing to int
+            if org_number:
+                number = int(org_number)
+            else:
+                number = 0
+
         except:
-            numeral_map = zip(
-                (1000, 900, 500, 400, 100, 90, 50, 40, 10, 9, 5, 4, 1),
-                ('M', 'CM', 'D', 'CD', 'C', 'XC', 'L', 'XL', 'X', 'IX', 'V', 'IV', 'I')
+            # on error try converting from Roman numerals
+            roman_to_int_map = (('M', 1000), ('CM', 900), ('D', 500), ('CD', 400), ('C', 100),
+                                ('XC', 90), ('L', 50), ('XL', 40), ('X', 10),
+                                ('IX', 9), ('V', 5), ('IV', 4), ('I', 1)
             )
 
-            n = unicode(number).upper()
+            roman_numeral = str(org_number).upper()
+            number = 0
+            index = 0
 
-            i = result = 0
-            for integer, numeral in numeral_map:
-                while n[i:i + len(numeral)] == numeral:
-                    result += integer
-                    i += len(numeral)
+            for numeral, integer in roman_to_int_map:
+                while roman_numeral[index:index + len(numeral)] == numeral:
+                    number += integer
+                    index += len(numeral)
 
-            return result
+        return number
 
     def parse(self, name, cache_result=True):
         name = self._unicodify(name)
@@ -319,18 +335,6 @@ class NameParser(object):
                 final_result.which_regex += dir_name_result.which_regex
 
         final_result.show = self._combine_results(file_name_result, dir_name_result, 'show')
-        if final_result.show and final_result.show.is_anime and final_result.is_anime:  # only need to to do another conversion if the scene2tvdb didn work
-            logger.log("Getting season and episodes from absolute numbers", logger.DEBUG)
-            try:
-                _actual_season, _actual_episodes = helpers.get_all_episodes_from_absolute_number(final_result.show,
-                                                                                                 None,
-                                                                                                 final_result.ab_episode_numbers)
-            except EpisodeNotFoundByAbsoluteNumberException:
-                logger.log(str(final_result.show.indexerid) + ": Indexer object absolute number " + str(
-                    final_result.ab_episode_numbers) + " is incomplete, cant determin season and episode numbers")
-            else:
-                final_result.season = _actual_season
-                final_result.episodes = _actual_episodes
 
         # if there's no useful info in it then raise an exception
         if final_result.season_number == None and not final_result.episode_numbers and final_result.air_date == None and not final_result.series_name:
@@ -341,141 +345,6 @@ class NameParser(object):
 
         return final_result
 
-
-    def scene2indexer(self, show, scene_name, season, episodes, absolute_numbers):
-        if not show: return self  # need show object
-
-        # TODO: check if adb and make scene2indexer useable with correct numbers
-        out_season = None
-        out_episodes = []
-        out_absolute_numbers = []
-
-        # is the scene name a special season ?
-        # TODO: define if we get scene seasons or indexer seasons ... for now they are mostly the same ... and i will use them as scene seasons
-        _possible_seasons = sickbeard.scene_exceptions.get_scene_exception_by_name_multiple(scene_name)
-        # filter possible_seasons
-        possible_seasons = []
-        for cur_scene_indexer_id, cur_scene_season in _possible_seasons:
-            if cur_scene_indexer_id and str(cur_scene_indexer_id) != str(show.indexerid):
-                logger.log("Indexer ID mismatch: " + str(show.indexerid) + " now: " + str(cur_scene_indexer_id),
-                           logger.ERROR)
-                raise MultipleSceneShowResults("indexerid mismatch")
-            # don't add season -1 since this is a generic name and not a real season... or if we get None
-            # if this was the only result possible_seasons will stay empty and the next parts will look in the general matter
-            if cur_scene_season == -1 or cur_scene_season == None:
-                continue
-            possible_seasons.append(cur_scene_season)
-        # if not possible_seasons: # no special season name was used or we could not find it
-        logger.log(
-            "possible seasons for '" + scene_name + "' (" + str(show.indexerid) + ") are " + str(possible_seasons),
-            logger.DEBUG)
-
-        # lets just get a db connection we will need it anyway
-        cacheDB = db.DBConnection('cache.db')
-        # should we use absolute_numbers -> anime or season, episodes -> normal show
-        if show.is_anime:
-            logger.log(
-                u"'" + show.name + "' is an anime i will scene convert the absolute numbers " + str(absolute_numbers),
-                logger.DEBUG)
-            if possible_seasons:
-                # check if we have a scene_absolute_number in the possible seasons
-                for cur_possible_season in possible_seasons:
-                    # and for all absolute numbers
-                    for cur_ab_number in absolute_numbers:
-                        namesSQlResult = cacheDB.select(
-                            "SELECT season, episode, absolute_number FROM xem_numbering WHERE indexer_id = ? and scene_season = ? and scene_absolute_number = ?",
-                            [show.indexerid, cur_possible_season, cur_ab_number])
-                        if len(namesSQlResult) > 1:
-                            logger.log(
-                                "Multiple episodes for a absolute number and season. check XEM numbering",
-                                logger.ERROR)
-                            raise MultipleSceneEpisodeResults("Multiple episodes for a absolute number and season")
-                        elif len(namesSQlResult) == 0:
-                            break  # break out of current absolute_numbers -> next season ... this is not a good sign
-                        # if we are here we found ONE episode for this season absolute number
-                        # logger.log(u"I found matching episode: " + namesSQlResult[0]['name'], logger.DEBUG)
-                        out_episodes.append(int(namesSQlResult[0]['episode']))
-                        out_absolute_numbers.append(int(namesSQlResult[0]['absolute_number']))
-                        out_season = int(namesSQlResult[0][
-                            'season'])  # note this will always use the last season we got ... this will be a problem on double episodes that break the season barrier
-                    if out_season:  # if we found a episode in the cur_possible_season we dont need / want to look at the other season possibilities
-                        break
-            else:  # no possible seasons from the scene names lets look at this more generic
-                for cur_ab_number in absolute_numbers:
-                    namesSQlResult = cacheDB.select(
-                        "SELECT season, episode, absolute_number FROM xem_numbering WHERE indexer_id = ? and scene_absolute_number = ?",
-                        [show.indexerid, cur_ab_number])
-                    if len(namesSQlResult) > 1:
-                        logger.log(
-                            "Multiple episodes for a absolute number. this might happend because we are missing a scene name for this season. xem lacking behind ?",
-                            logger.ERROR)
-                        raise MultipleSceneEpisodeResults("Multiple episodes for a absolute number")
-                    elif len(namesSQlResult) == 0:
-                        continue
-                    # if we are here we found ONE episode for this season absolute number
-                    # logger.log(u"I found matching episode: " + namesSQlResult[0]['name'], logger.DEBUG)
-                    out_episodes.append(int(namesSQlResult[0]['episode']))
-                    out_absolute_numbers.append(int(namesSQlResult[0]['absolute_number']))
-                    out_season = int(namesSQlResult[0][
-                        'season'])  # note this will always use the last season we got ... this will be a problem on double episodes that break the season barrier
-            if not out_season:  # we did not find anything in the loops ? damit there is no episode
-                logger.log("No episode found for these scene numbers. asuming indexer numbers", logger.DEBUG)
-                # we still have to convert the absolute number to sxxexx ... but that is done not here
-        else:
-            logger.log(u"'" + show.name + "' is a normal show i will scene convert the season and episodes " + str(
-                season) + "x" + str(episodes), logger.DEBUG)
-            out_absolute_numbers = None
-            if possible_seasons:
-                # check if we have a scene_absolute_number in the possible seasons
-                for cur_possible_season in possible_seasons:
-                    # and for all episode
-                    for cur_episode in episodes:
-                        namesSQlResult = cacheDB.select(
-                            "SELECT season, episode FROM xem_numbering WHERE indexer_id = ? and scene_season = ? and scene_episode = ?",
-                            [show.indexerid, cur_possible_season, cur_episode])
-                        if len(namesSQlResult) > 1:
-                            logger.log(
-                                "Multiple episodes for season episode number combination. this should not be check xem configuration",
-                                logger.ERROR)
-                            raise MultipleSceneEpisodeResults("Multiple episodes for season episode number combination")
-                        elif len(namesSQlResult) == 0:
-                            break  # break out of current episode -> next season ... this is not a good sign
-                        # if we are here we found ONE episode for this season absolute number
-                        # logger.log(u"I found matching episode: " + namesSQlResult[0]['name'], logger.DEBUG)
-                        out_episodes.append(int(namesSQlResult[0]['episode']))
-                        out_season = int(namesSQlResult[0][
-                            'season'])  # note this will always use the last season we got ... this will be a problem on double episodes that break the season barrier
-                    if out_season:  # if we found a episode in the cur_possible_season we dont need / want to look at the other posibilites
-                        break
-            else:  # no possible seasons from the scene names lets look at this more generic
-                for cur_episode in episodes:
-                    namesSQlResult = cacheDB.select(
-                        "SELECT season, episode FROM xem_numbering WHERE indexer_id = ? and scene_episode = ? and scene_season = ?",
-                        [show.indexerid, cur_episode, season])
-                    if len(namesSQlResult) > 1:
-                        logger.log(
-                            "Multiple episodes for season episode number combination. this might happend because we are missing a scene name for this season. xem lacking behind ?",
-                            logger.ERROR)
-                        raise MultipleSceneEpisodeResults("Multiple episodes for season episode number combination")
-                    elif len(namesSQlResult) == 0:
-                        continue
-                    # if we are here we found ONE episode for this season absolute number
-                    # logger.log(u"I found matching episode: " + namesSQlResult[0]['name'], logger.DEBUG)
-                    out_episodes.append(int(namesSQlResult[0]['episode']))
-                    out_season = int(namesSQlResult[0][
-                        'season'])  # note this will always use the last season we got ... this will be a problem on double episodes that break the season barrier
-            # this is only done for normal shows
-            if not out_season:  # we did not find anything in the loops ? darn there is no episode
-                logger.log("No episode found for these scene numbers. assuming these are valid indexer numbers",
-                           logger.DEBUG)
-                out_season = season
-                out_episodes = episodes
-                out_absolute_numbers = absolute_numbers
-
-        # okay that was easy we found the correct season and episode numbers
-        return (out_season, out_episodes, out_absolute_numbers)
-
-
 class ParseResult(object):
     def __init__(self,
                  original_name,
@@ -581,22 +450,31 @@ class ParseResult(object):
 
     def convert(self):
         if not self.show: return self  # need show object
-        if not self.season_number: return self  # can't work without a season
-        if not len(self.episode_numbers): return self  # need at least one episode
         if self.air_by_date or self.sports: return self  # scene numbering does not apply to air-by-date
 
+        # check if show is anime
+        if self.show.is_anime and not (len(self.episode_numbers) or self.season_number) and not len(self.ab_episode_numbers):
+            return self  # can't work without a season
+        elif not self.show._is_anime and not (len(self.episode_numbers) or self.season_number):
+            return self
+
         new_episode_numbers = []
         new_season_numbers = []
         new_absolute_numbers = []
 
-        for i, epNo in enumerate(self.episode_numbers):
-            abNo = None
-            if len(self.ab_episode_numbers):
-                abNo = self.ab_episode_numbers[i]
+        if len(self.ab_episode_numbers) and not len(self.episode_numbers):
+            for epAbNo in self.ab_episode_numbers:
+                (s, e, a) = scene_numbering.get_absolute_numbering(self.show.indexerid, self.show.indexer, epAbNo)
 
+                if (s or e or a):
+                    new_episode_numbers.append(e)
+                    new_season_numbers.append(s)
+                    new_absolute_numbers.append(a)
+        else:
+            for epNo in self.episode_numbers:
                 (s, e, a) = scene_numbering.get_indexer_numbering(self.show.indexerid, self.show.indexer,
                                                                   self.season_number,
-                                                              epNo, abNo)
+                                                                  epNo, None)
                 new_episode_numbers.append(e)
                 new_season_numbers.append(s)
                 new_absolute_numbers.append(a)

sickbeard/providers/btn.py

@@ -47,7 +47,7 @@ class BTNProvider(generic.TorrentProvider):
 
         self.cache = BTNCache(self)
 
-        self.url = "http://broadcasthe.net"
+        self.url = "http://api.btnapps.net"
 
     def isEnabled(self):
         return self.enabled
@@ -134,7 +134,7 @@ class BTNProvider(generic.TorrentProvider):
 
     def _api_call(self, apikey, params={}, results_per_page=1000, offset=0):
 
-        server = jsonrpclib.Server('http://api.btnapps.net')
+        server = jsonrpclib.Server(self.url)
         parsedJSON = {}
 
         try:
@@ -219,7 +219,8 @@ class BTNProvider(generic.TorrentProvider):
                 # Search for the year of the air by date show
                 whole_season_params['name'] = str(ep_obj.airdate).split('-')[0]
             elif ep_obj.show.is_anime:
-                whole_season_params['name'] = "%d" % ep_obj.scene_absolute_number
+                whole_season_params['name'] = 'S' + str(ep_obj.scene_season)
+                #whole_season_params['name'] = "%d" % ep_obj.scene_absolute_number
             else:
                 whole_season_params['name'] = 'Season ' + str(ep_obj.scene_season)
 
@@ -234,9 +235,9 @@ class BTNProvider(generic.TorrentProvider):
 
         search_params = {'category': 'Episode'}
 
-        if self.show.indexer == 1 and not self.show.is_anime:
+        if self.show.indexer == 1:
             search_params['tvdb'] = self.show.indexerid
-        elif self.show.indexer == 2 and not self.show.is_anime:
+        elif self.show.indexer == 2:
             search_params['tvrage'] = self.show.indexerid
         else:
             search_params['series'] = sanitizeSceneName(self.show.name)
@@ -253,9 +254,7 @@ class BTNProvider(generic.TorrentProvider):
             # BTN uses dots in dates, we just search for the date since that
             # combined with the series identifier should result in just one episode
             search_params['name'] = date_str.replace('-', '.')
-        elif self.show.is_anime:
-            search_params['name'] = "%i" % int(ep_obj.scene_absolute_number)
-        else:
+        elif not self.show.is_anime:
             # Do a general name search for the episode, formatted like SXXEYY
             search_params['name'] = "S%02dE%02d" % (ep_obj.scene_season, ep_obj.scene_episode)
 

sickbeard/providers/generic.py

@@ -68,7 +68,6 @@ class GenericProvider:
         self.session.headers.update({
             'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'})
 
-
     def getID(self):
         return GenericProvider.makeID(self.name)
 

sickbeard/scene_numbering.py

@@ -23,6 +23,9 @@
 
 import time
 import traceback
+import sickbeard
+
+from lib.tmdb_api import TMDB
 
 try:
     import json
@@ -110,6 +113,28 @@ def get_indexer_numbering(indexer_id, indexer, sceneSeason, sceneEpisode, sceneA
             return get_indexer_numbering_for_xem(indexer_id, indexer, sceneSeason, sceneEpisode, sceneAbsoluteNumber)
         return (sceneSeason, sceneEpisode, sceneAbsoluteNumber)
 
+def get_absolute_numbering(indexer_id, indexer, sceneAbsoluteNumber, fallback_to_xem=True):
+    """
+    Returns a tuple, (season, episode, absolute_number) with the TVDB and TVRAGE numbering for (sceneAbsoluteNumber)
+    (this works like the reverse of get_absolute_numbering)
+    """
+    if indexer_id is None or sceneAbsoluteNumber is None:
+        return (None, None, None)
+
+    indexer_id = int(indexer_id)
+    indexer = int(indexer)
+
+    myDB = db.DBConnection()
+
+    rows = myDB.select(
+        "SELECT season, episode, absolute_number FROM scene_numbering WHERE indexer = ? and indexer_id = ? and scene_absolute_number = ?",
+        [indexer, indexer_id, sceneAbsoluteNumber])
+    if rows:
+        return (int(rows[0]["season"]), int(rows[0]["episode"]), int(rows[0]["absolute_number"]))
+    else:
+        if fallback_to_xem:
+            return get_indexer_numbering_for_xem(indexer_id, indexer, sceneAbsoluteNumber)
+    return (None, None, None)
 
 def get_scene_numbering_for_show(indexer_id, indexer):
     """
@@ -137,9 +162,6 @@ def get_scene_numbering_for_show(indexer_id, indexer):
         scene_episode = int(row['scene_episode'])
         scene_absolute_number = int(row['scene_absolute_number'])
 
-        try:
-            result[(season, episode)]
-        except:
         result[(season, episode)] = (scene_season, scene_episode, scene_absolute_number)
 
     return result
@@ -191,25 +213,17 @@ def find_xem_numbering(indexer_id, indexer, season, episode, absolute_number):
     indexer_id = int(indexer_id)
     indexer = int(indexer)
 
-    if _xem_refresh_needed(indexer_id, indexer):
-        _xem_refresh(indexer_id, indexer)
+    if xem_refresh_needed(indexer_id, indexer):
+        xem_refresh(indexer_id, indexer)
 
-    cacheDB = db.DBConnection('cache.db')
+    myDB = db.DBConnection()
 
-    rows = cacheDB.select(
-        "SELECT scene_season, scene_episode, scene_absolute_number FROM xem_numbering WHERE indexer = ? and indexer_id = ? and season = ? and episode = ?",
+    rows = myDB.select(
+        "SELECT scene_season, scene_episode, scene_absolute_number FROM tv_episodes WHERE indexer = ? and showid = ? and season = ? and episode = ?",
         [indexer, indexer_id, season, episode])
 
     if rows:
         return (int(rows[0]["scene_season"]), int(rows[0]["scene_episode"]), int(rows[0]["scene_absolute_number"]))
-    elif cacheDB.select(
-            "SELECT * FROM xem_numbering WHERE indexer = ? and indexer_id = ?",
-            [indexer, indexer_id]):
-
-        return (0, 0, 0)
-    else:
-        return None
-
 
 def get_indexer_numbering_for_xem(indexer_id, indexer, sceneSeason, sceneEpisode, sceneAbsoluteNumber):
     """
@@ -226,19 +240,48 @@ def get_indexer_numbering_for_xem(indexer_id, indexer, sceneSeason, sceneEpisode
     indexer_id = int(indexer_id)
     indexer = int(indexer)
 
-    if _xem_refresh_needed(indexer_id, indexer):
-        _xem_refresh(indexer_id, indexer)
-    cacheDB = db.DBConnection('cache.db')
-    rows = cacheDB.select(
-        "SELECT season, episode, absolute_number FROM xem_numbering WHERE indexer = ? and indexer_id = ? and scene_season = ? and scene_episode = ?",
+    if xem_refresh_needed(indexer_id, indexer):
+        xem_refresh(indexer_id, indexer)
+
+    myDB = db.DBConnection()
+
+    rows = myDB.select(
+        "SELECT season, episode, absolute_number FROM tv_episodes WHERE indexer = ? and showid = ? and scene_season = ? and scene_episode = ?",
         [indexer, indexer_id, sceneSeason, sceneEpisode])
     if rows:
         return (int(rows[0]["season"]), int(rows[0]["episode"]), int(rows[0]["absolute_number"]))
-    else:
+
     return (sceneSeason, sceneEpisode, sceneAbsoluteNumber)
 
+def get_absolute_numbering_for_xem(indexer_id, indexer, sceneAbsoluteNumber):
+    """
+    Reverse of find_xem_numbering: lookup a tvdb season and episode using scene numbering
+
+    @param indexer_id: int
+    @param sceneSeason: int
+    @param sceneEpisode: int
+    @return: (int, int) a tuple of (season, episode)
+    """
+    if indexer_id is None or sceneAbsoluteNumber is None:
+        return None
+
+    indexer_id = int(indexer_id)
+    indexer = int(indexer)
+
+    if xem_refresh_needed(indexer_id, indexer):
+        xem_refresh(indexer_id, indexer)
+
+    myDB = db.DBConnection()
+
+    rows = myDB.select(
+        "SELECT season, episode, absolute_number FROM tv_episodes WHERE indexer = ? and showid = ? and scene_absolute_number = ?",
+        [indexer, indexer_id, sceneAbsoluteNumber])
+    if rows:
+        return (int(rows[0]["season"]), int(rows[0]["episode"]), int(rows[0]["absolute_number"]))
 
-def _xem_refresh_needed(indexer_id, indexer):
+    return (None, None, None)
+
+def xem_refresh_needed(indexer_id, indexer):
     """
     Is a refresh needed on a show?
     
@@ -251,8 +294,8 @@ def _xem_refresh_needed(indexer_id, indexer):
     indexer_id = int(indexer_id)
     indexer = int(indexer)
 
-    cacheDB = db.DBConnection('cache.db')
-    rows = cacheDB.select("SELECT last_refreshed FROM xem_refresh WHERE indexer = ? and indexer_id = ?",
+    myDB = db.DBConnection()
+    rows = myDB.select("SELECT last_refreshed FROM xem_refresh WHERE indexer = ? and indexer_id = ?",
                           [indexer, indexer_id])
     if rows:
         return time.time() > (int(rows[0]['last_refreshed']) + MAX_XEM_AGE_SECS)
@@ -260,7 +303,7 @@ def _xem_refresh_needed(indexer_id, indexer):
         return True
 
 
-def _xem_refresh(indexer_id, indexer):
+def xem_refresh(indexer_id, indexer):
     """
     Refresh data from xem for a tv show
     
@@ -290,35 +333,34 @@ def _xem_refresh(indexer_id, indexer):
                 return None
 
         result = data
-        cacheDB = db.DBConnection('cache.db')
+        myDB = db.DBConnection()
 
         ql = []
         if result:
             ql.append(["INSERT OR REPLACE INTO xem_refresh (indexer, indexer_id, last_refreshed) VALUES (?,?,?)",
                        [indexer, indexer_id, time.time()]])
             if 'success' in result['result']:
-                ql.append(["DELETE FROM xem_numbering where indexer = ? and indexer_id = ?", [indexer, indexer_id]])
                 for entry in result['data']:
                     if 'scene' in entry:
                         ql.append([
-                            "INSERT OR IGNORE INTO xem_numbering (indexer, indexer_id, season, episode, absolute_number, scene_season, scene_episode, scene_absolute_number) VALUES (?,?,?,?,?,?,?,?)",
-                            [indexer, indexer_id,
-                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['season'],
-                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['episode'],
-                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['absolute'],
-                             entry['scene']['season'],
+                            "UPDATE tv_episodes SET scene_season = ?, scene_episode = ?, scene_absolute_number = ? WHERE showid = ? AND season = ? AND episode = ?",
+                            [entry['scene']['season'],
                              entry['scene']['episode'],
-                             entry['scene']['absolute']]])
+                             entry['scene']['absolute'],
+                             indexer_id,
+                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['season'],
+                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['episode']
+                            ]])
                     if 'scene_2' in entry:  # for doubles
                         ql.append([
-                            "INSERT OR IGNORE INTO xem_numbering (indexer, indexer_id, season, episode, absolute_number, scene_season, scene_episode, scene_absolute_number) VALUES (?,?,?,?,?,?,?,?)",
-                            [indexer, indexer_id,
-                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['season'],
-                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['episode'],
-                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['absolute'],
-                             entry['scene_2']['season'],
+                            "UPDATE tv_episodes SET scene_season = ?, scene_episode = ?, scene_absolute_number = ? WHERE showid = ? AND season = ? AND episode = ?",
+                            [entry['scene_2']['season'],
                              entry['scene_2']['episode'],
-                             entry['scene_2']['absolute']]])
+                             entry['scene_2']['absolute'],
+                             indexer_id,
+                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['season'],
+                             entry[sickbeard.indexerApi(indexer).config['xem_origin']]['episode']
+                            ]])
             else:
                 logger.log(u'Failed to get XEM scene data for show %s from %s because "%s"' % (
                     indexer_id, sickbeard.indexerApi(indexer).name, result['message']), logger.DEBUG)
@@ -332,8 +374,10 @@ def _xem_refresh(indexer_id, indexer):
         return None
 
     if ql:
-        cacheDB.mass_action(ql)
+        myDB.mass_action(ql)
 
+    # fix xem scene numbering issues
+    #fix_xem_numbering(indexer_id, indexer)
 
 def get_xem_numbering_for_show(indexer_id, indexer):
     """
@@ -347,127 +391,186 @@ def get_xem_numbering_for_show(indexer_id, indexer):
     indexer_id = int(indexer_id)
     indexer = int(indexer)
 
-    if _xem_refresh_needed(indexer_id, indexer):
-        _xem_refresh(indexer_id, indexer)
+    if xem_refresh_needed(indexer_id, indexer):
+        xem_refresh(indexer_id, indexer)
 
-    cacheDB = db.DBConnection('cache.db')
+    myDB = db.DBConnection()
 
-    rows = cacheDB.select(
-        'SELECT season, episode, absolute_number, scene_season, scene_episode, scene_absolute_number FROM xem_numbering WHERE indexer = ? and indexer_id = ? ORDER BY season, episode',
+    rows = myDB.select(
+        'SELECT season, episode, absolute_number, scene_season, scene_episode, scene_absolute_number FROM tv_episodes WHERE indexer = ? and showid = ? ORDER BY season, episode',
         [indexer, indexer_id])
 
     result = {}
     for row in rows:
-        season = int(row['season'])
-        episode = int(row['episode'])
-        scene_season = int(row['scene_season'])
-        scene_episode = int(row['scene_episode'])
-        scene_absolute_number = int(row['scene_absolute_number'])
+        season = int(row['season'] or 0)
+        episode = int(row['episode'] or 0)
+        scene_season = int(row['scene_season'] or 0)
+        scene_episode = int(row['scene_episode'] or 0)
+        scene_absolute_number = int(row['scene_absolute_number'] or 0)
 
         result[(season, episode)] = (scene_season, scene_episode, scene_absolute_number)
 
     return result
 
-
-def get_xem_numbering_for_season(indexer_id, indexer, season):
+def fix_xem_numbering(indexer_id, indexer):
     """
     Returns a dict of (season, episode) : (sceneSeason, sceneEpisode) mappings
     for an entire show.  Both the keys and values of the dict are tuples.
-    Will be empty if there are no scene numbers set
+    Will be empty if there are no scene numbers set in xem
     """
-    if indexer_id is None or season is None:
+    if indexer_id is None:
         return {}
 
     indexer_id = int(indexer_id)
     indexer = int(indexer)
 
-    if _xem_refresh_needed(indexer_id, indexer):
-        _xem_refresh(indexer_id, indexer)
+    # query = [{
+    #              "name": self.show.name,
+    #              "seasons": [{
+    #                              "episodes": [{
+    #                                               "episode_number": None,
+    #                                               "name": None
+    #                                           }],
+    #                              "season_number": None,
+    #                          }],
+    #              "/tv/tv_program/number_of_seasons": [],
+    #              "/tv/tv_program/number_of_episodes": [],
+    #              "/tv/tv_program/thetvdb_id": [],
+    #              "/tv/tv_program/tvrage_id": [],
+    #              "type": "/tv/tv_program",
+    #          }]
+    #
+    #
+    # url = 'https://www.googleapis.com/freebase/v1/mqlread'
+    # api_key = "AIzaSyCCHNp4dhVHxJYzbLiCE4y4a1rgTnX4fDE"
+    # params = {
+    #     'query': json.dumps(query),
+    #     'key': api_key
+    # }
+    #
+    #
+    # def get_from_api(url, params=None):
+    #     """Build request and return results
+    #     """
+    #     import xmltodict
+    #
+    #     response = requests.get(url, params=params)
+    #     if response.status_code == 200:
+    #         try:
+    #             return response.json()
+    #         except ValueError:
+    #             return xmltodict.parse(response.text)['Data']
+    #
+    # # Get query results
+    # tmp = get_from_api(url, params=params)['result']
 
-    cacheDB = db.DBConnection('cache.db')
+    myDB = db.DBConnection()
 
-    rows = cacheDB.select(
-        'SELECT season, scene_season FROM xem_numbering WHERE indexer = ? and indexer_id = ? AND season = ? ORDER BY season',
-        [indexer, indexer_id, season])
+    rows = myDB.select(
+        'SELECT season, episode, absolute_number, scene_season, scene_episode, scene_absolute_number FROM tv_episodes WHERE indexer = ? and showid = ?',
+        [indexer, indexer_id])
 
-    result = {}
-    if rows:
-        for row in rows:
-            result.setdefault(int(row['season']), []).append(int(row['scene_season']))
-    else:
-        result.setdefault(int(season), []).append(int(season))
+    last_absolute_number = None
+    last_scene_season = None
+    last_scene_episode = None
+    last_scene_absolute_number = None
 
-    return result
+    update_absolute_number = False
+    update_scene_season = False
+    update_scene_episode = False
+    update_scene_absolute_number = False
+
+    logger.log(
+        u'Fixing any XEM scene mapping issues for show %s on %s' % (indexer_id, sickbeard.indexerApi(indexer).name,),
+        logger.DEBUG)
 
-def fix_scene_numbering():
     ql = []
+    for row in rows:
+        season = int(row['season'])
+        episode = int(row['episode'])
 
-    myDB = db.DBConnection()
+        if not int(row['scene_season']) and last_scene_season:
+            scene_season = last_scene_season + 1
+            update_scene_season = True
+        else:
+            scene_season = int(row['scene_season'])
+            if last_scene_season and scene_season < last_scene_season:
+                scene_season = last_scene_season + 1
+                update_scene_season = True
+
+        if not int(row['scene_episode']) and last_scene_episode:
+            scene_episode = last_scene_episode + 1
+            update_scene_episode = True
+        else:
+            scene_episode = int(row['scene_episode'])
+            if last_scene_episode and scene_episode < last_scene_episode:
+                scene_episode = last_scene_episode + 1
+                update_scene_episode = True
+
+        # check for unset values and correct them
+        if not int(row['absolute_number']) and last_absolute_number:
+            absolute_number = last_absolute_number + 1
+            update_absolute_number = True
+        else:
+            absolute_number = int(row['absolute_number'])
+            if last_absolute_number and absolute_number < last_absolute_number:
+                absolute_number = last_absolute_number + 1
+                update_absolute_number = True
+
+        if not int(row['scene_absolute_number']) and last_scene_absolute_number:
+            scene_absolute_number = last_scene_absolute_number + 1
+            update_scene_absolute_number = True
+        else:
+            scene_absolute_number = int(row['scene_absolute_number'])
+            if last_scene_absolute_number and scene_absolute_number < last_scene_absolute_number:
+                scene_absolute_number = last_scene_absolute_number + 1
+                update_scene_absolute_number = True
 
-    sqlResults = myDB.select(
-        "SELECT showid, indexerid, indexer, episode_id, season, episode FROM tv_episodes WHERE scene_season = -1 OR scene_episode = -1")
+        # store values for lookup on next iteration
+        last_absolute_number = absolute_number
+        last_scene_season = scene_season
+        last_scene_episode = scene_episode
+        last_scene_absolute_number = scene_absolute_number
 
-    for epResult in sqlResults:
-        indexerid = int(epResult["showid"])
-        indexer = int(epResult["indexer"])
-        season = int(epResult["season"])
-        episode = int(epResult["episode"])
-        absolute_number = int(epResult["absolute_number"])
+        if update_absolute_number:
+            ql.append([
+                "UPDATE tv_episodes SET absolute_number = ? WHERE showid = ? AND season = ? AND episode = ?",
+                [absolute_number,
+                 indexer_id,
+                 season,
+                 episode
+                ]])
+            update_absolute_number = False
 
-        logger.log(
-            u"Repairing any scene numbering issues for showid: " + str(epResult["showid"]) + u" season: " + str(
-                epResult["season"]) + u" episode: " + str(epResult["episode"]), logger.DEBUG)
+        if update_scene_season:
+            ql.append([
+                "UPDATE tv_episodes SET scene_season = ? WHERE showid = ? AND season = ? AND episode = ?",
+                [scene_season,
+                 indexer_id,
+                 season,
+                 episode
+                ]])
+            update_scene_season = False
 
-        scene_season, scene_episode, scene_absolute_number = sickbeard.scene_numbering.get_scene_numbering(indexerid,
-                                                                                                           indexer,
+        if update_scene_episode:
+            ql.append([
+                "UPDATE tv_episodes SET scene_episode = ? WHERE showid = ? AND season = ? AND episode = ?",
+                [scene_episode,
+                 indexer_id,
                  season,
-                                                                                                           episode,
-                                                                                                           absolute_number)
+                 episode
+                ]])
+            update_scene_episode = False
 
-        ql.append(
-            ["UPDATE tv_episodes SET scene_season = ? WHERE indexerid = ?", [scene_season, epResult["indexerid"]]])
-        ql.append(
-            ["UPDATE tv_episodes SET scene_episode = ? WHERE indexerid = ?", [scene_episode, epResult["indexerid"]]])
-        ql.append(
-            ["UPDATE tv_episodes SET scene_absolute_number = ? WHERE indexerid = ?",
-             [scene_absolute_number, epResult["indexerid"]]])
+        if update_scene_absolute_number:
+            ql.append([
+                "UPDATE tv_episodes SET scene_absolute_number = ? WHERE showid = ? AND season = ? AND episode = ?",
+                [scene_absolute_number,
+                 indexer_id,
+                 season,
+                 episode
+                ]])
+            update_scene_absolute_number = False
 
     if ql:
         myDB.mass_action(ql)
-
-def get_ep_mapping(epObj, parse_result):
-    # scores
-    indexer_numbering = 0
-    scene_numbering = 0
-    absolute_numbering = 0
-
-    _possible_seasons = sickbeard.scene_exceptions.get_scene_exception_by_name_multiple(parse_result.series_name)
-
-    # indexer numbering
-    if epObj.season == parse_result.season_number:
-        indexer_numbering += 1
-    elif epObj.episode in parse_result.episode_numbers:
-        indexer_numbering += 1
-
-    # scene numbering
-    if epObj.scene_season == parse_result.season_number:
-        scene_numbering += 1
-    elif epObj.scene_episode in parse_result.episode_numbers:
-        scene_numbering += 1
-
-    # absolute numbering
-    if epObj.show.is_anime and parse_result.is_anime:
-
-        if epObj.absolute_number in parse_result.ab_episode_numbers:
-            absolute_numbering +=1
-        elif epObj.scene_absolute_number in parse_result.ab_episode_numbers:
-            absolute_numbering += 1
-
-    if indexer_numbering == 2:
-        print "indexer numbering"
-    elif scene_numbering == 2:
-        print "scene numbering"
-    elif absolute_numbering == 1:
-        print "indexer numbering"
-    else:
-        print "could not determin numbering"

sickbeard/search.py

@@ -344,12 +344,12 @@ def searchForNeededEpisodes(show, episodes):
     origThreadName = threading.currentThread().name
     providers = [x for x in sickbeard.providers.sortedProviderList() if x.isActive() and not x.backlog_only]
     for curProviderCount, curProvider in enumerate(providers):
-        threading.currentThread().name = origThreadName + " :: [" + curProvider.name + "]"
-
         if curProvider.anime_only and not show.is_anime:
             logger.log(u"" + str(show.name) + " is not an anime skiping ...")
             continue
 
+        threading.currentThread().name = origThreadName + " :: [" + curProvider.name + "]"
+
         try:
             logger.log(u"Updating RSS cache ...")
             curProvider.cache.updateCache()
@@ -418,14 +418,14 @@ def searchProviders(show, season, episodes, manualSearch=False):
 
     origThreadName = threading.currentThread().name
     for providerNum, provider in enumerate(providers):
-        threading.currentThread().name = origThreadName + " :: [" + provider.name + "]"
-        foundResults.setdefault(provider.name, {})
-        searchCount = 0
-
         if provider.anime_only and not show.is_anime:
             logger.log(u"" + str(show.name) + " is not an anime skiping ...")
             continue
 
+        threading.currentThread().name = origThreadName + " :: [" + provider.name + "]"
+        foundResults.setdefault(provider.name, {})
+        searchCount = 0
+
         search_mode = 'eponly'
         if seasonSearch and provider.search_mode == 'sponly':
             search_mode = provider.search_mode

sickbeard/show_name_helpers.py

@@ -30,6 +30,7 @@ from sickbeard import db
 from sickbeard import encodingKludge as ek
 from name_parser.parser import NameParser, InvalidNameException
 from lib.unidecode import unidecode
+from sickbeard.blackandwhitelist import BlackAndWhiteList
 
 resultFilters = ["sub(bed|ed|pack|s)", "(dk|fin|heb|kor|nor|nordic|pl|swe)sub(bed|ed|s)?",
                  "(dir|sample|sub|nfo)fix", "sample", "(dvd)?extras",
@@ -58,7 +59,8 @@ def filterBadReleases(name):
     filters = [re.compile('(^|[\W_])%s($|[\W_])' % filter.strip(), re.I) for filter in resultFilters]
     for regfilter in filters:
         if regfilter.search(name):
-            logger.log(u"Invalid scene release: " + name + " contains pattern: " + regfilter.pattern + ", ignoring it", logger.DEBUG)
+            logger.log(u"Invalid scene release: " + name + " contains pattern: " + regfilter.pattern + ", ignoring it",
+                       logger.DEBUG)
             return False
 
     return True
@@ -135,7 +137,8 @@ def makeSceneSeasonSearchString(show, ep_obj, extraSearchType=None):
 
             # if we need a better one then add it to the list of episodes to fetch
             if (curStatus in (
-                    common.DOWNLOADED, common.SNATCHED) and curQuality < highestBestQuality) or curStatus == common.WANTED:
+                    common.DOWNLOADED,
+                    common.SNATCHED) and curQuality < highestBestQuality) or curStatus == common.WANTED:
                 ab_number = episode.scene_absolute_number
                 if ab_number > 0:
                     seasonStrings.append("%d" % ab_number)
@@ -148,6 +151,7 @@ def makeSceneSeasonSearchString(show, ep_obj, extraSearchType=None):
         numseasons = int(numseasonsSQlResult[0][0])
         seasonStrings = ["S%02d" % int(ep_obj.scene_season)]
 
+    bwl = BlackAndWhiteList(show.indexerid)
     showNames = set(makeSceneShowSearchStrings(show, ep_obj.scene_season))
 
     toReturn = []
@@ -162,8 +166,13 @@ def makeSceneSeasonSearchString(show, ep_obj, extraSearchType=None):
             # for providers that don't allow multiple searches in one request we only search for Sxx style stuff
             else:
                 for cur_season in seasonStrings:
+                    if len(bwl.whiteList) > 0:
+                        for keyword in bwl.whiteList:
+                            toReturn.append(keyword + '.' + curShow+ "." + cur_season)
+                    else:
                         toReturn.append(curShow + "." + cur_season)
 
+
     return toReturn
 
 
@@ -188,12 +197,17 @@ def makeSceneSearchString(show, ep_obj):
     if numseasons == 1 and not ep_obj.show.is_anime:
         epStrings = ['']
 
+    bwl = BlackAndWhiteList(ep_obj.show.indexerid)
     showNames = set(makeSceneShowSearchStrings(show, ep_obj.scene_season))
 
     toReturn = []
 
     for curShow in showNames:
         for curEpString in epStrings:
+            if len(bwl.whiteList) > 0:
+                for keyword in bwl.whiteList:
+                    toReturn.append(keyword + '.' + curShow + '.' + curEpString)
+            else:
                 toReturn.append(curShow + '.' + curEpString)
 
     return toReturn
@@ -228,7 +242,8 @@ def isGoodResult(name, show, log=True, season=-1):
             return True
 
     if log:
-        logger.log(u"Provider gave result " + name + " but that doesn't seem like a valid result for " + show.name + " so I'm ignoring it")
+        logger.log(
+            u"Provider gave result " + name + " but that doesn't seem like a valid result for " + show.name + " so I'm ignoring it")
     return False
 
 

sickbeard/show_queue.py

@@ -354,13 +354,6 @@ class QueueItemAdd(ShowQueueItem):
         # before we parse local files lets update exceptions
         sickbeard.scene_exceptions.retrieve_exceptions()
 
-        # and get scene numbers
-        logger.log(u"Attempting to load scene numbers", logger.DEBUG)
-        if self.show.loadEpisodeSceneNumbers():
-            logger.log(u"loading scene numbers successfull", logger.DEBUG)
-        else:
-            logger.log(u"loading scene numbers NOT successfull or no scene numbers available", logger.DEBUG)
-
         try:
             self.show.loadEpisodesFromDir()
         except Exception, e:
@@ -548,12 +541,6 @@ class QueueItemUpdate(ShowQueueItem):
                     except exceptions.EpisodeDeletedException:
                         pass
 
-        logger.log(u"Attempting to load scene numbers", logger.DEBUG)
-        if self.show.loadEpisodeSceneNumbers():
-            logger.log(u"loading scene numbers successfull", logger.DEBUG)
-        else:
-            logger.log(u"loading scene numbers NOT successfull or no scene numbers available", logger.DEBUG)
-
         sickbeard.showQueueScheduler.action.refreshShow(self.show, True)
 
 class QueueItemForceUpdate(QueueItemUpdate):

sickbeard/tv.py

@@ -17,6 +17,7 @@
 # along with SickRage.  If not, see <http://www.gnu.org/licenses/>.
 
 from __future__ import with_statement
+import json
 
 import os.path
 import datetime
@@ -24,6 +25,7 @@ import threading
 import re
 import glob
 import traceback
+import requests
 
 import sickbeard
 
@@ -186,6 +188,10 @@ class TVShow(object):
 
     def getEpisode(self, season, episode, file=None, noCreate=False, absolute_number=None):
 
+        # Load XEM data to DB for show
+        if sickbeard.scene_numbering.xem_refresh_needed(self.indexerid, self.indexer):
+            sickbeard.scene_numbering.xem_refresh(self.indexerid, self.indexer)
+
         if not season in self.episodes:
             self.episodes[season] = {}
 
@@ -213,12 +219,6 @@ class TVShow(object):
                     logger.DEBUG)
                 return None
 
-        def createCurSeasonDict():
-            if not season in self.episodes:
-                self.episodes[season] = {}
-
-        createCurSeasonDict()
-
         if not episode in self.episodes[season] or self.episodes[season][episode] == None:
             if noCreate:
                 return None
@@ -275,7 +275,6 @@ class TVShow(object):
         last_update_indexer = datetime.date.fromordinal(self.last_update_indexer)
 
         # in the first year after ended (last airdate), update every 30 days
-        # in the first year after ended (last airdate), update every 30 days
         if (update_date - last_airdate) < datetime.timedelta(days=450) and (
                     update_date - last_update_indexer) > datetime.timedelta(days=30):
             return True
@@ -532,33 +531,6 @@ class TVShow(object):
 
         return scannedEps
 
-    def loadEpisodeSceneNumbers(self):
-        epList = self.loadEpisodesFromDB()
-
-        sql_l = []
-        for curSeason in epList:
-            for curEp in epList[curSeason]:
-                epObj = self.getEpisode(curSeason, curEp)
-
-                with epObj.lock:
-                    (epObj.scene_season, epObj.scene_episode, epObj.scene_absolute_number) = \
-                        sickbeard.scene_numbering.get_scene_numbering(self.indexerid, self.indexer, epObj.season,
-                                                                      epObj.episode, epObj.absolute_number)
-                    logger.log(
-                        str(self.indexerid) + ": adding scene numbering. Indexer: " + str(epObj.season) + "x" + str(
-                            epObj.episode) + "| Scene: " + str(epObj.scene_season) + "x" + str(epObj.scene_episode),
-                        logger.DEBUG)
-
-                    # mass add to database
-                    if epObj.dirty:
-                        sql_l.append(epObj.get_sql())
-
-        if len(sql_l) > 0:
-            myDB = db.DBConnection()
-            myDB.mass_action(sql_l)
-
-        return True
-
     def getImages(self, fanart=None, poster=None):
         fanart_result = poster_result = banner_result = False
         season_posters_result = season_banners_result = season_all_poster_result = season_all_banner_result = False
@@ -1460,7 +1432,6 @@ class TVEpisode(object):
                             "Couldn't find episode " + str(season) + "x" + str(episode))
 
     def loadFromDB(self, season, episode):
-
         logger.log(
             str(self.show.indexerid) + u": Loading episode details from DB for episode " + str(season) + "x" + str(
                 episode), logger.DEBUG)
@@ -1521,15 +1492,6 @@ class TVEpisode(object):
             if sqlResults[0]["is_proper"]:
                 self.is_proper = int(sqlResults[0]["is_proper"])
 
-            if self.scene_season == 0 or self.scene_episode == 0 or self.scene_absolute_number == 0:
-                (self.scene_season, self.scene_episode, self.scene_absolute_number) = \
-                    sickbeard.scene_numbering.get_scene_numbering(
-                    self.show.indexerid,
-                    self.show.indexer,
-                    self.season,
-                    self.episode,
-                    self.absolute_number)
-
             self.dirty = False
             return True
 
@@ -2302,19 +2264,3 @@ class TVEpisode(object):
             self.saveToDB()
             for relEp in self.relatedEps:
                 relEp.saveToDB()
-
-    def convertToSceneNumbering(self):
-        (self.scene_season, self.scene_episode,
-         self.scene_absolute_number) = sickbeard.scene_numbering.get_scene_numbering(self.show.indexerid,
-                                                                                     self.show.indexer,
-                                                                                     self.season,
-                                                                                     self.episode,
-                                                                                     self.absolute_number)
-
-    def convertToIndexerNumbering(self):
-        (self.season, self.episode, self.absolute_number) = sickbeard.scene_numbering.get_indexer_numbering(
-            self.show.indexerid,
-            self.show.indexer,
-            self.scene_season,
-            self.scene_episode,
-            self.scene_absolute_number)