From 1edf7a1a4f0b7fbe7e3386adfa11a14682a8c72d Mon Sep 17 00:00:00 2001 From: bwq Date: Sat, 23 Jun 2012 00:55:09 +0200 Subject: [PATCH 1/4] - added global login func for all torrent providers (all private trackers need cookies) - added global download func (same, downloading also requires cookies) These 2 functions were necessary to allow the user to download other found releases at a later time. In the current version this doesn't work because it goes to the url directly, without any cookies which causes it to download the login page instead of a torrent. All 3 providers now properly use the cache and have much better error handling. --- couchpotato/core/providers/torrent/base.py | 26 +++++++++++++ .../core/providers/torrent/sceneaccess/main.py | 44 +++++++++------------- couchpotato/core/providers/torrent/scenehd/main.py | 40 +++++++++----------- .../core/providers/torrent/torrentleech/main.py | 44 +++++++++------------- 4 files changed, 77 insertions(+), 77 deletions(-) diff --git a/couchpotato/core/providers/torrent/base.py b/couchpotato/core/providers/torrent/base.py index e136a99..35c902c 100644 --- a/couchpotato/core/providers/torrent/base.py +++ b/couchpotato/core/providers/torrent/base.py @@ -1,5 +1,31 @@ from couchpotato.core.providers.base import YarrProvider +from couchpotato.core.logger import CPLog +import urllib2 +import cookielib + +log = CPLog(__name__) class TorrentProvider(YarrProvider): type = 'torrent' + + def login(self, params): + + try: + cookiejar = cookielib.CookieJar() + opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) + urllib2.install_opener(opener) + f = opener.open(self.urls['login'], params) + data = f.read() + f.close() + + except: + log.error('Failed to login.') + + return opener + + def download(self, url = '', nzb_id = ''): + loginParams = self.getLoginParams() + self.login(params = loginParams) + torrent = self.urlopen(url) + return torrent diff --git a/couchpotato/core/providers/torrent/sceneaccess/main.py b/couchpotato/core/providers/torrent/sceneaccess/main.py index 09d7dfa..2bb6944 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/main.py +++ b/couchpotato/core/providers/torrent/sceneaccess/main.py @@ -20,6 +20,7 @@ class SceneAccess(TorrentProvider): urls = { 'test': 'https://www.sceneaccess.eu/', + 'login' : 'https://www.sceneaccess.eu/login', 'detail': 'https://www.sceneaccess.eu/details?id=%s', 'search': 'https://www.sceneaccess.eu/browse?search=%s&method=2&c%d=%d', 'download': 'https://www.sceneaccess.eu/%s', @@ -32,6 +33,10 @@ class SceneAccess(TorrentProvider): ] http_time_between_calls = 1 #seconds + + def getLoginParams(self): + loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), submit='come on in')) + return loginParams def search(self, movie, quality): @@ -40,30 +45,21 @@ class SceneAccess(TorrentProvider): return results cache_key = 'sceneaccess.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0], self.getCatId(quality['identifier'])[0]) - data = self.getCache(cache_key, searchUrl) + searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0], self.getCatId(quality['identifier'])[0]) + loginParams = self.getLoginParams() - if data: + opener = self.login(params = loginParams) + if not opener: + log.info("Couldn't login at SceneAccess") + return results - cat_ids = self.getCatId(quality['identifier']) - - try: - cookiejar = cookielib.CookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) - urllib2.install_opener(opener) - params = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), submit='come on in')) - f = opener.open('https://www.sceneaccess.eu/login', params) - data = f.read() - f.close() - f = opener.open(searchUrl) - data = f.read() - f.close() - - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results + data = self.getCache(cache_key, searchUrl) - html = BeautifulSoup(data) + if data: + html = BeautifulSoup(data) + + else: + log.info("No results found at SceneAccess") try: resultsTable = html.find('table', attrs = {'id' : 'torrents-table'}) @@ -126,9 +122,3 @@ class SceneAccess(TorrentProvider): if 'imdb.com/title/' + imdbId in imdbDiv or 'imdb.com/title/' + imdbIdAlt in imdbDiv: return 50 return 0 - - def download(self, url = '', nzb_id = ''): - torrent = self.urlopen(url) - return torrent - - diff --git a/couchpotato/core/providers/torrent/scenehd/main.py b/couchpotato/core/providers/torrent/scenehd/main.py index 497f2d0..e55dc89 100644 --- a/couchpotato/core/providers/torrent/scenehd/main.py +++ b/couchpotato/core/providers/torrent/scenehd/main.py @@ -21,6 +21,7 @@ class SceneHD(TorrentProvider): urls = { 'test': 'http://scenehd.org/', + 'login' : 'http://scenehd.org/takelogin.php', 'detail': 'http://scenehd.org/details.php?id=%s', 'search': 'http://scenehd.org/browse.php?ajax&search=%s', 'download': 'http://scenehd.org/download.php?id=%s', @@ -28,6 +29,10 @@ class SceneHD(TorrentProvider): http_time_between_calls = 1 #seconds + def getLoginParams(self): + loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), ssl='yes')) + return loginParams + def search(self, movie, quality): results = [] @@ -35,28 +40,21 @@ class SceneHD(TorrentProvider): return results cache_key = 'scenehd.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']) + ' ' + quality['identifier'])) + searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier'])) + loginParams = self.getLoginParams() + + opener = self.login(params = loginParams) + if not opener: + log.error("Couldn't login at SceneHD") + return results + data = self.getCache(cache_key, searchUrl) if data: - - try: - cookiejar = cookielib.CookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) - urllib2.install_opener(opener) - params = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), ssl='yes')) - f = opener.open('http://scenehd.org/takelogin.php', params) - data = f.read() - f.close() - f = opener.open(searchUrl) - data = f.read() - f.close() - - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results - - html = BeautifulSoup(data) + html = BeautifulSoup(data) + + else: + log.info("No results found at SceneHD") try: resultsTable = html.findAll('table')[6] @@ -112,8 +110,4 @@ class SceneHD(TorrentProvider): return 50 return 0 - def download(self, url = '', nzb_id = ''): - torrent = self.urlopen(url) - return torrent - diff --git a/couchpotato/core/providers/torrent/torrentleech/main.py b/couchpotato/core/providers/torrent/torrentleech/main.py index 0a693dc..efe2017 100644 --- a/couchpotato/core/providers/torrent/torrentleech/main.py +++ b/couchpotato/core/providers/torrent/torrentleech/main.py @@ -22,6 +22,7 @@ class TorrentLeech(TorrentProvider): urls = { 'test' : 'http://torrentleech.org/', + 'login' : 'http://torrentleech.org/user/account/login/', 'detail' : 'http://torrentleech.org/torrent/%s', 'search' : 'http://torrentleech.org/torrents/browse/index/query/%s/categories/%d', 'download' : 'http://torrentleech.org%s', @@ -38,6 +39,10 @@ class TorrentLeech(TorrentProvider): ] http_time_between_calls = 1 #seconds + + def getLoginParams(self): + loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), remember_me='on', login='submit')) + return loginParams def search(self, movie, quality): @@ -46,30 +51,21 @@ class TorrentLeech(TorrentProvider): return results cache_key = 'torrentleech.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0]) + searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0]) + loginParams = self.getLoginParams() + + opener = self.login(params = loginParams) + if not opener: + log.info("Couldn't login at Torrentleech") + return results + data = self.getCache(cache_key, searchUrl) if data: - - cat_ids = self.getCatId(quality['identifier']) - - try: - cookiejar = cookielib.CookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) - urllib2.install_opener(opener) - params = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), remember_me='on', login='submit')) - f = opener.open('http://torrentleech.org/user/account/login/', params) - data = f.read() - f.close() - f = opener.open(searchUrl) - data = f.read() - f.close() - - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results - - html = BeautifulSoup(data) + html = BeautifulSoup(data) + + else: + log.info("No results found at Torrentleech") try: resultsTable = html.find('table', attrs = {'id' : 'torrenttable'}) @@ -125,9 +121,3 @@ class TorrentLeech(TorrentProvider): if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' + imdbIdAlt in data: return 50 return 0 - - def download(self, url = '', nzb_id = ''): - torrent = self.urlopen(url) - return torrent - - From 3bf3e4bb7f0b8c82c885e3dc0ca53ff599d9d6f7 Mon Sep 17 00:00:00 2001 From: bwq Date: Sat, 23 Jun 2012 04:24:13 +0200 Subject: [PATCH 2/4] - Added a new arg for getCache: 'opener', which is a urllib2 opener with a cookiejar installed - The torrent providers now use getCache with the 'opener' argument which they get after a succesful login, this way they can always access the site with proper authentication. --- couchpotato/core/plugins/base.py | 13 ++++++++++++- couchpotato/core/providers/torrent/base.py | 2 +- couchpotato/core/providers/torrent/sceneaccess/main.py | 2 +- couchpotato/core/providers/torrent/scenehd/main.py | 2 +- couchpotato/core/providers/torrent/torrentleech/main.py | 6 +++--- 5 files changed, 18 insertions(+), 7 deletions(-) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 2ad616b..68a24a2 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -215,7 +215,18 @@ class Plugin(object): cache_timeout = kwargs.get('cache_timeout') del kwargs['cache_timeout'] - data = self.urlopen(url, **kwargs) + if kwargs.get('opener'): + opener = kwargs.get('opener') + del kwargs['opener'] + + if opener: + log.info('Opening url: %s', url) + f = opener.open(url) + data = f.read() + f.close() + else: + data = self.urlopen(url, **kwargs) + if data: self.setCache(cache_key, data, timeout = cache_timeout) return data diff --git a/couchpotato/core/providers/torrent/base.py b/couchpotato/core/providers/torrent/base.py index 35c902c..a91af44 100644 --- a/couchpotato/core/providers/torrent/base.py +++ b/couchpotato/core/providers/torrent/base.py @@ -16,7 +16,7 @@ class TorrentProvider(YarrProvider): opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) urllib2.install_opener(opener) f = opener.open(self.urls['login'], params) - data = f.read() + loginData = f.read() f.close() except: diff --git a/couchpotato/core/providers/torrent/sceneaccess/main.py b/couchpotato/core/providers/torrent/sceneaccess/main.py index 2bb6944..aba475d 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/main.py +++ b/couchpotato/core/providers/torrent/sceneaccess/main.py @@ -53,7 +53,7 @@ class SceneAccess(TorrentProvider): log.info("Couldn't login at SceneAccess") return results - data = self.getCache(cache_key, searchUrl) + data = self.getCache(cache_key, searchUrl, opener = opener) if data: html = BeautifulSoup(data) diff --git a/couchpotato/core/providers/torrent/scenehd/main.py b/couchpotato/core/providers/torrent/scenehd/main.py index e55dc89..e7cb9db 100644 --- a/couchpotato/core/providers/torrent/scenehd/main.py +++ b/couchpotato/core/providers/torrent/scenehd/main.py @@ -48,7 +48,7 @@ class SceneHD(TorrentProvider): log.error("Couldn't login at SceneHD") return results - data = self.getCache(cache_key, searchUrl) + data = self.getCache(cache_key, searchUrl, opener = opener) if data: html = BeautifulSoup(data) diff --git a/couchpotato/core/providers/torrent/torrentleech/main.py b/couchpotato/core/providers/torrent/torrentleech/main.py index efe2017..88dfce9 100644 --- a/couchpotato/core/providers/torrent/torrentleech/main.py +++ b/couchpotato/core/providers/torrent/torrentleech/main.py @@ -22,7 +22,7 @@ class TorrentLeech(TorrentProvider): urls = { 'test' : 'http://torrentleech.org/', - 'login' : 'http://torrentleech.org/user/account/login/', + 'login' : 'http://torrentleech.org/user/account/login/', 'detail' : 'http://torrentleech.org/torrent/%s', 'search' : 'http://torrentleech.org/torrents/browse/index/query/%s/categories/%d', 'download' : 'http://torrentleech.org%s', @@ -42,7 +42,7 @@ class TorrentLeech(TorrentProvider): def getLoginParams(self): loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), remember_me='on', login='submit')) - return loginParams + return loginParams def search(self, movie, quality): @@ -59,7 +59,7 @@ class TorrentLeech(TorrentProvider): log.info("Couldn't login at Torrentleech") return results - data = self.getCache(cache_key, searchUrl) + data = self.getCache(cache_key, searchUrl, opener = opener) if data: html = BeautifulSoup(data) From 27d9225c3fb14db0b0a778fc21b28a8957c956b2 Mon Sep 17 00:00:00 2001 From: bwq Date: Sat, 23 Jun 2012 16:49:07 +0200 Subject: [PATCH 3/4] - fixed UnboundLocalError (forgot to init opener) --- couchpotato/core/plugins/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 68a24a2..469108d 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -215,6 +215,7 @@ class Plugin(object): cache_timeout = kwargs.get('cache_timeout') del kwargs['cache_timeout'] + opener = None if kwargs.get('opener'): opener = kwargs.get('opener') del kwargs['opener'] From a7bcc9bdfade3624e38b460ee04cc78e2c98dace Mon Sep 17 00:00:00 2001 From: bwq Date: Sat, 23 Jun 2012 22:01:13 +0200 Subject: [PATCH 4/4] Updated the 3 new providers to use imdb_results for is_correct_movie validation and left out the extra_score since a proper imdb match shouldn't be part of the extra_score. --- .../core/providers/torrent/sceneaccess/main.py | 17 +++++++---------- couchpotato/core/providers/torrent/scenehd/main.py | 22 ++++++++++------------ .../core/providers/torrent/torrentleech/main.py | 19 ++++++++----------- 3 files changed, 25 insertions(+), 33 deletions(-) diff --git a/couchpotato/core/providers/torrent/sceneaccess/main.py b/couchpotato/core/providers/torrent/sceneaccess/main.py index aba475d..4f413f4 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/main.py +++ b/couchpotato/core/providers/torrent/sceneaccess/main.py @@ -85,11 +85,12 @@ class SceneAccess(TorrentProvider): else: new['leechers'] = 0 - new['imdbid'] = movie['library']['identifier'] - new['extra_score'] = self.extra_score + details = self.urls['detail'] % new['id'] + imdb_results = self.imdbMatch(details, movie['library']['identifier']) + new['score'] = fireEvent('score.calculate', new, movie, single = True) is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = True, single_category = False, single = True) + imdb_results = imdb_results, single_category = False, single = True) if is_correct_movie: new['download'] = self.download @@ -101,10 +102,6 @@ class SceneAccess(TorrentProvider): log.info("No results found at SceneAccess") return [] - def extra_score(self, nzb): - url = self.urls['detail'] % nzb['id'] - imdbId = nzb['imdbid'] - return self.imdbMatch(url, imdbId) def imdbMatch(self, url, imdbId): try: @@ -112,7 +109,7 @@ class SceneAccess(TorrentProvider): pass except IOError: log.error('Failed to open %s.' % url) - return '' + return False html = BeautifulSoup(data) imdbDiv = html.find('span', attrs = {'class':'i_link'}) @@ -120,5 +117,5 @@ class SceneAccess(TorrentProvider): imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId) if 'imdb.com/title/' + imdbId in imdbDiv or 'imdb.com/title/' + imdbIdAlt in imdbDiv: - return 50 - return 0 + return True + return False diff --git a/couchpotato/core/providers/torrent/scenehd/main.py b/couchpotato/core/providers/torrent/scenehd/main.py index e7cb9db..443c7ad 100644 --- a/couchpotato/core/providers/torrent/scenehd/main.py +++ b/couchpotato/core/providers/torrent/scenehd/main.py @@ -82,17 +82,16 @@ class SceneHD(TorrentProvider): new['name'] = detailLink['title'] imdbLink = allCells[1].find('a') + imdb_results = False + if imdbLink: - new['imdbresult'] = imdbLink['href'].replace('http://www.imdb.com/title/','').rstrip('/') - else: - new['imdbresult'] = 'tt00000000' + imdbFound = imdbLink['href'].replace('http://www.imdb.com/title/','').rstrip('/') + imdb_results = self.imdbMatch(imdbFound, movie['library']['identifier']) new['url'] = self.urls['download'] % new['id'] - new['imdbid'] = movie['library']['identifier'] - new['extra_score'] = self.extra_score new['score'] = fireEvent('score.calculate', new, movie, single = True) is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = True, single_category = False, single = True) + imdb_results = imdb_results, single_category = False, single = True) if is_correct_movie: new['download'] = self.download @@ -104,10 +103,9 @@ class SceneHD(TorrentProvider): log.info("No results found at SceneHD") return [] - def extra_score(self, nzb): - imdbIdAlt = re.sub('tt[0]*', 'tt', nzb['imdbresult']) - if nzb['imdbresult'] == nzb['imdbid'] or imdbIdAlt == nzb['imdbid']: - return 50 - return 0 - + def imdbMatch(self, imdbFound, imdbId): + imdbIdAlt = re.sub('tt[0]*', 'tt', imdbFound) + if imdbFound == imdbId or imdbIdAlt == imdbId: + return True + return False diff --git a/couchpotato/core/providers/torrent/torrentleech/main.py b/couchpotato/core/providers/torrent/torrentleech/main.py index 88dfce9..3614ea7 100644 --- a/couchpotato/core/providers/torrent/torrentleech/main.py +++ b/couchpotato/core/providers/torrent/torrentleech/main.py @@ -85,13 +85,14 @@ class TorrentLeech(TorrentProvider): new['url'] = self.urls['download'] % url['href'] new['size'] = self.parseSize(result.findAll('td')[4].string) new['seeders'] = int(result.find('td', attrs = {'class' : 'seeders'}).string) - new['leechers'] = int(result.find('td', attrs = {'class' : 'leechers'}).string) - new['imdbid'] = movie['library']['identifier'] + new['leechers'] = int(result.find('td', attrs = {'class' : 'leechers'}).string) + + details = self.urls['detail'] % new['id'] + imdb_results = self.imdbMatch(details, movie['library']['identifier']) - new['extra_score'] = self.extra_score new['score'] = fireEvent('score.calculate', new, movie, single = True) is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = True, single_category = False, single = True) + imdb_results = imdb_results, single_category = False, single = True) if is_correct_movie: new['download'] = self.download @@ -103,10 +104,6 @@ class TorrentLeech(TorrentProvider): log.info("No results found at TorrentLeech") return [] - def extra_score(self, nzb): - url = self.urls['detail'] % nzb['id'] - imdbId = nzb['imdbid'] - return self.imdbMatch(url, imdbId) def imdbMatch(self, url, imdbId): try: @@ -114,10 +111,10 @@ class TorrentLeech(TorrentProvider): pass except IOError: log.error('Failed to open %s.' % url) - return '' + return False imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId) data = unicode(data, errors='ignore') if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' + imdbIdAlt in data: - return 50 - return 0 + return True + return False