diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 1c98868..67f36a8 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -215,7 +215,19 @@ class Plugin(object): cache_timeout = kwargs.get('cache_timeout') del kwargs['cache_timeout'] - data = self.urlopen(url, **kwargs) + opener = None + if kwargs.get('opener'): + opener = kwargs.get('opener') + del kwargs['opener'] + + if opener: + log.info('Opening url: %s', url) + f = opener.open(url) + data = f.read() + f.close() + else: + data = self.urlopen(url, **kwargs) + if data: self.setCache(cache_key, data, timeout = cache_timeout) return data diff --git a/couchpotato/core/providers/torrent/base.py b/couchpotato/core/providers/torrent/base.py index e136a99..a91af44 100644 --- a/couchpotato/core/providers/torrent/base.py +++ b/couchpotato/core/providers/torrent/base.py @@ -1,5 +1,31 @@ from couchpotato.core.providers.base import YarrProvider +from couchpotato.core.logger import CPLog +import urllib2 +import cookielib + +log = CPLog(__name__) class TorrentProvider(YarrProvider): type = 'torrent' + + def login(self, params): + + try: + cookiejar = cookielib.CookieJar() + opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) + urllib2.install_opener(opener) + f = opener.open(self.urls['login'], params) + loginData = f.read() + f.close() + + except: + log.error('Failed to login.') + + return opener + + def download(self, url = '', nzb_id = ''): + loginParams = self.getLoginParams() + self.login(params = loginParams) + torrent = self.urlopen(url) + return torrent diff --git a/couchpotato/core/providers/torrent/sceneaccess/main.py b/couchpotato/core/providers/torrent/sceneaccess/main.py index 09d7dfa..4f413f4 100644 --- a/couchpotato/core/providers/torrent/sceneaccess/main.py +++ b/couchpotato/core/providers/torrent/sceneaccess/main.py @@ -20,6 +20,7 @@ class SceneAccess(TorrentProvider): urls = { 'test': 'https://www.sceneaccess.eu/', + 'login' : 'https://www.sceneaccess.eu/login', 'detail': 'https://www.sceneaccess.eu/details?id=%s', 'search': 'https://www.sceneaccess.eu/browse?search=%s&method=2&c%d=%d', 'download': 'https://www.sceneaccess.eu/%s', @@ -32,6 +33,10 @@ class SceneAccess(TorrentProvider): ] http_time_between_calls = 1 #seconds + + def getLoginParams(self): + loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), submit='come on in')) + return loginParams def search(self, movie, quality): @@ -40,30 +45,21 @@ class SceneAccess(TorrentProvider): return results cache_key = 'sceneaccess.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0], self.getCatId(quality['identifier'])[0]) - data = self.getCache(cache_key, searchUrl) + searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0], self.getCatId(quality['identifier'])[0]) + loginParams = self.getLoginParams() - if data: + opener = self.login(params = loginParams) + if not opener: + log.info("Couldn't login at SceneAccess") + return results - cat_ids = self.getCatId(quality['identifier']) - - try: - cookiejar = cookielib.CookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) - urllib2.install_opener(opener) - params = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), submit='come on in')) - f = opener.open('https://www.sceneaccess.eu/login', params) - data = f.read() - f.close() - f = opener.open(searchUrl) - data = f.read() - f.close() - - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results + data = self.getCache(cache_key, searchUrl, opener = opener) - html = BeautifulSoup(data) + if data: + html = BeautifulSoup(data) + + else: + log.info("No results found at SceneAccess") try: resultsTable = html.find('table', attrs = {'id' : 'torrents-table'}) @@ -89,11 +85,12 @@ class SceneAccess(TorrentProvider): else: new['leechers'] = 0 - new['imdbid'] = movie['library']['identifier'] - new['extra_score'] = self.extra_score + details = self.urls['detail'] % new['id'] + imdb_results = self.imdbMatch(details, movie['library']['identifier']) + new['score'] = fireEvent('score.calculate', new, movie, single = True) is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = True, single_category = False, single = True) + imdb_results = imdb_results, single_category = False, single = True) if is_correct_movie: new['download'] = self.download @@ -105,10 +102,6 @@ class SceneAccess(TorrentProvider): log.info("No results found at SceneAccess") return [] - def extra_score(self, nzb): - url = self.urls['detail'] % nzb['id'] - imdbId = nzb['imdbid'] - return self.imdbMatch(url, imdbId) def imdbMatch(self, url, imdbId): try: @@ -116,7 +109,7 @@ class SceneAccess(TorrentProvider): pass except IOError: log.error('Failed to open %s.' % url) - return '' + return False html = BeautifulSoup(data) imdbDiv = html.find('span', attrs = {'class':'i_link'}) @@ -124,11 +117,5 @@ class SceneAccess(TorrentProvider): imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId) if 'imdb.com/title/' + imdbId in imdbDiv or 'imdb.com/title/' + imdbIdAlt in imdbDiv: - return 50 - return 0 - - def download(self, url = '', nzb_id = ''): - torrent = self.urlopen(url) - return torrent - - + return True + return False diff --git a/couchpotato/core/providers/torrent/scenehd/main.py b/couchpotato/core/providers/torrent/scenehd/main.py index 497f2d0..443c7ad 100644 --- a/couchpotato/core/providers/torrent/scenehd/main.py +++ b/couchpotato/core/providers/torrent/scenehd/main.py @@ -21,6 +21,7 @@ class SceneHD(TorrentProvider): urls = { 'test': 'http://scenehd.org/', + 'login' : 'http://scenehd.org/takelogin.php', 'detail': 'http://scenehd.org/details.php?id=%s', 'search': 'http://scenehd.org/browse.php?ajax&search=%s', 'download': 'http://scenehd.org/download.php?id=%s', @@ -28,6 +29,10 @@ class SceneHD(TorrentProvider): http_time_between_calls = 1 #seconds + def getLoginParams(self): + loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), ssl='yes')) + return loginParams + def search(self, movie, quality): results = [] @@ -35,28 +40,21 @@ class SceneHD(TorrentProvider): return results cache_key = 'scenehd.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']) + ' ' + quality['identifier'])) - data = self.getCache(cache_key, searchUrl) + searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier'])) + loginParams = self.getLoginParams() - if data: - - try: - cookiejar = cookielib.CookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) - urllib2.install_opener(opener) - params = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), ssl='yes')) - f = opener.open('http://scenehd.org/takelogin.php', params) - data = f.read() - f.close() - f = opener.open(searchUrl) - data = f.read() - f.close() - - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results + opener = self.login(params = loginParams) + if not opener: + log.error("Couldn't login at SceneHD") + return results + + data = self.getCache(cache_key, searchUrl, opener = opener) - html = BeautifulSoup(data) + if data: + html = BeautifulSoup(data) + + else: + log.info("No results found at SceneHD") try: resultsTable = html.findAll('table')[6] @@ -84,17 +82,16 @@ class SceneHD(TorrentProvider): new['name'] = detailLink['title'] imdbLink = allCells[1].find('a') + imdb_results = False + if imdbLink: - new['imdbresult'] = imdbLink['href'].replace('http://www.imdb.com/title/','').rstrip('/') - else: - new['imdbresult'] = 'tt00000000' + imdbFound = imdbLink['href'].replace('http://www.imdb.com/title/','').rstrip('/') + imdb_results = self.imdbMatch(imdbFound, movie['library']['identifier']) new['url'] = self.urls['download'] % new['id'] - new['imdbid'] = movie['library']['identifier'] - new['extra_score'] = self.extra_score new['score'] = fireEvent('score.calculate', new, movie, single = True) is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = True, single_category = False, single = True) + imdb_results = imdb_results, single_category = False, single = True) if is_correct_movie: new['download'] = self.download @@ -106,14 +103,9 @@ class SceneHD(TorrentProvider): log.info("No results found at SceneHD") return [] - def extra_score(self, nzb): - imdbIdAlt = re.sub('tt[0]*', 'tt', nzb['imdbresult']) - if nzb['imdbresult'] == nzb['imdbid'] or imdbIdAlt == nzb['imdbid']: - return 50 - return 0 - - def download(self, url = '', nzb_id = ''): - torrent = self.urlopen(url) - return torrent - + def imdbMatch(self, imdbFound, imdbId): + imdbIdAlt = re.sub('tt[0]*', 'tt', imdbFound) + if imdbFound == imdbId or imdbIdAlt == imdbId: + return True + return False diff --git a/couchpotato/core/providers/torrent/torrentleech/main.py b/couchpotato/core/providers/torrent/torrentleech/main.py index 0a693dc..3614ea7 100644 --- a/couchpotato/core/providers/torrent/torrentleech/main.py +++ b/couchpotato/core/providers/torrent/torrentleech/main.py @@ -22,6 +22,7 @@ class TorrentLeech(TorrentProvider): urls = { 'test' : 'http://torrentleech.org/', + 'login' : 'http://torrentleech.org/user/account/login/', 'detail' : 'http://torrentleech.org/torrent/%s', 'search' : 'http://torrentleech.org/torrents/browse/index/query/%s/categories/%d', 'download' : 'http://torrentleech.org%s', @@ -38,6 +39,10 @@ class TorrentLeech(TorrentProvider): ] http_time_between_calls = 1 #seconds + + def getLoginParams(self): + loginParams = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), remember_me='on', login='submit')) + return loginParams def search(self, movie, quality): @@ -46,30 +51,21 @@ class TorrentLeech(TorrentProvider): return results cache_key = 'torrentleech.%s.%s' % (movie['library']['identifier'], quality.get('identifier')) - searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0]) - data = self.getCache(cache_key, searchUrl) + searchUrl = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':','') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0]) + loginParams = self.getLoginParams() + + opener = self.login(params = loginParams) + if not opener: + log.info("Couldn't login at Torrentleech") + return results - if data: + data = self.getCache(cache_key, searchUrl, opener = opener) - cat_ids = self.getCatId(quality['identifier']) - - try: - cookiejar = cookielib.CookieJar() - opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) - urllib2.install_opener(opener) - params = urllib.urlencode(dict(username=''+self.conf('username'), password=''+self.conf('password'), remember_me='on', login='submit')) - f = opener.open('http://torrentleech.org/user/account/login/', params) - data = f.read() - f.close() - f = opener.open(searchUrl) - data = f.read() - f.close() - - except (IOError, URLError): - log.error('Failed to open %s.' % url) - return results - - html = BeautifulSoup(data) + if data: + html = BeautifulSoup(data) + + else: + log.info("No results found at Torrentleech") try: resultsTable = html.find('table', attrs = {'id' : 'torrenttable'}) @@ -89,13 +85,14 @@ class TorrentLeech(TorrentProvider): new['url'] = self.urls['download'] % url['href'] new['size'] = self.parseSize(result.findAll('td')[4].string) new['seeders'] = int(result.find('td', attrs = {'class' : 'seeders'}).string) - new['leechers'] = int(result.find('td', attrs = {'class' : 'leechers'}).string) - new['imdbid'] = movie['library']['identifier'] + new['leechers'] = int(result.find('td', attrs = {'class' : 'leechers'}).string) + + details = self.urls['detail'] % new['id'] + imdb_results = self.imdbMatch(details, movie['library']['identifier']) - new['extra_score'] = self.extra_score new['score'] = fireEvent('score.calculate', new, movie, single = True) is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality, - imdb_results = True, single_category = False, single = True) + imdb_results = imdb_results, single_category = False, single = True) if is_correct_movie: new['download'] = self.download @@ -107,10 +104,6 @@ class TorrentLeech(TorrentProvider): log.info("No results found at TorrentLeech") return [] - def extra_score(self, nzb): - url = self.urls['detail'] % nzb['id'] - imdbId = nzb['imdbid'] - return self.imdbMatch(url, imdbId) def imdbMatch(self, url, imdbId): try: @@ -118,16 +111,10 @@ class TorrentLeech(TorrentProvider): pass except IOError: log.error('Failed to open %s.' % url) - return '' + return False imdbIdAlt = re.sub('tt[0]*', 'tt', imdbId) data = unicode(data, errors='ignore') if 'imdb.com/title/' + imdbId in data or 'imdb.com/title/' + imdbIdAlt in data: - return 50 - return 0 - - def download(self, url = '', nzb_id = ''): - torrent = self.urlopen(url) - return torrent - - + return True + return False