Browse Source

Cleanup torrent providers

pull/1178/merge
Ruud 13 years ago
parent
commit
5ab0d7a97b
  1. 2
      couchpotato/core/providers/nzb/ftdworld/main.py
  2. 4
      couchpotato/core/providers/nzb/newznab/main.py
  3. 1
      couchpotato/core/providers/nzb/omgwtfnzbs/main.py
  4. 6
      couchpotato/core/providers/torrent/base.py
  5. 98
      couchpotato/core/providers/torrent/kickasstorrents/main.py
  6. 30
      couchpotato/core/providers/torrent/passthepopcorn/main.py
  7. 36
      couchpotato/core/providers/torrent/publichd/main.py
  8. 39
      couchpotato/core/providers/torrent/sceneaccess/main.py
  9. 42
      couchpotato/core/providers/torrent/scenehd/main.py
  10. 39
      couchpotato/core/providers/torrent/thepiratebay/main.py
  11. 48
      couchpotato/core/providers/torrent/torrentleech/main.py

2
couchpotato/core/providers/nzb/ftdworld/main.py

@ -86,8 +86,6 @@ class FTDWorld(NZBProvider):
'score': (tryInt(up.attrs['title'].split(' ')[0]) * 3) - (tryInt(down.attrs['title'].split(' ')[0]) * 3) if up else 0,
})
return results
except:
log.error('Failed to parse HTML response from FTDWorld')

4
couchpotato/core/providers/nzb/newznab/main.py

@ -35,12 +35,12 @@ class Newznab(NZBProvider, RSS):
results = ResultList(self, movie, quality, imdb_result = True)
for host in hosts:
result = self.singleSearch(host, movie, quality)
result = self._search(host, movie, quality)
results.extend(result)
return results
def singleSearch(self, host, movie, quality):
def _search(self, host, movie, quality):
if self.isDisabled(host):
return []

1
couchpotato/core/providers/nzb/omgwtfnzbs/main.py

@ -9,7 +9,6 @@ from couchpotato.core.providers.nzb.base import NZBProvider
from dateutil.parser import parse
from urlparse import urlparse, parse_qs
import time
import xml.etree.ElementTree as XMLTree
log = CPLog(__name__)

6
couchpotato/core/providers/torrent/base.py

@ -24,3 +24,9 @@ class TorrentProvider(YarrProvider):
return getImdb(data) == imdbId
return False
class TorrentMagnetProvider(TorrentProvider):
type = 'torrent_magnet'
download = None

98
couchpotato/core/providers/torrent/kickasstorrents/main.py

@ -1,16 +1,15 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import simplifyString
from couchpotato.core.helpers.variable import tryInt, getTitle
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentMagnetProvider
import re
import traceback
log = CPLog(__name__)
class KickAssTorrents(TorrentProvider):
class KickAssTorrents(TorrentMagnetProvider):
urls = {
'test': 'https://kat.ph/',
@ -32,14 +31,13 @@ class KickAssTorrents(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
title = simplifyString(getTitle(movie['library'])).replace(' ', '-')
results = ResultList(self, movie, quality, imdb_results = True)
data = self.getHTMLData(self.urls['search'] % ('m', movie['library']['identifier'].replace('tt', '')))
cache_key = 'kickasstorrents.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, self.urls['search'] % (title, movie['library']['identifier'].replace('tt', '')))
if data:
cat_ids = self.getCatId(quality['identifier'])
@ -53,57 +51,39 @@ class KickAssTorrents(TorrentProvider):
continue
try:
try:
for temp in result.find_all('tr'):
if temp['class'] is 'firstr' or not temp.get('id'):
continue
new = {
'type': 'torrent_magnet',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'score': 0,
}
nr = 0
for td in temp.find_all('td'):
column_name = table_order[nr]
if column_name:
if column_name is 'name':
link = td.find('div', {'class': 'torrentname'}).find_all('a')[1]
new['id'] = temp.get('id')[-8:]
new['name'] = link.text
new['url'] = td.find('a', 'imagnet')['href']
new['detail_url'] = self.urls['detail'] % link['href'][1:]
new['score'] = 20 if td.find('a', 'iverif') else 0
elif column_name is 'size':
new['size'] = self.parseSize(td.text)
elif column_name is 'age':
new['age'] = self.ageToDays(td.text)
elif column_name is 'seeds':
new['seeders'] = tryInt(td.text)
elif column_name is 'leechers':
new['leechers'] = tryInt(td.text)
nr += 1
new['score'] += fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie',
nzb = new, movie = movie, quality = quality,
imdb_results = True, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
except:
log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc())
for temp in result.find_all('tr'):
if temp['class'] is 'firstr' or not temp.get('id'):
continue
new = {}
nr = 0
for td in temp.find_all('td'):
column_name = table_order[nr]
if column_name:
if column_name is 'name':
link = td.find('div', {'class': 'torrentname'}).find_all('a')[1]
new['id'] = temp.get('id')[-8:]
new['name'] = link.text
new['url'] = td.find('a', 'imagnet')['href']
new['detail_url'] = self.urls['detail'] % link['href'][1:]
new['score'] = 20 if td.find('a', 'iverif') else 0
elif column_name is 'size':
new['size'] = self.parseSize(td.text)
elif column_name is 'age':
new['age'] = self.ageToDays(td.text)
elif column_name is 'seeds':
new['seeders'] = tryInt(td.text)
elif column_name is 'leechers':
new['leechers'] = tryInt(td.text)
nr += 1
results.append(new)
except:
pass
log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc())
return results
except AttributeError:
log.debug('No search results found.')

30
couchpotato/core/providers/torrent/passthepopcorn/main.py

@ -2,6 +2,7 @@ from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import getTitle, tryInt, mergeDicts
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentProvider
from dateutil.parser import parse
import cookielib
@ -67,10 +68,10 @@ class PassThePopcorn(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
results = ResultList(self, movie, quality, imdb_results = True)
movie_title = getTitle(movie['library'])
quality_id = quality['identifier']
@ -118,7 +119,6 @@ class PassThePopcorn(TorrentProvider):
if 'Scene' in torrent and torrent['Scene']:
torrentdesc += ' Scene'
if 'RemasterTitle' in torrent and torrent['RemasterTitle']:
# eliminate odd characters...
torrentdesc += self.htmlToASCII(' %s' % torrent['RemasterTitle'])
torrentdesc += ' (%s)' % quality_id
@ -127,38 +127,24 @@ class PassThePopcorn(TorrentProvider):
def extra_check(item):
return self.torrentMeetsQualitySpec(item, type)
def extra_score(item):
return 50 if torrent['GoldenPopcorn'] else 0
new = {
results.append({
'id': torrent_id,
'type': 'torrent',
'provider': self.getName(),
'name': torrent_name,
'description': '',
'url': '%s?action=download&id=%d&authkey=%s&torrent_pass=%s' % (self.urls['torrent'], torrent_id, authkey, passkey),
'detail_url': self.urls['detail'] % torrent_id,
'date': tryInt(time.mktime(parse(torrent['UploadTime']).timetuple())),
'size': tryInt(torrent['Size']) / (1024 * 1024),
'provider': self.getName(),
'seeders': tryInt(torrent['Seeders']),
'leechers': tryInt(torrent['Leechers']),
'extra_score': extra_score,
'score': 50 if torrent['GoldenPopcorn'] else 0,
'extra_check': extra_check,
'download': self.loginDownload,
}
new['score'] = fireEvent('score.calculate', new, movie, single = True)
})
if fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality):
results.append(new)
self.found(new)
return results
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def login(self):

36
couchpotato/core/providers/torrent/publichd/main.py

@ -1,9 +1,9 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentMagnetProvider
from urlparse import parse_qs
import re
import traceback
@ -11,7 +11,7 @@ import traceback
log = CPLog(__name__)
class PublicHD(TorrentProvider):
class PublicHD(TorrentMagnetProvider):
urls = {
'test': 'https://publichd.se',
@ -22,20 +22,18 @@ class PublicHD(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled() or not quality.get('hd', False):
return results
return []
results = ResultList(self, movie, quality)
params = tryUrlencode({
'page':'torrents',
'search': '%s %s' % (getTitle(movie['library']), movie['library']['year']),
'active': 1,
})
url = '%s?%s' % (self.urls['search'], params)
cache_key = 'publichd.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, url)
data = self.getHTMLData('%s?%s' % (self.urls['search'], params))
if data:
@ -53,35 +51,21 @@ class PublicHD(TorrentProvider):
url = parse_qs(info_url['href'])
new = {
results.append({
'id': url['id'][0],
'name': info_url.string,
'type': 'torrent_magnet',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'url': download['href'],
'detail_url': self.urls['detail'] % url['id'][0],
'size': self.parseSize(result.find_all('td')[7].string),
'seeders': tryInt(result.find_all('td')[4].string),
'leechers': tryInt(result.find_all('td')[5].string),
'get_more_info': self.getMoreInfo
}
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = False, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
return results
})
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def getMoreInfo(self, item):
full_description = self.getCache('publichd.%s' % item['id'], item['detail_url'], cache_timeout = 25920000)

39
couchpotato/core/providers/torrent/sceneaccess/main.py

@ -1,8 +1,8 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import tryUrlencode, toUnicode
from couchpotato.core.helpers.variable import tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentProvider
import traceback
@ -29,9 +29,8 @@ class SceneAccess(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
url = self.urls['search'] % (
self.getCatId(quality['identifier'])[0],
@ -47,10 +46,11 @@ class SceneAccess(TorrentProvider):
# Do login for the cookies
if not self.login_opener and not self.login():
return results
return []
cache_key = 'sceneaccess.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, url, opener = self.login_opener)
results = ResultList(self, movie, quality, imdb_results = True)
data = self.getHTMLData(url, opener = self.login_opener)
if data:
html = BeautifulSoup(data)
@ -66,37 +66,24 @@ class SceneAccess(TorrentProvider):
link = result.find('td', attrs = {'class' : 'ttr_name'}).find('a')
url = result.find('td', attrs = {'class' : 'td_dl'}).find('a')
leechers = result.find('td', attrs = {'class' : 'ttr_leechers'}).find('a')
id = link['href'].replace('details?id=', '')
new = {
'id': id,
'type': 'torrent',
'check_nzb': False,
'description': '',
'provider': self.getName(),
torrent_id = link['href'].replace('details?id=', '')
results.append({
'id': torrent_id,
'name': link['title'],
'url': self.urls['download'] % url['href'],
'detail_url': self.urls['detail'] % id,
'detail_url': self.urls['detail'] % torrent_id,
'size': self.parseSize(result.find('td', attrs = {'class' : 'ttr_size'}).contents[0]),
'seeders': tryInt(result.find('td', attrs = {'class' : 'ttr_seeders'}).find('a').string),
'leechers': tryInt(leechers.string) if leechers else 0,
'download': self.loginDownload,
'get_more_info': self.getMoreInfo,
}
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = False, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
})
return results
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def getLoginParams(self):
return tryUrlencode({

42
couchpotato/core/providers/torrent/scenehd/main.py

@ -1,8 +1,8 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import simplifyString, tryUrlencode
from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentProvider
import traceback
@ -23,9 +23,8 @@ class SceneHD(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
q = '"%s %s" %s' % (simplifyString(getTitle(movie['library'])), movie['library']['year'], quality.get('identifier'))
arguments = tryUrlencode({
@ -35,10 +34,11 @@ class SceneHD(TorrentProvider):
# Cookie login
if not self.login_opener and not self.login():
return results
return []
cache_key = 'scenehd.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
data = self.getCache(cache_key, url, opener = self.login_opener)
results = ResultList(self, movie, quality, imdb_results = True)
data = self.getHTMLData(url, opener = self.login_opener)
if data:
html = BeautifulSoup(data)
@ -52,7 +52,7 @@ class SceneHD(TorrentProvider):
detail_link = all_cells[2].find('a')
details = detail_link['href']
id = details.replace('details.php?id=', '')
torrent_id = details.replace('details.php?id=', '')
leechers = all_cells[11].find('a')
if leechers:
@ -60,37 +60,21 @@ class SceneHD(TorrentProvider):
else:
leechers = all_cells[11].string
new = {
'id': id,
results.append({
'id': torrent_id,
'name': detail_link['title'],
'type': 'torrent',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'size': self.parseSize(all_cells[7].string),
'seeders': tryInt(all_cells[10].find('a').string),
'leechers': tryInt(leechers),
'url': self.urls['download'] % id,
'url': self.urls['download'] % torrent_id,
'download': self.loginDownload,
}
imdb_link = all_cells[1].find('a')
imdb_results = self.imdbMatch(imdb_link['href'], movie['library']['identifier']) if imdb_link else False
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = imdb_results, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
return results
'description': all_cells[1].find('a')['href'],
})
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def getLoginParams(self, params):

39
couchpotato/core/providers/torrent/thepiratebay/main.py

@ -1,11 +1,10 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import toUnicode
from couchpotato.core.helpers.encoding import toUnicode, tryUrlencode
from couchpotato.core.helpers.variable import getTitle, tryInt, cleanHost
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.torrent.base import TorrentProvider
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentMagnetProvider
from couchpotato.environment import Env
from urllib import quote_plus
import re
import time
import traceback
@ -13,7 +12,7 @@ import traceback
log = CPLog(__name__)
class ThePirateBay(TorrentProvider):
class ThePirateBay(TorrentMagnetProvider):
urls = {
'detail': '%s/torrent/%s',
@ -76,13 +75,14 @@ class ThePirateBay(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled() or not self.getDomain():
return results
return []
cache_key = 'thepiratebay.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
search_url = self.urls['search'] % (self.getDomain(), quote_plus(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
data = self.getCache(cache_key, search_url)
results = ResultList(self, movie, quality)
search_url = self.urls['search'] % (self.getDomain(), tryUrlencode(getTitle(movie['library']) + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
data = self.getHTMLData(search_url)
if data:
try:
@ -90,7 +90,7 @@ class ThePirateBay(TorrentProvider):
results_table = soup.find('table', attrs = {'id': 'searchResult'})
if not results_table:
return results
return []
entries = results_table.find_all('tr')
for result in entries[2:]:
@ -112,13 +112,9 @@ class ThePirateBay(TorrentProvider):
return confirmed + trusted + vip + moderated
new = {
results.append({
'id': re.search('/(?P<id>\d+)/', link['href']).group('id'),
'type': 'torrent_magnet',
'name': link.string,
'check_nzb': False,
'description': '',
'provider': self.getName(),
'url': download['href'],
'detail_url': self.getDomain(link['href']),
'size': self.parseSize(size),
@ -126,17 +122,8 @@ class ThePirateBay(TorrentProvider):
'leechers': tryInt(result.find_all('td')[3].string),
'extra_score': extra_score,
'get_more_info': self.getMoreInfo
}
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = False, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
})
return results
except:
log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))

48
couchpotato/core/providers/torrent/torrentleech/main.py

@ -1,10 +1,9 @@
from bs4 import BeautifulSoup
from couchpotato.core.event import fireEvent
from couchpotato.core.helpers.encoding import tryUrlencode
from couchpotato.core.helpers.variable import getTitle, tryInt
from couchpotato.core.logger import CPLog
from couchpotato.core.providers.base import ResultList
from couchpotato.core.providers.torrent.base import TorrentProvider
from urllib import quote_plus
import traceback
@ -14,11 +13,11 @@ log = CPLog(__name__)
class TorrentLeech(TorrentProvider):
urls = {
'test' : 'http://torrentleech.org/',
'login' : 'http://torrentleech.org/user/account/login/',
'detail' : 'http://torrentleech.org/torrent/%s',
'search' : 'http://torrentleech.org/torrents/browse/index/query/%s/categories/%d',
'download' : 'http://torrentleech.org%s',
'test' : 'http://www.torrentleech.org/',
'login' : 'http://www.torrentleech.org/user/account/login/',
'detail' : 'http://www.torrentleech.org/torrent/%s',
'search' : 'http://www.torrentleech.org/torrents/browse/index/query/%s/categories/%d',
'download' : 'http://www.torrentleech.org%s',
}
cat_ids = [
@ -36,17 +35,17 @@ class TorrentLeech(TorrentProvider):
def search(self, movie, quality):
results = []
if self.isDisabled():
return results
return []
# Cookie login
if not self.login_opener and not self.login():
return results
return []
cache_key = 'torrentleech.%s.%s' % (movie['library']['identifier'], quality.get('identifier'))
url = self.urls['search'] % (quote_plus(getTitle(movie['library']).replace(':', '') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
data = self.getCache(cache_key, url, opener = self.login_opener)
results = ResultList(self, movie, quality)
url = self.urls['search'] % (tryUrlencode(getTitle(movie['library']).replace(':', '') + ' ' + quality['identifier']), self.getCatId(quality['identifier'])[0])
data = self.getHTMLData(url, opener = self.login_opener)
if data:
html = BeautifulSoup(data)
@ -54,7 +53,7 @@ class TorrentLeech(TorrentProvider):
try:
result_table = html.find('table', attrs = {'id' : 'torrenttable'})
if not result_table:
return results
return []
entries = result_table.find_all('tr')
@ -64,36 +63,21 @@ class TorrentLeech(TorrentProvider):
url = result.find('td', attrs = {'class' : 'quickdownload'}).find('a')
details = result.find('td', attrs = {'class' : 'name'}).find('a')
new = {
results.append({
'id': link['href'].replace('/torrent/', ''),
'name': link.string,
'type': 'torrent',
'check_nzb': False,
'description': '',
'provider': self.getName(),
'url': self.urls['download'] % url['href'],
'detail_url': self.urls['download'] % details['href'],
'download': self.loginDownload,
'size': self.parseSize(result.find_all('td')[4].string),
'seeders': tryInt(result.find('td', attrs = {'class' : 'seeders'}).string),
'leechers': tryInt(result.find('td', attrs = {'class' : 'leechers'}).string),
}
imdb_results = self.imdbMatch(self.urls['detail'] % new['id'], movie['library']['identifier'])
new['score'] = fireEvent('score.calculate', new, movie, single = True)
is_correct_movie = fireEvent('searcher.correct_movie', nzb = new, movie = movie, quality = quality,
imdb_results = imdb_results, single = True)
if is_correct_movie:
results.append(new)
self.found(new)
})
return results
except:
log.error('Failed to parsing %s: %s', (self.getName(), traceback.format_exc()))
return []
return results
def getLoginParams(self):
return tryUrlencode({

Loading…
Cancel
Save