Browse Source
Change improve probability selecting most seeded release. Change add the TorrentDay x265 category to search. Change torrent provider code PEP8 and refactoring. Add BTScene torrent provider. Add Extratorrent provider. Add Limetorrents provider. Add nCore torrent provider. Remove Usenet Crawler provider.pull/764/head
JackDandy
9 years ago
56 changed files with 987 additions and 529 deletions
|
After Width: | Height: | Size: 548 B |
|
After Width: | Height: | Size: 497 B |
|
After Width: | Height: | Size: 682 B |
|
After Width: | Height: | Size: 482 B |
@ -0,0 +1,117 @@ |
|||||
|
# coding=utf-8 |
||||
|
# |
||||
|
# This file is part of SickGear. |
||||
|
# |
||||
|
# SickGear is free software: you can redistribute it and/or modify |
||||
|
# it under the terms of the GNU General Public License as published by |
||||
|
# the Free Software Foundation, either version 3 of the License, or |
||||
|
# (at your option) any later version. |
||||
|
# |
||||
|
# SickGear is distributed in the hope that it will be useful, |
||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
|
# GNU General Public License for more details. |
||||
|
# |
||||
|
# You should have received a copy of the GNU General Public License |
||||
|
# along with SickGear. If not, see <http://www.gnu.org/licenses/>. |
||||
|
|
||||
|
import re |
||||
|
import traceback |
||||
|
import urllib |
||||
|
|
||||
|
from . import generic |
||||
|
from sickbeard import logger |
||||
|
from sickbeard.bs4_parser import BS4Parser |
||||
|
from sickbeard.helpers import tryInt |
||||
|
from lib.unidecode import unidecode |
||||
|
|
||||
|
|
||||
|
class BTSceneProvider(generic.TorrentProvider): |
||||
|
|
||||
|
def __init__(self): |
||||
|
generic.TorrentProvider.__init__(self, 'BTScene') |
||||
|
|
||||
|
self.url_home = ['http://www.btstorrent.cc/', 'http://bittorrentstart.com/', |
||||
|
'http://diriri.xyz/', 'http://mytorrentz.tv/'] |
||||
|
|
||||
|
self.url_vars = {'search': 'results.php?q=%s&category=series&order=1', 'browse': 'lastdaycat/type/Series/', |
||||
|
'get': 'torrentdownload.php?id=%s'} |
||||
|
self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'search': '%(home)s%(vars)s', |
||||
|
'browse': '%(home)s%(vars)s', 'get': '%(home)s%(vars)s'} |
||||
|
|
||||
|
self.minseed, self.minleech = 2 * [None] |
||||
|
self.confirmed = False |
||||
|
|
||||
|
@staticmethod |
||||
|
def _has_signature(data=None): |
||||
|
return data and re.search(r'(?i)(?:btscene|bts[-]official|full\sindex)', data) |
||||
|
|
||||
|
def _search_provider(self, search_params, **kwargs): |
||||
|
|
||||
|
results = [] |
||||
|
if not self.url: |
||||
|
return results |
||||
|
|
||||
|
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} |
||||
|
|
||||
|
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in { |
||||
|
'info': '\w+?(\d+)[.]html', 'verified': 'Verified'}.iteritems()) |
||||
|
for mode in search_params.keys(): |
||||
|
for search_string in search_params[mode]: |
||||
|
|
||||
|
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string |
||||
|
|
||||
|
search_url = self.urls['browse'] if 'Cache' == mode \ |
||||
|
else self.urls['search'] % (urllib.quote_plus(search_string)) |
||||
|
|
||||
|
html = self.get_url(search_url) |
||||
|
|
||||
|
cnt = len(items[mode]) |
||||
|
try: |
||||
|
if not html or self._has_no_results(html): |
||||
|
raise generic.HaltParseException |
||||
|
with BS4Parser(html, features=['html5lib', 'permissive']) as soup: |
||||
|
torrent_rows = soup.select('tr[class$="_tr"]') |
||||
|
|
||||
|
if not len(torrent_rows): |
||||
|
raise generic.HaltParseException |
||||
|
|
||||
|
for tr in torrent_rows: |
||||
|
try: |
||||
|
seeders, leechers, size = [tryInt(n, n) for n in [ |
||||
|
tr.find_all('td')[x].get_text().strip() for x in -4, -3, -5]] |
||||
|
if self._peers_fail(mode, seeders, leechers) or \ |
||||
|
self.confirmed and not (tr.find('img', src=rc['verified']) |
||||
|
or tr.find('img', title=rc['verified'])): |
||||
|
continue |
||||
|
|
||||
|
info = tr.find('a', href=rc['info']) |
||||
|
title = info and info.get_text().strip() |
||||
|
tid_href = info and rc['info'].findall(info['href']) |
||||
|
tid_href = tid_href and tryInt(tid_href[0], 0) or 0 |
||||
|
tid_tr = tryInt(tr['id'].strip('_'), 0) |
||||
|
tid = (tid_tr, tid_href)[tid_href > tid_tr] |
||||
|
|
||||
|
download_url = info and (self.urls['get'] % tid) |
||||
|
except (AttributeError, TypeError, ValueError, IndexError): |
||||
|
continue |
||||
|
|
||||
|
if title and download_url: |
||||
|
items[mode].append((title, download_url, seeders, self._bytesizer(size))) |
||||
|
|
||||
|
except generic.HaltParseException: |
||||
|
pass |
||||
|
except (StandardError, Exception): |
||||
|
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) |
||||
|
|
||||
|
self._log_search(mode, len(items[mode]) - cnt, search_url) |
||||
|
|
||||
|
results = self._sort_seeding(mode, results + items[mode]) |
||||
|
|
||||
|
return results |
||||
|
|
||||
|
def _episode_strings(self, ep_obj, **kwargs): |
||||
|
return generic.TorrentProvider._episode_strings(self, ep_obj, sep_date='.', **kwargs) |
||||
|
|
||||
|
|
||||
|
provider = BTSceneProvider() |
||||
@ -0,0 +1,108 @@ |
|||||
|
# coding=utf-8 |
||||
|
# |
||||
|
# This file is part of SickGear. |
||||
|
# |
||||
|
# SickGear is free software: you can redistribute it and/or modify |
||||
|
# it under the terms of the GNU General Public License as published by |
||||
|
# the Free Software Foundation, either version 3 of the License, or |
||||
|
# (at your option) any later version. |
||||
|
# |
||||
|
# SickGear is distributed in the hope that it will be useful, |
||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
|
# GNU General Public License for more details. |
||||
|
# |
||||
|
# You should have received a copy of the GNU General Public License |
||||
|
# along with SickGear. If not, see <http://www.gnu.org/licenses/>. |
||||
|
|
||||
|
import re |
||||
|
import traceback |
||||
|
import urllib |
||||
|
|
||||
|
from . import generic |
||||
|
from sickbeard import logger |
||||
|
from sickbeard.bs4_parser import BS4Parser |
||||
|
from sickbeard.helpers import tryInt |
||||
|
from lib.unidecode import unidecode |
||||
|
|
||||
|
|
||||
|
class ExtraTorrentProvider(generic.TorrentProvider): |
||||
|
|
||||
|
def __init__(self): |
||||
|
generic.TorrentProvider.__init__(self, 'ExtraTorrent') |
||||
|
|
||||
|
self.url_home = ['https://www.extratorrent%s/' % u for u in '.works', 'live.com', 'online.com', '.cc'] + \ |
||||
|
['https://etmirror.com/', 'https://etproxy.com/', 'https://extratorrent.usbypass.xyz/'] |
||||
|
|
||||
|
self.url_vars = {'search': 'search/?new=1&search=%s&s_cat=8', 'browse': 'view/today/TV.html', |
||||
|
'get': '%s'} |
||||
|
self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'search': '%(home)s%(vars)s', |
||||
|
'browse': '%(home)s%(vars)s', 'get': '%(home)s%(vars)s'} |
||||
|
|
||||
|
self.minseed, self.minleech = 2 * [None] |
||||
|
|
||||
|
@staticmethod |
||||
|
def _has_signature(data=None): |
||||
|
return data and re.search(r'(?i)ExtraTorrent', data[33:1024:]) |
||||
|
|
||||
|
def _search_provider(self, search_params, **kwargs): |
||||
|
|
||||
|
results = [] |
||||
|
if not self.url: |
||||
|
return results |
||||
|
|
||||
|
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} |
||||
|
|
||||
|
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in { |
||||
|
'get': 'download', 'title': '(?:^download|torrent$)', 'get_url': '^/(torrent_)?'}.iteritems()) |
||||
|
|
||||
|
for mode in search_params.keys(): |
||||
|
for search_string in search_params[mode]: |
||||
|
|
||||
|
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string |
||||
|
|
||||
|
search_url = self.urls['browse'] if 'Cache' == mode \ |
||||
|
else self.urls['search'] % (urllib.quote_plus(search_string)) |
||||
|
|
||||
|
html = self.get_url(search_url) |
||||
|
|
||||
|
cnt = len(items[mode]) |
||||
|
try: |
||||
|
if not html or self._has_no_results(html): |
||||
|
raise generic.HaltParseException |
||||
|
with BS4Parser(html, features=['html5lib', 'permissive']) as soup: |
||||
|
torrent_table = soup.find('table', class_='tl') |
||||
|
torrent_rows = [] if not torrent_table else torrent_table.find_all('tr') |
||||
|
|
||||
|
if 2 > len(torrent_rows): |
||||
|
raise generic.HaltParseException |
||||
|
|
||||
|
for tr in torrent_rows[1:]: |
||||
|
try: |
||||
|
seeders, leechers, size = [tryInt(n.replace('---', '0'), n) for n in [ |
||||
|
tr.find_all('td')[x].get_text().strip() for x in -3, -2, -4]] |
||||
|
if self._peers_fail(mode, seeders, leechers): |
||||
|
continue |
||||
|
|
||||
|
info = tr.find('a', title=rc['get']) or {} |
||||
|
title = rc['title'].sub('', info.get('title') or '').strip() |
||||
|
download_url = self._link(rc['get_url'].sub('', info['href'])) |
||||
|
except (AttributeError, TypeError, ValueError, IndexError): |
||||
|
continue |
||||
|
|
||||
|
if title and download_url: |
||||
|
items[mode].append((title, download_url, seeders, self._bytesizer(size))) |
||||
|
|
||||
|
except generic.HaltParseException: |
||||
|
pass |
||||
|
except (StandardError, Exception): |
||||
|
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) |
||||
|
|
||||
|
self._log_search(mode, len(items[mode]) - cnt, search_url) |
||||
|
|
||||
|
results = self._sort_seeding(mode, results + items[mode]) |
||||
|
|
||||
|
return results |
||||
|
|
||||
|
|
||||
|
provider = ExtraTorrentProvider() |
||||
@ -0,0 +1,109 @@ |
|||||
|
# coding=utf-8 |
||||
|
# |
||||
|
# This file is part of SickGear. |
||||
|
# |
||||
|
# SickGear is free software: you can redistribute it and/or modify |
||||
|
# it under the terms of the GNU General Public License as published by |
||||
|
# the Free Software Foundation, either version 3 of the License, or |
||||
|
# (at your option) any later version. |
||||
|
# |
||||
|
# SickGear is distributed in the hope that it will be useful, |
||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
|
# GNU General Public License for more details. |
||||
|
# |
||||
|
# You should have received a copy of the GNU General Public License |
||||
|
# along with SickGear. If not, see <http://www.gnu.org/licenses/>. |
||||
|
|
||||
|
import re |
||||
|
import traceback |
||||
|
import urllib |
||||
|
|
||||
|
from . import generic |
||||
|
from sickbeard import logger |
||||
|
from sickbeard.bs4_parser import BS4Parser |
||||
|
from sickbeard.helpers import tryInt |
||||
|
from lib.unidecode import unidecode |
||||
|
|
||||
|
|
||||
|
class LimeTorrentsProvider(generic.TorrentProvider): |
||||
|
|
||||
|
def __init__(self): |
||||
|
generic.TorrentProvider.__init__(self, 'LimeTorrents') |
||||
|
|
||||
|
self.url_home = ['https://www.limetorrents.cc/', 'https://limetorrents.usbypass.xyz/'] |
||||
|
|
||||
|
self.url_vars = {'search': 'search/tv/%s/', 'browse': 'browse-torrents/TV-shows/'} |
||||
|
self.url_tmpl = {'config_provider_home_uri': '%(home)s', 'search': '%(home)s%(vars)s', |
||||
|
'browse': '%(home)s%(vars)s'} |
||||
|
|
||||
|
self.minseed, self.minleech = 2 * [None] |
||||
|
|
||||
|
@staticmethod |
||||
|
def _has_signature(data=None): |
||||
|
return data and re.search(r'(?i)LimeTorrents', data[33:1024:]) |
||||
|
|
||||
|
def _search_provider(self, search_params, **kwargs): |
||||
|
|
||||
|
results = [] |
||||
|
if not self.url: |
||||
|
return results |
||||
|
|
||||
|
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} |
||||
|
|
||||
|
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'get': 'dl'}.iteritems()) |
||||
|
|
||||
|
for mode in search_params.keys(): |
||||
|
for search_string in search_params[mode]: |
||||
|
|
||||
|
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string |
||||
|
|
||||
|
search_url = self.urls['browse'] if 'Cache' == mode \ |
||||
|
else self.urls['search'] % (urllib.quote_plus(search_string)) |
||||
|
|
||||
|
html = self.get_url(search_url) |
||||
|
|
||||
|
cnt = len(items[mode]) |
||||
|
try: |
||||
|
if not html or self._has_no_results(html): |
||||
|
raise generic.HaltParseException |
||||
|
with BS4Parser(html, features=['html5lib', 'permissive']) as soup: |
||||
|
torrent_table = soup.find_all('table', class_='table2') |
||||
|
torrent_rows = [] if not torrent_table else [ |
||||
|
t.select('tr[bgcolor]') for t in torrent_table if |
||||
|
all([x in ' '.join(x.get_text() for x in t.find_all('th')).lower() for x in |
||||
|
['torrent', 'size']])] |
||||
|
|
||||
|
if not len(torrent_rows): |
||||
|
raise generic.HaltParseException |
||||
|
|
||||
|
for tr in torrent_rows[0]: # 0 = all rows |
||||
|
try: |
||||
|
seeders, leechers, size = [tryInt(n.replace(',', ''), n) for n in [ |
||||
|
tr.find_all('td')[x].get_text().strip() for x in -3, -2, -4]] |
||||
|
if self._peers_fail(mode, seeders, leechers): |
||||
|
continue |
||||
|
|
||||
|
anchors = tr.td.find_all('a') |
||||
|
stats = anchors and [len(a.get_text()) for a in anchors] |
||||
|
title = stats and anchors[stats.index(max(stats))].get_text().strip() |
||||
|
download_url = self._link((tr.td.find('a', class_=rc['get']) or {}).get('href')) |
||||
|
except (AttributeError, TypeError, ValueError, IndexError): |
||||
|
continue |
||||
|
|
||||
|
if title and download_url: |
||||
|
items[mode].append((title, download_url, seeders, self._bytesizer(size))) |
||||
|
|
||||
|
except generic.HaltParseException: |
||||
|
pass |
||||
|
except (StandardError, Exception): |
||||
|
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) |
||||
|
|
||||
|
self._log_search(mode, len(items[mode]) - cnt, search_url) |
||||
|
|
||||
|
results = self._sort_seeding(mode, results + items[mode]) |
||||
|
|
||||
|
return results |
||||
|
|
||||
|
|
||||
|
provider = LimeTorrentsProvider() |
||||
@ -0,0 +1,112 @@ |
|||||
|
# coding=utf-8 |
||||
|
# |
||||
|
# Author: SickGear |
||||
|
# |
||||
|
# This file is part of SickGear. |
||||
|
# |
||||
|
# SickGear is free software: you can redistribute it and/or modify |
||||
|
# it under the terms of the GNU General Public License as published by |
||||
|
# the Free Software Foundation, either version 3 of the License, or |
||||
|
# (at your option) any later version. |
||||
|
# |
||||
|
# SickGear is distributed in the hope that it will be useful, |
||||
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
|
# GNU General Public License for more details. |
||||
|
# |
||||
|
# You should have received a copy of the GNU General Public License |
||||
|
# along with SickGear. If not, see <http://www.gnu.org/licenses/>. |
||||
|
|
||||
|
import re |
||||
|
import traceback |
||||
|
|
||||
|
from . import generic |
||||
|
from sickbeard import logger |
||||
|
from sickbeard.bs4_parser import BS4Parser |
||||
|
from sickbeard.helpers import tryInt |
||||
|
from lib.unidecode import unidecode |
||||
|
|
||||
|
|
||||
|
class NcoreProvider(generic.TorrentProvider): |
||||
|
|
||||
|
def __init__(self): |
||||
|
generic.TorrentProvider.__init__(self, 'nCore') |
||||
|
|
||||
|
self.url_base = 'https://ncore.cc/' |
||||
|
self.urls = {'config_provider_home_uri': self.url_base, |
||||
|
'login_action': self.url_base + 'login.php', |
||||
|
'search': self.url_base + 'torrents.php?mire=%s&' + '&'.join([ |
||||
|
'miszerint=fid', 'hogyan=DESC', 'tipus=kivalasztottak_kozott', |
||||
|
'kivalasztott_tipus=xvidser,dvdser,hdser', 'miben=name']), |
||||
|
'get': self.url_base + '%s'} |
||||
|
|
||||
|
self.url = self.urls['config_provider_home_uri'] |
||||
|
|
||||
|
self.username, self.password, self.minseed, self.minleech = 4 * [None] |
||||
|
self.chk_td = True |
||||
|
|
||||
|
def _authorised(self, **kwargs): |
||||
|
|
||||
|
return super(NcoreProvider, self)._authorised( |
||||
|
logged_in=(lambda y='': all([bool(y), 'action="login' not in y, self.has_all_cookies('PHPSESSID')])), |
||||
|
post_params={'nev': self.username, 'pass': self.password, 'form_tmpl': 'name=[\'"]login[\'"]'}) |
||||
|
|
||||
|
def _search_provider(self, search_params, **kwargs): |
||||
|
|
||||
|
results = [] |
||||
|
if not self._authorised(): |
||||
|
return results |
||||
|
|
||||
|
items = {'Cache': [], 'Season': [], 'Episode': [], 'Propers': []} |
||||
|
|
||||
|
rc = dict((k, re.compile('(?i)' + v)) for (k, v) in {'list': '.*?torrent_all', 'info': 'details'}.iteritems()) |
||||
|
for mode in search_params.keys(): |
||||
|
for search_string in search_params[mode]: |
||||
|
search_string = isinstance(search_string, unicode) and unidecode(search_string) or search_string |
||||
|
search_url = self.urls['search'] % search_string |
||||
|
|
||||
|
# fetches 15 results by default, and up to 100 if allowed in user profile |
||||
|
html = self.get_url(search_url) |
||||
|
|
||||
|
cnt = len(items[mode]) |
||||
|
try: |
||||
|
if not html or self._has_no_results(html): |
||||
|
raise generic.HaltParseException |
||||
|
|
||||
|
with BS4Parser(html, features=['html5lib', 'permissive']) as soup: |
||||
|
torrent_table = soup.find('div', class_=rc['list']) |
||||
|
torrent_rows = [] if not torrent_table else torrent_table.find_all('div', class_='box_torrent') |
||||
|
|
||||
|
if not len(torrent_rows): |
||||
|
raise generic.HaltParseException |
||||
|
|
||||
|
for tr in torrent_rows: |
||||
|
try: |
||||
|
seeders, leechers, size = [tryInt(n, n) for n in [ |
||||
|
tr.find('div', class_=x).get_text().strip() |
||||
|
for x in 'box_s2', 'box_l2', 'box_meret2']] |
||||
|
if self._peers_fail(mode, seeders, leechers): |
||||
|
continue |
||||
|
|
||||
|
anchor = tr.find('a', href=rc['info']) |
||||
|
title = (anchor.get('title') or anchor.get_text()).strip() |
||||
|
download_url = self._link(anchor.get('href').replace('details', 'download')) |
||||
|
except (AttributeError, TypeError, ValueError): |
||||
|
continue |
||||
|
|
||||
|
if title and download_url: |
||||
|
items[mode].append((title, download_url, seeders, self._bytesizer(size))) |
||||
|
|
||||
|
except generic.HaltParseException: |
||||
|
pass |
||||
|
except (StandardError, Exception): |
||||
|
logger.log(u'Failed to parse. Traceback: %s' % traceback.format_exc(), logger.ERROR) |
||||
|
|
||||
|
self._log_search(mode, len(items[mode]) - cnt, search_url) |
||||
|
|
||||
|
results = self._sort_seeding(mode, results + items[mode]) |
||||
|
|
||||
|
return results |
||||
|
|
||||
|
|
||||
|
provider = NcoreProvider() |
||||
Loading…
Reference in new issue