From 9049e3baf69945e7239b3482228c6c07e4ea0f09 Mon Sep 17 00:00:00 2001 From: James White Date: Fri, 30 Dec 2016 10:36:08 +0000 Subject: [PATCH 1/9] Remove kat.cr and fix trailing slash --- couchpotato/core/media/_base/providers/torrent/kickasstorrents.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py index 825dd64..8a3ea60 100644 --- a/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py +++ b/couchpotato/core/media/_base/providers/torrent/kickasstorrents.py @@ -45,8 +45,7 @@ class Base(TorrentMagnetProvider): 'https://kickass.bypassed.live', 'https://kickass.bypassed.video', 'https://kickass.bypassed.red', - 'https://kat.cr', - 'https://kickass.unblocked.pw/', + 'https://kickass.unblocked.pw', 'https://katproxy.com' ] From 7ba69047ca472bb5d387529b469b3c68b3c850f0 Mon Sep 17 00:00:00 2001 From: jwvanderbeck Date: Mon, 9 Jan 2017 20:27:48 -0800 Subject: [PATCH 2/9] Add option to configure search categories for Newznab providers --- couchpotato/core/media/_base/providers/nzb/newznab.py | 14 +++++++++++++- couchpotato/core/media/movie/providers/nzb/newznab.py | 3 +++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/nzb/newznab.py b/couchpotato/core/media/_base/providers/nzb/newznab.py index 84b0e45..0f28db8 100644 --- a/couchpotato/core/media/_base/providers/nzb/newznab.py +++ b/couchpotato/core/media/_base/providers/nzb/newznab.py @@ -128,6 +128,7 @@ class Base(NZBProvider, RSS): api_keys = splitString(self.conf('api_key'), clean = False) extra_score = splitString(self.conf('extra_score'), clean = False) custom_tags = splitString(self.conf('custom_tag'), clean = False) + custom_categories = splitString(self.conf('custom_categories'), clean = False) list = [] for nr in range(len(hosts)): @@ -144,12 +145,16 @@ class Base(NZBProvider, RSS): try: custom_tag = custom_tags[nr] except: custom_tag = '' + try: custom_category = custom_categories[nr].replace(" ", ",") + except: custom_category = '' + list.append({ 'use': uses[nr], 'host': host, 'api_key': key, 'extra_score': score, - 'custom_tag': custom_tag + 'custom_tag': custom_tag, + 'custom_category' : custom_category }) return list @@ -266,6 +271,13 @@ config = [{ 'description': 'Add custom tags, for example add rls=1 to get only scene releases from nzbs.org', }, { + 'name': 'custom_categories', + 'advanced': True, + 'label': 'Custom Categories', + 'default': '2000,2000,2000,2000,2000,2000', + 'description': 'Specify categories to search in seperated by a single space, defaults to all movies. EG: "2030 2040 2060" would only search in HD, SD, and 3D movie categories', + }, + { 'name': 'api_key', 'default': ',,,,,', 'label': 'Api Key', diff --git a/couchpotato/core/media/movie/providers/nzb/newznab.py b/couchpotato/core/media/movie/providers/nzb/newznab.py index fc94acb..3392b91 100644 --- a/couchpotato/core/media/movie/providers/nzb/newznab.py +++ b/couchpotato/core/media/movie/providers/nzb/newznab.py @@ -23,4 +23,7 @@ class Newznab(MovieProvider, Base): if len(host.get('custom_tag', '')) > 0: query = '%s&%s' % (query, host.get('custom_tag')) + if len(host['custom_category']) > 0: + query = '%s&cat=%s' % (query, host['custom_category']) + return query From a340655848d16a7f10bdb0d8c9cefc5d380ce929 Mon Sep 17 00:00:00 2001 From: James White Date: Fri, 30 Dec 2016 09:08:31 +0000 Subject: [PATCH 3/9] Clean TPB proxy list --- .../media/_base/providers/torrent/thepiratebay.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/thepiratebay.py b/couchpotato/core/media/_base/providers/torrent/thepiratebay.py index 4cd180b..4e84ceb 100644 --- a/couchpotato/core/media/_base/providers/torrent/thepiratebay.py +++ b/couchpotato/core/media/_base/providers/torrent/thepiratebay.py @@ -25,35 +25,18 @@ class Base(TorrentMagnetProvider): http_time_between_calls = 0 proxy_list = [ - 'https://thepiratebay.mn', - 'https://thepiratebay.gd', - 'https://thepiratebay.la', - 'https://pirateproxy.sx', - 'https://piratebay.host', - 'https://thepiratebay.expert', + 'https://pirateproxy.cat', 'https://pirateproxy.wf', 'https://pirateproxy.tf', 'https://urbanproxy.eu', - 'https://pirate.guru', 'https://piratebays.co', 'https://pirateproxy.yt', 'https://thepiratebay.uk.net', - 'https://tpb.ninja', - 'https://thehiddenbay.me', - 'https://ukunlocked.com', 'https://thebay.tv', - 'https://tpb.freed0m4all.net', - 'https://piratebays.eu', 'https://thepirateproxy.co', - 'https://thepiratebayz.com', - 'https://zaatoka.eu', - 'https://piratemirror.net', 'https://theproxypirate.pw', - 'https://torrentdr.com', - 'https://tpbproxy.co', 'https://arrr.xyz', - 'https://www.cleantpbproxy.com', - 'http://tpb.dashitz.com', + 'https://tpb.dashitz.com' ] def __init__(self): From 8c8d74f299df4da8b943c844a6a879c32cff86b0 Mon Sep 17 00:00:00 2001 From: TheQwertiest Date: Sun, 2 Apr 2017 14:06:17 +0300 Subject: [PATCH 4/9] OmdbApi Fix Fixed OmdbApi parsing: omdapi json also contains list (under 'Ratings' key), trying to call lower() on this list resulted in error. --- couchpotato/core/media/movie/providers/info/omdbapi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/movie/providers/info/omdbapi.py b/couchpotato/core/media/movie/providers/info/omdbapi.py index 16f30ad..3f257c6 100644 --- a/couchpotato/core/media/movie/providers/info/omdbapi.py +++ b/couchpotato/core/media/movie/providers/info/omdbapi.py @@ -88,7 +88,8 @@ class OMDBAPI(MovieProvider): tmp_movie = movie.copy() for key in tmp_movie: - if tmp_movie.get(key).lower() == 'n/a': + tmp_movie_elem = tmp_movie.get(key) + if not isinstance(tmp_movie_elem, str) or tmp_movie_elem.lower() == 'n/a': del movie[key] year = tryInt(movie.get('Year', '')) From 2086d9acda2b070752bd0d3f9682ecd99845f6db Mon Sep 17 00:00:00 2001 From: Filip Andre Larsen Tomren Date: Thu, 13 Apr 2017 17:12:19 +0200 Subject: [PATCH 5/9] Changed where to look for matches for Bit-HDTV due to site changes https://www.bit-hdtv.com/forums/viewtopic.php?pid=96375 --- couchpotato/core/media/_base/providers/torrent/bithdtv.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/couchpotato/core/media/_base/providers/torrent/bithdtv.py b/couchpotato/core/media/_base/providers/torrent/bithdtv.py index 69c030e..149d7c1 100644 --- a/couchpotato/core/media/_base/providers/torrent/bithdtv.py +++ b/couchpotato/core/media/_base/providers/torrent/bithdtv.py @@ -42,7 +42,12 @@ class Base(TorrentProvider): html = BeautifulSoup(data, 'html.parser') try: - result_table = html.find('table', attrs = {'width': '750', 'class': ''}) + result_tables = html.find_all('table', attrs = {'width': '750', 'class': ''}) + if result_tables is None: + return + + result_table = result_tables[1] + if result_table is None: return From 04688dd41ce72ce4f45ca3d4bab10b83fd4ff273 Mon Sep 17 00:00:00 2001 From: ofir123 Date: Sat, 22 Apr 2017 14:36:30 +0300 Subject: [PATCH 6/9] Added support for the Hebrew subtitles website - TheWiz. --- couchpotato/core/plugins/subtitle.py | 2 +- libs/subliminal/core.py | 3 +- libs/subliminal/services/thewiz.py | 140 +++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 2 deletions(-) create mode 100644 libs/subliminal/services/thewiz.py diff --git a/couchpotato/core/plugins/subtitle.py b/couchpotato/core/plugins/subtitle.py index 135ec2d..ef4a806 100644 --- a/couchpotato/core/plugins/subtitle.py +++ b/couchpotato/core/plugins/subtitle.py @@ -16,7 +16,7 @@ autoload = 'Subtitle' class Subtitle(Plugin): - services = ['opensubtitles', 'thesubdb', 'subswiki', 'subscenter'] + services = ['opensubtitles', 'thesubdb', 'subswiki', 'subscenter', 'thewiz'] def __init__(self): addEvent('renamer.before', self.searchSingle) diff --git a/libs/subliminal/core.py b/libs/subliminal/core.py index 6661968..fcecd06 100755 --- a/libs/subliminal/core.py +++ b/libs/subliminal/core.py @@ -32,7 +32,8 @@ __all__ = ['SERVICES', 'LANGUAGE_INDEX', 'SERVICE_INDEX', 'SERVICE_CONFIDENCE', 'create_list_tasks', 'create_download_tasks', 'consume_task', 'matching_confidence', 'key_subtitles', 'group_by_video'] logger = logging.getLogger(__name__) -SERVICES = ['opensubtitles', 'bierdopje', 'subswiki', 'subtitulos', 'thesubdb', 'addic7ed', 'tvsubtitles', 'subscenter'] +SERVICES = ['opensubtitles', 'bierdopje', 'subswiki', 'subtitulos', 'thesubdb', 'addic7ed', 'tvsubtitles', + 'subscenter', 'thewiz'] LANGUAGE_INDEX, SERVICE_INDEX, SERVICE_CONFIDENCE, MATCHING_CONFIDENCE = range(4) diff --git a/libs/subliminal/services/thewiz.py b/libs/subliminal/services/thewiz.py new file mode 100644 index 0000000..da351a8 --- /dev/null +++ b/libs/subliminal/services/thewiz.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Ofir123 +# +# This file is part of subliminal. +# +# subliminal is free software; you can redistribute it and/or modify it under +# the terms of the GNU Lesser General Public License as published by +# the Free Software Foundation; either version 3 of the License, or +# (at your option) any later version. +# +# subliminal is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with subliminal. If not, see . +from . import ServiceBase +from ..exceptions import ServiceError +from ..language import language_set +from ..subtitles import get_subtitle_path, ResultSubtitle +from ..videos import Episode, Movie +from ..utils import to_unicode +import bisect +import logging +import os + +logger = logging.getLogger(__name__) + + +class TheWiz(ServiceBase): + server = 'http://subs.thewiz.info/' + api_based = True + languages = language_set(['he']) + videos = [Episode, Movie] + require_video = False + + _tmdb_api_key = 'f7f51775877e0bb6703520952b3c7840' + + def _search_imdb_id(self, title, year, is_movie): + """Search the IMDB ID for the given `title` and `year`. + + :param str title: title to search for. + :param int year: year to search for (or 0 if not relevant). + :param bool is_movie: If True, IMDB ID will be searched for in TMDB instead of TheWiz. + :return: the IMDB ID for the given title and year (or None if not found). + :rtype: str + """ + # make the search + logger.info('Searching IMDB ID for %r%r', title, '' if not year else ' ({})'.format(year)) + title = title.replace('\'', '') + if is_movie: + # get TMDB ID first + r = self.session.get('http://api.tmdb.org/3/search/movie?api_key={}&query={}{}&language=en'.format( + self._tmdb_api_key, title, '' if not year else '&year={}'.format(year))) + r.raise_for_status() + tmdb_results = r.json().get('results') + if tmdb_results: + tmdb_id = tmdb_results[0].get('id') + if tmdb_id: + # get actual IMDB ID from TMDB + r = self.session.get('http://api.tmdb.org/3/movie/{}?api_key={}&language=en'.format( + tmdb_id, self._tmdb_api_key)) + r.raise_for_status() + return str(r.json().get('imdb_id', '')) or None + return None + + # handle TV series + r = self.session.get(self.server_url + 'search.tv.php', params={'name': title}, timeout=10) + r.raise_for_status() + return r.text or None + + def list_checked(self, video, languages): + series = None + season = None + episode = None + title = video.title + imdb_id = video.imdbid + year = video.year + if isinstance(video, Episode): + series = video.series + season = video.season + episode = video.episode + return self.query(video.path or video.release, languages, series, season, + episode, title, imdb_id, year) + + def query(self, filepath, languages=None, series=None, season=None, episode=None, title=None, imdbid=None, year=None): + logger.debug(u'Getting subtitles for {0} season {1} episode {2} with languages {3}'.format( + series, season, episode, languages)) + # search for the IMDB ID if needed + is_movie = not (series and season and episode) + if is_movie and not title: + raise ServiceError('One or more parameters are missing') + # for tv series, we need the series IMDB ID, and not the specific episode ID + imdb_id = (is_movie and imdbid) or self._search_imdb_id(title, year, is_movie) + # get search parameters + season = season or 0 + episode = episode or 0 + version = os.path.splitext(os.path.basename(filepath))[0] if filepath else 0 + + # search + logger.debug(u'Using IMDB ID {0}'.format(imdb_id)) + url = 'http://subs.thewiz.info/search.id.php?imdb={}&season={}&episode={}&version={}'.format( + imdb_id, season, episode, version) + + # get the list of subtitles + logger.debug(u'Getting the list of subtitles') + r = self.session.get(url) + r.raise_for_status() + results = r.json() + + # loop over results + + subtitles = dict() + for result in results: + language_object = self.get_language('heb') + subtitle_id = result['id'] + release = result['versioname'] + subtitle_path = get_subtitle_path(filepath, language_object, self.config.multi) + download_link = self.server_url + 'zip/{0}.zip'.format(subtitle_id) + # add the release and increment downloaded count if we already have the subtitle + if subtitle_id in subtitles: + logger.debug(u'Found additional release {0} for subtitle {1}'.format(release, subtitle_id)) + bisect.insort_left(subtitles[subtitle_id].releases, release) # deterministic order + subtitles[subtitle_id].downloaded += 1 + continue + # otherwise create it + subtitle = ResultSubtitle(subtitle_path, language_object, self.__class__.__name__.lower(), + download_link, release=to_unicode(release)) + logger.debug(u'Found subtitle {0}'.format(subtitle)) + subtitles[subtitle_id] = subtitle + + return subtitles.values() + + def download(self, subtitle): + self.download_zip_file(subtitle.link, subtitle.path) + return subtitle + + +Service = TheWiz From f1b993745c41094fa54ae0a22be355bbc1fd19ef Mon Sep 17 00:00:00 2001 From: ofir123 Date: Sat, 22 Apr 2017 23:48:58 +0300 Subject: [PATCH 7/9] SubsCenter service now supports the new API. --- libs/subliminal/services/subscenter.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/libs/subliminal/services/subscenter.py b/libs/subliminal/services/subscenter.py index 9a5511c..253d925 100644 --- a/libs/subliminal/services/subscenter.py +++ b/libs/subliminal/services/subscenter.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2012 Ofir Brukner +# Copyright 2012 Ofir123 # # This file is part of subliminal. # @@ -16,7 +16,7 @@ # You should have received a copy of the GNU Lesser General Public License # along with subliminal. If not, see . from . import ServiceBase -from ..exceptions import ServiceError +from ..exceptions import DownloadFailedError, ServiceError from ..language import language_set from ..subtitles import get_subtitle_path, ResultSubtitle from ..videos import Episode, Movie @@ -30,7 +30,7 @@ logger = logging.getLogger(__name__) class Subscenter(ServiceBase): - server = 'http://www.subscenter.co/he/' + server = 'http://www.subscenter.org/he/' api_based = False languages = language_set(['he']) videos = [Episode, Movie] @@ -110,10 +110,11 @@ class Subscenter(ServiceBase): # Read the item. subtitle_id = subtitle_item['id'] subtitle_key = subtitle_item['key'] + subtitle_version = subtitle_item['h_version'] release = subtitle_item['subtitle_version'] subtitle_path = get_subtitle_path(filepath, language_object, self.config.multi) download_link = self.server_url + 'subtitle/download/{0}/{1}/?v={2}&key={3}'.format( - language_code, subtitle_id, release, subtitle_key) + language_code, subtitle_id, subtitle_version, subtitle_key) # Add the release and increment downloaded count if we already have the subtitle. if subtitle_id in subtitles: logger.debug('Found additional release {0} for subtitle {1}'.format( @@ -128,7 +129,11 @@ class Subscenter(ServiceBase): return subtitles.values() def download(self, subtitle): - self.download_zip_file(subtitle.link, subtitle.path) + try: + self.download_zip_file(subtitle.link, subtitle.path) + except DownloadFailedError: + # If no zip file was retrieved, daily downloads limit has exceeded. + raise ServiceError('Daily limit exceeded') return subtitle From 714a942bc5d8cf72b45369c9bb73566cdacebd68 Mon Sep 17 00:00:00 2001 From: Fmstrat Date: Thu, 11 May 2017 07:15:18 -0400 Subject: [PATCH 8/9] Updated script notification to use Popen instead of call. --- couchpotato/core/notifications/script.py | 34 +++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/couchpotato/core/notifications/script.py b/couchpotato/core/notifications/script.py index 9d8f753..1736c33 100644 --- a/couchpotato/core/notifications/script.py +++ b/couchpotato/core/notifications/script.py @@ -1,38 +1,48 @@ import traceback import subprocess +import os from couchpotato.core.helpers.encoding import toUnicode from couchpotato.core.helpers.variable import getIdentifier +from couchpotato.api import addApiView +from couchpotato.core.event import addEvent from couchpotato.core.logger import CPLog from couchpotato.core.notifications.base import Notification + + log = CPLog(__name__) autoload = 'Script' class Script(Notification): - def notify(self, message = '', data = None, listener = None): - if not data: data = {} + def __init__(self): + addApiView(self.testNotifyName(), self.test) - script_data = { - 'message': toUnicode(message) - } + addEvent('renamer.after', self.runScript) - if getIdentifier(data): - script_data.update({ - 'imdb_id': getIdentifier(data) - }) + def runScript(self, message = None, group = None): + if self.isDisabled(): return + if not group: group = {} + command = [self.conf('path'), group.get('destination_dir')] + log.info('Executing script command: %s ', command) try: - subprocess.call([self.conf('path'), message]) + p = subprocess.Popen(command, stdout = subprocess.PIPE, stderr = subprocess.STDOUT) + out = p.communicate() + log.info('Result from script: %s', str(out)) return True - except: - log.error('Script notification failed: %s', traceback.format_exc()) + except OSError as e: + log.error('Unable to run script: %s', e) return False + def test(self, **kwargs): + return { + 'success': os.path.isfile(self.conf('path')) + } config = [{ 'name': 'script', From 8b7934e50a50bab420a29ea36552cc03c02fa176 Mon Sep 17 00:00:00 2001 From: jbarr13 Date: Thu, 25 May 2017 09:41:09 -0500 Subject: [PATCH 9/9] Use IMDb codes for search and comparison Using the movie title to search was inaccurate and often failed. Also, the original title is not always correct. YTS offers lookup by imdb code and always finds an exact match if they have it. --- .../core/media/_base/providers/torrent/yts.py | 62 ++++++++++------------ 1 file changed, 28 insertions(+), 34 deletions(-) diff --git a/couchpotato/core/media/_base/providers/torrent/yts.py b/couchpotato/core/media/_base/providers/torrent/yts.py index f4567a9..188f9e5 100644 --- a/couchpotato/core/media/_base/providers/torrent/yts.py +++ b/couchpotato/core/media/_base/providers/torrent/yts.py @@ -1,7 +1,6 @@ from datetime import datetime -from couchpotato.core.helpers.variable import tryInt +from couchpotato.core.helpers.variable import tryInt, getIdentifier from couchpotato.core.logger import CPLog -from couchpotato.core.helpers.variable import getTitle from couchpotato.core.media._base.providers.torrent.base import TorrentMagnetProvider import random @@ -19,7 +18,7 @@ class Base(TorrentMagnetProvider): def _search(self, movie, quality, results): limit = 10 page = 1 - data = self.getJsonData(self.urls['search'] % (getTitle(movie), limit, page)) + data = self.getJsonData(self.urls['search'] % (getIdentifier(movie), limit, page)) if data: movie_count = tryInt(data['data']['movie_count']) @@ -32,37 +31,32 @@ class Base(TorrentMagnetProvider): for i in range(0,len(movie_results)): result = data['data']['movies'][i] name = result['title'] - - t = movie['info']['original_title'].split(' ') - - if all(word in name for word in t) and movie['info']['year'] == result['year']: - - year = result['year'] - detail_url = result['url'] - - for torrent in result['torrents']: - t_quality = torrent['quality'] - - if t_quality in quality['label']: - hash = torrent['hash'] - size = tryInt(torrent['size_bytes'] / 1048576) - seeders = tryInt(torrent['seeds']) - leechers = tryInt(torrent['peers']) - pubdate = torrent['date_uploaded'] # format: 2017-02-17 18:40:03 - pubdate = datetime.strptime(pubdate, '%Y-%m-%d %H:%M:%S') - age = (datetime.now() - pubdate).days - - results.append({ - 'id': random.randint(100, 9999), - 'name': '%s (%s) %s %s %s' % (name, year, 'YTS', t_quality, 'BR-Rip'), - 'url': self.make_magnet(hash, name), - 'size': size, - 'seeders': seeders, - 'leechers': leechers, - 'age': age, - 'detail_url': detail_url, - 'score': 1 - }) + year = result['year'] + detail_url = result['url'] + + for torrent in result['torrents']: + t_quality = torrent['quality'] + + if t_quality in quality['label']: + hash = torrent['hash'] + size = tryInt(torrent['size_bytes'] / 1048576) + seeders = tryInt(torrent['seeds']) + leechers = tryInt(torrent['peers']) + pubdate = torrent['date_uploaded'] # format: 2017-02-17 18:40:03 + pubdate = datetime.strptime(pubdate, '%Y-%m-%d %H:%M:%S') + age = (datetime.now() - pubdate).days + + results.append({ + 'id': random.randint(100, 9999), + 'name': '%s (%s) %s %s %s' % (name, year, 'YTS', t_quality, 'BR-Rip'), + 'url': self.make_magnet(hash, name), + 'size': size, + 'seeders': seeders, + 'leechers': leechers, + 'age': age, + 'detail_url': detail_url, + 'score': 1 + }) return