diff --git a/CHANGES.md b/CHANGES.md index 69cca59..cf85a38 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,4 +1,15 @@ -### 0.23.13 (2021-02-26 19:05:00 UTC) +### 0.23.14 (2021-03-10 01:40:00 UTC) + +* Add config/Search/Search Tasks/"Host running FlareSolverr" to handle CloudFlare providers +* Change the cf_clearance cookie to an undocumented optional config instead of a requirement +* Change where cf_clearance does not exist or expires, config/Search/Search Tasks/"Host running FlareSolverr" is required +* Fix saving magnet from PAs as files under py3 +* Fix SkyTorrents provider +* Fix Torlock provider +* Fix TBP provider + + +### 0.23.13 (2021-02-26 19:05:00 UTC) * Add Newznab providers can use API only or API + RSS cache fallback. Tip added to Newznab config/Media Providers/API key * Add correct user entry mistakes for nzbs2go api url diff --git a/gui/slick/interfaces/default/config_search.tmpl b/gui/slick/interfaces/default/config_search.tmpl index 8b861d0..106e14c 100755 --- a/gui/slick/interfaces/default/config_search.tmpl +++ b/gui/slick/interfaces/default/config_search.tmpl @@ -1,6 +1,6 @@ #import sickbeard #from sickbeard import clients -#from sickbeard.helpers import starify, generate_word_str +#from sickbeard.helpers import anon_url, starify, generate_word_str <% def sg_var(varname, default=False): return getattr(sickbeard, varname, default) %>#slurp# <% def sg_str(varname, default=''): return getattr(sickbeard, varname, default) %>#slurp# ## @@ -237,7 +237,21 @@ - +
+ + + + connection result + +
+ + diff --git a/gui/slick/js/configSearch.js b/gui/slick/js/configSearch.js index 3d58481..04759bd 100644 --- a/gui/slick/js/configSearch.js +++ b/gui/slick/js/configSearch.js @@ -9,6 +9,29 @@ $(document).ready(function(){ /^\s*scgi:\/\//.test($('#torrent_host').val()) ? opts$.hide() && $('#torrent_label').select() && '' : opts$.show() && $('#torrent_username').select() && $(this).prop('defaultValue'));});}; + $('#test-flaresolverr').click(function(){ + var host$ = $('#flaresolverr'), host = $.trim(host$.val()), result$ = $('#test-flaresolverr-result'), + valid = !!host && -1 !== host.indexOf('http'); + if (!valid) { + result$.html('Please correct the field above'); + if (!valid) { + host$.addClass('warning'); + } else { + host$.removeClass('warning'); + } + return; + } + host$.removeClass('warning'); + $(this).prop('disabled', !0); + result$.html(loading); + $.get(sbRoot + '/home/test-flaresolverr', + {host: host}) + .done(function (data) { + $('#test-flaresolverr-result').html(data); + $('#test-flaresolverr').prop('disabled', !1); + }); + }); + function toggleTorrentTitle() { var noTorrent$ = $('#no_torrents'); if ($('#use_torrents').prop('checked')) diff --git a/lib/cfscrape/__init__.py b/lib/cfscrape/__init__.py index ebb26fb..50a5302 100644 --- a/lib/cfscrape/__init__.py +++ b/lib/cfscrape/__init__.py @@ -1,18 +1,13 @@ -from requests.adapters import HTTPAdapter from requests.exceptions import RequestException from requests.models import Response from requests.sessions import Session -from urllib3.util.ssl_ import create_urllib3_context import logging import random import re -import ssl import time -import js2py - -from _23 import b64decodestring, b64encodestring, urlparse, urlunparse +from _23 import b64encodestring, urlparse DEFAULT_USER_AGENTS = [ @@ -46,25 +41,27 @@ class CloudflareScraper(Session): self.delay = kwargs.pop('delay', self.default_delay) self.start_time = None - self.cipher_suite = self.load_cipher_suite() - self.mount('https://', CloudflareAdapter(self.cipher_suite)) self.trust_env = False def request(self, method, url, *args, **kwargs): - resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs) + url_solver = kwargs.pop('url_solver', None) + + if not kwargs.pop('proxy_browser', None): + resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs) + else: + resp = self.get_content(method, url, url_solver, + user_agent=self.headers.get('User-Agent'), proxy_browser=True, **kwargs) - # Check if anti-bot is on if (isinstance(resp, type(Response())) and resp.status_code in (503, 429, 403)): self.start_time = time.time() if (re.search('(?i)cloudflare', resp.headers.get('Server', '')) and b'jschl_vc' in resp.content and b'jschl_answer' in resp.content): - resp = self.solve_cf_challenge(resp, **kwargs) + resp = self.solve_cf_challenge(resp, url_solver, **kwargs) elif b'ddgu' in resp.content: resp = self.solve_ddg_challenge(resp, **kwargs) - # Otherwise, no anti-bot detected return resp def wait(self): @@ -87,142 +84,74 @@ class CloudflareScraper(Session): pass return resp - def solve_cf_challenge(self, resp, **original_kwargs): + def test_flaresolverr(self, url_solver): + # test if FlareSolverr software is running + response_test = super(CloudflareScraper, self).request('GET', url_solver) + fs_ver = None + if 200 == response_test.status_code and response_test.ok: + json_data = response_test.json() + if 'ok' == json_data.get('status'): + fs_ver = json_data.get('version') + if None is fs_ver: + raise ValueError('FlareSolverr software not found (is it running?)') + return fs_ver + + def get_content(self, method, url, url_solver, user_agent, proxy_browser=False, **kwargs): + + url_solver = url_solver and re.sub(r'(/|v1)*$', '', url_solver) or 'http://localhost:8191' + if not self.test_flaresolverr(url_solver): + raise ValueError('No FlareSolverr software running %sat %s' % (('to solve Cloudflare challenge ', + '')[proxy_browser], url_solver)) + try: + response = super(CloudflareScraper, self).request('POST', '%s/v1' % url_solver, json=dict( + cmd='request.%s' % method.lower(), userAgent=user_agent, url=url, + cookies=[{'name': cur_ckee.name, 'value': cur_ckee.value, + 'domain': cur_ckee.domain, 'path': cur_ckee.path} for cur_ckee in self.cookies])) + except(BaseException, Exception) as e: + raise ValueError('FlareSolverr software unable to %s: %r' % (('solve Cloudflare anti-bot IUAM challenge', + 'fetch content')[proxy_browser], e)) + if None is not response: + data_json = response.json() + result = ({}, data_json)[isinstance(data_json, (dict, list))] + if response.ok: + if 'ok' == result.get('status'): + self.cookies.clear() + for cur_ckee in result.get('solution', {}).get('cookies', []): + if cur_ckee.get('value') and cur_ckee.get('name') not in ('', None, '_gid', '_ga', '_gat'): + self.cookies.set( + cur_ckee['name'], cur_ckee['value'], + rest={'httpOnly': cur_ckee.get('httpOnly'), 'session': cur_ckee.get('session')}, + **dict([(k, cur_ckee.get(k)) for k in ('expires', 'domain', 'path', 'secure')])) + else: + response = None + elif 'error' == result.get('status'): + raise ValueError('Failure with FlareSolverr: %s' % result.get('message', 'See the FlareSolver output')) + + return response + + def solve_cf_challenge(self, resp, url_solver, **original_kwargs): body = resp.text parsed_url = urlparse(resp.url) domain = parsed_url.netloc if '/cdn-cgi/l/chk_captcha' in body or 'cf_chl_captcha' in body: raise CloudflareError( - 'Cloudflare captcha presented for %s, please notify SickGear for an update, ua: %s' % + 'Cloudflare captcha presented for %s, safe to ignore as this shouldn\'t happen every time, ua: %s' % (domain, self.cf_ua), response=resp) - try: - action, method = re.findall(r'(?sim)]+?hidden[^>]+?>)', re.sub(r'(?sim)