diff --git a/CHANGES.md b/CHANGES.md
index 69cca59..cf85a38 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -1,4 +1,15 @@
-### 0.23.13 (2021-02-26 19:05:00 UTC)
+### 0.23.14 (2021-03-10 01:40:00 UTC)
+
+* Add config/Search/Search Tasks/"Host running FlareSolverr" to handle CloudFlare providers
+* Change the cf_clearance cookie to an undocumented optional config instead of a requirement
+* Change where cf_clearance does not exist or expires, config/Search/Search Tasks/"Host running FlareSolverr" is required
+* Fix saving magnet from PAs as files under py3
+* Fix SkyTorrents provider
+* Fix Torlock provider
+* Fix TBP provider
+
+
+### 0.23.13 (2021-02-26 19:05:00 UTC)
* Add Newznab providers can use API only or API + RSS cache fallback. Tip added to Newznab config/Media Providers/API key
* Add correct user entry mistakes for nzbs2go api url
diff --git a/gui/slick/interfaces/default/config_search.tmpl b/gui/slick/interfaces/default/config_search.tmpl
index 8b861d0..106e14c 100755
--- a/gui/slick/interfaces/default/config_search.tmpl
+++ b/gui/slick/interfaces/default/config_search.tmpl
@@ -1,6 +1,6 @@
#import sickbeard
#from sickbeard import clients
-#from sickbeard.helpers import starify, generate_word_str
+#from sickbeard.helpers import anon_url, starify, generate_word_str
<% def sg_var(varname, default=False): return getattr(sickbeard, varname, default) %>#slurp#
<% def sg_str(varname, default=''): return getattr(sickbeard, varname, default) %>#slurp#
##
@@ -237,7 +237,21 @@
-
+
+
+
+
+ connection result
+
+
+
+
diff --git a/gui/slick/js/configSearch.js b/gui/slick/js/configSearch.js
index 3d58481..04759bd 100644
--- a/gui/slick/js/configSearch.js
+++ b/gui/slick/js/configSearch.js
@@ -9,6 +9,29 @@ $(document).ready(function(){
/^\s*scgi:\/\//.test($('#torrent_host').val()) ? opts$.hide() && $('#torrent_label').select() && ''
: opts$.show() && $('#torrent_username').select() && $(this).prop('defaultValue'));});};
+ $('#test-flaresolverr').click(function(){
+ var host$ = $('#flaresolverr'), host = $.trim(host$.val()), result$ = $('#test-flaresolverr-result'),
+ valid = !!host && -1 !== host.indexOf('http');
+ if (!valid) {
+ result$.html('Please correct the field above');
+ if (!valid) {
+ host$.addClass('warning');
+ } else {
+ host$.removeClass('warning');
+ }
+ return;
+ }
+ host$.removeClass('warning');
+ $(this).prop('disabled', !0);
+ result$.html(loading);
+ $.get(sbRoot + '/home/test-flaresolverr',
+ {host: host})
+ .done(function (data) {
+ $('#test-flaresolverr-result').html(data);
+ $('#test-flaresolverr').prop('disabled', !1);
+ });
+ });
+
function toggleTorrentTitle() {
var noTorrent$ = $('#no_torrents');
if ($('#use_torrents').prop('checked'))
diff --git a/lib/cfscrape/__init__.py b/lib/cfscrape/__init__.py
index ebb26fb..50a5302 100644
--- a/lib/cfscrape/__init__.py
+++ b/lib/cfscrape/__init__.py
@@ -1,18 +1,13 @@
-from requests.adapters import HTTPAdapter
from requests.exceptions import RequestException
from requests.models import Response
from requests.sessions import Session
-from urllib3.util.ssl_ import create_urllib3_context
import logging
import random
import re
-import ssl
import time
-import js2py
-
-from _23 import b64decodestring, b64encodestring, urlparse, urlunparse
+from _23 import b64encodestring, urlparse
DEFAULT_USER_AGENTS = [
@@ -46,25 +41,27 @@ class CloudflareScraper(Session):
self.delay = kwargs.pop('delay', self.default_delay)
self.start_time = None
- self.cipher_suite = self.load_cipher_suite()
- self.mount('https://', CloudflareAdapter(self.cipher_suite))
self.trust_env = False
def request(self, method, url, *args, **kwargs):
- resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs)
+ url_solver = kwargs.pop('url_solver', None)
+
+ if not kwargs.pop('proxy_browser', None):
+ resp = super(CloudflareScraper, self).request(method, url, *args, **kwargs)
+ else:
+ resp = self.get_content(method, url, url_solver,
+ user_agent=self.headers.get('User-Agent'), proxy_browser=True, **kwargs)
- # Check if anti-bot is on
if (isinstance(resp, type(Response()))
and resp.status_code in (503, 429, 403)):
self.start_time = time.time()
if (re.search('(?i)cloudflare', resp.headers.get('Server', ''))
and b'jschl_vc' in resp.content
and b'jschl_answer' in resp.content):
- resp = self.solve_cf_challenge(resp, **kwargs)
+ resp = self.solve_cf_challenge(resp, url_solver, **kwargs)
elif b'ddgu' in resp.content:
resp = self.solve_ddg_challenge(resp, **kwargs)
- # Otherwise, no anti-bot detected
return resp
def wait(self):
@@ -87,142 +84,74 @@ class CloudflareScraper(Session):
pass
return resp
- def solve_cf_challenge(self, resp, **original_kwargs):
+ def test_flaresolverr(self, url_solver):
+ # test if FlareSolverr software is running
+ response_test = super(CloudflareScraper, self).request('GET', url_solver)
+ fs_ver = None
+ if 200 == response_test.status_code and response_test.ok:
+ json_data = response_test.json()
+ if 'ok' == json_data.get('status'):
+ fs_ver = json_data.get('version')
+ if None is fs_ver:
+ raise ValueError('FlareSolverr software not found (is it running?)')
+ return fs_ver
+
+ def get_content(self, method, url, url_solver, user_agent, proxy_browser=False, **kwargs):
+
+ url_solver = url_solver and re.sub(r'(/|v1)*$', '', url_solver) or 'http://localhost:8191'
+ if not self.test_flaresolverr(url_solver):
+ raise ValueError('No FlareSolverr software running %sat %s' % (('to solve Cloudflare challenge ',
+ '')[proxy_browser], url_solver))
+ try:
+ response = super(CloudflareScraper, self).request('POST', '%s/v1' % url_solver, json=dict(
+ cmd='request.%s' % method.lower(), userAgent=user_agent, url=url,
+ cookies=[{'name': cur_ckee.name, 'value': cur_ckee.value,
+ 'domain': cur_ckee.domain, 'path': cur_ckee.path} for cur_ckee in self.cookies]))
+ except(BaseException, Exception) as e:
+ raise ValueError('FlareSolverr software unable to %s: %r' % (('solve Cloudflare anti-bot IUAM challenge',
+ 'fetch content')[proxy_browser], e))
+ if None is not response:
+ data_json = response.json()
+ result = ({}, data_json)[isinstance(data_json, (dict, list))]
+ if response.ok:
+ if 'ok' == result.get('status'):
+ self.cookies.clear()
+ for cur_ckee in result.get('solution', {}).get('cookies', []):
+ if cur_ckee.get('value') and cur_ckee.get('name') not in ('', None, '_gid', '_ga', '_gat'):
+ self.cookies.set(
+ cur_ckee['name'], cur_ckee['value'],
+ rest={'httpOnly': cur_ckee.get('httpOnly'), 'session': cur_ckee.get('session')},
+ **dict([(k, cur_ckee.get(k)) for k in ('expires', 'domain', 'path', 'secure')]))
+ else:
+ response = None
+ elif 'error' == result.get('status'):
+ raise ValueError('Failure with FlareSolverr: %s' % result.get('message', 'See the FlareSolver output'))
+
+ return response
+
+ def solve_cf_challenge(self, resp, url_solver, **original_kwargs):
body = resp.text
parsed_url = urlparse(resp.url)
domain = parsed_url.netloc
if '/cdn-cgi/l/chk_captcha' in body or 'cf_chl_captcha' in body:
raise CloudflareError(
- 'Cloudflare captcha presented for %s, please notify SickGear for an update, ua: %s' %
+ 'Cloudflare captcha presented for %s, safe to ignore as this shouldn\'t happen every time, ua: %s' %
(domain, self.cf_ua), response=resp)
- try:
- action, method = re.findall(r'(?sim)]+?hidden[^>]+?>)', re.sub(r'(?sim)