dockeruser
/
sabnzbd
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

Duplicate check based on md5 was performed before md5 was calculated 

The side effect of this change is also that if you have an nzb-backup dir with the file already present that this second will be found duplicate, even before the first job has finished in the queue. Relates to #727
pull/1920/head
Safihre 4 years ago
parent
dc1f9bb252
commit
a7a3334c9a
5 changed files with 28 additions and 20 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      sabnzbd/__init__.py
  2. 4
      sabnzbd/database.py
  3. 4
      sabnzbd/nzbparser.py
  4. 36
      sabnzbd/nzbstuff.py
  5. 2
      tests/test_functional_adding_nzbs.py

2
sabnzbd/__init__.py
View File

@ -642,7 +642,7 @@ def add_nzbfile(
script: Optional[str] = None, script: Optional[str] = None,
cat: Optional[str] = None, cat: Optional[str] = None,
catdir: Optional[str] = None, catdir: Optional[str] = None,
priority: Optional[Union[Status, str]] = DEFAULT_PRIORITY, priority: Optional[Union[int, str]] = DEFAULT_PRIORITY,
nzbname: Optional[str] = None, nzbname: Optional[str] = None,
nzo_info=None, nzo_info=None,
url: Optional[str] = None, url: Optional[str] = None,

4
sabnzbd/database.py
View File

@ -343,7 +343,7 @@ class HistoryDB:
return items, total_items return items, total_items
def have_episode(self, series, season, episode): def have_episode(self, series: str, season: str, episode: str) -> bool:
"""Check whether History contains this series episode""" """Check whether History contains this series episode"""
total = 0 total = 0
if series and season and episode: if series and season and episode:
@ -354,7 +354,7 @@ class HistoryDB:
total = self.c.fetchone()["COUNT(*)"] total = self.c.fetchone()["COUNT(*)"]
return total > 0 return total > 0
def have_name_or_md5sum(self, name, md5sum): def have_name_or_md5sum(self, name: str, md5sum: str) -> bool:
"""Check whether this name or md5sum is already in History""" """Check whether this name or md5sum is already in History"""
total = 0 total = 0
if self.execute( if self.execute(

4
sabnzbd/nzbparser.py
View File

@ -163,7 +163,7 @@ def process_nzb_archive_file(
cat: Optional[str] = None, cat: Optional[str] = None,
catdir: Optional[str] = None, catdir: Optional[str] = None,
keep: bool = False, keep: bool = False,
priority: Optional[Union[Status, str]] = None, priority: Optional[Union[int, str]] = None,
nzbname: Optional[str] = None, nzbname: Optional[str] = None,
reuse: Optional[str] = None, reuse: Optional[str] = None,
nzo_info: Optional[Dict[str, Any]] = None, nzo_info: Optional[Dict[str, Any]] = None,
@ -264,7 +264,7 @@ def process_single_nzb(
cat: Optional[str] = None, cat: Optional[str] = None,
catdir: Optional[str] = None, catdir: Optional[str] = None,
keep: bool = False, keep: bool = False,
priority: Optional[Union[Status, str]] = None, priority: Optional[Union[int, str]] = None,
nzbname: Optional[str] = None, nzbname: Optional[str] = None,
reuse: Optional[str] = None, reuse: Optional[str] = None,
nzo_info: Optional[Dict[str, Any]] = None, nzo_info: Optional[Dict[str, Any]] = None,

36
sabnzbd/nzbstuff.py
View File

@ -591,7 +591,7 @@ class NzbObject(TryList):
futuretype: bool = False, futuretype: bool = False,
cat: Optional[str] = None, cat: Optional[str] = None,
url: Optional[str] = None, url: Optional[str] = None,
priority: Optional[Union[Status, str]] = DEFAULT_PRIORITY, priority: Optional[Union[int, str]] = DEFAULT_PRIORITY,
nzbname: Optional[str] = None, nzbname: Optional[str] = None,
status: Status = Status.QUEUED, status: Status = Status.QUEUED,
nzo_info: Optional[Dict[str, Any]] = None, nzo_info: Optional[Dict[str, Any]] = None,
@ -721,7 +721,7 @@ class NzbObject(TryList):
self.url_wait: Optional[float] = None self.url_wait: Optional[float] = None
self.url_tries = 0 self.url_tries = 0
self.pp_active = False # Signals active post-processing (not saved) self.pp_active = False # Signals active post-processing (not saved)
self.md5sum: Optional[bytes] = None self.md5sum: Optional[str] = None
if nzb_data is None and not reuse: if nzb_data is None and not reuse:
# This is a slot for a future NZB, ready now # This is a slot for a future NZB, ready now
@ -736,12 +736,6 @@ class NzbObject(TryList):
logging.info("Replacing dots with spaces in %s", self.final_name) logging.info("Replacing dots with spaces in %s", self.final_name)
self.final_name = self.final_name.replace(".", " ") self.final_name = self.final_name.replace(".", " ")
# Check against identical checksum or series/season/episode
if (not reuse) and nzb_data and dup_check and self.priority != REPAIR_PRIORITY:
duplicate, series = self.has_duplicates()
else:
duplicate = series = 0
# Reuse the existing directory # Reuse the existing directory
if reuse and os.path.exists(reuse): if reuse and os.path.exists(reuse):
work_dir = long_path(reuse) work_dir = long_path(reuse)
@ -874,13 +868,23 @@ class NzbObject(TryList):
self.oversized = True self.oversized = True
self.priority = LOW_PRIORITY self.priority = LOW_PRIORITY
if duplicate and ((not series and cfg.no_dupes() == 1) or (series and cfg.no_series_dupes() == 1)): # Check against identical checksum or series/season/episode
if (not reuse) and nzb_data and dup_check and self.priority != REPAIR_PRIORITY:
duplicate, series_duplicate = self.has_duplicates()
else:
duplicate = series_duplicate = False
if duplicate and (
(not series_duplicate and cfg.no_dupes() == 1) or (series_duplicate and cfg.no_series_dupes() == 1)
):
if cfg.warn_dupl_jobs(): if cfg.warn_dupl_jobs():
logging.warning(T('Ignoring duplicate NZB "%s"'), filename) logging.warning(T('Ignoring duplicate NZB "%s"'), filename)
self.purge_data() self.purge_data()
raise TypeError raise TypeError
if duplicate and ((not series and cfg.no_dupes() == 3) or (series and cfg.no_series_dupes() == 3)): if duplicate and (
(not series_duplicate and cfg.no_dupes() == 3) or (series_duplicate and cfg.no_series_dupes() == 3)
):
if cfg.warn_dupl_jobs(): if cfg.warn_dupl_jobs():
logging.warning(T('Failing duplicate NZB "%s"'), filename) logging.warning(T('Failing duplicate NZB "%s"'), filename)
# Move to history, utilizing the same code as accept&fail from pre-queue script # Move to history, utilizing the same code as accept&fail from pre-queue script
@ -1304,7 +1308,7 @@ class NzbObject(TryList):
if not self.unpack: if not self.unpack:
self.abort_direct_unpacker() self.abort_direct_unpacker()
def set_priority(self, value: Optional[Union[Status, str]]): def set_priority(self, value: Optional[Union[int, str]]):
"""Check if this is a valid priority""" """Check if this is a valid priority"""
# When unknown (0 is a known one), set to DEFAULT # When unknown (0 is a known one), set to DEFAULT
if value == "" or value is None: if value == "" or value is None:
@ -1962,7 +1966,7 @@ class NzbObject(TryList):
else: else:
nzf_ids.remove(nzf_id) nzf_ids.remove(nzf_id)
def has_duplicates(self): def has_duplicates(self) -> Tuple[bool, bool]:
"""Return (res, series) """Return (res, series)
where "res" is True when this is a duplicate where "res" is True when this is a duplicate
where "series" is True when this is an episode where "series" is True when this is an episode
@ -1984,11 +1988,15 @@ class NzbObject(TryList):
if no_dupes: if no_dupes:
res = history_db.have_name_or_md5sum(self.final_name, self.md5sum) res = history_db.have_name_or_md5sum(self.final_name, self.md5sum)
logging.debug( logging.debug(
"Dupe checking NZB in history: filename=%s, md5sum=%s, result=%s", self.filename, self.md5sum, res "Duplicate checked NZB in history: filename=%s, md5sum=%s, result=%s",
self.filename,
self.md5sum,
res,
) )
if not res and cfg.backup_for_duplicates(): if not res and cfg.backup_for_duplicates():
res = sabnzbd.backup_exists(self.filename) res = sabnzbd.backup_exists(self.filename)
logging.debug("Dupe checking NZB against backup: filename=%s, result=%s", self.filename, res) logging.debug("Duplicate checked NZB against backup: filename=%s, result=%s", self.filename, res)
# Dupe check off nzb filename # Dupe check off nzb filename
if not res and no_series_dupes: if not res and no_series_dupes:
series, season, episode, _, is_proper = sabnzbd.newsunpack.analyse_show(self.final_name) series, season, episode, _, is_proper = sabnzbd.newsunpack.analyse_show(self.final_name)

2
tests/test_functional_adding_nzbs.py
View File

@ -612,7 +612,7 @@ class TestAddingNZBs:
assert job["status"] == "Paused" if expected_prio != FORCE_PRIORITY else "Downloading" assert job["status"] == "Paused" if expected_prio != FORCE_PRIORITY else "Downloading"
# Reset duplicate handling (0), nzb_backup_dir ("") # Reset duplicate handling (0), nzb_backup_dir ("")
get_api_result(mode="set_config_default", extra_arguments={"keyword": "no_dupes", "keyword": "nzb_backup_dir"}) get_api_result(mode="set_config_default", extra_arguments={"keyword": ["no_dupes", "nzb_backup_dir"]})
# Remove backup_dir # Remove backup_dir
for timer in range(0, 5): for timer in range(0, 5):

Write Preview
Loading…
Cancel
Save