Warn when a server has downloaded a certain amount of bytes or a given date is reached (#1762)

* First working version

* Remove pprint

* Black

* Use date type and move to 5 minute polling

* Give hints about intended usage in explain text

* Use scheduled tasks and some smaller changes

* Black

* Remove hidden fields from form

* Cleanup

* This is not the easiest part to get right

* Black hook take 3

* Rework the server check tasks

* Show quota left for server

* Move Server description

Co-authored-by: Safihre <safihre@sabnzbd.org>

Closes #1455

interfaces/Config/templates/config_server.tmpl

@@ -5,7 +5,7 @@
 <!--#import json#-->
 <!--#import datetime#-->
 
-<script type="text/javascript">
+<script type="text/javascript" xmlns="http://www.w3.org/1999/html">
     // Define variable needed for the server-statistics
     var serverBandwithData = {}
     var serverArticleTries = {}
@@ -40,6 +40,10 @@
                         <input type="checkbox" name="enable" id="enable" value="1" checked="checked" />
                         <span class="desc">$T('srv-enable')</span>
                     </div>
+                    <div class="field-pair advanced-settings">
+                        <label class="config" for="displayname">$T('srv-displayname')</label>
+                        <input type="text" name="displayname" id="displayname" />
+                    </div>
                     <div class="field-pair">
                         <label class="config" for="host">$T('srv-host')</label>
                         <input type="text" name="host" id="host" required />
@@ -104,8 +108,14 @@
                         <span class="desc">$T('explain-optional')</span>
                     </div>
                     <div class="field-pair advanced-settings">
-                        <label class="config" for="displayname">$T('srv-displayname')</label>
-                        <input type="text" name="displayname" id="displayname" />
+                        <label class="config" for="expire_date">$T('srv-expire_date')</label>
+                        <input type="date" name="expire_date" id="expire_date" />
+                        <span class="desc">$T('srv-explain-expire_date')</span>
+                    </div>
+                    <div class="field-pair advanced-settings">
+                        <label class="config" for="quota">$T('swtag-quota')</label>
+                        <input type="text" name="quota" id="quota" class="smaller_input" />
+                        <span class="desc">$T('srv-explain-quota')</span>
                     </div>
                     <div class="field-pair advanced-settings">
                         <label class="config" for="notes">$T('srv-notes')</label>
@@ -155,6 +165,10 @@
                 <div class="col1" style="display:none;">
                     <input type="hidden" name="enable" id="enable$cur" value="$int($server['enable'])" />
                     <fieldset>
+                        <div class="field-pair advanced-settings">
+                            <label class="config" for="displayname$cur">$T('srv-displayname')</label>
+                            <input type="text" name="displayname" id="displayname$cur" value="$server['displayname']" />
+                        </div>
                         <div class="field-pair">
                             <label class="config" for="host$cur">$T('srv-host')</label>
                             <input type="text" name="host" id="host$cur" value="$server['host']" required />
@@ -220,8 +234,14 @@
                             <span class="desc">$T('srv-explain-send_group')</span>
                         </div>
                         <div class="field-pair advanced-settings">
-                            <label class="config" for="displayname$cur">$T('srv-displayname')</label>
-                            <input type="text" name="displayname" id="displayname$cur" value="$server['displayname']" />
+                            <label class="config" for="expire_date$cur">$T('srv-expire_date')</label>
+                            <input type="date" name="expire_date" id="expire_date$cur"  value="$server['expire_date']" />
+                            <span class="desc">$T('srv-explain-expire_date')</span>
+                        </div>
+                        <div class="field-pair advanced-settings">
+                            <label class="config" for="quota$cur">$T('swtag-quota')</label>
+                            <input type="text" name="quota" id="quota$cur" value="$server['quota']" class="smaller_input" />
+                            <span class="desc">$T('srv-explain-quota')</span>
                         </div>
                         <div class="field-pair advanced-settings">
                             <label class="config" for="notes$cur">$T('srv-notes')</label>
@@ -248,10 +268,14 @@
                                 $T('thisMonth'): $(server['amounts'][1])B<br/>
                                 $T('selectedDates'): <span id="server-bandwith-value-${cur}"></span>
                             </p>
+
                             <p title="$T('readwiki')">
                                 <b>$T('srv-article-availability'):</b><br/>
                                 $T('selectedDates'): <span id="server-article-value-${cur}"></span>
                             </p>
+                            <!--#if $server['quota']#-->
+                            <p><b>$T('quota-left'):</b> $(server['quota_left'])B</p>
+                            <!--#end if#-->
                         </div>
                         <div class="server-chart" data-serverid="${cur}">
                             <div id="server-chart-${cur}" class="ct-chart"></div>

sabnzbd/config.py

@@ -411,6 +411,9 @@ class ConfigServer:
         self.enable = OptionBool(name, "enable", True, add=False)
         self.optional = OptionBool(name, "optional", False, add=False)
         self.retention = OptionNumber(name, "retention", 0, add=False)
+        self.expire_date = OptionStr(name, "expire_date", add=False)
+        self.quota = OptionStr(name, "quota", add=False)
+        self.usage_at_start = OptionNumber(name, "usage_at_start", add=False)
         self.send_group = OptionBool(name, "send_group", False, add=False)
         self.priority = OptionNumber(name, "priority", 0, 0, 99, add=False)
         self.notes = OptionStr(name, "notes", add=False)
@@ -420,6 +423,12 @@ class ConfigServer:
 
     def set_dict(self, values: Dict[str, Any]):
         """ Set one or more fields, passed as dictionary """
+        # Replace usage_at_start value with most recent statistics if the user changes the quota value
+        # Only when we are updating it from the Config
+        if sabnzbd.WEBUI_READY and values.get("quota", "") != self.quota():
+            values["usage_at_start"] = sabnzbd.BPSMeter.grand_total.get(self.__name, 0)
+
+        # Store all values
         for kw in (
             "displayname",
             "host",
@@ -435,6 +444,9 @@ class ConfigServer:
             "enable",
             "optional",
             "retention",
+            "expire_date",
+            "quota",
+            "usage_at_start",
             "priority",
             "notes",
         ):
@@ -466,6 +478,9 @@ class ConfigServer:
         output_dict["enable"] = self.enable()
         output_dict["optional"] = self.optional()
         output_dict["retention"] = self.retention()
+        output_dict["expire_date"] = self.expire_date()
+        output_dict["quota"] = self.quota()
+        output_dict["usage_at_start"] = self.usage_at_start()
         output_dict["send_group"] = self.send_group()
         output_dict["priority"] = self.priority()
         output_dict["notes"] = self.notes()

sabnzbd/downloader.py

@@ -22,6 +22,7 @@ sabnzbd.downloader - download engine
 import time
 import select
 import logging
+from math import ceil
 from threading import Thread, RLock
 from nntplib import NNTPPermanentError
 import socket
@@ -469,6 +470,9 @@ class Downloader(Thread):
         # Kick BPS-Meter to check quota
         sabnzbd.BPSMeter.update()
 
+        # Check server expiration dates
+        check_server_expiration()
+
         while 1:
             now = time.time()
 
@@ -1014,3 +1018,29 @@ def clues_pay(text: str) -> bool:
         if clue in text:
             return True
     return False
+
+
+def check_server_expiration():
+    """Check if user should get warning about server date expiration"""
+    for server in config.get_servers().values():
+        if server.expire_date():
+            days_to_expire = ceil(
+                (time.mktime(time.strptime(server.expire_date(), "%Y-%m-%d")) - time.time()) / (60 * 60 * 24)
+            )
+            # Notify from 5 days in advance
+            if days_to_expire < 6:
+                logging.warning(T("Server %s is expiring in %s day(s)"), server.displayname(), days_to_expire)
+                # Reset on the day of expiration
+                if days_to_expire <= 0:
+                    server.expire_date.set("")
+                    config.save_config()
+
+
+def check_server_quota():
+    """Check quota on servers"""
+    for srv, server in config.get_servers().items():
+        if server.quota():
+            if server.quota.get_int() + server.usage_at_start() < sabnzbd.BPSMeter.grand_total.get(srv, 0):
+                logging.warning(T("Server %s has used the specified quota"), server.displayname())
+                server.quota.set("")
+                config.save_config()

sabnzbd/interface.py

@@ -1575,6 +1575,10 @@ class ConfigServer:
                     articles_tried,
                     articles_success,
                 )
+            new[-1]["quota_left"] = to_units(
+                servers[svr].quota.get_int() - sabnzbd.BPSMeter.grand_total.get(svr, 0) + servers[svr].usage_at_start()
+            )
+
         conf["servers"] = new
         conf["cats"] = list_cats(default=True)
         conf["certificate_validation"] = sabnzbd.CERTIFICATE_VALIDATION

sabnzbd/scheduler.py

@@ -34,6 +34,8 @@ import sabnzbd.cfg as cfg
 from sabnzbd.filesystem import diskspace
 from sabnzbd.constants import LOW_PRIORITY, NORMAL_PRIORITY, HIGH_PRIORITY
 
+DAILY_RANGE = list(range(1, 8))
+
 
 class Scheduler:
     def __init__(self):
@@ -105,7 +107,7 @@ class Scheduler:
             if d.isdigit():
                 d = [int(i) for i in d]
             else:
-                d = list(range(1, 8))
+                d = DAILY_RANGE
 
             if action_name == "resume":
                 action = self.scheduled_resume
@@ -202,16 +204,29 @@ class Scheduler:
         action, hour, minute = sabnzbd.BPSMeter.get_quota()
         if action:
             logging.info("Setting schedule for quota check daily at %s:%s", hour, minute)
-            self.scheduler.add_daytime_task(action, "quota_reset", list(range(1, 8)), None, (hour, minute))
+            self.scheduler.add_daytime_task(action, "quota_reset", DAILY_RANGE, None, (hour, minute))
 
         if sabnzbd.misc.int_conv(cfg.history_retention()) > 0:
             logging.info("Setting schedule for midnight auto history-purge")
             self.scheduler.add_daytime_task(
-                sabnzbd.database.midnight_history_purge, "midnight_history_purge", list(range(1, 8)), None, (0, 0)
+                sabnzbd.database.midnight_history_purge, "midnight_history_purge", DAILY_RANGE, None, (0, 0)
             )
 
         logging.info("Setting schedule for midnight BPS reset")
-        self.scheduler.add_daytime_task(sabnzbd.BPSMeter.midnight, "midnight_bps", list(range(1, 8)), None, (0, 0))
+        self.scheduler.add_daytime_task(sabnzbd.BPSMeter.midnight, "midnight_bps", DAILY_RANGE, None, (0, 0))
+
+        logging.info("Setting schedule for server expiration check")
+        self.scheduler.add_daytime_task(
+            sabnzbd.downloader.check_server_expiration, "check_server_expiration", DAILY_RANGE, None, (0, 0)
+        )
+
+        logging.info("Setting scheduler for server quota check")
+        self.scheduler.add_interval_task(
+            sabnzbd.downloader.check_server_quota,
+            "check_server_quota",
+            0,
+            10 * 60,
+        )
 
         # Subscribe to special schedule changes
         cfg.rss_rate.callback(self.scheduler_restart_guard)
@@ -377,6 +392,7 @@ class Scheduler:
         )
 
     def cancel_resume_task(self):
+        """ Cancel the current auto resume task """
         if self.resume_task:
             logging.debug("Cancelling existing resume_task '%s'", self.resume_task.name)
             self.scheduler.cancel(self.resume_task)

sabnzbd/skintext.py

@@ -624,6 +624,11 @@ SKIN_TEXT = {
     "srv-password": TT("Password"),  #: Server password
     "srv-timeout": TT("Timeout"),  #: Server timeout
     "srv-connections": TT("Connections"),  #: Server: amount of connections
+    "srv-expire_date": TT("Account expiration date"),
+    "srv-explain-expire_date": TT("Warn 5 days in advance of account expiration date."),
+    "srv-explain-quota": TT(
+        "Quota for this account, counted from the time it is set. In bytes, optionally follow with K,M,G.<br />Warn when it reaches 0, checked every few minutes."
+    ),
     "srv-retention": TT("Retention time"),  #: Server's retention time in days
     "srv-ssl": TT("SSL"),  #: Server SSL tickbox
     "explain-ssl": TT("Secure connection to server"),  #: Server SSL tickbox