Diskspace macOS large drives (#1838)

* disk_free_macos_clib_statfs64() to report correct available disk space on MacOS

* disk_free_macos_clib_statfs64() ... correct call

* feedback processed into better code, and improved comments

* MACOSLIBC into __init__. And some comments about gnu libc

* import ctypes.util

* log ctypes.get_errno() in case of problems

* more cleanup and clarifications based on feedback

* mention python bug report in comment

* ... to trigger the CI again

* ... typo

sabnzbd/__init__.py

@@ -21,6 +21,7 @@ import datetime
 import tempfile
 import pickle
 import ctypes
+import ctypes.util
 import gzip
 import time
 import socket
@@ -35,7 +36,7 @@ from typing import Any, AnyStr
 # Determine platform flags
 ##############################################################################
 WIN32 = DARWIN = FOUNDATION = WIN64 = DOCKER = False
-KERNEL32 = LIBC = None
+KERNEL32 = LIBC = MACOSLIBC = None
 
 if os.name == "nt":
     WIN32 = True
@@ -56,12 +57,12 @@ elif os.name == "posix":
     except:
         pass
 
-    # See if we have Linux memory functions
+    # See if we have the GNU glibc malloc_trim() memory release function
     try:
         LIBC = ctypes.CDLL("libc.so.6")
-        LIBC.malloc_trim(0)
+        LIBC.malloc_trim(0)  # try the malloc_trim() call, which is a GNU extension
     except:
-        # No malloc_trim(), probably because no libc
+        # No malloc_trim(), probably because no glibc
         LIBC = None
         pass
 
@@ -70,6 +71,7 @@ elif os.name == "posix":
         DARWIN = True
         # 12 = Sierra, 11 = ElCaptain, 10 = Yosemite, 9 = Mavericks, 8 = MountainLion
         DARWIN_VERSION = int(platform.mac_ver()[0].split(".")[1])
+        MACOSLIBC = ctypes.CDLL(ctypes.util.find_library("c"), use_errno=True)  # the MacOS C library
         try:
             import Foundation
             import sabnzbd.utils.sleepless as sleepless
@@ -78,6 +80,7 @@ elif os.name == "posix":
         except:
             pass
 
+
 # Imported to be referenced from other files directly
 from sabnzbd.version import __version__, __baseline__
 

sabnzbd/filesystem.py

@@ -29,6 +29,7 @@ import time
 import fnmatch
 import stat
 import zipfile
+import ctypes
 from typing import Union, List, Tuple, Any, Dict, Optional
 
 try:
@@ -40,7 +41,7 @@ except ImportError:
 import sabnzbd
 from sabnzbd.decorators import synchronized
 from sabnzbd.constants import FUTURE_Q_FOLDER, JOB_ADMIN, GIGI, DEF_FILE_MAX
-from sabnzbd.encoding import correct_unknown_encoding
+from sabnzbd.encoding import correct_unknown_encoding, utob
 from sabnzbd.utils import rarfile
 
 # For Windows: determine executable extensions
@@ -944,6 +945,50 @@ def remove_all(path: str, pattern: str = "*", keep_folder: bool = False, recursi
 ##############################################################################
 # Diskfree
 ##############################################################################
+def disk_free_macos_clib_statfs64(directory: str) -> Tuple[int, int]:
+    # MacOS only!
+    # direct system call to c-lib's statfs(), not python's os.statvfs()
+    # because statvfs() on MacOS has a rollover at 4TB (possibly a 32bit rollover with 10bit block size)
+    # See https://bugs.python.org/issue43638
+    # Based on code of pudquick and blackntan
+    # Input: directory.
+    # Output: disksize and available space, in bytes
+
+    # format & parameters: on MacOS, see "man statfs", lines starting at
+    # "struct statfs { /* when _DARWIN_FEATURE_64_BIT_INODE is defined */"
+    class statfs64(ctypes.Structure):
+        _fields_ = [
+            ("f_bsize", ctypes.c_uint32),
+            ("f_iosize", ctypes.c_int32),
+            ("f_blocks", ctypes.c_uint64),
+            ("f_bfree", ctypes.c_uint64),
+            ("f_bavail", ctypes.c_uint64),
+            ("f_files", ctypes.c_uint64),
+            ("f_ffree", ctypes.c_uint64),
+            ("f_fsid", ctypes.c_uint64),
+            ("f_owner", ctypes.c_uint32),
+            ("f_type", ctypes.c_uint32),
+            ("f_flags", ctypes.c_uint32),
+            ("f_fssubtype", ctypes.c_uint32),
+            ("f_fstypename", ctypes.c_char * 16),
+            ("f_mntonname", ctypes.c_char * 1024),
+            ("f_mntfromname", ctypes.c_char * 1024),
+            ("f_reserved", ctypes.c_uint32 * 8),
+        ]
+
+    fs_info64 = statfs64()  # set up the parameters to be filled out
+    result = sabnzbd.MACOSLIBC.statfs64(
+        ctypes.create_string_buffer(utob(directory)), ctypes.byref(fs_info64)
+    )  # fs_info64 gets filled out via the byref()
+    if result == 0:
+        # result = 0: "Upon successful completion, a value of 0 is returned."
+        return fs_info64.f_blocks * fs_info64.f_bsize, fs_info64.f_bavail * fs_info64.f_bsize
+    else:
+        # result = -1: "Otherwise, -1 is returned and the global variable errno is set to indicate the error."
+        logging.debug("Call to MACOSLIBC.statfs64 not successful. Value of errno is %s", ctypes.get_errno())
+        return 0, 0
+
+
 def diskspace_base(dir_to_check: str) -> Tuple[float, float]:
     """ Return amount of free and used diskspace in GBytes """
     # Find first folder level that exists in the path
@@ -958,6 +1003,10 @@ def diskspace_base(dir_to_check: str) -> Tuple[float, float]:
             return disk_size / GIGI, available / GIGI
         except:
             return 0.0, 0.0
+    elif sabnzbd.DARWIN:
+        # MacOS diskfree ... via c-lib call statfs()
+        disk_size, available = disk_free_macos_clib_statfs64(dir_to_check)
+        return disk_size / GIGI, available / GIGI
     elif hasattr(os, "statvfs"):
         # posix diskfree
         try: