diff --git a/couchpotato/core/_base/updater/main.py b/couchpotato/core/_base/updater/main.py index 9ebecce..8164dc5 100644 --- a/couchpotato/core/_base/updater/main.py +++ b/couchpotato/core/_base/updater/main.py @@ -16,7 +16,6 @@ from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env from dateutil.parser import parse from git.repository import LocalRepository -from scandir import scandir import version from six.moves import filter @@ -182,7 +181,7 @@ class BaseUpdater(Plugin): def deletePyc(self, only_excess = True, show_logs = True): - for root, dirs, files in scandir.walk(Env.get('app_dir')): + for root, dirs, files in os.walk(Env.get('app_dir')): pyc_files = filter(lambda filename: filename.endswith('.pyc'), files) py_files = set(filter(lambda filename: filename.endswith('.py'), files)) @@ -329,11 +328,11 @@ class SourceUpdater(BaseUpdater): # Get list of files we want to overwrite self.deletePyc() existing_files = [] - for root, subfiles, filenames in scandir.walk(app_dir): + for root, subfiles, filenames in os.walk(app_dir): for filename in filenames: existing_files.append(os.path.join(root, filename)) - for root, subfiles, filenames in scandir.walk(path): + for root, subfiles, filenames in os.walk(path): for filename in filenames: fromfile = os.path.join(root, filename) tofile = os.path.join(app_dir, fromfile.replace(path + os.path.sep, '')) diff --git a/couchpotato/core/downloaders/rtorrent_.py b/couchpotato/core/downloaders/rtorrent_.py index 5ea3a4d..822501a 100644 --- a/couchpotato/core/downloaders/rtorrent_.py +++ b/couchpotato/core/downloaders/rtorrent_.py @@ -12,7 +12,6 @@ from couchpotato.core.helpers.variable import cleanHost, splitString from couchpotato.core.logger import CPLog from bencode import bencode, bdecode from rtorrent import RTorrent -from scandir import scandir log = CPLog(__name__) @@ -238,7 +237,7 @@ class rTorrent(DownloaderBase): if torrent.is_multi_file() and torrent.directory.endswith(torrent.name): # Remove empty directories bottom up try: - for path, _, _ in scandir.walk(sp(torrent.directory), topdown = False): + for path, _, _ in os.walk(sp(torrent.directory), topdown = False): os.rmdir(path) except OSError: log.info('Directory "%s" contains extra files, unable to remove', torrent.directory) diff --git a/couchpotato/core/plugins/base.py b/couchpotato/core/plugins/base.py index 1d9c0c4..f548cb2 100644 --- a/couchpotato/core/plugins/base.py +++ b/couchpotato/core/plugins/base.py @@ -16,7 +16,6 @@ from couchpotato.environment import Env import requests from requests.packages.urllib3 import Timeout from requests.packages.urllib3.exceptions import MaxRetryError -from scandir import scandir from tornado import template from tornado.web import StaticFileHandler @@ -144,7 +143,7 @@ class Plugin(object): def deleteEmptyFolder(self, folder, show_error = True): folder = sp(folder) - for root, dirs, files in scandir.walk(folder): + for root, dirs, files in os.walk(folder): for dir_name in dirs: full_path = os.path.join(root, dir_name) diff --git a/couchpotato/core/plugins/file.py b/couchpotato/core/plugins/file.py index 92d85c5..0d6483b 100644 --- a/couchpotato/core/plugins/file.py +++ b/couchpotato/core/plugins/file.py @@ -9,7 +9,6 @@ from couchpotato.core.helpers.variable import md5, getExt from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env -from scandir import scandir from tornado.web import StaticFileHandler @@ -49,7 +48,7 @@ class FileManager(Plugin): for x in file_dict.keys(): files.extend(file_dict[x]) - for f in scandir.scandir(cache_dir): + for f in os.listdir(cache_dir): if os.path.splitext(f.name)[1] in ['.png', '.jpg', '.jpeg']: file_path = os.path.join(cache_dir, f.name) if toUnicode(file_path) not in files: diff --git a/couchpotato/core/plugins/renamer.py b/couchpotato/core/plugins/renamer.py index 50c55d2..4ba07a2 100644 --- a/couchpotato/core/plugins/renamer.py +++ b/couchpotato/core/plugins/renamer.py @@ -14,7 +14,6 @@ from couchpotato.core.helpers.variable import getExt, mergeDicts, getTitle, \ from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from couchpotato.environment import Env -from scandir import scandir from unrar2 import RarFile import six from six.moves import filter @@ -195,7 +194,7 @@ class Renamer(Plugin): else: # Get all files from the specified folder try: - for root, folders, names in scandir.walk(media_folder): + for root, folders, names in os.walk(media_folder): files.extend([sp(os.path.join(root, name)) for name in names]) except: log.error('Failed getting files from %s: %s', (media_folder, traceback.format_exc())) @@ -664,7 +663,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Tag all files in release folder elif release_download['folder']: - for root, folders, names in scandir.walk(sp(release_download['folder'])): + for root, folders, names in os.walk(sp(release_download['folder'])): tag_files.extend([os.path.join(root, name) for name in names]) for filename in tag_files: @@ -704,7 +703,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Untag all files in release folder else: - for root, folders, names in scandir.walk(folder): + for root, folders, names in os.walk(folder): tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore']) if not folder: @@ -712,7 +711,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Find all .ignore files in folder ignore_files = [] - for root, dirnames, filenames in scandir.walk(folder): + for root, dirnames, filenames in os.walk(folder): ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag)) # Match all found ignore files with the tag_files and delete if found @@ -741,11 +740,11 @@ Remove it if you want it to be renamed (again, or at least let it try again) # Find tag on all files in release folder else: - for root, folders, names in scandir.walk(folder): + for root, folders, names in os.walk(folder): tag_files.extend([sp(os.path.join(root, name)) for name in names if not os.path.splitext(name)[1] == '.ignore']) # Find all .ignore files in folder - for root, dirnames, filenames in scandir.walk(folder): + for root, dirnames, filenames in os.walk(folder): ignore_files.extend(fnmatch.filter([sp(os.path.join(root, filename)) for filename in filenames], '*%s.ignore' % tag)) # Match all found ignore files with the tag_files and return True found @@ -1102,7 +1101,7 @@ Remove it if you want it to be renamed (again, or at least let it try again) check_file_date = False if not files: - for root, folders, names in scandir.walk(folder): + for root, folders, names in os.walk(folder): files.extend([sp(os.path.join(root, name)) for name in names]) # Find all archive files diff --git a/couchpotato/core/plugins/scanner.py b/couchpotato/core/plugins/scanner.py index b8eca4c..c24d64f 100644 --- a/couchpotato/core/plugins/scanner.py +++ b/couchpotato/core/plugins/scanner.py @@ -13,7 +13,6 @@ from couchpotato.core.logger import CPLog from couchpotato.core.plugins.base import Plugin from enzyme.exceptions import NoParserError, ParseError from guessit import guess_movie_info -from scandir import scandir from subliminal.videos import Video import enzyme from six.moves import filter, map, zip @@ -150,7 +149,7 @@ class Scanner(Plugin): check_file_date = True try: files = [] - for root, dirs, walk_files in scandir.walk(folder, followlinks=True): + for root, dirs, walk_files in os.walk(folder, followlinks=True): files.extend([sp(os.path.join(sp(root), ss(filename))) for filename in walk_files]) # Break if CP wants to shut down diff --git a/couchpotato/runner.py b/couchpotato/runner.py index d22f8fc..3ddd4ac 100644 --- a/couchpotato/runner.py +++ b/couchpotato/runner.py @@ -18,7 +18,6 @@ from couchpotato.api import NonBlockHandler, ApiHandler from couchpotato.core.event import fireEventAsync, fireEvent from couchpotato.core.helpers.encoding import sp from couchpotato.core.helpers.variable import getDataDir, tryInt -from scandir import scandir from tornado.httpserver import HTTPServer from tornado.web import Application, StaticFileHandler, RedirectHandler @@ -99,7 +98,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En existing_backups = [] if not os.path.isdir(backup_path): os.makedirs(backup_path) - for root, dirs, files in scandir.walk(backup_path): + for root, dirs, files in os.walk(backup_path): for backup_file in files: ints = re.findall('\d+', backup_file) @@ -116,7 +115,7 @@ def runCouchPotato(options, base_path, args, data_dir = None, log_dir = None, En # Create new backup new_backup = sp(os.path.join(backup_path, '%s.tar.gz' % int(time.time()))) zipf = tarfile.open(new_backup, 'w:gz') - for root, dirs, files in scandir.walk(db_path): + for root, dirs, files in os.walk(db_path): for zfilename in files: zipf.add(os.path.join(root, zfilename), arcname = 'database/%s' % os.path.join(root[len(db_path) + 1:], zfilename)) zipf.close() diff --git a/libs/scandir/.gitattributes b/libs/scandir/.gitattributes deleted file mode 100644 index 176a458..0000000 --- a/libs/scandir/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -* text=auto diff --git a/libs/scandir/.gitignore b/libs/scandir/.gitignore deleted file mode 100644 index 48878c7..0000000 --- a/libs/scandir/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*.pyc -*.pyd -benchtree -build diff --git a/libs/scandir/__init__.py b/libs/scandir/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/libs/scandir/_scandir.c b/libs/scandir/_scandir.c deleted file mode 100644 index 26c81ed..0000000 --- a/libs/scandir/_scandir.c +++ /dev/null @@ -1,373 +0,0 @@ -// scandir C speedups -// -// TODO: this is a work in progress! -// -// There's a fair bit of PY_MAJOR_VERSION boilerplate to support both Python 2 -// and Python 3 -- the structure of this is taken from here: -// http://docs.python.org/3.3/howto/cporting.html - -#include -#include - -#ifdef MS_WINDOWS -#include -#endif - -#if PY_MAJOR_VERSION >= 3 -#define INITERROR return NULL -#define FROM_LONG PyLong_FromLong -#define FROM_STRING PyUnicode_FromStringAndSize -#else -#define INITERROR return -#define FROM_LONG PyInt_FromLong -#define FROM_STRING PyString_FromStringAndSize -#endif - -#ifdef MS_WINDOWS - -static PyObject * -win32_error_unicode(char* function, Py_UNICODE* filename) -{ - errno = GetLastError(); - if (filename) - return PyErr_SetFromWindowsErrWithUnicodeFilename(errno, filename); - else - return PyErr_SetFromWindowsErr(errno); -} - -/* Below, we *know* that ugo+r is 0444 */ -#if _S_IREAD != 0400 -#error Unsupported C library -#endif -static int -attributes_to_mode(DWORD attr) -{ - int m = 0; - if (attr & FILE_ATTRIBUTE_DIRECTORY) - m |= _S_IFDIR | 0111; /* IFEXEC for user,group,other */ - else - m |= _S_IFREG; - if (attr & FILE_ATTRIBUTE_READONLY) - m |= 0444; - else - m |= 0666; - if (attr & FILE_ATTRIBUTE_REPARSE_POINT) - m |= 0120000; // S_IFLNK - return m; -} - -double -filetime_to_time(FILETIME *filetime) -{ - const double SECONDS_BETWEEN_EPOCHS = 11644473600.0; - - unsigned long long total = (unsigned long long)filetime->dwHighDateTime << 32 | - (unsigned long long)filetime->dwLowDateTime; - return (double)total / 10000000.0 - SECONDS_BETWEEN_EPOCHS; -} - -static PyTypeObject StatResultType; - -static PyObject * -find_data_to_statresult(WIN32_FIND_DATAW *data) -{ - PY_LONG_LONG size; - PyObject *v = PyStructSequence_New(&StatResultType); - if (v == NULL) - return NULL; - - size = (PY_LONG_LONG)data->nFileSizeHigh << 32 | - (PY_LONG_LONG)data->nFileSizeLow; - - PyStructSequence_SET_ITEM(v, 0, FROM_LONG(attributes_to_mode(data->dwFileAttributes))); - PyStructSequence_SET_ITEM(v, 1, FROM_LONG(0)); - PyStructSequence_SET_ITEM(v, 2, FROM_LONG(0)); - PyStructSequence_SET_ITEM(v, 3, FROM_LONG(0)); - PyStructSequence_SET_ITEM(v, 4, FROM_LONG(0)); - PyStructSequence_SET_ITEM(v, 5, FROM_LONG(0)); - PyStructSequence_SET_ITEM(v, 6, PyLong_FromLongLong((PY_LONG_LONG)size)); - PyStructSequence_SET_ITEM(v, 7, PyFloat_FromDouble(filetime_to_time(&data->ftLastAccessTime))); - PyStructSequence_SET_ITEM(v, 8, PyFloat_FromDouble(filetime_to_time(&data->ftLastWriteTime))); - PyStructSequence_SET_ITEM(v, 9, PyFloat_FromDouble(filetime_to_time(&data->ftCreationTime))); - - if (PyErr_Occurred()) { - Py_DECREF(v); - return NULL; - } - - return v; -} - -static PyStructSequence_Field stat_result_fields[] = { - {"st_mode", "protection bits"}, - {"st_ino", "inode"}, - {"st_dev", "device"}, - {"st_nlink", "number of hard links"}, - {"st_uid", "user ID of owner"}, - {"st_gid", "group ID of owner"}, - {"st_size", "total size, in bytes"}, - {"st_atime", "time of last access"}, - {"st_mtime", "time of last modification"}, - {"st_ctime", "time of last change"}, - {0} -}; - -static PyStructSequence_Desc stat_result_desc = { - "stat_result", /* name */ - NULL, /* doc */ - stat_result_fields, - 10 -}; - -static PyObject * -scandir_helper(PyObject *self, PyObject *args) -{ - PyObject *d, *v; - HANDLE hFindFile; - BOOL result; - WIN32_FIND_DATAW wFileData; - Py_UNICODE *wnamebuf; - Py_ssize_t len; - PyObject *po; - PyObject *name_stat; - - if (!PyArg_ParseTuple(args, "U:scandir_helper", &po)) - return NULL; - - /* Overallocate for \\*.*\0 */ - len = PyUnicode_GET_SIZE(po); - wnamebuf = malloc((len + 5) * sizeof(wchar_t)); - if (!wnamebuf) { - PyErr_NoMemory(); - return NULL; - } - wcscpy(wnamebuf, PyUnicode_AS_UNICODE(po)); - if (len > 0) { - Py_UNICODE wch = wnamebuf[len-1]; - if (wch != L'/' && wch != L'\\' && wch != L':') - wnamebuf[len++] = L'\\'; - wcscpy(wnamebuf + len, L"*.*"); - } - if ((d = PyList_New(0)) == NULL) { - free(wnamebuf); - return NULL; - } - Py_BEGIN_ALLOW_THREADS - hFindFile = FindFirstFileW(wnamebuf, &wFileData); - Py_END_ALLOW_THREADS - if (hFindFile == INVALID_HANDLE_VALUE) { - int error = GetLastError(); - if (error == ERROR_FILE_NOT_FOUND) { - free(wnamebuf); - return d; - } - Py_DECREF(d); - win32_error_unicode("FindFirstFileW", wnamebuf); - free(wnamebuf); - return NULL; - } - do { - /* Skip over . and .. */ - if (wcscmp(wFileData.cFileName, L".") != 0 && - wcscmp(wFileData.cFileName, L"..") != 0) { - v = PyUnicode_FromUnicode(wFileData.cFileName, wcslen(wFileData.cFileName)); - if (v == NULL) { - Py_DECREF(d); - d = NULL; - break; - } - name_stat = Py_BuildValue("ON", v, find_data_to_statresult(&wFileData)); - if (name_stat == NULL) { - Py_DECREF(v); - Py_DECREF(d); - d = NULL; - break; - } - if (PyList_Append(d, name_stat) != 0) { - Py_DECREF(v); - Py_DECREF(d); - Py_DECREF(name_stat); - d = NULL; - break; - } - Py_DECREF(name_stat); - Py_DECREF(v); - } - Py_BEGIN_ALLOW_THREADS - result = FindNextFileW(hFindFile, &wFileData); - Py_END_ALLOW_THREADS - /* FindNextFile sets error to ERROR_NO_MORE_FILES if - it got to the end of the directory. */ - if (!result && GetLastError() != ERROR_NO_MORE_FILES) { - Py_DECREF(d); - win32_error_unicode("FindNextFileW", wnamebuf); - FindClose(hFindFile); - free(wnamebuf); - return NULL; - } - } while (result == TRUE); - - if (FindClose(hFindFile) == FALSE) { - Py_DECREF(d); - win32_error_unicode("FindClose", wnamebuf); - free(wnamebuf); - return NULL; - } - free(wnamebuf); - return d; -} - -#else // Linux / OS X - -#include -#define NAMLEN(dirent) strlen((dirent)->d_name) - -static PyObject * -posix_error_with_allocated_filename(char* name) -{ - PyObject *rc = PyErr_SetFromErrnoWithFilename(PyExc_OSError, name); - PyMem_Free(name); - return rc; -} - -static PyObject * -scandir_helper(PyObject *self, PyObject *args) -{ - char *name = NULL; - PyObject *d, *v, *name_type; - DIR *dirp; - struct dirent *ep; - int arg_is_unicode = 1; - - errno = 0; - if (!PyArg_ParseTuple(args, "U:scandir_helper", &v)) { - arg_is_unicode = 0; - PyErr_Clear(); - } - if (!PyArg_ParseTuple(args, "et:scandir_helper", Py_FileSystemDefaultEncoding, &name)) - return NULL; - Py_BEGIN_ALLOW_THREADS - dirp = opendir(name); - Py_END_ALLOW_THREADS - if (dirp == NULL) { - return posix_error_with_allocated_filename(name); - } - if ((d = PyList_New(0)) == NULL) { - Py_BEGIN_ALLOW_THREADS - closedir(dirp); - Py_END_ALLOW_THREADS - PyMem_Free(name); - return NULL; - } - for (;;) { - errno = 0; - Py_BEGIN_ALLOW_THREADS - ep = readdir(dirp); - Py_END_ALLOW_THREADS - if (ep == NULL) { - if (errno == 0) { - break; - } else { - Py_BEGIN_ALLOW_THREADS - closedir(dirp); - Py_END_ALLOW_THREADS - Py_DECREF(d); - return posix_error_with_allocated_filename(name); - } - } - if (ep->d_name[0] == '.' && - (NAMLEN(ep) == 1 || - (ep->d_name[1] == '.' && NAMLEN(ep) == 2))) - continue; - v = FROM_STRING(ep->d_name, NAMLEN(ep)); - if (v == NULL) { - Py_DECREF(d); - d = NULL; - break; - } - if (arg_is_unicode) { - PyObject *w; - - w = PyUnicode_FromEncodedObject(v, - Py_FileSystemDefaultEncoding, - "strict"); - if (w != NULL) { - Py_DECREF(v); - v = w; - } - else { - /* fall back to the original byte string, as - discussed in patch #683592 */ - PyErr_Clear(); - } - } - name_type = Py_BuildValue("ON", v, FROM_LONG(ep->d_type)); - if (name_type == NULL) { - Py_DECREF(v); - Py_DECREF(d); - d = NULL; - break; - } - if (PyList_Append(d, name_type) != 0) { - Py_DECREF(v); - Py_DECREF(d); - Py_DECREF(name_type); - d = NULL; - break; - } - Py_DECREF(name_type); - Py_DECREF(v); - } - Py_BEGIN_ALLOW_THREADS - closedir(dirp); - Py_END_ALLOW_THREADS - PyMem_Free(name); - - return d; -} - -#endif - -static PyMethodDef scandir_methods[] = { - {"scandir_helper", (PyCFunction)scandir_helper, METH_VARARGS, NULL}, - {NULL, NULL}, -}; - -#if PY_MAJOR_VERSION >= 3 -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - "_scandir", - NULL, - 0, - scandir_methods, - NULL, - NULL, - NULL, - NULL, -}; -#endif - -#if PY_MAJOR_VERSION >= 3 -PyObject * -PyInit__scandir(void) -{ - PyObject *module = PyModule_Create(&moduledef); -#else -void -init_scandir(void) -{ - PyObject *module = Py_InitModule("_scandir", scandir_methods); -#endif - if (module == NULL) { - INITERROR; - } - -#ifdef MS_WINDOWS - stat_result_desc.name = "scandir.stat_result"; - PyStructSequence_InitType(&StatResultType, &stat_result_desc); -#endif - -#if PY_MAJOR_VERSION >= 3 - return module; -#endif -} diff --git a/libs/scandir/scandir.py b/libs/scandir/scandir.py deleted file mode 100644 index 1e34f8e..0000000 --- a/libs/scandir/scandir.py +++ /dev/null @@ -1,456 +0,0 @@ -"""scandir, a better directory iterator that exposes all file info OS provides - -scandir is a generator version of os.listdir() that returns an iterator over -files in a directory, and also exposes the extra information most OSes provide -while iterating files in a directory. - -See README.md or https://github.com/benhoyt/scandir for rationale and docs. - -scandir is released under the new BSD 3-clause license. See LICENSE.txt for -the full license text. -""" - -from __future__ import division - -import ctypes -import os -import stat -import sys - -__version__ = '0.3' -__all__ = ['scandir', 'walk'] - -# Shortcuts to these functions for speed and ease -join = os.path.join -lstat = os.lstat - -S_IFDIR = stat.S_IFDIR -S_IFREG = stat.S_IFREG -S_IFLNK = stat.S_IFLNK - -# 'unicode' isn't defined on Python 3 -try: - unicode -except NameError: - unicode = str - -_scandir = None - - -class GenericDirEntry(object): - __slots__ = ('name', '_lstat', '_path') - - def __init__(self, path, name): - self._path = path - self.name = name - self._lstat = None - - def lstat(self): - if self._lstat is None: - self._lstat = lstat(join(self._path, self.name)) - return self._lstat - - def is_dir(self): - try: - self.lstat() - except OSError: - return False - return self._lstat.st_mode & 0o170000 == S_IFDIR - - def is_file(self): - try: - self.lstat() - except OSError: - return False - return self._lstat.st_mode & 0o170000 == S_IFREG - - def is_symlink(self): - try: - self.lstat() - except OSError: - return False - return self._lstat.st_mode & 0o170000 == S_IFLNK - - def __str__(self): - return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) - - __repr__ = __str__ - - -if sys.platform == 'win32': - from ctypes import wintypes - - # Various constants from windows.h - INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value - ERROR_FILE_NOT_FOUND = 2 - ERROR_NO_MORE_FILES = 18 - FILE_ATTRIBUTE_READONLY = 1 - FILE_ATTRIBUTE_DIRECTORY = 16 - FILE_ATTRIBUTE_REPARSE_POINT = 1024 - - # Numer of seconds between 1601-01-01 and 1970-01-01 - SECONDS_BETWEEN_EPOCHS = 11644473600 - - kernel32 = ctypes.windll.kernel32 - - # ctypes wrappers for (wide string versions of) FindFirstFile, - # FindNextFile, and FindClose - FindFirstFile = kernel32.FindFirstFileW - FindFirstFile.argtypes = [ - wintypes.LPCWSTR, - ctypes.POINTER(wintypes.WIN32_FIND_DATAW), - ] - FindFirstFile.restype = wintypes.HANDLE - - FindNextFile = kernel32.FindNextFileW - FindNextFile.argtypes = [ - wintypes.HANDLE, - ctypes.POINTER(wintypes.WIN32_FIND_DATAW), - ] - FindNextFile.restype = wintypes.BOOL - - FindClose = kernel32.FindClose - FindClose.argtypes = [wintypes.HANDLE] - FindClose.restype = wintypes.BOOL - - def filetime_to_time(filetime): - """Convert Win32 FILETIME to time since Unix epoch in seconds.""" - total = filetime.dwHighDateTime << 32 | filetime.dwLowDateTime - return total / 10000000 - SECONDS_BETWEEN_EPOCHS - - def find_data_to_stat(data): - """Convert Win32 FIND_DATA struct to stat_result.""" - # First convert Win32 dwFileAttributes to st_mode - attributes = data.dwFileAttributes - st_mode = 0 - if attributes & FILE_ATTRIBUTE_DIRECTORY: - st_mode |= S_IFDIR | 0o111 - else: - st_mode |= S_IFREG - if attributes & FILE_ATTRIBUTE_READONLY: - st_mode |= 0o444 - else: - st_mode |= 0o666 - if attributes & FILE_ATTRIBUTE_REPARSE_POINT: - st_mode |= S_IFLNK - - st_size = data.nFileSizeHigh << 32 | data.nFileSizeLow - st_atime = filetime_to_time(data.ftLastAccessTime) - st_mtime = filetime_to_time(data.ftLastWriteTime) - st_ctime = filetime_to_time(data.ftCreationTime) - - # Some fields set to zero per CPython's posixmodule.c: st_ino, st_dev, - # st_nlink, st_uid, st_gid - return os.stat_result((st_mode, 0, 0, 0, 0, 0, st_size, st_atime, - st_mtime, st_ctime)) - - class Win32DirEntry(object): - __slots__ = ('name', '_lstat', '_find_data') - - def __init__(self, name, find_data): - self.name = name - self._lstat = None - self._find_data = find_data - - def lstat(self): - if self._lstat is None: - # Lazily convert to stat object, because it's slow, and often - # we only need is_dir() etc - self._lstat = find_data_to_stat(self._find_data) - return self._lstat - - def is_dir(self): - return (self._find_data.dwFileAttributes & - FILE_ATTRIBUTE_DIRECTORY != 0) - - def is_file(self): - return (self._find_data.dwFileAttributes & - FILE_ATTRIBUTE_DIRECTORY == 0) - - def is_symlink(self): - return (self._find_data.dwFileAttributes & - FILE_ATTRIBUTE_REPARSE_POINT != 0) - - def __str__(self): - return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) - - __repr__ = __str__ - - def win_error(error, filename): - exc = WindowsError(error, ctypes.FormatError(error)) - exc.filename = filename - return exc - - def scandir(path='.', windows_wildcard='*.*'): - """Like os.listdir(), but yield DirEntry objects instead of returning - a list of names. - """ - # Call FindFirstFile and handle errors - data = wintypes.WIN32_FIND_DATAW() - data_p = ctypes.byref(data) - filename = join(path, windows_wildcard) - handle = FindFirstFile(filename, data_p) - if handle == INVALID_HANDLE_VALUE: - error = ctypes.GetLastError() - if error == ERROR_FILE_NOT_FOUND: - # No files, don't yield anything - return - raise win_error(error, path) - - # Call FindNextFile in a loop, stopping when no more files - try: - while True: - # Skip '.' and '..' (current and parent directory), but - # otherwise yield (filename, stat_result) tuple - name = data.cFileName - if name not in ('.', '..'): - yield Win32DirEntry(name, data) - - data = wintypes.WIN32_FIND_DATAW() - data_p = ctypes.byref(data) - success = FindNextFile(handle, data_p) - if not success: - error = ctypes.GetLastError() - if error == ERROR_NO_MORE_FILES: - break - raise win_error(error, path) - finally: - if not FindClose(handle): - raise win_error(ctypes.GetLastError(), path) - - try: - import _scandir - - scandir_helper = _scandir.scandir_helper - - class Win32DirEntry(object): - __slots__ = ('name', '_lstat') - - def __init__(self, name, lstat): - self.name = name - self._lstat = lstat - - def lstat(self): - return self._lstat - - def is_dir(self): - return self._lstat.st_mode & 0o170000 == S_IFDIR - - def is_file(self): - return self._lstat.st_mode & 0o170000 == S_IFREG - - def is_symlink(self): - return self._lstat.st_mode & 0o170000 == S_IFLNK - - def __str__(self): - return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) - - __repr__ = __str__ - - def scandir(path='.'): - for name, stat in scandir_helper(unicode(path)): - yield Win32DirEntry(name, stat) - - except ImportError: - pass - - -# Linux, OS X, and BSD implementation -elif sys.platform.startswith(('linux', 'darwin')) or 'bsd' in sys.platform: - import ctypes.util - - DIR_p = ctypes.c_void_p - - # Rather annoying how the dirent struct is slightly different on each - # platform. The only fields we care about are d_name and d_type. - class Dirent(ctypes.Structure): - if sys.platform.startswith('linux'): - _fields_ = ( - ('d_ino', ctypes.c_ulong), - ('d_off', ctypes.c_long), - ('d_reclen', ctypes.c_ushort), - ('d_type', ctypes.c_byte), - ('d_name', ctypes.c_char * 256), - ) - else: - _fields_ = ( - ('d_ino', ctypes.c_uint32), # must be uint32, not ulong - ('d_reclen', ctypes.c_ushort), - ('d_type', ctypes.c_byte), - ('d_namlen', ctypes.c_byte), - ('d_name', ctypes.c_char * 256), - ) - - DT_UNKNOWN = 0 - DT_DIR = 4 - DT_REG = 8 - DT_LNK = 10 - - Dirent_p = ctypes.POINTER(Dirent) - Dirent_pp = ctypes.POINTER(Dirent_p) - - libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) - opendir = libc.opendir - opendir.argtypes = [ctypes.c_char_p] - opendir.restype = DIR_p - - readdir_r = libc.readdir_r - readdir_r.argtypes = [DIR_p, Dirent_p, Dirent_pp] - readdir_r.restype = ctypes.c_int - - closedir = libc.closedir - closedir.argtypes = [DIR_p] - closedir.restype = ctypes.c_int - - file_system_encoding = sys.getfilesystemencoding() - - class PosixDirEntry(object): - __slots__ = ('name', '_d_type', '_lstat', '_path') - - def __init__(self, path, name, d_type): - self._path = path - self.name = name - self._d_type = d_type - self._lstat = None - - def lstat(self): - if self._lstat is None: - self._lstat = lstat(join(self._path, self.name)) - return self._lstat - - # Ridiculous duplication between these is* functions -- helps a little - # bit with os.walk() performance compared to calling another function. - def is_dir(self): - d_type = self._d_type - if d_type != DT_UNKNOWN: - return d_type == DT_DIR - try: - self.lstat() - except OSError: - return False - return self._lstat.st_mode & 0o170000 == S_IFDIR - - def is_file(self): - d_type = self._d_type - if d_type != DT_UNKNOWN: - return d_type == DT_REG - try: - self.lstat() - except OSError: - return False - return self._lstat.st_mode & 0o170000 == S_IFREG - - def is_symlink(self): - d_type = self._d_type - if d_type != DT_UNKNOWN: - return d_type == DT_LNK - try: - self.lstat() - except OSError: - return False - return self._lstat.st_mode & 0o170000 == S_IFLNK - - def __str__(self): - return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) - - __repr__ = __str__ - - def posix_error(filename): - errno = ctypes.get_errno() - exc = OSError(errno, os.strerror(errno)) - exc.filename = filename - return exc - - def scandir(path='.'): - """Like os.listdir(), but yield DirEntry objects instead of returning - a list of names. - """ - dir_p = opendir(path.encode(file_system_encoding)) - if not dir_p: - raise posix_error(path) - try: - result = Dirent_p() - while True: - entry = Dirent() - if readdir_r(dir_p, entry, result): - raise posix_error(path) - if not result: - break - name = entry.d_name.decode(file_system_encoding) - if name not in ('.', '..'): - yield PosixDirEntry(path, name, entry.d_type) - finally: - if closedir(dir_p): - raise posix_error(path) - - try: - import _scandir - - scandir_helper = _scandir.scandir_helper - - def scandir(path='.'): - for name, d_type in scandir_helper(unicode(path)): - yield PosixDirEntry(path, name, d_type) - - except ImportError: - pass - - -# Some other system -- no d_type or stat information -else: - def scandir(path='.'): - """Like os.listdir(), but yield DirEntry objects instead of returning - a list of names. - """ - for name in os.listdir(path): - yield GenericDirEntry(path, name) - - -def walk(top, topdown=True, onerror=None, followlinks=False): - """Like os.walk(), but faster, as it uses scandir() internally.""" - # Determine which are files and which are directories - dirs = [] - nondirs = [] - try: - for entry in scandir(top): - if entry.is_dir(): - dirs.append(entry) - else: - nondirs.append(entry) - except OSError as error: - if onerror is not None: - onerror(error) - return - - # Yield before recursion if going top down - if topdown: - # Need to do some fancy footwork here as caller is allowed to modify - # dir_names, and we really want them to modify dirs (list of DirEntry - # objects) instead. Keep a mapping of entries keyed by name. - dir_names = [] - entries_by_name = {} - for entry in dirs: - dir_names.append(entry.name) - entries_by_name[entry.name] = entry - - yield top, dir_names, [e.name for e in nondirs] - - dirs = [] - for dir_name in dir_names: - entry = entries_by_name.get(dir_name) - if entry is None: - # Only happens when caller creates a new directory and adds it - # to dir_names - entry = GenericDirEntry(top, dir_name) - dirs.append(entry) - - # Recurse into sub-directories, following symbolic links if "followlinks" - for entry in dirs: - if followlinks or not entry.is_symlink(): - new_path = join(top, entry.name) - for x in walk(new_path, topdown, onerror, followlinks): - yield x - - # Yield before recursion if going bottom up - if not topdown: - yield top, [e.name for e in dirs], [e.name for e in nondirs]