Ruud
11 years ago
12 changed files with 16 additions and 857 deletions
@ -1 +0,0 @@ |
|||||
* text=auto |
|
||||
@ -1,4 +0,0 @@ |
|||||
*.pyc |
|
||||
*.pyd |
|
||||
benchtree |
|
||||
build |
|
||||
@ -1,373 +0,0 @@ |
|||||
// scandir C speedups
|
|
||||
//
|
|
||||
// TODO: this is a work in progress!
|
|
||||
//
|
|
||||
// There's a fair bit of PY_MAJOR_VERSION boilerplate to support both Python 2
|
|
||||
// and Python 3 -- the structure of this is taken from here:
|
|
||||
// http://docs.python.org/3.3/howto/cporting.html
|
|
||||
|
|
||||
#include <Python.h> |
|
||||
#include <structseq.h> |
|
||||
|
|
||||
#ifdef MS_WINDOWS |
|
||||
#include <windows.h> |
|
||||
#endif |
|
||||
|
|
||||
#if PY_MAJOR_VERSION >= 3 |
|
||||
#define INITERROR return NULL |
|
||||
#define FROM_LONG PyLong_FromLong |
|
||||
#define FROM_STRING PyUnicode_FromStringAndSize |
|
||||
#else |
|
||||
#define INITERROR return |
|
||||
#define FROM_LONG PyInt_FromLong |
|
||||
#define FROM_STRING PyString_FromStringAndSize |
|
||||
#endif |
|
||||
|
|
||||
#ifdef MS_WINDOWS |
|
||||
|
|
||||
static PyObject * |
|
||||
win32_error_unicode(char* function, Py_UNICODE* filename) |
|
||||
{ |
|
||||
errno = GetLastError(); |
|
||||
if (filename) |
|
||||
return PyErr_SetFromWindowsErrWithUnicodeFilename(errno, filename); |
|
||||
else |
|
||||
return PyErr_SetFromWindowsErr(errno); |
|
||||
} |
|
||||
|
|
||||
/* Below, we *know* that ugo+r is 0444 */ |
|
||||
#if _S_IREAD != 0400 |
|
||||
#error Unsupported C library |
|
||||
#endif |
|
||||
static int |
|
||||
attributes_to_mode(DWORD attr) |
|
||||
{ |
|
||||
int m = 0; |
|
||||
if (attr & FILE_ATTRIBUTE_DIRECTORY) |
|
||||
m |= _S_IFDIR | 0111; /* IFEXEC for user,group,other */ |
|
||||
else |
|
||||
m |= _S_IFREG; |
|
||||
if (attr & FILE_ATTRIBUTE_READONLY) |
|
||||
m |= 0444; |
|
||||
else |
|
||||
m |= 0666; |
|
||||
if (attr & FILE_ATTRIBUTE_REPARSE_POINT) |
|
||||
m |= 0120000; // S_IFLNK
|
|
||||
return m; |
|
||||
} |
|
||||
|
|
||||
double |
|
||||
filetime_to_time(FILETIME *filetime) |
|
||||
{ |
|
||||
const double SECONDS_BETWEEN_EPOCHS = 11644473600.0; |
|
||||
|
|
||||
unsigned long long total = (unsigned long long)filetime->dwHighDateTime << 32 | |
|
||||
(unsigned long long)filetime->dwLowDateTime; |
|
||||
return (double)total / 10000000.0 - SECONDS_BETWEEN_EPOCHS; |
|
||||
} |
|
||||
|
|
||||
static PyTypeObject StatResultType; |
|
||||
|
|
||||
static PyObject * |
|
||||
find_data_to_statresult(WIN32_FIND_DATAW *data) |
|
||||
{ |
|
||||
PY_LONG_LONG size; |
|
||||
PyObject *v = PyStructSequence_New(&StatResultType); |
|
||||
if (v == NULL) |
|
||||
return NULL; |
|
||||
|
|
||||
size = (PY_LONG_LONG)data->nFileSizeHigh << 32 | |
|
||||
(PY_LONG_LONG)data->nFileSizeLow; |
|
||||
|
|
||||
PyStructSequence_SET_ITEM(v, 0, FROM_LONG(attributes_to_mode(data->dwFileAttributes))); |
|
||||
PyStructSequence_SET_ITEM(v, 1, FROM_LONG(0)); |
|
||||
PyStructSequence_SET_ITEM(v, 2, FROM_LONG(0)); |
|
||||
PyStructSequence_SET_ITEM(v, 3, FROM_LONG(0)); |
|
||||
PyStructSequence_SET_ITEM(v, 4, FROM_LONG(0)); |
|
||||
PyStructSequence_SET_ITEM(v, 5, FROM_LONG(0)); |
|
||||
PyStructSequence_SET_ITEM(v, 6, PyLong_FromLongLong((PY_LONG_LONG)size)); |
|
||||
PyStructSequence_SET_ITEM(v, 7, PyFloat_FromDouble(filetime_to_time(&data->ftLastAccessTime))); |
|
||||
PyStructSequence_SET_ITEM(v, 8, PyFloat_FromDouble(filetime_to_time(&data->ftLastWriteTime))); |
|
||||
PyStructSequence_SET_ITEM(v, 9, PyFloat_FromDouble(filetime_to_time(&data->ftCreationTime))); |
|
||||
|
|
||||
if (PyErr_Occurred()) { |
|
||||
Py_DECREF(v); |
|
||||
return NULL; |
|
||||
} |
|
||||
|
|
||||
return v; |
|
||||
} |
|
||||
|
|
||||
static PyStructSequence_Field stat_result_fields[] = { |
|
||||
{"st_mode", "protection bits"}, |
|
||||
{"st_ino", "inode"}, |
|
||||
{"st_dev", "device"}, |
|
||||
{"st_nlink", "number of hard links"}, |
|
||||
{"st_uid", "user ID of owner"}, |
|
||||
{"st_gid", "group ID of owner"}, |
|
||||
{"st_size", "total size, in bytes"}, |
|
||||
{"st_atime", "time of last access"}, |
|
||||
{"st_mtime", "time of last modification"}, |
|
||||
{"st_ctime", "time of last change"}, |
|
||||
{0} |
|
||||
}; |
|
||||
|
|
||||
static PyStructSequence_Desc stat_result_desc = { |
|
||||
"stat_result", /* name */ |
|
||||
NULL, /* doc */ |
|
||||
stat_result_fields, |
|
||||
10 |
|
||||
}; |
|
||||
|
|
||||
static PyObject * |
|
||||
scandir_helper(PyObject *self, PyObject *args) |
|
||||
{ |
|
||||
PyObject *d, *v; |
|
||||
HANDLE hFindFile; |
|
||||
BOOL result; |
|
||||
WIN32_FIND_DATAW wFileData; |
|
||||
Py_UNICODE *wnamebuf; |
|
||||
Py_ssize_t len; |
|
||||
PyObject *po; |
|
||||
PyObject *name_stat; |
|
||||
|
|
||||
if (!PyArg_ParseTuple(args, "U:scandir_helper", &po)) |
|
||||
return NULL; |
|
||||
|
|
||||
/* Overallocate for \\*.*\0 */ |
|
||||
len = PyUnicode_GET_SIZE(po); |
|
||||
wnamebuf = malloc((len + 5) * sizeof(wchar_t)); |
|
||||
if (!wnamebuf) { |
|
||||
PyErr_NoMemory(); |
|
||||
return NULL; |
|
||||
} |
|
||||
wcscpy(wnamebuf, PyUnicode_AS_UNICODE(po)); |
|
||||
if (len > 0) { |
|
||||
Py_UNICODE wch = wnamebuf[len-1]; |
|
||||
if (wch != L'/' && wch != L'\\' && wch != L':') |
|
||||
wnamebuf[len++] = L'\\'; |
|
||||
wcscpy(wnamebuf + len, L"*.*"); |
|
||||
} |
|
||||
if ((d = PyList_New(0)) == NULL) { |
|
||||
free(wnamebuf); |
|
||||
return NULL; |
|
||||
} |
|
||||
Py_BEGIN_ALLOW_THREADS |
|
||||
hFindFile = FindFirstFileW(wnamebuf, &wFileData); |
|
||||
Py_END_ALLOW_THREADS |
|
||||
if (hFindFile == INVALID_HANDLE_VALUE) { |
|
||||
int error = GetLastError(); |
|
||||
if (error == ERROR_FILE_NOT_FOUND) { |
|
||||
free(wnamebuf); |
|
||||
return d; |
|
||||
} |
|
||||
Py_DECREF(d); |
|
||||
win32_error_unicode("FindFirstFileW", wnamebuf); |
|
||||
free(wnamebuf); |
|
||||
return NULL; |
|
||||
} |
|
||||
do { |
|
||||
/* Skip over . and .. */ |
|
||||
if (wcscmp(wFileData.cFileName, L".") != 0 && |
|
||||
wcscmp(wFileData.cFileName, L"..") != 0) { |
|
||||
v = PyUnicode_FromUnicode(wFileData.cFileName, wcslen(wFileData.cFileName)); |
|
||||
if (v == NULL) { |
|
||||
Py_DECREF(d); |
|
||||
d = NULL; |
|
||||
break; |
|
||||
} |
|
||||
name_stat = Py_BuildValue("ON", v, find_data_to_statresult(&wFileData)); |
|
||||
if (name_stat == NULL) { |
|
||||
Py_DECREF(v); |
|
||||
Py_DECREF(d); |
|
||||
d = NULL; |
|
||||
break; |
|
||||
} |
|
||||
if (PyList_Append(d, name_stat) != 0) { |
|
||||
Py_DECREF(v); |
|
||||
Py_DECREF(d); |
|
||||
Py_DECREF(name_stat); |
|
||||
d = NULL; |
|
||||
break; |
|
||||
} |
|
||||
Py_DECREF(name_stat); |
|
||||
Py_DECREF(v); |
|
||||
} |
|
||||
Py_BEGIN_ALLOW_THREADS |
|
||||
result = FindNextFileW(hFindFile, &wFileData); |
|
||||
Py_END_ALLOW_THREADS |
|
||||
/* FindNextFile sets error to ERROR_NO_MORE_FILES if
|
|
||||
it got to the end of the directory. */ |
|
||||
if (!result && GetLastError() != ERROR_NO_MORE_FILES) { |
|
||||
Py_DECREF(d); |
|
||||
win32_error_unicode("FindNextFileW", wnamebuf); |
|
||||
FindClose(hFindFile); |
|
||||
free(wnamebuf); |
|
||||
return NULL; |
|
||||
} |
|
||||
} while (result == TRUE); |
|
||||
|
|
||||
if (FindClose(hFindFile) == FALSE) { |
|
||||
Py_DECREF(d); |
|
||||
win32_error_unicode("FindClose", wnamebuf); |
|
||||
free(wnamebuf); |
|
||||
return NULL; |
|
||||
} |
|
||||
free(wnamebuf); |
|
||||
return d; |
|
||||
} |
|
||||
|
|
||||
#else // Linux / OS X
|
|
||||
|
|
||||
#include <dirent.h> |
|
||||
#define NAMLEN(dirent) strlen((dirent)->d_name) |
|
||||
|
|
||||
static PyObject * |
|
||||
posix_error_with_allocated_filename(char* name) |
|
||||
{ |
|
||||
PyObject *rc = PyErr_SetFromErrnoWithFilename(PyExc_OSError, name); |
|
||||
PyMem_Free(name); |
|
||||
return rc; |
|
||||
} |
|
||||
|
|
||||
static PyObject * |
|
||||
scandir_helper(PyObject *self, PyObject *args) |
|
||||
{ |
|
||||
char *name = NULL; |
|
||||
PyObject *d, *v, *name_type; |
|
||||
DIR *dirp; |
|
||||
struct dirent *ep; |
|
||||
int arg_is_unicode = 1; |
|
||||
|
|
||||
errno = 0; |
|
||||
if (!PyArg_ParseTuple(args, "U:scandir_helper", &v)) { |
|
||||
arg_is_unicode = 0; |
|
||||
PyErr_Clear(); |
|
||||
} |
|
||||
if (!PyArg_ParseTuple(args, "et:scandir_helper", Py_FileSystemDefaultEncoding, &name)) |
|
||||
return NULL; |
|
||||
Py_BEGIN_ALLOW_THREADS |
|
||||
dirp = opendir(name); |
|
||||
Py_END_ALLOW_THREADS |
|
||||
if (dirp == NULL) { |
|
||||
return posix_error_with_allocated_filename(name); |
|
||||
} |
|
||||
if ((d = PyList_New(0)) == NULL) { |
|
||||
Py_BEGIN_ALLOW_THREADS |
|
||||
closedir(dirp); |
|
||||
Py_END_ALLOW_THREADS |
|
||||
PyMem_Free(name); |
|
||||
return NULL; |
|
||||
} |
|
||||
for (;;) { |
|
||||
errno = 0; |
|
||||
Py_BEGIN_ALLOW_THREADS |
|
||||
ep = readdir(dirp); |
|
||||
Py_END_ALLOW_THREADS |
|
||||
if (ep == NULL) { |
|
||||
if (errno == 0) { |
|
||||
break; |
|
||||
} else { |
|
||||
Py_BEGIN_ALLOW_THREADS |
|
||||
closedir(dirp); |
|
||||
Py_END_ALLOW_THREADS |
|
||||
Py_DECREF(d); |
|
||||
return posix_error_with_allocated_filename(name); |
|
||||
} |
|
||||
} |
|
||||
if (ep->d_name[0] == '.' && |
|
||||
(NAMLEN(ep) == 1 || |
|
||||
(ep->d_name[1] == '.' && NAMLEN(ep) == 2))) |
|
||||
continue; |
|
||||
v = FROM_STRING(ep->d_name, NAMLEN(ep)); |
|
||||
if (v == NULL) { |
|
||||
Py_DECREF(d); |
|
||||
d = NULL; |
|
||||
break; |
|
||||
} |
|
||||
if (arg_is_unicode) { |
|
||||
PyObject *w; |
|
||||
|
|
||||
w = PyUnicode_FromEncodedObject(v, |
|
||||
Py_FileSystemDefaultEncoding, |
|
||||
"strict"); |
|
||||
if (w != NULL) { |
|
||||
Py_DECREF(v); |
|
||||
v = w; |
|
||||
} |
|
||||
else { |
|
||||
/* fall back to the original byte string, as
|
|
||||
discussed in patch #683592 */ |
|
||||
PyErr_Clear(); |
|
||||
} |
|
||||
} |
|
||||
name_type = Py_BuildValue("ON", v, FROM_LONG(ep->d_type)); |
|
||||
if (name_type == NULL) { |
|
||||
Py_DECREF(v); |
|
||||
Py_DECREF(d); |
|
||||
d = NULL; |
|
||||
break; |
|
||||
} |
|
||||
if (PyList_Append(d, name_type) != 0) { |
|
||||
Py_DECREF(v); |
|
||||
Py_DECREF(d); |
|
||||
Py_DECREF(name_type); |
|
||||
d = NULL; |
|
||||
break; |
|
||||
} |
|
||||
Py_DECREF(name_type); |
|
||||
Py_DECREF(v); |
|
||||
} |
|
||||
Py_BEGIN_ALLOW_THREADS |
|
||||
closedir(dirp); |
|
||||
Py_END_ALLOW_THREADS |
|
||||
PyMem_Free(name); |
|
||||
|
|
||||
return d; |
|
||||
} |
|
||||
|
|
||||
#endif |
|
||||
|
|
||||
static PyMethodDef scandir_methods[] = { |
|
||||
{"scandir_helper", (PyCFunction)scandir_helper, METH_VARARGS, NULL}, |
|
||||
{NULL, NULL}, |
|
||||
}; |
|
||||
|
|
||||
#if PY_MAJOR_VERSION >= 3 |
|
||||
static struct PyModuleDef moduledef = { |
|
||||
PyModuleDef_HEAD_INIT, |
|
||||
"_scandir", |
|
||||
NULL, |
|
||||
0, |
|
||||
scandir_methods, |
|
||||
NULL, |
|
||||
NULL, |
|
||||
NULL, |
|
||||
NULL, |
|
||||
}; |
|
||||
#endif |
|
||||
|
|
||||
#if PY_MAJOR_VERSION >= 3 |
|
||||
PyObject * |
|
||||
PyInit__scandir(void) |
|
||||
{ |
|
||||
PyObject *module = PyModule_Create(&moduledef); |
|
||||
#else |
|
||||
void |
|
||||
init_scandir(void) |
|
||||
{ |
|
||||
PyObject *module = Py_InitModule("_scandir", scandir_methods); |
|
||||
#endif |
|
||||
if (module == NULL) { |
|
||||
INITERROR; |
|
||||
} |
|
||||
|
|
||||
#ifdef MS_WINDOWS |
|
||||
stat_result_desc.name = "scandir.stat_result"; |
|
||||
PyStructSequence_InitType(&StatResultType, &stat_result_desc); |
|
||||
#endif |
|
||||
|
|
||||
#if PY_MAJOR_VERSION >= 3 |
|
||||
return module; |
|
||||
#endif |
|
||||
} |
|
||||
@ -1,456 +0,0 @@ |
|||||
"""scandir, a better directory iterator that exposes all file info OS provides |
|
||||
|
|
||||
scandir is a generator version of os.listdir() that returns an iterator over |
|
||||
files in a directory, and also exposes the extra information most OSes provide |
|
||||
while iterating files in a directory. |
|
||||
|
|
||||
See README.md or https://github.com/benhoyt/scandir for rationale and docs. |
|
||||
|
|
||||
scandir is released under the new BSD 3-clause license. See LICENSE.txt for |
|
||||
the full license text. |
|
||||
""" |
|
||||
|
|
||||
from __future__ import division |
|
||||
|
|
||||
import ctypes |
|
||||
import os |
|
||||
import stat |
|
||||
import sys |
|
||||
|
|
||||
__version__ = '0.3' |
|
||||
__all__ = ['scandir', 'walk'] |
|
||||
|
|
||||
# Shortcuts to these functions for speed and ease |
|
||||
join = os.path.join |
|
||||
lstat = os.lstat |
|
||||
|
|
||||
S_IFDIR = stat.S_IFDIR |
|
||||
S_IFREG = stat.S_IFREG |
|
||||
S_IFLNK = stat.S_IFLNK |
|
||||
|
|
||||
# 'unicode' isn't defined on Python 3 |
|
||||
try: |
|
||||
unicode |
|
||||
except NameError: |
|
||||
unicode = str |
|
||||
|
|
||||
_scandir = None |
|
||||
|
|
||||
|
|
||||
class GenericDirEntry(object): |
|
||||
__slots__ = ('name', '_lstat', '_path') |
|
||||
|
|
||||
def __init__(self, path, name): |
|
||||
self._path = path |
|
||||
self.name = name |
|
||||
self._lstat = None |
|
||||
|
|
||||
def lstat(self): |
|
||||
if self._lstat is None: |
|
||||
self._lstat = lstat(join(self._path, self.name)) |
|
||||
return self._lstat |
|
||||
|
|
||||
def is_dir(self): |
|
||||
try: |
|
||||
self.lstat() |
|
||||
except OSError: |
|
||||
return False |
|
||||
return self._lstat.st_mode & 0o170000 == S_IFDIR |
|
||||
|
|
||||
def is_file(self): |
|
||||
try: |
|
||||
self.lstat() |
|
||||
except OSError: |
|
||||
return False |
|
||||
return self._lstat.st_mode & 0o170000 == S_IFREG |
|
||||
|
|
||||
def is_symlink(self): |
|
||||
try: |
|
||||
self.lstat() |
|
||||
except OSError: |
|
||||
return False |
|
||||
return self._lstat.st_mode & 0o170000 == S_IFLNK |
|
||||
|
|
||||
def __str__(self): |
|
||||
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) |
|
||||
|
|
||||
__repr__ = __str__ |
|
||||
|
|
||||
|
|
||||
if sys.platform == 'win32': |
|
||||
from ctypes import wintypes |
|
||||
|
|
||||
# Various constants from windows.h |
|
||||
INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value |
|
||||
ERROR_FILE_NOT_FOUND = 2 |
|
||||
ERROR_NO_MORE_FILES = 18 |
|
||||
FILE_ATTRIBUTE_READONLY = 1 |
|
||||
FILE_ATTRIBUTE_DIRECTORY = 16 |
|
||||
FILE_ATTRIBUTE_REPARSE_POINT = 1024 |
|
||||
|
|
||||
# Numer of seconds between 1601-01-01 and 1970-01-01 |
|
||||
SECONDS_BETWEEN_EPOCHS = 11644473600 |
|
||||
|
|
||||
kernel32 = ctypes.windll.kernel32 |
|
||||
|
|
||||
# ctypes wrappers for (wide string versions of) FindFirstFile, |
|
||||
# FindNextFile, and FindClose |
|
||||
FindFirstFile = kernel32.FindFirstFileW |
|
||||
FindFirstFile.argtypes = [ |
|
||||
wintypes.LPCWSTR, |
|
||||
ctypes.POINTER(wintypes.WIN32_FIND_DATAW), |
|
||||
] |
|
||||
FindFirstFile.restype = wintypes.HANDLE |
|
||||
|
|
||||
FindNextFile = kernel32.FindNextFileW |
|
||||
FindNextFile.argtypes = [ |
|
||||
wintypes.HANDLE, |
|
||||
ctypes.POINTER(wintypes.WIN32_FIND_DATAW), |
|
||||
] |
|
||||
FindNextFile.restype = wintypes.BOOL |
|
||||
|
|
||||
FindClose = kernel32.FindClose |
|
||||
FindClose.argtypes = [wintypes.HANDLE] |
|
||||
FindClose.restype = wintypes.BOOL |
|
||||
|
|
||||
def filetime_to_time(filetime): |
|
||||
"""Convert Win32 FILETIME to time since Unix epoch in seconds.""" |
|
||||
total = filetime.dwHighDateTime << 32 | filetime.dwLowDateTime |
|
||||
return total / 10000000 - SECONDS_BETWEEN_EPOCHS |
|
||||
|
|
||||
def find_data_to_stat(data): |
|
||||
"""Convert Win32 FIND_DATA struct to stat_result.""" |
|
||||
# First convert Win32 dwFileAttributes to st_mode |
|
||||
attributes = data.dwFileAttributes |
|
||||
st_mode = 0 |
|
||||
if attributes & FILE_ATTRIBUTE_DIRECTORY: |
|
||||
st_mode |= S_IFDIR | 0o111 |
|
||||
else: |
|
||||
st_mode |= S_IFREG |
|
||||
if attributes & FILE_ATTRIBUTE_READONLY: |
|
||||
st_mode |= 0o444 |
|
||||
else: |
|
||||
st_mode |= 0o666 |
|
||||
if attributes & FILE_ATTRIBUTE_REPARSE_POINT: |
|
||||
st_mode |= S_IFLNK |
|
||||
|
|
||||
st_size = data.nFileSizeHigh << 32 | data.nFileSizeLow |
|
||||
st_atime = filetime_to_time(data.ftLastAccessTime) |
|
||||
st_mtime = filetime_to_time(data.ftLastWriteTime) |
|
||||
st_ctime = filetime_to_time(data.ftCreationTime) |
|
||||
|
|
||||
# Some fields set to zero per CPython's posixmodule.c: st_ino, st_dev, |
|
||||
# st_nlink, st_uid, st_gid |
|
||||
return os.stat_result((st_mode, 0, 0, 0, 0, 0, st_size, st_atime, |
|
||||
st_mtime, st_ctime)) |
|
||||
|
|
||||
class Win32DirEntry(object): |
|
||||
__slots__ = ('name', '_lstat', '_find_data') |
|
||||
|
|
||||
def __init__(self, name, find_data): |
|
||||
self.name = name |
|
||||
self._lstat = None |
|
||||
self._find_data = find_data |
|
||||
|
|
||||
def lstat(self): |
|
||||
if self._lstat is None: |
|
||||
# Lazily convert to stat object, because it's slow, and often |
|
||||
# we only need is_dir() etc |
|
||||
self._lstat = find_data_to_stat(self._find_data) |
|
||||
return self._lstat |
|
||||
|
|
||||
def is_dir(self): |
|
||||
return (self._find_data.dwFileAttributes & |
|
||||
FILE_ATTRIBUTE_DIRECTORY != 0) |
|
||||
|
|
||||
def is_file(self): |
|
||||
return (self._find_data.dwFileAttributes & |
|
||||
FILE_ATTRIBUTE_DIRECTORY == 0) |
|
||||
|
|
||||
def is_symlink(self): |
|
||||
return (self._find_data.dwFileAttributes & |
|
||||
FILE_ATTRIBUTE_REPARSE_POINT != 0) |
|
||||
|
|
||||
def __str__(self): |
|
||||
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) |
|
||||
|
|
||||
__repr__ = __str__ |
|
||||
|
|
||||
def win_error(error, filename): |
|
||||
exc = WindowsError(error, ctypes.FormatError(error)) |
|
||||
exc.filename = filename |
|
||||
return exc |
|
||||
|
|
||||
def scandir(path='.', windows_wildcard='*.*'): |
|
||||
"""Like os.listdir(), but yield DirEntry objects instead of returning |
|
||||
a list of names. |
|
||||
""" |
|
||||
# Call FindFirstFile and handle errors |
|
||||
data = wintypes.WIN32_FIND_DATAW() |
|
||||
data_p = ctypes.byref(data) |
|
||||
filename = join(path, windows_wildcard) |
|
||||
handle = FindFirstFile(filename, data_p) |
|
||||
if handle == INVALID_HANDLE_VALUE: |
|
||||
error = ctypes.GetLastError() |
|
||||
if error == ERROR_FILE_NOT_FOUND: |
|
||||
# No files, don't yield anything |
|
||||
return |
|
||||
raise win_error(error, path) |
|
||||
|
|
||||
# Call FindNextFile in a loop, stopping when no more files |
|
||||
try: |
|
||||
while True: |
|
||||
# Skip '.' and '..' (current and parent directory), but |
|
||||
# otherwise yield (filename, stat_result) tuple |
|
||||
name = data.cFileName |
|
||||
if name not in ('.', '..'): |
|
||||
yield Win32DirEntry(name, data) |
|
||||
|
|
||||
data = wintypes.WIN32_FIND_DATAW() |
|
||||
data_p = ctypes.byref(data) |
|
||||
success = FindNextFile(handle, data_p) |
|
||||
if not success: |
|
||||
error = ctypes.GetLastError() |
|
||||
if error == ERROR_NO_MORE_FILES: |
|
||||
break |
|
||||
raise win_error(error, path) |
|
||||
finally: |
|
||||
if not FindClose(handle): |
|
||||
raise win_error(ctypes.GetLastError(), path) |
|
||||
|
|
||||
try: |
|
||||
import _scandir |
|
||||
|
|
||||
scandir_helper = _scandir.scandir_helper |
|
||||
|
|
||||
class Win32DirEntry(object): |
|
||||
__slots__ = ('name', '_lstat') |
|
||||
|
|
||||
def __init__(self, name, lstat): |
|
||||
self.name = name |
|
||||
self._lstat = lstat |
|
||||
|
|
||||
def lstat(self): |
|
||||
return self._lstat |
|
||||
|
|
||||
def is_dir(self): |
|
||||
return self._lstat.st_mode & 0o170000 == S_IFDIR |
|
||||
|
|
||||
def is_file(self): |
|
||||
return self._lstat.st_mode & 0o170000 == S_IFREG |
|
||||
|
|
||||
def is_symlink(self): |
|
||||
return self._lstat.st_mode & 0o170000 == S_IFLNK |
|
||||
|
|
||||
def __str__(self): |
|
||||
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) |
|
||||
|
|
||||
__repr__ = __str__ |
|
||||
|
|
||||
def scandir(path='.'): |
|
||||
for name, stat in scandir_helper(unicode(path)): |
|
||||
yield Win32DirEntry(name, stat) |
|
||||
|
|
||||
except ImportError: |
|
||||
pass |
|
||||
|
|
||||
|
|
||||
# Linux, OS X, and BSD implementation |
|
||||
elif sys.platform.startswith(('linux', 'darwin')) or 'bsd' in sys.platform: |
|
||||
import ctypes.util |
|
||||
|
|
||||
DIR_p = ctypes.c_void_p |
|
||||
|
|
||||
# Rather annoying how the dirent struct is slightly different on each |
|
||||
# platform. The only fields we care about are d_name and d_type. |
|
||||
class Dirent(ctypes.Structure): |
|
||||
if sys.platform.startswith('linux'): |
|
||||
_fields_ = ( |
|
||||
('d_ino', ctypes.c_ulong), |
|
||||
('d_off', ctypes.c_long), |
|
||||
('d_reclen', ctypes.c_ushort), |
|
||||
('d_type', ctypes.c_byte), |
|
||||
('d_name', ctypes.c_char * 256), |
|
||||
) |
|
||||
else: |
|
||||
_fields_ = ( |
|
||||
('d_ino', ctypes.c_uint32), # must be uint32, not ulong |
|
||||
('d_reclen', ctypes.c_ushort), |
|
||||
('d_type', ctypes.c_byte), |
|
||||
('d_namlen', ctypes.c_byte), |
|
||||
('d_name', ctypes.c_char * 256), |
|
||||
) |
|
||||
|
|
||||
DT_UNKNOWN = 0 |
|
||||
DT_DIR = 4 |
|
||||
DT_REG = 8 |
|
||||
DT_LNK = 10 |
|
||||
|
|
||||
Dirent_p = ctypes.POINTER(Dirent) |
|
||||
Dirent_pp = ctypes.POINTER(Dirent_p) |
|
||||
|
|
||||
libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) |
|
||||
opendir = libc.opendir |
|
||||
opendir.argtypes = [ctypes.c_char_p] |
|
||||
opendir.restype = DIR_p |
|
||||
|
|
||||
readdir_r = libc.readdir_r |
|
||||
readdir_r.argtypes = [DIR_p, Dirent_p, Dirent_pp] |
|
||||
readdir_r.restype = ctypes.c_int |
|
||||
|
|
||||
closedir = libc.closedir |
|
||||
closedir.argtypes = [DIR_p] |
|
||||
closedir.restype = ctypes.c_int |
|
||||
|
|
||||
file_system_encoding = sys.getfilesystemencoding() |
|
||||
|
|
||||
class PosixDirEntry(object): |
|
||||
__slots__ = ('name', '_d_type', '_lstat', '_path') |
|
||||
|
|
||||
def __init__(self, path, name, d_type): |
|
||||
self._path = path |
|
||||
self.name = name |
|
||||
self._d_type = d_type |
|
||||
self._lstat = None |
|
||||
|
|
||||
def lstat(self): |
|
||||
if self._lstat is None: |
|
||||
self._lstat = lstat(join(self._path, self.name)) |
|
||||
return self._lstat |
|
||||
|
|
||||
# Ridiculous duplication between these is* functions -- helps a little |
|
||||
# bit with os.walk() performance compared to calling another function. |
|
||||
def is_dir(self): |
|
||||
d_type = self._d_type |
|
||||
if d_type != DT_UNKNOWN: |
|
||||
return d_type == DT_DIR |
|
||||
try: |
|
||||
self.lstat() |
|
||||
except OSError: |
|
||||
return False |
|
||||
return self._lstat.st_mode & 0o170000 == S_IFDIR |
|
||||
|
|
||||
def is_file(self): |
|
||||
d_type = self._d_type |
|
||||
if d_type != DT_UNKNOWN: |
|
||||
return d_type == DT_REG |
|
||||
try: |
|
||||
self.lstat() |
|
||||
except OSError: |
|
||||
return False |
|
||||
return self._lstat.st_mode & 0o170000 == S_IFREG |
|
||||
|
|
||||
def is_symlink(self): |
|
||||
d_type = self._d_type |
|
||||
if d_type != DT_UNKNOWN: |
|
||||
return d_type == DT_LNK |
|
||||
try: |
|
||||
self.lstat() |
|
||||
except OSError: |
|
||||
return False |
|
||||
return self._lstat.st_mode & 0o170000 == S_IFLNK |
|
||||
|
|
||||
def __str__(self): |
|
||||
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) |
|
||||
|
|
||||
__repr__ = __str__ |
|
||||
|
|
||||
def posix_error(filename): |
|
||||
errno = ctypes.get_errno() |
|
||||
exc = OSError(errno, os.strerror(errno)) |
|
||||
exc.filename = filename |
|
||||
return exc |
|
||||
|
|
||||
def scandir(path='.'): |
|
||||
"""Like os.listdir(), but yield DirEntry objects instead of returning |
|
||||
a list of names. |
|
||||
""" |
|
||||
dir_p = opendir(path.encode(file_system_encoding)) |
|
||||
if not dir_p: |
|
||||
raise posix_error(path) |
|
||||
try: |
|
||||
result = Dirent_p() |
|
||||
while True: |
|
||||
entry = Dirent() |
|
||||
if readdir_r(dir_p, entry, result): |
|
||||
raise posix_error(path) |
|
||||
if not result: |
|
||||
break |
|
||||
name = entry.d_name.decode(file_system_encoding) |
|
||||
if name not in ('.', '..'): |
|
||||
yield PosixDirEntry(path, name, entry.d_type) |
|
||||
finally: |
|
||||
if closedir(dir_p): |
|
||||
raise posix_error(path) |
|
||||
|
|
||||
try: |
|
||||
import _scandir |
|
||||
|
|
||||
scandir_helper = _scandir.scandir_helper |
|
||||
|
|
||||
def scandir(path='.'): |
|
||||
for name, d_type in scandir_helper(unicode(path)): |
|
||||
yield PosixDirEntry(path, name, d_type) |
|
||||
|
|
||||
except ImportError: |
|
||||
pass |
|
||||
|
|
||||
|
|
||||
# Some other system -- no d_type or stat information |
|
||||
else: |
|
||||
def scandir(path='.'): |
|
||||
"""Like os.listdir(), but yield DirEntry objects instead of returning |
|
||||
a list of names. |
|
||||
""" |
|
||||
for name in os.listdir(path): |
|
||||
yield GenericDirEntry(path, name) |
|
||||
|
|
||||
|
|
||||
def walk(top, topdown=True, onerror=None, followlinks=False): |
|
||||
"""Like os.walk(), but faster, as it uses scandir() internally.""" |
|
||||
# Determine which are files and which are directories |
|
||||
dirs = [] |
|
||||
nondirs = [] |
|
||||
try: |
|
||||
for entry in scandir(top): |
|
||||
if entry.is_dir(): |
|
||||
dirs.append(entry) |
|
||||
else: |
|
||||
nondirs.append(entry) |
|
||||
except OSError as error: |
|
||||
if onerror is not None: |
|
||||
onerror(error) |
|
||||
return |
|
||||
|
|
||||
# Yield before recursion if going top down |
|
||||
if topdown: |
|
||||
# Need to do some fancy footwork here as caller is allowed to modify |
|
||||
# dir_names, and we really want them to modify dirs (list of DirEntry |
|
||||
# objects) instead. Keep a mapping of entries keyed by name. |
|
||||
dir_names = [] |
|
||||
entries_by_name = {} |
|
||||
for entry in dirs: |
|
||||
dir_names.append(entry.name) |
|
||||
entries_by_name[entry.name] = entry |
|
||||
|
|
||||
yield top, dir_names, [e.name for e in nondirs] |
|
||||
|
|
||||
dirs = [] |
|
||||
for dir_name in dir_names: |
|
||||
entry = entries_by_name.get(dir_name) |
|
||||
if entry is None: |
|
||||
# Only happens when caller creates a new directory and adds it |
|
||||
# to dir_names |
|
||||
entry = GenericDirEntry(top, dir_name) |
|
||||
dirs.append(entry) |
|
||||
|
|
||||
# Recurse into sub-directories, following symbolic links if "followlinks" |
|
||||
for entry in dirs: |
|
||||
if followlinks or not entry.is_symlink(): |
|
||||
new_path = join(top, entry.name) |
|
||||
for x in walk(new_path, topdown, onerror, followlinks): |
|
||||
yield x |
|
||||
|
|
||||
# Yield before recursion if going bottom up |
|
||||
if not topdown: |
|
||||
yield top, [e.name for e in dirs], [e.name for e in nondirs] |
|
||||
Loading…
Reference in new issue