Ruud
11 years ago
12 changed files with 16 additions and 857 deletions
@ -1 +0,0 @@ |
|||
* text=auto |
|||
@ -1,4 +0,0 @@ |
|||
*.pyc |
|||
*.pyd |
|||
benchtree |
|||
build |
|||
@ -1,373 +0,0 @@ |
|||
// scandir C speedups
|
|||
//
|
|||
// TODO: this is a work in progress!
|
|||
//
|
|||
// There's a fair bit of PY_MAJOR_VERSION boilerplate to support both Python 2
|
|||
// and Python 3 -- the structure of this is taken from here:
|
|||
// http://docs.python.org/3.3/howto/cporting.html
|
|||
|
|||
#include <Python.h> |
|||
#include <structseq.h> |
|||
|
|||
#ifdef MS_WINDOWS |
|||
#include <windows.h> |
|||
#endif |
|||
|
|||
#if PY_MAJOR_VERSION >= 3 |
|||
#define INITERROR return NULL |
|||
#define FROM_LONG PyLong_FromLong |
|||
#define FROM_STRING PyUnicode_FromStringAndSize |
|||
#else |
|||
#define INITERROR return |
|||
#define FROM_LONG PyInt_FromLong |
|||
#define FROM_STRING PyString_FromStringAndSize |
|||
#endif |
|||
|
|||
#ifdef MS_WINDOWS |
|||
|
|||
static PyObject * |
|||
win32_error_unicode(char* function, Py_UNICODE* filename) |
|||
{ |
|||
errno = GetLastError(); |
|||
if (filename) |
|||
return PyErr_SetFromWindowsErrWithUnicodeFilename(errno, filename); |
|||
else |
|||
return PyErr_SetFromWindowsErr(errno); |
|||
} |
|||
|
|||
/* Below, we *know* that ugo+r is 0444 */ |
|||
#if _S_IREAD != 0400 |
|||
#error Unsupported C library |
|||
#endif |
|||
static int |
|||
attributes_to_mode(DWORD attr) |
|||
{ |
|||
int m = 0; |
|||
if (attr & FILE_ATTRIBUTE_DIRECTORY) |
|||
m |= _S_IFDIR | 0111; /* IFEXEC for user,group,other */ |
|||
else |
|||
m |= _S_IFREG; |
|||
if (attr & FILE_ATTRIBUTE_READONLY) |
|||
m |= 0444; |
|||
else |
|||
m |= 0666; |
|||
if (attr & FILE_ATTRIBUTE_REPARSE_POINT) |
|||
m |= 0120000; // S_IFLNK
|
|||
return m; |
|||
} |
|||
|
|||
double |
|||
filetime_to_time(FILETIME *filetime) |
|||
{ |
|||
const double SECONDS_BETWEEN_EPOCHS = 11644473600.0; |
|||
|
|||
unsigned long long total = (unsigned long long)filetime->dwHighDateTime << 32 | |
|||
(unsigned long long)filetime->dwLowDateTime; |
|||
return (double)total / 10000000.0 - SECONDS_BETWEEN_EPOCHS; |
|||
} |
|||
|
|||
static PyTypeObject StatResultType; |
|||
|
|||
static PyObject * |
|||
find_data_to_statresult(WIN32_FIND_DATAW *data) |
|||
{ |
|||
PY_LONG_LONG size; |
|||
PyObject *v = PyStructSequence_New(&StatResultType); |
|||
if (v == NULL) |
|||
return NULL; |
|||
|
|||
size = (PY_LONG_LONG)data->nFileSizeHigh << 32 | |
|||
(PY_LONG_LONG)data->nFileSizeLow; |
|||
|
|||
PyStructSequence_SET_ITEM(v, 0, FROM_LONG(attributes_to_mode(data->dwFileAttributes))); |
|||
PyStructSequence_SET_ITEM(v, 1, FROM_LONG(0)); |
|||
PyStructSequence_SET_ITEM(v, 2, FROM_LONG(0)); |
|||
PyStructSequence_SET_ITEM(v, 3, FROM_LONG(0)); |
|||
PyStructSequence_SET_ITEM(v, 4, FROM_LONG(0)); |
|||
PyStructSequence_SET_ITEM(v, 5, FROM_LONG(0)); |
|||
PyStructSequence_SET_ITEM(v, 6, PyLong_FromLongLong((PY_LONG_LONG)size)); |
|||
PyStructSequence_SET_ITEM(v, 7, PyFloat_FromDouble(filetime_to_time(&data->ftLastAccessTime))); |
|||
PyStructSequence_SET_ITEM(v, 8, PyFloat_FromDouble(filetime_to_time(&data->ftLastWriteTime))); |
|||
PyStructSequence_SET_ITEM(v, 9, PyFloat_FromDouble(filetime_to_time(&data->ftCreationTime))); |
|||
|
|||
if (PyErr_Occurred()) { |
|||
Py_DECREF(v); |
|||
return NULL; |
|||
} |
|||
|
|||
return v; |
|||
} |
|||
|
|||
static PyStructSequence_Field stat_result_fields[] = { |
|||
{"st_mode", "protection bits"}, |
|||
{"st_ino", "inode"}, |
|||
{"st_dev", "device"}, |
|||
{"st_nlink", "number of hard links"}, |
|||
{"st_uid", "user ID of owner"}, |
|||
{"st_gid", "group ID of owner"}, |
|||
{"st_size", "total size, in bytes"}, |
|||
{"st_atime", "time of last access"}, |
|||
{"st_mtime", "time of last modification"}, |
|||
{"st_ctime", "time of last change"}, |
|||
{0} |
|||
}; |
|||
|
|||
static PyStructSequence_Desc stat_result_desc = { |
|||
"stat_result", /* name */ |
|||
NULL, /* doc */ |
|||
stat_result_fields, |
|||
10 |
|||
}; |
|||
|
|||
static PyObject * |
|||
scandir_helper(PyObject *self, PyObject *args) |
|||
{ |
|||
PyObject *d, *v; |
|||
HANDLE hFindFile; |
|||
BOOL result; |
|||
WIN32_FIND_DATAW wFileData; |
|||
Py_UNICODE *wnamebuf; |
|||
Py_ssize_t len; |
|||
PyObject *po; |
|||
PyObject *name_stat; |
|||
|
|||
if (!PyArg_ParseTuple(args, "U:scandir_helper", &po)) |
|||
return NULL; |
|||
|
|||
/* Overallocate for \\*.*\0 */ |
|||
len = PyUnicode_GET_SIZE(po); |
|||
wnamebuf = malloc((len + 5) * sizeof(wchar_t)); |
|||
if (!wnamebuf) { |
|||
PyErr_NoMemory(); |
|||
return NULL; |
|||
} |
|||
wcscpy(wnamebuf, PyUnicode_AS_UNICODE(po)); |
|||
if (len > 0) { |
|||
Py_UNICODE wch = wnamebuf[len-1]; |
|||
if (wch != L'/' && wch != L'\\' && wch != L':') |
|||
wnamebuf[len++] = L'\\'; |
|||
wcscpy(wnamebuf + len, L"*.*"); |
|||
} |
|||
if ((d = PyList_New(0)) == NULL) { |
|||
free(wnamebuf); |
|||
return NULL; |
|||
} |
|||
Py_BEGIN_ALLOW_THREADS |
|||
hFindFile = FindFirstFileW(wnamebuf, &wFileData); |
|||
Py_END_ALLOW_THREADS |
|||
if (hFindFile == INVALID_HANDLE_VALUE) { |
|||
int error = GetLastError(); |
|||
if (error == ERROR_FILE_NOT_FOUND) { |
|||
free(wnamebuf); |
|||
return d; |
|||
} |
|||
Py_DECREF(d); |
|||
win32_error_unicode("FindFirstFileW", wnamebuf); |
|||
free(wnamebuf); |
|||
return NULL; |
|||
} |
|||
do { |
|||
/* Skip over . and .. */ |
|||
if (wcscmp(wFileData.cFileName, L".") != 0 && |
|||
wcscmp(wFileData.cFileName, L"..") != 0) { |
|||
v = PyUnicode_FromUnicode(wFileData.cFileName, wcslen(wFileData.cFileName)); |
|||
if (v == NULL) { |
|||
Py_DECREF(d); |
|||
d = NULL; |
|||
break; |
|||
} |
|||
name_stat = Py_BuildValue("ON", v, find_data_to_statresult(&wFileData)); |
|||
if (name_stat == NULL) { |
|||
Py_DECREF(v); |
|||
Py_DECREF(d); |
|||
d = NULL; |
|||
break; |
|||
} |
|||
if (PyList_Append(d, name_stat) != 0) { |
|||
Py_DECREF(v); |
|||
Py_DECREF(d); |
|||
Py_DECREF(name_stat); |
|||
d = NULL; |
|||
break; |
|||
} |
|||
Py_DECREF(name_stat); |
|||
Py_DECREF(v); |
|||
} |
|||
Py_BEGIN_ALLOW_THREADS |
|||
result = FindNextFileW(hFindFile, &wFileData); |
|||
Py_END_ALLOW_THREADS |
|||
/* FindNextFile sets error to ERROR_NO_MORE_FILES if
|
|||
it got to the end of the directory. */ |
|||
if (!result && GetLastError() != ERROR_NO_MORE_FILES) { |
|||
Py_DECREF(d); |
|||
win32_error_unicode("FindNextFileW", wnamebuf); |
|||
FindClose(hFindFile); |
|||
free(wnamebuf); |
|||
return NULL; |
|||
} |
|||
} while (result == TRUE); |
|||
|
|||
if (FindClose(hFindFile) == FALSE) { |
|||
Py_DECREF(d); |
|||
win32_error_unicode("FindClose", wnamebuf); |
|||
free(wnamebuf); |
|||
return NULL; |
|||
} |
|||
free(wnamebuf); |
|||
return d; |
|||
} |
|||
|
|||
#else // Linux / OS X
|
|||
|
|||
#include <dirent.h> |
|||
#define NAMLEN(dirent) strlen((dirent)->d_name) |
|||
|
|||
static PyObject * |
|||
posix_error_with_allocated_filename(char* name) |
|||
{ |
|||
PyObject *rc = PyErr_SetFromErrnoWithFilename(PyExc_OSError, name); |
|||
PyMem_Free(name); |
|||
return rc; |
|||
} |
|||
|
|||
static PyObject * |
|||
scandir_helper(PyObject *self, PyObject *args) |
|||
{ |
|||
char *name = NULL; |
|||
PyObject *d, *v, *name_type; |
|||
DIR *dirp; |
|||
struct dirent *ep; |
|||
int arg_is_unicode = 1; |
|||
|
|||
errno = 0; |
|||
if (!PyArg_ParseTuple(args, "U:scandir_helper", &v)) { |
|||
arg_is_unicode = 0; |
|||
PyErr_Clear(); |
|||
} |
|||
if (!PyArg_ParseTuple(args, "et:scandir_helper", Py_FileSystemDefaultEncoding, &name)) |
|||
return NULL; |
|||
Py_BEGIN_ALLOW_THREADS |
|||
dirp = opendir(name); |
|||
Py_END_ALLOW_THREADS |
|||
if (dirp == NULL) { |
|||
return posix_error_with_allocated_filename(name); |
|||
} |
|||
if ((d = PyList_New(0)) == NULL) { |
|||
Py_BEGIN_ALLOW_THREADS |
|||
closedir(dirp); |
|||
Py_END_ALLOW_THREADS |
|||
PyMem_Free(name); |
|||
return NULL; |
|||
} |
|||
for (;;) { |
|||
errno = 0; |
|||
Py_BEGIN_ALLOW_THREADS |
|||
ep = readdir(dirp); |
|||
Py_END_ALLOW_THREADS |
|||
if (ep == NULL) { |
|||
if (errno == 0) { |
|||
break; |
|||
} else { |
|||
Py_BEGIN_ALLOW_THREADS |
|||
closedir(dirp); |
|||
Py_END_ALLOW_THREADS |
|||
Py_DECREF(d); |
|||
return posix_error_with_allocated_filename(name); |
|||
} |
|||
} |
|||
if (ep->d_name[0] == '.' && |
|||
(NAMLEN(ep) == 1 || |
|||
(ep->d_name[1] == '.' && NAMLEN(ep) == 2))) |
|||
continue; |
|||
v = FROM_STRING(ep->d_name, NAMLEN(ep)); |
|||
if (v == NULL) { |
|||
Py_DECREF(d); |
|||
d = NULL; |
|||
break; |
|||
} |
|||
if (arg_is_unicode) { |
|||
PyObject *w; |
|||
|
|||
w = PyUnicode_FromEncodedObject(v, |
|||
Py_FileSystemDefaultEncoding, |
|||
"strict"); |
|||
if (w != NULL) { |
|||
Py_DECREF(v); |
|||
v = w; |
|||
} |
|||
else { |
|||
/* fall back to the original byte string, as
|
|||
discussed in patch #683592 */ |
|||
PyErr_Clear(); |
|||
} |
|||
} |
|||
name_type = Py_BuildValue("ON", v, FROM_LONG(ep->d_type)); |
|||
if (name_type == NULL) { |
|||
Py_DECREF(v); |
|||
Py_DECREF(d); |
|||
d = NULL; |
|||
break; |
|||
} |
|||
if (PyList_Append(d, name_type) != 0) { |
|||
Py_DECREF(v); |
|||
Py_DECREF(d); |
|||
Py_DECREF(name_type); |
|||
d = NULL; |
|||
break; |
|||
} |
|||
Py_DECREF(name_type); |
|||
Py_DECREF(v); |
|||
} |
|||
Py_BEGIN_ALLOW_THREADS |
|||
closedir(dirp); |
|||
Py_END_ALLOW_THREADS |
|||
PyMem_Free(name); |
|||
|
|||
return d; |
|||
} |
|||
|
|||
#endif |
|||
|
|||
static PyMethodDef scandir_methods[] = { |
|||
{"scandir_helper", (PyCFunction)scandir_helper, METH_VARARGS, NULL}, |
|||
{NULL, NULL}, |
|||
}; |
|||
|
|||
#if PY_MAJOR_VERSION >= 3 |
|||
static struct PyModuleDef moduledef = { |
|||
PyModuleDef_HEAD_INIT, |
|||
"_scandir", |
|||
NULL, |
|||
0, |
|||
scandir_methods, |
|||
NULL, |
|||
NULL, |
|||
NULL, |
|||
NULL, |
|||
}; |
|||
#endif |
|||
|
|||
#if PY_MAJOR_VERSION >= 3 |
|||
PyObject * |
|||
PyInit__scandir(void) |
|||
{ |
|||
PyObject *module = PyModule_Create(&moduledef); |
|||
#else |
|||
void |
|||
init_scandir(void) |
|||
{ |
|||
PyObject *module = Py_InitModule("_scandir", scandir_methods); |
|||
#endif |
|||
if (module == NULL) { |
|||
INITERROR; |
|||
} |
|||
|
|||
#ifdef MS_WINDOWS |
|||
stat_result_desc.name = "scandir.stat_result"; |
|||
PyStructSequence_InitType(&StatResultType, &stat_result_desc); |
|||
#endif |
|||
|
|||
#if PY_MAJOR_VERSION >= 3 |
|||
return module; |
|||
#endif |
|||
} |
|||
@ -1,456 +0,0 @@ |
|||
"""scandir, a better directory iterator that exposes all file info OS provides |
|||
|
|||
scandir is a generator version of os.listdir() that returns an iterator over |
|||
files in a directory, and also exposes the extra information most OSes provide |
|||
while iterating files in a directory. |
|||
|
|||
See README.md or https://github.com/benhoyt/scandir for rationale and docs. |
|||
|
|||
scandir is released under the new BSD 3-clause license. See LICENSE.txt for |
|||
the full license text. |
|||
""" |
|||
|
|||
from __future__ import division |
|||
|
|||
import ctypes |
|||
import os |
|||
import stat |
|||
import sys |
|||
|
|||
__version__ = '0.3' |
|||
__all__ = ['scandir', 'walk'] |
|||
|
|||
# Shortcuts to these functions for speed and ease |
|||
join = os.path.join |
|||
lstat = os.lstat |
|||
|
|||
S_IFDIR = stat.S_IFDIR |
|||
S_IFREG = stat.S_IFREG |
|||
S_IFLNK = stat.S_IFLNK |
|||
|
|||
# 'unicode' isn't defined on Python 3 |
|||
try: |
|||
unicode |
|||
except NameError: |
|||
unicode = str |
|||
|
|||
_scandir = None |
|||
|
|||
|
|||
class GenericDirEntry(object): |
|||
__slots__ = ('name', '_lstat', '_path') |
|||
|
|||
def __init__(self, path, name): |
|||
self._path = path |
|||
self.name = name |
|||
self._lstat = None |
|||
|
|||
def lstat(self): |
|||
if self._lstat is None: |
|||
self._lstat = lstat(join(self._path, self.name)) |
|||
return self._lstat |
|||
|
|||
def is_dir(self): |
|||
try: |
|||
self.lstat() |
|||
except OSError: |
|||
return False |
|||
return self._lstat.st_mode & 0o170000 == S_IFDIR |
|||
|
|||
def is_file(self): |
|||
try: |
|||
self.lstat() |
|||
except OSError: |
|||
return False |
|||
return self._lstat.st_mode & 0o170000 == S_IFREG |
|||
|
|||
def is_symlink(self): |
|||
try: |
|||
self.lstat() |
|||
except OSError: |
|||
return False |
|||
return self._lstat.st_mode & 0o170000 == S_IFLNK |
|||
|
|||
def __str__(self): |
|||
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) |
|||
|
|||
__repr__ = __str__ |
|||
|
|||
|
|||
if sys.platform == 'win32': |
|||
from ctypes import wintypes |
|||
|
|||
# Various constants from windows.h |
|||
INVALID_HANDLE_VALUE = ctypes.c_void_p(-1).value |
|||
ERROR_FILE_NOT_FOUND = 2 |
|||
ERROR_NO_MORE_FILES = 18 |
|||
FILE_ATTRIBUTE_READONLY = 1 |
|||
FILE_ATTRIBUTE_DIRECTORY = 16 |
|||
FILE_ATTRIBUTE_REPARSE_POINT = 1024 |
|||
|
|||
# Numer of seconds between 1601-01-01 and 1970-01-01 |
|||
SECONDS_BETWEEN_EPOCHS = 11644473600 |
|||
|
|||
kernel32 = ctypes.windll.kernel32 |
|||
|
|||
# ctypes wrappers for (wide string versions of) FindFirstFile, |
|||
# FindNextFile, and FindClose |
|||
FindFirstFile = kernel32.FindFirstFileW |
|||
FindFirstFile.argtypes = [ |
|||
wintypes.LPCWSTR, |
|||
ctypes.POINTER(wintypes.WIN32_FIND_DATAW), |
|||
] |
|||
FindFirstFile.restype = wintypes.HANDLE |
|||
|
|||
FindNextFile = kernel32.FindNextFileW |
|||
FindNextFile.argtypes = [ |
|||
wintypes.HANDLE, |
|||
ctypes.POINTER(wintypes.WIN32_FIND_DATAW), |
|||
] |
|||
FindNextFile.restype = wintypes.BOOL |
|||
|
|||
FindClose = kernel32.FindClose |
|||
FindClose.argtypes = [wintypes.HANDLE] |
|||
FindClose.restype = wintypes.BOOL |
|||
|
|||
def filetime_to_time(filetime): |
|||
"""Convert Win32 FILETIME to time since Unix epoch in seconds.""" |
|||
total = filetime.dwHighDateTime << 32 | filetime.dwLowDateTime |
|||
return total / 10000000 - SECONDS_BETWEEN_EPOCHS |
|||
|
|||
def find_data_to_stat(data): |
|||
"""Convert Win32 FIND_DATA struct to stat_result.""" |
|||
# First convert Win32 dwFileAttributes to st_mode |
|||
attributes = data.dwFileAttributes |
|||
st_mode = 0 |
|||
if attributes & FILE_ATTRIBUTE_DIRECTORY: |
|||
st_mode |= S_IFDIR | 0o111 |
|||
else: |
|||
st_mode |= S_IFREG |
|||
if attributes & FILE_ATTRIBUTE_READONLY: |
|||
st_mode |= 0o444 |
|||
else: |
|||
st_mode |= 0o666 |
|||
if attributes & FILE_ATTRIBUTE_REPARSE_POINT: |
|||
st_mode |= S_IFLNK |
|||
|
|||
st_size = data.nFileSizeHigh << 32 | data.nFileSizeLow |
|||
st_atime = filetime_to_time(data.ftLastAccessTime) |
|||
st_mtime = filetime_to_time(data.ftLastWriteTime) |
|||
st_ctime = filetime_to_time(data.ftCreationTime) |
|||
|
|||
# Some fields set to zero per CPython's posixmodule.c: st_ino, st_dev, |
|||
# st_nlink, st_uid, st_gid |
|||
return os.stat_result((st_mode, 0, 0, 0, 0, 0, st_size, st_atime, |
|||
st_mtime, st_ctime)) |
|||
|
|||
class Win32DirEntry(object): |
|||
__slots__ = ('name', '_lstat', '_find_data') |
|||
|
|||
def __init__(self, name, find_data): |
|||
self.name = name |
|||
self._lstat = None |
|||
self._find_data = find_data |
|||
|
|||
def lstat(self): |
|||
if self._lstat is None: |
|||
# Lazily convert to stat object, because it's slow, and often |
|||
# we only need is_dir() etc |
|||
self._lstat = find_data_to_stat(self._find_data) |
|||
return self._lstat |
|||
|
|||
def is_dir(self): |
|||
return (self._find_data.dwFileAttributes & |
|||
FILE_ATTRIBUTE_DIRECTORY != 0) |
|||
|
|||
def is_file(self): |
|||
return (self._find_data.dwFileAttributes & |
|||
FILE_ATTRIBUTE_DIRECTORY == 0) |
|||
|
|||
def is_symlink(self): |
|||
return (self._find_data.dwFileAttributes & |
|||
FILE_ATTRIBUTE_REPARSE_POINT != 0) |
|||
|
|||
def __str__(self): |
|||
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) |
|||
|
|||
__repr__ = __str__ |
|||
|
|||
def win_error(error, filename): |
|||
exc = WindowsError(error, ctypes.FormatError(error)) |
|||
exc.filename = filename |
|||
return exc |
|||
|
|||
def scandir(path='.', windows_wildcard='*.*'): |
|||
"""Like os.listdir(), but yield DirEntry objects instead of returning |
|||
a list of names. |
|||
""" |
|||
# Call FindFirstFile and handle errors |
|||
data = wintypes.WIN32_FIND_DATAW() |
|||
data_p = ctypes.byref(data) |
|||
filename = join(path, windows_wildcard) |
|||
handle = FindFirstFile(filename, data_p) |
|||
if handle == INVALID_HANDLE_VALUE: |
|||
error = ctypes.GetLastError() |
|||
if error == ERROR_FILE_NOT_FOUND: |
|||
# No files, don't yield anything |
|||
return |
|||
raise win_error(error, path) |
|||
|
|||
# Call FindNextFile in a loop, stopping when no more files |
|||
try: |
|||
while True: |
|||
# Skip '.' and '..' (current and parent directory), but |
|||
# otherwise yield (filename, stat_result) tuple |
|||
name = data.cFileName |
|||
if name not in ('.', '..'): |
|||
yield Win32DirEntry(name, data) |
|||
|
|||
data = wintypes.WIN32_FIND_DATAW() |
|||
data_p = ctypes.byref(data) |
|||
success = FindNextFile(handle, data_p) |
|||
if not success: |
|||
error = ctypes.GetLastError() |
|||
if error == ERROR_NO_MORE_FILES: |
|||
break |
|||
raise win_error(error, path) |
|||
finally: |
|||
if not FindClose(handle): |
|||
raise win_error(ctypes.GetLastError(), path) |
|||
|
|||
try: |
|||
import _scandir |
|||
|
|||
scandir_helper = _scandir.scandir_helper |
|||
|
|||
class Win32DirEntry(object): |
|||
__slots__ = ('name', '_lstat') |
|||
|
|||
def __init__(self, name, lstat): |
|||
self.name = name |
|||
self._lstat = lstat |
|||
|
|||
def lstat(self): |
|||
return self._lstat |
|||
|
|||
def is_dir(self): |
|||
return self._lstat.st_mode & 0o170000 == S_IFDIR |
|||
|
|||
def is_file(self): |
|||
return self._lstat.st_mode & 0o170000 == S_IFREG |
|||
|
|||
def is_symlink(self): |
|||
return self._lstat.st_mode & 0o170000 == S_IFLNK |
|||
|
|||
def __str__(self): |
|||
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) |
|||
|
|||
__repr__ = __str__ |
|||
|
|||
def scandir(path='.'): |
|||
for name, stat in scandir_helper(unicode(path)): |
|||
yield Win32DirEntry(name, stat) |
|||
|
|||
except ImportError: |
|||
pass |
|||
|
|||
|
|||
# Linux, OS X, and BSD implementation |
|||
elif sys.platform.startswith(('linux', 'darwin')) or 'bsd' in sys.platform: |
|||
import ctypes.util |
|||
|
|||
DIR_p = ctypes.c_void_p |
|||
|
|||
# Rather annoying how the dirent struct is slightly different on each |
|||
# platform. The only fields we care about are d_name and d_type. |
|||
class Dirent(ctypes.Structure): |
|||
if sys.platform.startswith('linux'): |
|||
_fields_ = ( |
|||
('d_ino', ctypes.c_ulong), |
|||
('d_off', ctypes.c_long), |
|||
('d_reclen', ctypes.c_ushort), |
|||
('d_type', ctypes.c_byte), |
|||
('d_name', ctypes.c_char * 256), |
|||
) |
|||
else: |
|||
_fields_ = ( |
|||
('d_ino', ctypes.c_uint32), # must be uint32, not ulong |
|||
('d_reclen', ctypes.c_ushort), |
|||
('d_type', ctypes.c_byte), |
|||
('d_namlen', ctypes.c_byte), |
|||
('d_name', ctypes.c_char * 256), |
|||
) |
|||
|
|||
DT_UNKNOWN = 0 |
|||
DT_DIR = 4 |
|||
DT_REG = 8 |
|||
DT_LNK = 10 |
|||
|
|||
Dirent_p = ctypes.POINTER(Dirent) |
|||
Dirent_pp = ctypes.POINTER(Dirent_p) |
|||
|
|||
libc = ctypes.CDLL(ctypes.util.find_library('c'), use_errno=True) |
|||
opendir = libc.opendir |
|||
opendir.argtypes = [ctypes.c_char_p] |
|||
opendir.restype = DIR_p |
|||
|
|||
readdir_r = libc.readdir_r |
|||
readdir_r.argtypes = [DIR_p, Dirent_p, Dirent_pp] |
|||
readdir_r.restype = ctypes.c_int |
|||
|
|||
closedir = libc.closedir |
|||
closedir.argtypes = [DIR_p] |
|||
closedir.restype = ctypes.c_int |
|||
|
|||
file_system_encoding = sys.getfilesystemencoding() |
|||
|
|||
class PosixDirEntry(object): |
|||
__slots__ = ('name', '_d_type', '_lstat', '_path') |
|||
|
|||
def __init__(self, path, name, d_type): |
|||
self._path = path |
|||
self.name = name |
|||
self._d_type = d_type |
|||
self._lstat = None |
|||
|
|||
def lstat(self): |
|||
if self._lstat is None: |
|||
self._lstat = lstat(join(self._path, self.name)) |
|||
return self._lstat |
|||
|
|||
# Ridiculous duplication between these is* functions -- helps a little |
|||
# bit with os.walk() performance compared to calling another function. |
|||
def is_dir(self): |
|||
d_type = self._d_type |
|||
if d_type != DT_UNKNOWN: |
|||
return d_type == DT_DIR |
|||
try: |
|||
self.lstat() |
|||
except OSError: |
|||
return False |
|||
return self._lstat.st_mode & 0o170000 == S_IFDIR |
|||
|
|||
def is_file(self): |
|||
d_type = self._d_type |
|||
if d_type != DT_UNKNOWN: |
|||
return d_type == DT_REG |
|||
try: |
|||
self.lstat() |
|||
except OSError: |
|||
return False |
|||
return self._lstat.st_mode & 0o170000 == S_IFREG |
|||
|
|||
def is_symlink(self): |
|||
d_type = self._d_type |
|||
if d_type != DT_UNKNOWN: |
|||
return d_type == DT_LNK |
|||
try: |
|||
self.lstat() |
|||
except OSError: |
|||
return False |
|||
return self._lstat.st_mode & 0o170000 == S_IFLNK |
|||
|
|||
def __str__(self): |
|||
return '<{0}: {1!r}>'.format(self.__class__.__name__, self.name) |
|||
|
|||
__repr__ = __str__ |
|||
|
|||
def posix_error(filename): |
|||
errno = ctypes.get_errno() |
|||
exc = OSError(errno, os.strerror(errno)) |
|||
exc.filename = filename |
|||
return exc |
|||
|
|||
def scandir(path='.'): |
|||
"""Like os.listdir(), but yield DirEntry objects instead of returning |
|||
a list of names. |
|||
""" |
|||
dir_p = opendir(path.encode(file_system_encoding)) |
|||
if not dir_p: |
|||
raise posix_error(path) |
|||
try: |
|||
result = Dirent_p() |
|||
while True: |
|||
entry = Dirent() |
|||
if readdir_r(dir_p, entry, result): |
|||
raise posix_error(path) |
|||
if not result: |
|||
break |
|||
name = entry.d_name.decode(file_system_encoding) |
|||
if name not in ('.', '..'): |
|||
yield PosixDirEntry(path, name, entry.d_type) |
|||
finally: |
|||
if closedir(dir_p): |
|||
raise posix_error(path) |
|||
|
|||
try: |
|||
import _scandir |
|||
|
|||
scandir_helper = _scandir.scandir_helper |
|||
|
|||
def scandir(path='.'): |
|||
for name, d_type in scandir_helper(unicode(path)): |
|||
yield PosixDirEntry(path, name, d_type) |
|||
|
|||
except ImportError: |
|||
pass |
|||
|
|||
|
|||
# Some other system -- no d_type or stat information |
|||
else: |
|||
def scandir(path='.'): |
|||
"""Like os.listdir(), but yield DirEntry objects instead of returning |
|||
a list of names. |
|||
""" |
|||
for name in os.listdir(path): |
|||
yield GenericDirEntry(path, name) |
|||
|
|||
|
|||
def walk(top, topdown=True, onerror=None, followlinks=False): |
|||
"""Like os.walk(), but faster, as it uses scandir() internally.""" |
|||
# Determine which are files and which are directories |
|||
dirs = [] |
|||
nondirs = [] |
|||
try: |
|||
for entry in scandir(top): |
|||
if entry.is_dir(): |
|||
dirs.append(entry) |
|||
else: |
|||
nondirs.append(entry) |
|||
except OSError as error: |
|||
if onerror is not None: |
|||
onerror(error) |
|||
return |
|||
|
|||
# Yield before recursion if going top down |
|||
if topdown: |
|||
# Need to do some fancy footwork here as caller is allowed to modify |
|||
# dir_names, and we really want them to modify dirs (list of DirEntry |
|||
# objects) instead. Keep a mapping of entries keyed by name. |
|||
dir_names = [] |
|||
entries_by_name = {} |
|||
for entry in dirs: |
|||
dir_names.append(entry.name) |
|||
entries_by_name[entry.name] = entry |
|||
|
|||
yield top, dir_names, [e.name for e in nondirs] |
|||
|
|||
dirs = [] |
|||
for dir_name in dir_names: |
|||
entry = entries_by_name.get(dir_name) |
|||
if entry is None: |
|||
# Only happens when caller creates a new directory and adds it |
|||
# to dir_names |
|||
entry = GenericDirEntry(top, dir_name) |
|||
dirs.append(entry) |
|||
|
|||
# Recurse into sub-directories, following symbolic links if "followlinks" |
|||
for entry in dirs: |
|||
if followlinks or not entry.is_symlink(): |
|||
new_path = join(top, entry.name) |
|||
for x in walk(new_path, topdown, onerror, followlinks): |
|||
yield x |
|||
|
|||
# Yield before recursion if going bottom up |
|||
if not topdown: |
|||
yield top, [e.name for e in dirs], [e.name for e in nondirs] |
|||
Loading…
Reference in new issue