From d31497bb34ec7f350fa6be115ec5956b76217785 Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Thu, 6 Aug 2020 20:12:40 +0900 Subject: [PATCH 01/10] Copy test_os.py from CPython 3.8.3 --- Lib/test/test_os.py | 4081 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 4081 insertions(+) create mode 100644 Lib/test/test_os.py diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py new file mode 100644 index 000000000..2a4ae1573 --- /dev/null +++ b/Lib/test/test_os.py @@ -0,0 +1,4081 @@ +# As a test suite for the os module, this is woefully inadequate, but this +# does add tests for a few functions which have been determined to be more +# portable than they had been thought to be. + +import asynchat +import asyncore +import codecs +import contextlib +import decimal +import errno +import fnmatch +import fractions +import itertools +import locale +import mmap +import os +import pickle +import shutil +import signal +import socket +import stat +import subprocess +import sys +import sysconfig +import tempfile +import threading +import time +import unittest +import uuid +import warnings +from test import support +from platform import win32_is_iot + +try: + import resource +except ImportError: + resource = None +try: + import fcntl +except ImportError: + fcntl = None +try: + import _winapi +except ImportError: + _winapi = None +try: + import pwd + all_users = [u.pw_uid for u in pwd.getpwall()] +except (ImportError, AttributeError): + all_users = [] +try: + from _testcapi import INT_MAX, PY_SSIZE_T_MAX +except ImportError: + INT_MAX = PY_SSIZE_T_MAX = sys.maxsize + +from test.support.script_helper import assert_python_ok +from test.support import unix_shell, FakePath + + +root_in_posix = False +if hasattr(os, 'geteuid'): + root_in_posix = (os.geteuid() == 0) + +# Detect whether we're on a Linux system that uses the (now outdated +# and unmaintained) linuxthreads threading library. There's an issue +# when combining linuxthreads with a failed execv call: see +# http://bugs.python.org/issue4970. +if hasattr(sys, 'thread_info') and sys.thread_info.version: + USING_LINUXTHREADS = sys.thread_info.version.startswith("linuxthreads") +else: + USING_LINUXTHREADS = False + +# Issue #14110: Some tests fail on FreeBSD if the user is in the wheel group. +HAVE_WHEEL_GROUP = sys.platform.startswith('freebsd') and os.getgid() == 0 + + +def requires_os_func(name): + return unittest.skipUnless(hasattr(os, name), 'requires os.%s' % name) + + +def create_file(filename, content=b'content'): + with open(filename, "xb", 0) as fp: + fp.write(content) + + +class MiscTests(unittest.TestCase): + def test_getcwd(self): + cwd = os.getcwd() + self.assertIsInstance(cwd, str) + + def test_getcwd_long_path(self): + # bpo-37412: On Linux, PATH_MAX is usually around 4096 bytes. On + # Windows, MAX_PATH is defined as 260 characters, but Windows supports + # longer path if longer paths support is enabled. Internally, the os + # module uses MAXPATHLEN which is at least 1024. + # + # Use a directory name of 200 characters to fit into Windows MAX_PATH + # limit. + # + # On Windows, the test can stop when trying to create a path longer + # than MAX_PATH if long paths support is disabled: + # see RtlAreLongPathsEnabled(). + min_len = 2000 # characters + dirlen = 200 # characters + dirname = 'python_test_dir_' + dirname = dirname + ('a' * (dirlen - len(dirname))) + + with tempfile.TemporaryDirectory() as tmpdir: + with support.change_cwd(tmpdir) as path: + expected = path + + while True: + cwd = os.getcwd() + self.assertEqual(cwd, expected) + + need = min_len - (len(cwd) + len(os.path.sep)) + if need <= 0: + break + if len(dirname) > need and need > 0: + dirname = dirname[:need] + + path = os.path.join(path, dirname) + try: + os.mkdir(path) + # On Windows, chdir() can fail + # even if mkdir() succeeded + os.chdir(path) + except FileNotFoundError: + # On Windows, catch ERROR_PATH_NOT_FOUND (3) and + # ERROR_FILENAME_EXCED_RANGE (206) errors + # ("The filename or extension is too long") + break + except OSError as exc: + if exc.errno == errno.ENAMETOOLONG: + break + else: + raise + + expected = path + + if support.verbose: + print(f"Tested current directory length: {len(cwd)}") + + def test_getcwdb(self): + cwd = os.getcwdb() + self.assertIsInstance(cwd, bytes) + self.assertEqual(os.fsdecode(cwd), os.getcwd()) + + +# Tests creating TESTFN +class FileTests(unittest.TestCase): + def setUp(self): + if os.path.lexists(support.TESTFN): + os.unlink(support.TESTFN) + tearDown = setUp + + def test_access(self): + f = os.open(support.TESTFN, os.O_CREAT|os.O_RDWR) + os.close(f) + self.assertTrue(os.access(support.TESTFN, os.W_OK)) + + def test_closerange(self): + first = os.open(support.TESTFN, os.O_CREAT|os.O_RDWR) + # We must allocate two consecutive file descriptors, otherwise + # it will mess up other file descriptors (perhaps even the three + # standard ones). + second = os.dup(first) + try: + retries = 0 + while second != first + 1: + os.close(first) + retries += 1 + if retries > 10: + # XXX test skipped + self.skipTest("couldn't allocate two consecutive fds") + first, second = second, os.dup(second) + finally: + os.close(second) + # close a fd that is open, and one that isn't + os.closerange(first, first + 2) + self.assertRaises(OSError, os.write, first, b"a") + + @support.cpython_only + def test_rename(self): + path = support.TESTFN + old = sys.getrefcount(path) + self.assertRaises(TypeError, os.rename, path, 0) + new = sys.getrefcount(path) + self.assertEqual(old, new) + + def test_read(self): + with open(support.TESTFN, "w+b") as fobj: + fobj.write(b"spam") + fobj.flush() + fd = fobj.fileno() + os.lseek(fd, 0, 0) + s = os.read(fd, 4) + self.assertEqual(type(s), bytes) + self.assertEqual(s, b"spam") + + @support.cpython_only + # Skip the test on 32-bit platforms: the number of bytes must fit in a + # Py_ssize_t type + @unittest.skipUnless(INT_MAX < PY_SSIZE_T_MAX, + "needs INT_MAX < PY_SSIZE_T_MAX") + @support.bigmemtest(size=INT_MAX + 10, memuse=1, dry_run=False) + def test_large_read(self, size): + self.addCleanup(support.unlink, support.TESTFN) + create_file(support.TESTFN, b'test') + + # Issue #21932: Make sure that os.read() does not raise an + # OverflowError for size larger than INT_MAX + with open(support.TESTFN, "rb") as fp: + data = os.read(fp.fileno(), size) + + # The test does not try to read more than 2 GiB at once because the + # operating system is free to return less bytes than requested. + self.assertEqual(data, b'test') + + def test_write(self): + # os.write() accepts bytes- and buffer-like objects but not strings + fd = os.open(support.TESTFN, os.O_CREAT | os.O_WRONLY) + self.assertRaises(TypeError, os.write, fd, "beans") + os.write(fd, b"bacon\n") + os.write(fd, bytearray(b"eggs\n")) + os.write(fd, memoryview(b"spam\n")) + os.close(fd) + with open(support.TESTFN, "rb") as fobj: + self.assertEqual(fobj.read().splitlines(), + [b"bacon", b"eggs", b"spam"]) + + def write_windows_console(self, *args): + retcode = subprocess.call(args, + # use a new console to not flood the test output + creationflags=subprocess.CREATE_NEW_CONSOLE, + # use a shell to hide the console window (SW_HIDE) + shell=True) + self.assertEqual(retcode, 0) + + @unittest.skipUnless(sys.platform == 'win32', + 'test specific to the Windows console') + def test_write_windows_console(self): + # Issue #11395: the Windows console returns an error (12: not enough + # space error) on writing into stdout if stdout mode is binary and the + # length is greater than 66,000 bytes (or less, depending on heap + # usage). + code = "print('x' * 100000)" + self.write_windows_console(sys.executable, "-c", code) + self.write_windows_console(sys.executable, "-u", "-c", code) + + def fdopen_helper(self, *args): + fd = os.open(support.TESTFN, os.O_RDONLY) + f = os.fdopen(fd, *args) + f.close() + + def test_fdopen(self): + fd = os.open(support.TESTFN, os.O_CREAT|os.O_RDWR) + os.close(fd) + + self.fdopen_helper() + self.fdopen_helper('r') + self.fdopen_helper('r', 100) + + def test_replace(self): + TESTFN2 = support.TESTFN + ".2" + self.addCleanup(support.unlink, support.TESTFN) + self.addCleanup(support.unlink, TESTFN2) + + create_file(support.TESTFN, b"1") + create_file(TESTFN2, b"2") + + os.replace(support.TESTFN, TESTFN2) + self.assertRaises(FileNotFoundError, os.stat, support.TESTFN) + with open(TESTFN2, 'r') as f: + self.assertEqual(f.read(), "1") + + def test_open_keywords(self): + f = os.open(path=__file__, flags=os.O_RDONLY, mode=0o777, + dir_fd=None) + os.close(f) + + def test_symlink_keywords(self): + symlink = support.get_attribute(os, "symlink") + try: + symlink(src='target', dst=support.TESTFN, + target_is_directory=False, dir_fd=None) + except (NotImplementedError, OSError): + pass # No OS support or unprivileged user + + @unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()') + def test_copy_file_range_invalid_values(self): + with self.assertRaises(ValueError): + os.copy_file_range(0, 1, -10) + + @unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()') + def test_copy_file_range(self): + TESTFN2 = support.TESTFN + ".3" + data = b'0123456789' + + create_file(support.TESTFN, data) + self.addCleanup(support.unlink, support.TESTFN) + + in_file = open(support.TESTFN, 'rb') + self.addCleanup(in_file.close) + in_fd = in_file.fileno() + + out_file = open(TESTFN2, 'w+b') + self.addCleanup(support.unlink, TESTFN2) + self.addCleanup(out_file.close) + out_fd = out_file.fileno() + + try: + i = os.copy_file_range(in_fd, out_fd, 5) + except OSError as e: + # Handle the case in which Python was compiled + # in a system with the syscall but without support + # in the kernel. + if e.errno != errno.ENOSYS: + raise + self.skipTest(e) + else: + # The number of copied bytes can be less than + # the number of bytes originally requested. + self.assertIn(i, range(0, 6)); + + with open(TESTFN2, 'rb') as in_file: + self.assertEqual(in_file.read(), data[:i]) + + @unittest.skipUnless(hasattr(os, 'copy_file_range'), 'test needs os.copy_file_range()') + def test_copy_file_range_offset(self): + TESTFN4 = support.TESTFN + ".4" + data = b'0123456789' + bytes_to_copy = 6 + in_skip = 3 + out_seek = 5 + + create_file(support.TESTFN, data) + self.addCleanup(support.unlink, support.TESTFN) + + in_file = open(support.TESTFN, 'rb') + self.addCleanup(in_file.close) + in_fd = in_file.fileno() + + out_file = open(TESTFN4, 'w+b') + self.addCleanup(support.unlink, TESTFN4) + self.addCleanup(out_file.close) + out_fd = out_file.fileno() + + try: + i = os.copy_file_range(in_fd, out_fd, bytes_to_copy, + offset_src=in_skip, + offset_dst=out_seek) + except OSError as e: + # Handle the case in which Python was compiled + # in a system with the syscall but without support + # in the kernel. + if e.errno != errno.ENOSYS: + raise + self.skipTest(e) + else: + # The number of copied bytes can be less than + # the number of bytes originally requested. + self.assertIn(i, range(0, bytes_to_copy+1)); + + with open(TESTFN4, 'rb') as in_file: + read = in_file.read() + # seeked bytes (5) are zero'ed + self.assertEqual(read[:out_seek], b'\x00'*out_seek) + # 012 are skipped (in_skip) + # 345678 are copied in the file (in_skip + bytes_to_copy) + self.assertEqual(read[out_seek:], + data[in_skip:in_skip+i]) + +# Test attributes on return values from os.*stat* family. +class StatAttributeTests(unittest.TestCase): + def setUp(self): + self.fname = support.TESTFN + self.addCleanup(support.unlink, self.fname) + create_file(self.fname, b"ABC") + + def check_stat_attributes(self, fname): + result = os.stat(fname) + + # Make sure direct access works + self.assertEqual(result[stat.ST_SIZE], 3) + self.assertEqual(result.st_size, 3) + + # Make sure all the attributes are there + members = dir(result) + for name in dir(stat): + if name[:3] == 'ST_': + attr = name.lower() + if name.endswith("TIME"): + def trunc(x): return int(x) + else: + def trunc(x): return x + self.assertEqual(trunc(getattr(result, attr)), + result[getattr(stat, name)]) + self.assertIn(attr, members) + + # Make sure that the st_?time and st_?time_ns fields roughly agree + # (they should always agree up to around tens-of-microseconds) + for name in 'st_atime st_mtime st_ctime'.split(): + floaty = int(getattr(result, name) * 100000) + nanosecondy = getattr(result, name + "_ns") // 10000 + self.assertAlmostEqual(floaty, nanosecondy, delta=2) + + try: + result[200] + self.fail("No exception raised") + except IndexError: + pass + + # Make sure that assignment fails + try: + result.st_mode = 1 + self.fail("No exception raised") + except AttributeError: + pass + + try: + result.st_rdev = 1 + self.fail("No exception raised") + except (AttributeError, TypeError): + pass + + try: + result.parrot = 1 + self.fail("No exception raised") + except AttributeError: + pass + + # Use the stat_result constructor with a too-short tuple. + try: + result2 = os.stat_result((10,)) + self.fail("No exception raised") + except TypeError: + pass + + # Use the constructor with a too-long tuple. + try: + result2 = os.stat_result((0,1,2,3,4,5,6,7,8,9,10,11,12,13,14)) + except TypeError: + pass + + def test_stat_attributes(self): + self.check_stat_attributes(self.fname) + + def test_stat_attributes_bytes(self): + try: + fname = self.fname.encode(sys.getfilesystemencoding()) + except UnicodeEncodeError: + self.skipTest("cannot encode %a for the filesystem" % self.fname) + self.check_stat_attributes(fname) + + def test_stat_result_pickle(self): + result = os.stat(self.fname) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + p = pickle.dumps(result, proto) + self.assertIn(b'stat_result', p) + if proto < 4: + self.assertIn(b'cos\nstat_result\n', p) + unpickled = pickle.loads(p) + self.assertEqual(result, unpickled) + + @unittest.skipUnless(hasattr(os, 'statvfs'), 'test needs os.statvfs()') + def test_statvfs_attributes(self): + result = os.statvfs(self.fname) + + # Make sure direct access works + self.assertEqual(result.f_bfree, result[3]) + + # Make sure all the attributes are there. + members = ('bsize', 'frsize', 'blocks', 'bfree', 'bavail', 'files', + 'ffree', 'favail', 'flag', 'namemax') + for value, member in enumerate(members): + self.assertEqual(getattr(result, 'f_' + member), result[value]) + + self.assertTrue(isinstance(result.f_fsid, int)) + + # Test that the size of the tuple doesn't change + self.assertEqual(len(result), 10) + + # Make sure that assignment really fails + try: + result.f_bfree = 1 + self.fail("No exception raised") + except AttributeError: + pass + + try: + result.parrot = 1 + self.fail("No exception raised") + except AttributeError: + pass + + # Use the constructor with a too-short tuple. + try: + result2 = os.statvfs_result((10,)) + self.fail("No exception raised") + except TypeError: + pass + + # Use the constructor with a too-long tuple. + try: + result2 = os.statvfs_result((0,1,2,3,4,5,6,7,8,9,10,11,12,13,14)) + except TypeError: + pass + + @unittest.skipUnless(hasattr(os, 'statvfs'), + "need os.statvfs()") + def test_statvfs_result_pickle(self): + result = os.statvfs(self.fname) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + p = pickle.dumps(result, proto) + self.assertIn(b'statvfs_result', p) + if proto < 4: + self.assertIn(b'cos\nstatvfs_result\n', p) + unpickled = pickle.loads(p) + self.assertEqual(result, unpickled) + + @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") + def test_1686475(self): + # Verify that an open file can be stat'ed + try: + os.stat(r"c:\pagefile.sys") + except FileNotFoundError: + self.skipTest(r'c:\pagefile.sys does not exist') + except OSError as e: + self.fail("Could not stat pagefile.sys") + + @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") + @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") + def test_15261(self): + # Verify that stat'ing a closed fd does not cause crash + r, w = os.pipe() + try: + os.stat(r) # should not raise error + finally: + os.close(r) + os.close(w) + with self.assertRaises(OSError) as ctx: + os.stat(r) + self.assertEqual(ctx.exception.errno, errno.EBADF) + + def check_file_attributes(self, result): + self.assertTrue(hasattr(result, 'st_file_attributes')) + self.assertTrue(isinstance(result.st_file_attributes, int)) + self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) + + @unittest.skipUnless(sys.platform == "win32", + "st_file_attributes is Win32 specific") + def test_file_attributes(self): + # test file st_file_attributes (FILE_ATTRIBUTE_DIRECTORY not set) + result = os.stat(self.fname) + self.check_file_attributes(result) + self.assertEqual( + result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, + 0) + + # test directory st_file_attributes (FILE_ATTRIBUTE_DIRECTORY set) + dirname = support.TESTFN + "dir" + os.mkdir(dirname) + self.addCleanup(os.rmdir, dirname) + + result = os.stat(dirname) + self.check_file_attributes(result) + self.assertEqual( + result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, + stat.FILE_ATTRIBUTE_DIRECTORY) + + @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") + def test_access_denied(self): + # Default to FindFirstFile WIN32_FIND_DATA when access is + # denied. See issue 28075. + # os.environ['TEMP'] should be located on a volume that + # supports file ACLs. + fname = os.path.join(os.environ['TEMP'], self.fname) + self.addCleanup(support.unlink, fname) + create_file(fname, b'ABC') + # Deny the right to [S]YNCHRONIZE on the file to + # force CreateFile to fail with ERROR_ACCESS_DENIED. + DETACHED_PROCESS = 8 + subprocess.check_call( + # bpo-30584: Use security identifier *S-1-5-32-545 instead + # of localized "Users" to not depend on the locale. + ['icacls.exe', fname, '/deny', '*S-1-5-32-545:(S)'], + creationflags=DETACHED_PROCESS + ) + result = os.stat(fname) + self.assertNotEqual(result.st_size, 0) + + @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") + def test_stat_block_device(self): + # bpo-38030: os.stat fails for block devices + # Test a filename like "//./C:" + fname = "//./" + os.path.splitdrive(os.getcwd())[0] + result = os.stat(fname) + self.assertEqual(result.st_mode, stat.S_IFBLK) + + +class UtimeTests(unittest.TestCase): + def setUp(self): + self.dirname = support.TESTFN + self.fname = os.path.join(self.dirname, "f1") + + self.addCleanup(support.rmtree, self.dirname) + os.mkdir(self.dirname) + create_file(self.fname) + + def support_subsecond(self, filename): + # Heuristic to check if the filesystem supports timestamp with + # subsecond resolution: check if float and int timestamps are different + st = os.stat(filename) + return ((st.st_atime != st[7]) + or (st.st_mtime != st[8]) + or (st.st_ctime != st[9])) + + def _test_utime(self, set_time, filename=None): + if not filename: + filename = self.fname + + support_subsecond = self.support_subsecond(filename) + if support_subsecond: + # Timestamp with a resolution of 1 microsecond (10^-6). + # + # The resolution of the C internal function used by os.utime() + # depends on the platform: 1 sec, 1 us, 1 ns. Writing a portable + # test with a resolution of 1 ns requires more work: + # see the issue #15745. + atime_ns = 1002003000 # 1.002003 seconds + mtime_ns = 4005006000 # 4.005006 seconds + else: + # use a resolution of 1 second + atime_ns = 5 * 10**9 + mtime_ns = 8 * 10**9 + + set_time(filename, (atime_ns, mtime_ns)) + st = os.stat(filename) + + if support_subsecond: + self.assertAlmostEqual(st.st_atime, atime_ns * 1e-9, delta=1e-6) + self.assertAlmostEqual(st.st_mtime, mtime_ns * 1e-9, delta=1e-6) + else: + self.assertEqual(st.st_atime, atime_ns * 1e-9) + self.assertEqual(st.st_mtime, mtime_ns * 1e-9) + self.assertEqual(st.st_atime_ns, atime_ns) + self.assertEqual(st.st_mtime_ns, mtime_ns) + + def test_utime(self): + def set_time(filename, ns): + # test the ns keyword parameter + os.utime(filename, ns=ns) + self._test_utime(set_time) + + @staticmethod + def ns_to_sec(ns): + # Convert a number of nanosecond (int) to a number of seconds (float). + # Round towards infinity by adding 0.5 nanosecond to avoid rounding + # issue, os.utime() rounds towards minus infinity. + return (ns * 1e-9) + 0.5e-9 + + def test_utime_by_indexed(self): + # pass times as floating point seconds as the second indexed parameter + def set_time(filename, ns): + atime_ns, mtime_ns = ns + atime = self.ns_to_sec(atime_ns) + mtime = self.ns_to_sec(mtime_ns) + # test utimensat(timespec), utimes(timeval), utime(utimbuf) + # or utime(time_t) + os.utime(filename, (atime, mtime)) + self._test_utime(set_time) + + def test_utime_by_times(self): + def set_time(filename, ns): + atime_ns, mtime_ns = ns + atime = self.ns_to_sec(atime_ns) + mtime = self.ns_to_sec(mtime_ns) + # test the times keyword parameter + os.utime(filename, times=(atime, mtime)) + self._test_utime(set_time) + + @unittest.skipUnless(os.utime in os.supports_follow_symlinks, + "follow_symlinks support for utime required " + "for this test.") + def test_utime_nofollow_symlinks(self): + def set_time(filename, ns): + # use follow_symlinks=False to test utimensat(timespec) + # or lutimes(timeval) + os.utime(filename, ns=ns, follow_symlinks=False) + self._test_utime(set_time) + + @unittest.skipUnless(os.utime in os.supports_fd, + "fd support for utime required for this test.") + def test_utime_fd(self): + def set_time(filename, ns): + with open(filename, 'wb', 0) as fp: + # use a file descriptor to test futimens(timespec) + # or futimes(timeval) + os.utime(fp.fileno(), ns=ns) + self._test_utime(set_time) + + @unittest.skipUnless(os.utime in os.supports_dir_fd, + "dir_fd support for utime required for this test.") + def test_utime_dir_fd(self): + def set_time(filename, ns): + dirname, name = os.path.split(filename) + dirfd = os.open(dirname, os.O_RDONLY) + try: + # pass dir_fd to test utimensat(timespec) or futimesat(timeval) + os.utime(name, dir_fd=dirfd, ns=ns) + finally: + os.close(dirfd) + self._test_utime(set_time) + + def test_utime_directory(self): + def set_time(filename, ns): + # test calling os.utime() on a directory + os.utime(filename, ns=ns) + self._test_utime(set_time, filename=self.dirname) + + def _test_utime_current(self, set_time): + # Get the system clock + current = time.time() + + # Call os.utime() to set the timestamp to the current system clock + set_time(self.fname) + + if not self.support_subsecond(self.fname): + delta = 1.0 + else: + # On Windows, the usual resolution of time.time() is 15.6 ms. + # bpo-30649: Tolerate 50 ms for slow Windows buildbots. + # + # x86 Gentoo Refleaks 3.x once failed with dt=20.2 ms. So use + # also 50 ms on other platforms. + delta = 0.050 + st = os.stat(self.fname) + msg = ("st_time=%r, current=%r, dt=%r" + % (st.st_mtime, current, st.st_mtime - current)) + self.assertAlmostEqual(st.st_mtime, current, + delta=delta, msg=msg) + + def test_utime_current(self): + def set_time(filename): + # Set to the current time in the new way + os.utime(self.fname) + self._test_utime_current(set_time) + + def test_utime_current_old(self): + def set_time(filename): + # Set to the current time in the old explicit way. + os.utime(self.fname, None) + self._test_utime_current(set_time) + + def get_file_system(self, path): + if sys.platform == 'win32': + root = os.path.splitdrive(os.path.abspath(path))[0] + '\\' + import ctypes + kernel32 = ctypes.windll.kernel32 + buf = ctypes.create_unicode_buffer("", 100) + ok = kernel32.GetVolumeInformationW(root, None, 0, + None, None, None, + buf, len(buf)) + if ok: + return buf.value + # return None if the filesystem is unknown + + def test_large_time(self): + # Many filesystems are limited to the year 2038. At least, the test + # pass with NTFS filesystem. + if self.get_file_system(self.dirname) != "NTFS": + self.skipTest("requires NTFS") + + large = 5000000000 # some day in 2128 + os.utime(self.fname, (large, large)) + self.assertEqual(os.stat(self.fname).st_mtime, large) + + def test_utime_invalid_arguments(self): + # seconds and nanoseconds parameters are mutually exclusive + with self.assertRaises(ValueError): + os.utime(self.fname, (5, 5), ns=(5, 5)) + with self.assertRaises(TypeError): + os.utime(self.fname, [5, 5]) + with self.assertRaises(TypeError): + os.utime(self.fname, (5,)) + with self.assertRaises(TypeError): + os.utime(self.fname, (5, 5, 5)) + with self.assertRaises(TypeError): + os.utime(self.fname, ns=[5, 5]) + with self.assertRaises(TypeError): + os.utime(self.fname, ns=(5,)) + with self.assertRaises(TypeError): + os.utime(self.fname, ns=(5, 5, 5)) + + if os.utime not in os.supports_follow_symlinks: + with self.assertRaises(NotImplementedError): + os.utime(self.fname, (5, 5), follow_symlinks=False) + if os.utime not in os.supports_fd: + with open(self.fname, 'wb', 0) as fp: + with self.assertRaises(TypeError): + os.utime(fp.fileno(), (5, 5)) + if os.utime not in os.supports_dir_fd: + with self.assertRaises(NotImplementedError): + os.utime(self.fname, (5, 5), dir_fd=0) + + @support.cpython_only + def test_issue31577(self): + # The interpreter shouldn't crash in case utime() received a bad + # ns argument. + def get_bad_int(divmod_ret_val): + class BadInt: + def __divmod__(*args): + return divmod_ret_val + return BadInt() + with self.assertRaises(TypeError): + os.utime(self.fname, ns=(get_bad_int(42), 1)) + with self.assertRaises(TypeError): + os.utime(self.fname, ns=(get_bad_int(()), 1)) + with self.assertRaises(TypeError): + os.utime(self.fname, ns=(get_bad_int((1, 2, 3)), 1)) + + +from test import mapping_tests + +class EnvironTests(mapping_tests.BasicTestMappingProtocol): + """check that os.environ object conform to mapping protocol""" + type2test = None + + def setUp(self): + self.__save = dict(os.environ) + if os.supports_bytes_environ: + self.__saveb = dict(os.environb) + for key, value in self._reference().items(): + os.environ[key] = value + + def tearDown(self): + os.environ.clear() + os.environ.update(self.__save) + if os.supports_bytes_environ: + os.environb.clear() + os.environb.update(self.__saveb) + + def _reference(self): + return {"KEY1":"VALUE1", "KEY2":"VALUE2", "KEY3":"VALUE3"} + + def _empty_mapping(self): + os.environ.clear() + return os.environ + + # Bug 1110478 + @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), + 'requires a shell') + def test_update2(self): + os.environ.clear() + os.environ.update(HELLO="World") + with os.popen("%s -c 'echo $HELLO'" % unix_shell) as popen: + value = popen.read().strip() + self.assertEqual(value, "World") + + @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), + 'requires a shell') + def test_os_popen_iter(self): + with os.popen("%s -c 'echo \"line1\nline2\nline3\"'" + % unix_shell) as popen: + it = iter(popen) + self.assertEqual(next(it), "line1\n") + self.assertEqual(next(it), "line2\n") + self.assertEqual(next(it), "line3\n") + self.assertRaises(StopIteration, next, it) + + # Verify environ keys and values from the OS are of the + # correct str type. + def test_keyvalue_types(self): + for key, val in os.environ.items(): + self.assertEqual(type(key), str) + self.assertEqual(type(val), str) + + def test_items(self): + for key, value in self._reference().items(): + self.assertEqual(os.environ.get(key), value) + + # Issue 7310 + def test___repr__(self): + """Check that the repr() of os.environ looks like environ({...}).""" + env = os.environ + self.assertEqual(repr(env), 'environ({{{}}})'.format(', '.join( + '{!r}: {!r}'.format(key, value) + for key, value in env.items()))) + + def test_get_exec_path(self): + defpath_list = os.defpath.split(os.pathsep) + test_path = ['/monty', '/python', '', '/flying/circus'] + test_env = {'PATH': os.pathsep.join(test_path)} + + saved_environ = os.environ + try: + os.environ = dict(test_env) + # Test that defaulting to os.environ works. + self.assertSequenceEqual(test_path, os.get_exec_path()) + self.assertSequenceEqual(test_path, os.get_exec_path(env=None)) + finally: + os.environ = saved_environ + + # No PATH environment variable + self.assertSequenceEqual(defpath_list, os.get_exec_path({})) + # Empty PATH environment variable + self.assertSequenceEqual(('',), os.get_exec_path({'PATH':''})) + # Supplied PATH environment variable + self.assertSequenceEqual(test_path, os.get_exec_path(test_env)) + + if os.supports_bytes_environ: + # env cannot contain 'PATH' and b'PATH' keys + try: + # ignore BytesWarning warning + with warnings.catch_warnings(record=True): + mixed_env = {'PATH': '1', b'PATH': b'2'} + except BytesWarning: + # mixed_env cannot be created with python -bb + pass + else: + self.assertRaises(ValueError, os.get_exec_path, mixed_env) + + # bytes key and/or value + self.assertSequenceEqual(os.get_exec_path({b'PATH': b'abc'}), + ['abc']) + self.assertSequenceEqual(os.get_exec_path({b'PATH': 'abc'}), + ['abc']) + self.assertSequenceEqual(os.get_exec_path({'PATH': b'abc'}), + ['abc']) + + @unittest.skipUnless(os.supports_bytes_environ, + "os.environb required for this test.") + def test_environb(self): + # os.environ -> os.environb + value = 'euro\u20ac' + try: + value_bytes = value.encode(sys.getfilesystemencoding(), + 'surrogateescape') + except UnicodeEncodeError: + msg = "U+20AC character is not encodable to %s" % ( + sys.getfilesystemencoding(),) + self.skipTest(msg) + os.environ['unicode'] = value + self.assertEqual(os.environ['unicode'], value) + self.assertEqual(os.environb[b'unicode'], value_bytes) + + # os.environb -> os.environ + value = b'\xff' + os.environb[b'bytes'] = value + self.assertEqual(os.environb[b'bytes'], value) + value_str = value.decode(sys.getfilesystemencoding(), 'surrogateescape') + self.assertEqual(os.environ['bytes'], value_str) + + # On OS X < 10.6, unsetenv() doesn't return a value (bpo-13415). + @support.requires_mac_ver(10, 6) + def test_unset_error(self): + if sys.platform == "win32": + # an environment variable is limited to 32,767 characters + key = 'x' * 50000 + self.assertRaises(ValueError, os.environ.__delitem__, key) + else: + # "=" is not allowed in a variable name + key = 'key=' + self.assertRaises(OSError, os.environ.__delitem__, key) + + def test_key_type(self): + missing = 'missingkey' + self.assertNotIn(missing, os.environ) + + with self.assertRaises(KeyError) as cm: + os.environ[missing] + self.assertIs(cm.exception.args[0], missing) + self.assertTrue(cm.exception.__suppress_context__) + + with self.assertRaises(KeyError) as cm: + del os.environ[missing] + self.assertIs(cm.exception.args[0], missing) + self.assertTrue(cm.exception.__suppress_context__) + + def _test_environ_iteration(self, collection): + iterator = iter(collection) + new_key = "__new_key__" + + next(iterator) # start iteration over os.environ.items + + # add a new key in os.environ mapping + os.environ[new_key] = "test_environ_iteration" + + try: + next(iterator) # force iteration over modified mapping + self.assertEqual(os.environ[new_key], "test_environ_iteration") + finally: + del os.environ[new_key] + + def test_iter_error_when_changing_os_environ(self): + self._test_environ_iteration(os.environ) + + def test_iter_error_when_changing_os_environ_items(self): + self._test_environ_iteration(os.environ.items()) + + def test_iter_error_when_changing_os_environ_values(self): + self._test_environ_iteration(os.environ.values()) + + +class WalkTests(unittest.TestCase): + """Tests for os.walk().""" + + # Wrapper to hide minor differences between os.walk and os.fwalk + # to tests both functions with the same code base + def walk(self, top, **kwargs): + if 'follow_symlinks' in kwargs: + kwargs['followlinks'] = kwargs.pop('follow_symlinks') + return os.walk(top, **kwargs) + + def setUp(self): + join = os.path.join + self.addCleanup(support.rmtree, support.TESTFN) + + # Build: + # TESTFN/ + # TEST1/ a file kid and two directory kids + # tmp1 + # SUB1/ a file kid and a directory kid + # tmp2 + # SUB11/ no kids + # SUB2/ a file kid and a dirsymlink kid + # tmp3 + # SUB21/ not readable + # tmp5 + # link/ a symlink to TESTFN.2 + # broken_link + # broken_link2 + # broken_link3 + # TEST2/ + # tmp4 a lone file + self.walk_path = join(support.TESTFN, "TEST1") + self.sub1_path = join(self.walk_path, "SUB1") + self.sub11_path = join(self.sub1_path, "SUB11") + sub2_path = join(self.walk_path, "SUB2") + sub21_path = join(sub2_path, "SUB21") + tmp1_path = join(self.walk_path, "tmp1") + tmp2_path = join(self.sub1_path, "tmp2") + tmp3_path = join(sub2_path, "tmp3") + tmp5_path = join(sub21_path, "tmp3") + self.link_path = join(sub2_path, "link") + t2_path = join(support.TESTFN, "TEST2") + tmp4_path = join(support.TESTFN, "TEST2", "tmp4") + broken_link_path = join(sub2_path, "broken_link") + broken_link2_path = join(sub2_path, "broken_link2") + broken_link3_path = join(sub2_path, "broken_link3") + + # Create stuff. + os.makedirs(self.sub11_path) + os.makedirs(sub2_path) + os.makedirs(sub21_path) + os.makedirs(t2_path) + + for path in tmp1_path, tmp2_path, tmp3_path, tmp4_path, tmp5_path: + with open(path, "x") as f: + f.write("I'm " + path + " and proud of it. Blame test_os.\n") + + if support.can_symlink(): + os.symlink(os.path.abspath(t2_path), self.link_path) + os.symlink('broken', broken_link_path, True) + os.symlink(join('tmp3', 'broken'), broken_link2_path, True) + os.symlink(join('SUB21', 'tmp5'), broken_link3_path, True) + self.sub2_tree = (sub2_path, ["SUB21", "link"], + ["broken_link", "broken_link2", "broken_link3", + "tmp3"]) + else: + self.sub2_tree = (sub2_path, ["SUB21"], ["tmp3"]) + + os.chmod(sub21_path, 0) + try: + os.listdir(sub21_path) + except PermissionError: + self.addCleanup(os.chmod, sub21_path, stat.S_IRWXU) + else: + os.chmod(sub21_path, stat.S_IRWXU) + os.unlink(tmp5_path) + os.rmdir(sub21_path) + del self.sub2_tree[1][:1] + + def test_walk_topdown(self): + # Walk top-down. + all = list(self.walk(self.walk_path)) + + self.assertEqual(len(all), 4) + # We can't know which order SUB1 and SUB2 will appear in. + # Not flipped: TESTFN, SUB1, SUB11, SUB2 + # flipped: TESTFN, SUB2, SUB1, SUB11 + flipped = all[0][1][0] != "SUB1" + all[0][1].sort() + all[3 - 2 * flipped][-1].sort() + all[3 - 2 * flipped][1].sort() + self.assertEqual(all[0], (self.walk_path, ["SUB1", "SUB2"], ["tmp1"])) + self.assertEqual(all[1 + flipped], (self.sub1_path, ["SUB11"], ["tmp2"])) + self.assertEqual(all[2 + flipped], (self.sub11_path, [], [])) + self.assertEqual(all[3 - 2 * flipped], self.sub2_tree) + + def test_walk_prune(self, walk_path=None): + if walk_path is None: + walk_path = self.walk_path + # Prune the search. + all = [] + for root, dirs, files in self.walk(walk_path): + all.append((root, dirs, files)) + # Don't descend into SUB1. + if 'SUB1' in dirs: + # Note that this also mutates the dirs we appended to all! + dirs.remove('SUB1') + + self.assertEqual(len(all), 2) + self.assertEqual(all[0], (self.walk_path, ["SUB2"], ["tmp1"])) + + all[1][-1].sort() + all[1][1].sort() + self.assertEqual(all[1], self.sub2_tree) + + def test_file_like_path(self): + self.test_walk_prune(FakePath(self.walk_path)) + + def test_walk_bottom_up(self): + # Walk bottom-up. + all = list(self.walk(self.walk_path, topdown=False)) + + self.assertEqual(len(all), 4, all) + # We can't know which order SUB1 and SUB2 will appear in. + # Not flipped: SUB11, SUB1, SUB2, TESTFN + # flipped: SUB2, SUB11, SUB1, TESTFN + flipped = all[3][1][0] != "SUB1" + all[3][1].sort() + all[2 - 2 * flipped][-1].sort() + all[2 - 2 * flipped][1].sort() + self.assertEqual(all[3], + (self.walk_path, ["SUB1", "SUB2"], ["tmp1"])) + self.assertEqual(all[flipped], + (self.sub11_path, [], [])) + self.assertEqual(all[flipped + 1], + (self.sub1_path, ["SUB11"], ["tmp2"])) + self.assertEqual(all[2 - 2 * flipped], + self.sub2_tree) + + def test_walk_symlink(self): + if not support.can_symlink(): + self.skipTest("need symlink support") + + # Walk, following symlinks. + walk_it = self.walk(self.walk_path, follow_symlinks=True) + for root, dirs, files in walk_it: + if root == self.link_path: + self.assertEqual(dirs, []) + self.assertEqual(files, ["tmp4"]) + break + else: + self.fail("Didn't follow symlink with followlinks=True") + + def test_walk_bad_dir(self): + # Walk top-down. + errors = [] + walk_it = self.walk(self.walk_path, onerror=errors.append) + root, dirs, files = next(walk_it) + self.assertEqual(errors, []) + dir1 = 'SUB1' + path1 = os.path.join(root, dir1) + path1new = os.path.join(root, dir1 + '.new') + os.rename(path1, path1new) + try: + roots = [r for r, d, f in walk_it] + self.assertTrue(errors) + self.assertNotIn(path1, roots) + self.assertNotIn(path1new, roots) + for dir2 in dirs: + if dir2 != dir1: + self.assertIn(os.path.join(root, dir2), roots) + finally: + os.rename(path1new, path1) + + def test_walk_many_open_files(self): + depth = 30 + base = os.path.join(support.TESTFN, 'deep') + p = os.path.join(base, *(['d']*depth)) + os.makedirs(p) + + iters = [self.walk(base, topdown=False) for j in range(100)] + for i in range(depth + 1): + expected = (p, ['d'] if i else [], []) + for it in iters: + self.assertEqual(next(it), expected) + p = os.path.dirname(p) + + iters = [self.walk(base, topdown=True) for j in range(100)] + p = base + for i in range(depth + 1): + expected = (p, ['d'] if i < depth else [], []) + for it in iters: + self.assertEqual(next(it), expected) + p = os.path.join(p, 'd') + + +@unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") +class FwalkTests(WalkTests): + """Tests for os.fwalk().""" + + def walk(self, top, **kwargs): + for root, dirs, files, root_fd in self.fwalk(top, **kwargs): + yield (root, dirs, files) + + def fwalk(self, *args, **kwargs): + return os.fwalk(*args, **kwargs) + + def _compare_to_walk(self, walk_kwargs, fwalk_kwargs): + """ + compare with walk() results. + """ + walk_kwargs = walk_kwargs.copy() + fwalk_kwargs = fwalk_kwargs.copy() + for topdown, follow_symlinks in itertools.product((True, False), repeat=2): + walk_kwargs.update(topdown=topdown, followlinks=follow_symlinks) + fwalk_kwargs.update(topdown=topdown, follow_symlinks=follow_symlinks) + + expected = {} + for root, dirs, files in os.walk(**walk_kwargs): + expected[root] = (set(dirs), set(files)) + + for root, dirs, files, rootfd in self.fwalk(**fwalk_kwargs): + self.assertIn(root, expected) + self.assertEqual(expected[root], (set(dirs), set(files))) + + def test_compare_to_walk(self): + kwargs = {'top': support.TESTFN} + self._compare_to_walk(kwargs, kwargs) + + def test_dir_fd(self): + try: + fd = os.open(".", os.O_RDONLY) + walk_kwargs = {'top': support.TESTFN} + fwalk_kwargs = walk_kwargs.copy() + fwalk_kwargs['dir_fd'] = fd + self._compare_to_walk(walk_kwargs, fwalk_kwargs) + finally: + os.close(fd) + + def test_yields_correct_dir_fd(self): + # check returned file descriptors + for topdown, follow_symlinks in itertools.product((True, False), repeat=2): + args = support.TESTFN, topdown, None + for root, dirs, files, rootfd in self.fwalk(*args, follow_symlinks=follow_symlinks): + # check that the FD is valid + os.fstat(rootfd) + # redundant check + os.stat(rootfd) + # check that listdir() returns consistent information + self.assertEqual(set(os.listdir(rootfd)), set(dirs) | set(files)) + + def test_fd_leak(self): + # Since we're opening a lot of FDs, we must be careful to avoid leaks: + # we both check that calling fwalk() a large number of times doesn't + # yield EMFILE, and that the minimum allocated FD hasn't changed. + minfd = os.dup(1) + os.close(minfd) + for i in range(256): + for x in self.fwalk(support.TESTFN): + pass + newfd = os.dup(1) + self.addCleanup(os.close, newfd) + self.assertEqual(newfd, minfd) + + # fwalk() keeps file descriptors open + test_walk_many_open_files = None + + +class BytesWalkTests(WalkTests): + """Tests for os.walk() with bytes.""" + def walk(self, top, **kwargs): + if 'follow_symlinks' in kwargs: + kwargs['followlinks'] = kwargs.pop('follow_symlinks') + for broot, bdirs, bfiles in os.walk(os.fsencode(top), **kwargs): + root = os.fsdecode(broot) + dirs = list(map(os.fsdecode, bdirs)) + files = list(map(os.fsdecode, bfiles)) + yield (root, dirs, files) + bdirs[:] = list(map(os.fsencode, dirs)) + bfiles[:] = list(map(os.fsencode, files)) + +@unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") +class BytesFwalkTests(FwalkTests): + """Tests for os.walk() with bytes.""" + def fwalk(self, top='.', *args, **kwargs): + for broot, bdirs, bfiles, topfd in os.fwalk(os.fsencode(top), *args, **kwargs): + root = os.fsdecode(broot) + dirs = list(map(os.fsdecode, bdirs)) + files = list(map(os.fsdecode, bfiles)) + yield (root, dirs, files, topfd) + bdirs[:] = list(map(os.fsencode, dirs)) + bfiles[:] = list(map(os.fsencode, files)) + + +class MakedirTests(unittest.TestCase): + def setUp(self): + os.mkdir(support.TESTFN) + + def test_makedir(self): + base = support.TESTFN + path = os.path.join(base, 'dir1', 'dir2', 'dir3') + os.makedirs(path) # Should work + path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4') + os.makedirs(path) + + # Try paths with a '.' in them + self.assertRaises(OSError, os.makedirs, os.curdir) + path = os.path.join(base, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', os.curdir) + os.makedirs(path) + path = os.path.join(base, 'dir1', os.curdir, 'dir2', 'dir3', 'dir4', + 'dir5', 'dir6') + os.makedirs(path) + + def test_mode(self): + with support.temp_umask(0o002): + base = support.TESTFN + parent = os.path.join(base, 'dir1') + path = os.path.join(parent, 'dir2') + os.makedirs(path, 0o555) + self.assertTrue(os.path.exists(path)) + self.assertTrue(os.path.isdir(path)) + if os.name != 'nt': + self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) + self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) + + def test_exist_ok_existing_directory(self): + path = os.path.join(support.TESTFN, 'dir1') + mode = 0o777 + old_mask = os.umask(0o022) + os.makedirs(path, mode) + self.assertRaises(OSError, os.makedirs, path, mode) + self.assertRaises(OSError, os.makedirs, path, mode, exist_ok=False) + os.makedirs(path, 0o776, exist_ok=True) + os.makedirs(path, mode=mode, exist_ok=True) + os.umask(old_mask) + + # Issue #25583: A drive root could raise PermissionError on Windows + os.makedirs(os.path.abspath('/'), exist_ok=True) + + def test_exist_ok_s_isgid_directory(self): + path = os.path.join(support.TESTFN, 'dir1') + S_ISGID = stat.S_ISGID + mode = 0o777 + old_mask = os.umask(0o022) + try: + existing_testfn_mode = stat.S_IMODE( + os.lstat(support.TESTFN).st_mode) + try: + os.chmod(support.TESTFN, existing_testfn_mode | S_ISGID) + except PermissionError: + raise unittest.SkipTest('Cannot set S_ISGID for dir.') + if (os.lstat(support.TESTFN).st_mode & S_ISGID != S_ISGID): + raise unittest.SkipTest('No support for S_ISGID dir mode.') + # The os should apply S_ISGID from the parent dir for us, but + # this test need not depend on that behavior. Be explicit. + os.makedirs(path, mode | S_ISGID) + # http://bugs.python.org/issue14992 + # Should not fail when the bit is already set. + os.makedirs(path, mode, exist_ok=True) + # remove the bit. + os.chmod(path, stat.S_IMODE(os.lstat(path).st_mode) & ~S_ISGID) + # May work even when the bit is not already set when demanded. + os.makedirs(path, mode | S_ISGID, exist_ok=True) + finally: + os.umask(old_mask) + + def test_exist_ok_existing_regular_file(self): + base = support.TESTFN + path = os.path.join(support.TESTFN, 'dir1') + with open(path, 'w') as f: + f.write('abc') + self.assertRaises(OSError, os.makedirs, path) + self.assertRaises(OSError, os.makedirs, path, exist_ok=False) + self.assertRaises(OSError, os.makedirs, path, exist_ok=True) + os.remove(path) + + def tearDown(self): + path = os.path.join(support.TESTFN, 'dir1', 'dir2', 'dir3', + 'dir4', 'dir5', 'dir6') + # If the tests failed, the bottom-most directory ('../dir6') + # may not have been created, so we look for the outermost directory + # that exists. + while not os.path.exists(path) and path != support.TESTFN: + path = os.path.dirname(path) + + os.removedirs(path) + + +@unittest.skipUnless(hasattr(os, 'chown'), "Test needs chown") +class ChownFileTests(unittest.TestCase): + + @classmethod + def setUpClass(cls): + os.mkdir(support.TESTFN) + + def test_chown_uid_gid_arguments_must_be_index(self): + stat = os.stat(support.TESTFN) + uid = stat.st_uid + gid = stat.st_gid + for value in (-1.0, -1j, decimal.Decimal(-1), fractions.Fraction(-2, 2)): + self.assertRaises(TypeError, os.chown, support.TESTFN, value, gid) + self.assertRaises(TypeError, os.chown, support.TESTFN, uid, value) + self.assertIsNone(os.chown(support.TESTFN, uid, gid)) + self.assertIsNone(os.chown(support.TESTFN, -1, -1)) + + @unittest.skipUnless(hasattr(os, 'getgroups'), 'need os.getgroups') + def test_chown_gid(self): + groups = os.getgroups() + if len(groups) < 2: + self.skipTest("test needs at least 2 groups") + + gid_1, gid_2 = groups[:2] + uid = os.stat(support.TESTFN).st_uid + + os.chown(support.TESTFN, uid, gid_1) + gid = os.stat(support.TESTFN).st_gid + self.assertEqual(gid, gid_1) + + os.chown(support.TESTFN, uid, gid_2) + gid = os.stat(support.TESTFN).st_gid + self.assertEqual(gid, gid_2) + + @unittest.skipUnless(root_in_posix and len(all_users) > 1, + "test needs root privilege and more than one user") + def test_chown_with_root(self): + uid_1, uid_2 = all_users[:2] + gid = os.stat(support.TESTFN).st_gid + os.chown(support.TESTFN, uid_1, gid) + uid = os.stat(support.TESTFN).st_uid + self.assertEqual(uid, uid_1) + os.chown(support.TESTFN, uid_2, gid) + uid = os.stat(support.TESTFN).st_uid + self.assertEqual(uid, uid_2) + + @unittest.skipUnless(not root_in_posix and len(all_users) > 1, + "test needs non-root account and more than one user") + def test_chown_without_permission(self): + uid_1, uid_2 = all_users[:2] + gid = os.stat(support.TESTFN).st_gid + with self.assertRaises(PermissionError): + os.chown(support.TESTFN, uid_1, gid) + os.chown(support.TESTFN, uid_2, gid) + + @classmethod + def tearDownClass(cls): + os.rmdir(support.TESTFN) + + +class RemoveDirsTests(unittest.TestCase): + def setUp(self): + os.makedirs(support.TESTFN) + + def tearDown(self): + support.rmtree(support.TESTFN) + + def test_remove_all(self): + dira = os.path.join(support.TESTFN, 'dira') + os.mkdir(dira) + dirb = os.path.join(dira, 'dirb') + os.mkdir(dirb) + os.removedirs(dirb) + self.assertFalse(os.path.exists(dirb)) + self.assertFalse(os.path.exists(dira)) + self.assertFalse(os.path.exists(support.TESTFN)) + + def test_remove_partial(self): + dira = os.path.join(support.TESTFN, 'dira') + os.mkdir(dira) + dirb = os.path.join(dira, 'dirb') + os.mkdir(dirb) + create_file(os.path.join(dira, 'file.txt')) + os.removedirs(dirb) + self.assertFalse(os.path.exists(dirb)) + self.assertTrue(os.path.exists(dira)) + self.assertTrue(os.path.exists(support.TESTFN)) + + def test_remove_nothing(self): + dira = os.path.join(support.TESTFN, 'dira') + os.mkdir(dira) + dirb = os.path.join(dira, 'dirb') + os.mkdir(dirb) + create_file(os.path.join(dirb, 'file.txt')) + with self.assertRaises(OSError): + os.removedirs(dirb) + self.assertTrue(os.path.exists(dirb)) + self.assertTrue(os.path.exists(dira)) + self.assertTrue(os.path.exists(support.TESTFN)) + + +class DevNullTests(unittest.TestCase): + def test_devnull(self): + with open(os.devnull, 'wb', 0) as f: + f.write(b'hello') + f.close() + with open(os.devnull, 'rb') as f: + self.assertEqual(f.read(), b'') + + +class URandomTests(unittest.TestCase): + def test_urandom_length(self): + self.assertEqual(len(os.urandom(0)), 0) + self.assertEqual(len(os.urandom(1)), 1) + self.assertEqual(len(os.urandom(10)), 10) + self.assertEqual(len(os.urandom(100)), 100) + self.assertEqual(len(os.urandom(1000)), 1000) + + def test_urandom_value(self): + data1 = os.urandom(16) + self.assertIsInstance(data1, bytes) + data2 = os.urandom(16) + self.assertNotEqual(data1, data2) + + def get_urandom_subprocess(self, count): + code = '\n'.join(( + 'import os, sys', + 'data = os.urandom(%s)' % count, + 'sys.stdout.buffer.write(data)', + 'sys.stdout.buffer.flush()')) + out = assert_python_ok('-c', code) + stdout = out[1] + self.assertEqual(len(stdout), count) + return stdout + + def test_urandom_subprocess(self): + data1 = self.get_urandom_subprocess(16) + data2 = self.get_urandom_subprocess(16) + self.assertNotEqual(data1, data2) + + +@unittest.skipUnless(hasattr(os, 'getrandom'), 'need os.getrandom()') +class GetRandomTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + try: + os.getrandom(1) + except OSError as exc: + if exc.errno == errno.ENOSYS: + # Python compiled on a more recent Linux version + # than the current Linux kernel + raise unittest.SkipTest("getrandom() syscall fails with ENOSYS") + else: + raise + + def test_getrandom_type(self): + data = os.getrandom(16) + self.assertIsInstance(data, bytes) + self.assertEqual(len(data), 16) + + def test_getrandom0(self): + empty = os.getrandom(0) + self.assertEqual(empty, b'') + + def test_getrandom_random(self): + self.assertTrue(hasattr(os, 'GRND_RANDOM')) + + # Don't test os.getrandom(1, os.GRND_RANDOM) to not consume the rare + # resource /dev/random + + def test_getrandom_nonblock(self): + # The call must not fail. Check also that the flag exists + try: + os.getrandom(1, os.GRND_NONBLOCK) + except BlockingIOError: + # System urandom is not initialized yet + pass + + def test_getrandom_value(self): + data1 = os.getrandom(16) + data2 = os.getrandom(16) + self.assertNotEqual(data1, data2) + + +# os.urandom() doesn't use a file descriptor when it is implemented with the +# getentropy() function, the getrandom() function or the getrandom() syscall +OS_URANDOM_DONT_USE_FD = ( + sysconfig.get_config_var('HAVE_GETENTROPY') == 1 + or sysconfig.get_config_var('HAVE_GETRANDOM') == 1 + or sysconfig.get_config_var('HAVE_GETRANDOM_SYSCALL') == 1) + +@unittest.skipIf(OS_URANDOM_DONT_USE_FD , + "os.random() does not use a file descriptor") +@unittest.skipIf(sys.platform == "vxworks", + "VxWorks can't set RLIMIT_NOFILE to 1") +class URandomFDTests(unittest.TestCase): + @unittest.skipUnless(resource, "test requires the resource module") + def test_urandom_failure(self): + # Check urandom() failing when it is not able to open /dev/random. + # We spawn a new process to make the test more robust (if getrlimit() + # failed to restore the file descriptor limit after this, the whole + # test suite would crash; this actually happened on the OS X Tiger + # buildbot). + code = """if 1: + import errno + import os + import resource + + soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + resource.setrlimit(resource.RLIMIT_NOFILE, (1, hard_limit)) + try: + os.urandom(16) + except OSError as e: + assert e.errno == errno.EMFILE, e.errno + else: + raise AssertionError("OSError not raised") + """ + assert_python_ok('-c', code) + + def test_urandom_fd_closed(self): + # Issue #21207: urandom() should reopen its fd to /dev/urandom if + # closed. + code = """if 1: + import os + import sys + import test.support + os.urandom(4) + with test.support.SuppressCrashReport(): + os.closerange(3, 256) + sys.stdout.buffer.write(os.urandom(4)) + """ + rc, out, err = assert_python_ok('-Sc', code) + + def test_urandom_fd_reopened(self): + # Issue #21207: urandom() should detect its fd to /dev/urandom + # changed to something else, and reopen it. + self.addCleanup(support.unlink, support.TESTFN) + create_file(support.TESTFN, b"x" * 256) + + code = """if 1: + import os + import sys + import test.support + os.urandom(4) + with test.support.SuppressCrashReport(): + for fd in range(3, 256): + try: + os.close(fd) + except OSError: + pass + else: + # Found the urandom fd (XXX hopefully) + break + os.closerange(3, 256) + with open({TESTFN!r}, 'rb') as f: + new_fd = f.fileno() + # Issue #26935: posix allows new_fd and fd to be equal but + # some libc implementations have dup2 return an error in this + # case. + if new_fd != fd: + os.dup2(new_fd, fd) + sys.stdout.buffer.write(os.urandom(4)) + sys.stdout.buffer.write(os.urandom(4)) + """.format(TESTFN=support.TESTFN) + rc, out, err = assert_python_ok('-Sc', code) + self.assertEqual(len(out), 8) + self.assertNotEqual(out[0:4], out[4:8]) + rc, out2, err2 = assert_python_ok('-Sc', code) + self.assertEqual(len(out2), 8) + self.assertNotEqual(out2, out) + + +@contextlib.contextmanager +def _execvpe_mockup(defpath=None): + """ + Stubs out execv and execve functions when used as context manager. + Records exec calls. The mock execv and execve functions always raise an + exception as they would normally never return. + """ + # A list of tuples containing (function name, first arg, args) + # of calls to execv or execve that have been made. + calls = [] + + def mock_execv(name, *args): + calls.append(('execv', name, args)) + raise RuntimeError("execv called") + + def mock_execve(name, *args): + calls.append(('execve', name, args)) + raise OSError(errno.ENOTDIR, "execve called") + + try: + orig_execv = os.execv + orig_execve = os.execve + orig_defpath = os.defpath + os.execv = mock_execv + os.execve = mock_execve + if defpath is not None: + os.defpath = defpath + yield calls + finally: + os.execv = orig_execv + os.execve = orig_execve + os.defpath = orig_defpath + +@unittest.skipUnless(hasattr(os, 'execv'), + "need os.execv()") +class ExecTests(unittest.TestCase): + @unittest.skipIf(USING_LINUXTHREADS, + "avoid triggering a linuxthreads bug: see issue #4970") + def test_execvpe_with_bad_program(self): + self.assertRaises(OSError, os.execvpe, 'no such app-', + ['no such app-'], None) + + def test_execv_with_bad_arglist(self): + self.assertRaises(ValueError, os.execv, 'notepad', ()) + self.assertRaises(ValueError, os.execv, 'notepad', []) + self.assertRaises(ValueError, os.execv, 'notepad', ('',)) + self.assertRaises(ValueError, os.execv, 'notepad', ['']) + + def test_execvpe_with_bad_arglist(self): + self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) + self.assertRaises(ValueError, os.execvpe, 'notepad', [], {}) + self.assertRaises(ValueError, os.execvpe, 'notepad', [''], {}) + + @unittest.skipUnless(hasattr(os, '_execvpe'), + "No internal os._execvpe function to test.") + def _test_internal_execvpe(self, test_type): + program_path = os.sep + 'absolutepath' + if test_type is bytes: + program = b'executable' + fullpath = os.path.join(os.fsencode(program_path), program) + native_fullpath = fullpath + arguments = [b'progname', 'arg1', 'arg2'] + else: + program = 'executable' + arguments = ['progname', 'arg1', 'arg2'] + fullpath = os.path.join(program_path, program) + if os.name != "nt": + native_fullpath = os.fsencode(fullpath) + else: + native_fullpath = fullpath + env = {'spam': 'beans'} + + # test os._execvpe() with an absolute path + with _execvpe_mockup() as calls: + self.assertRaises(RuntimeError, + os._execvpe, fullpath, arguments) + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0], ('execv', fullpath, (arguments,))) + + # test os._execvpe() with a relative path: + # os.get_exec_path() returns defpath + with _execvpe_mockup(defpath=program_path) as calls: + self.assertRaises(OSError, + os._execvpe, program, arguments, env=env) + self.assertEqual(len(calls), 1) + self.assertSequenceEqual(calls[0], + ('execve', native_fullpath, (arguments, env))) + + # test os._execvpe() with a relative path: + # os.get_exec_path() reads the 'PATH' variable + with _execvpe_mockup() as calls: + env_path = env.copy() + if test_type is bytes: + env_path[b'PATH'] = program_path + else: + env_path['PATH'] = program_path + self.assertRaises(OSError, + os._execvpe, program, arguments, env=env_path) + self.assertEqual(len(calls), 1) + self.assertSequenceEqual(calls[0], + ('execve', native_fullpath, (arguments, env_path))) + + def test_internal_execvpe_str(self): + self._test_internal_execvpe(str) + if os.name != "nt": + self._test_internal_execvpe(bytes) + + def test_execve_invalid_env(self): + args = [sys.executable, '-c', 'pass'] + + # null character in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + with self.assertRaises(ValueError): + os.execve(args[0], args, newenv) + + # null character in the environment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + with self.assertRaises(ValueError): + os.execve(args[0], args, newenv) + + # equal character in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + with self.assertRaises(ValueError): + os.execve(args[0], args, newenv) + + @unittest.skipUnless(sys.platform == "win32", "Win32-specific test") + def test_execve_with_empty_path(self): + # bpo-32890: Check GetLastError() misuse + try: + os.execve('', ['arg'], {}) + except OSError as e: + self.assertTrue(e.winerror is None or e.winerror != 0) + else: + self.fail('No OSError raised') + + +@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") +class Win32ErrorTests(unittest.TestCase): + def setUp(self): + try: + os.stat(support.TESTFN) + except FileNotFoundError: + exists = False + except OSError as exc: + exists = True + self.fail("file %s must not exist; os.stat failed with %s" + % (support.TESTFN, exc)) + else: + self.fail("file %s must not exist" % support.TESTFN) + + def test_rename(self): + self.assertRaises(OSError, os.rename, support.TESTFN, support.TESTFN+".bak") + + def test_remove(self): + self.assertRaises(OSError, os.remove, support.TESTFN) + + def test_chdir(self): + self.assertRaises(OSError, os.chdir, support.TESTFN) + + def test_mkdir(self): + self.addCleanup(support.unlink, support.TESTFN) + + with open(support.TESTFN, "x") as f: + self.assertRaises(OSError, os.mkdir, support.TESTFN) + + def test_utime(self): + self.assertRaises(OSError, os.utime, support.TESTFN, None) + + def test_chmod(self): + self.assertRaises(OSError, os.chmod, support.TESTFN, 0) + + +class TestInvalidFD(unittest.TestCase): + singles = ["fchdir", "dup", "fdopen", "fdatasync", "fstat", + "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] + #singles.append("close") + #We omit close because it doesn't raise an exception on some platforms + def get_single(f): + def helper(self): + if hasattr(os, f): + self.check(getattr(os, f)) + return helper + for f in singles: + locals()["test_"+f] = get_single(f) + + def check(self, f, *args): + try: + f(support.make_bad_fd(), *args) + except OSError as e: + self.assertEqual(e.errno, errno.EBADF) + else: + self.fail("%r didn't raise an OSError with a bad file descriptor" + % f) + + @unittest.skipUnless(hasattr(os, 'isatty'), 'test needs os.isatty()') + def test_isatty(self): + self.assertEqual(os.isatty(support.make_bad_fd()), False) + + @unittest.skipUnless(hasattr(os, 'closerange'), 'test needs os.closerange()') + def test_closerange(self): + fd = support.make_bad_fd() + # Make sure none of the descriptors we are about to close are + # currently valid (issue 6542). + for i in range(10): + try: os.fstat(fd+i) + except OSError: + pass + else: + break + if i < 2: + raise unittest.SkipTest( + "Unable to acquire a range of invalid file descriptors") + self.assertEqual(os.closerange(fd, fd + i-1), None) + + @unittest.skipUnless(hasattr(os, 'dup2'), 'test needs os.dup2()') + def test_dup2(self): + self.check(os.dup2, 20) + + @unittest.skipUnless(hasattr(os, 'fchmod'), 'test needs os.fchmod()') + def test_fchmod(self): + self.check(os.fchmod, 0) + + @unittest.skipUnless(hasattr(os, 'fchown'), 'test needs os.fchown()') + def test_fchown(self): + self.check(os.fchown, -1, -1) + + @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') + def test_fpathconf(self): + self.check(os.pathconf, "PC_NAME_MAX") + self.check(os.fpathconf, "PC_NAME_MAX") + + @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') + def test_ftruncate(self): + self.check(os.truncate, 0) + self.check(os.ftruncate, 0) + + @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') + def test_lseek(self): + self.check(os.lseek, 0, 0) + + @unittest.skipUnless(hasattr(os, 'read'), 'test needs os.read()') + def test_read(self): + self.check(os.read, 1) + + @unittest.skipUnless(hasattr(os, 'readv'), 'test needs os.readv()') + def test_readv(self): + buf = bytearray(10) + self.check(os.readv, [buf]) + + @unittest.skipUnless(hasattr(os, 'tcsetpgrp'), 'test needs os.tcsetpgrp()') + def test_tcsetpgrpt(self): + self.check(os.tcsetpgrp, 0) + + @unittest.skipUnless(hasattr(os, 'write'), 'test needs os.write()') + def test_write(self): + self.check(os.write, b" ") + + @unittest.skipUnless(hasattr(os, 'writev'), 'test needs os.writev()') + def test_writev(self): + self.check(os.writev, [b'abc']) + + def test_inheritable(self): + self.check(os.get_inheritable) + self.check(os.set_inheritable, True) + + @unittest.skipUnless(hasattr(os, 'get_blocking'), + 'needs os.get_blocking() and os.set_blocking()') + def test_blocking(self): + self.check(os.get_blocking) + self.check(os.set_blocking, True) + + +class LinkTests(unittest.TestCase): + def setUp(self): + self.file1 = support.TESTFN + self.file2 = os.path.join(support.TESTFN + "2") + + def tearDown(self): + for file in (self.file1, self.file2): + if os.path.exists(file): + os.unlink(file) + + def _test_link(self, file1, file2): + create_file(file1) + + try: + os.link(file1, file2) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) + with open(file1, "r") as f1, open(file2, "r") as f2: + self.assertTrue(os.path.sameopenfile(f1.fileno(), f2.fileno())) + + def test_link(self): + self._test_link(self.file1, self.file2) + + def test_link_bytes(self): + self._test_link(bytes(self.file1, sys.getfilesystemencoding()), + bytes(self.file2, sys.getfilesystemencoding())) + + def test_unicode_name(self): + try: + os.fsencode("\xf1") + except UnicodeError: + raise unittest.SkipTest("Unable to encode for this platform.") + + self.file1 += "\xf1" + self.file2 = self.file1 + "2" + self._test_link(self.file1, self.file2) + +@unittest.skipIf(sys.platform == "win32", "Posix specific tests") +class PosixUidGidTests(unittest.TestCase): + # uid_t and gid_t are 32-bit unsigned integers on Linux + UID_OVERFLOW = (1 << 32) + GID_OVERFLOW = (1 << 32) + + @unittest.skipUnless(hasattr(os, 'setuid'), 'test needs os.setuid()') + def test_setuid(self): + if os.getuid() != 0: + self.assertRaises(OSError, os.setuid, 0) + self.assertRaises(TypeError, os.setuid, 'not an int') + self.assertRaises(OverflowError, os.setuid, self.UID_OVERFLOW) + + @unittest.skipUnless(hasattr(os, 'setgid'), 'test needs os.setgid()') + def test_setgid(self): + if os.getuid() != 0 and not HAVE_WHEEL_GROUP: + self.assertRaises(OSError, os.setgid, 0) + self.assertRaises(TypeError, os.setgid, 'not an int') + self.assertRaises(OverflowError, os.setgid, self.GID_OVERFLOW) + + @unittest.skipUnless(hasattr(os, 'seteuid'), 'test needs os.seteuid()') + def test_seteuid(self): + if os.getuid() != 0: + self.assertRaises(OSError, os.seteuid, 0) + self.assertRaises(TypeError, os.setegid, 'not an int') + self.assertRaises(OverflowError, os.seteuid, self.UID_OVERFLOW) + + @unittest.skipUnless(hasattr(os, 'setegid'), 'test needs os.setegid()') + def test_setegid(self): + if os.getuid() != 0 and not HAVE_WHEEL_GROUP: + self.assertRaises(OSError, os.setegid, 0) + self.assertRaises(TypeError, os.setegid, 'not an int') + self.assertRaises(OverflowError, os.setegid, self.GID_OVERFLOW) + + @unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()') + def test_setreuid(self): + if os.getuid() != 0: + self.assertRaises(OSError, os.setreuid, 0, 0) + self.assertRaises(TypeError, os.setreuid, 'not an int', 0) + self.assertRaises(TypeError, os.setreuid, 0, 'not an int') + self.assertRaises(OverflowError, os.setreuid, self.UID_OVERFLOW, 0) + self.assertRaises(OverflowError, os.setreuid, 0, self.UID_OVERFLOW) + + @unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()') + def test_setreuid_neg1(self): + # Needs to accept -1. We run this in a subprocess to avoid + # altering the test runner's process state (issue8045). + subprocess.check_call([ + sys.executable, '-c', + 'import os,sys;os.setreuid(-1,-1);sys.exit(0)']) + + @unittest.skipUnless(hasattr(os, 'setregid'), 'test needs os.setregid()') + def test_setregid(self): + if os.getuid() != 0 and not HAVE_WHEEL_GROUP: + self.assertRaises(OSError, os.setregid, 0, 0) + self.assertRaises(TypeError, os.setregid, 'not an int', 0) + self.assertRaises(TypeError, os.setregid, 0, 'not an int') + self.assertRaises(OverflowError, os.setregid, self.GID_OVERFLOW, 0) + self.assertRaises(OverflowError, os.setregid, 0, self.GID_OVERFLOW) + + @unittest.skipUnless(hasattr(os, 'setregid'), 'test needs os.setregid()') + def test_setregid_neg1(self): + # Needs to accept -1. We run this in a subprocess to avoid + # altering the test runner's process state (issue8045). + subprocess.check_call([ + sys.executable, '-c', + 'import os,sys;os.setregid(-1,-1);sys.exit(0)']) + +@unittest.skipIf(sys.platform == "win32", "Posix specific tests") +class Pep383Tests(unittest.TestCase): + def setUp(self): + if support.TESTFN_UNENCODABLE: + self.dir = support.TESTFN_UNENCODABLE + elif support.TESTFN_NONASCII: + self.dir = support.TESTFN_NONASCII + else: + self.dir = support.TESTFN + self.bdir = os.fsencode(self.dir) + + bytesfn = [] + def add_filename(fn): + try: + fn = os.fsencode(fn) + except UnicodeEncodeError: + return + bytesfn.append(fn) + add_filename(support.TESTFN_UNICODE) + if support.TESTFN_UNENCODABLE: + add_filename(support.TESTFN_UNENCODABLE) + if support.TESTFN_NONASCII: + add_filename(support.TESTFN_NONASCII) + if not bytesfn: + self.skipTest("couldn't create any non-ascii filename") + + self.unicodefn = set() + os.mkdir(self.dir) + try: + for fn in bytesfn: + support.create_empty_file(os.path.join(self.bdir, fn)) + fn = os.fsdecode(fn) + if fn in self.unicodefn: + raise ValueError("duplicate filename") + self.unicodefn.add(fn) + except: + shutil.rmtree(self.dir) + raise + + def tearDown(self): + shutil.rmtree(self.dir) + + def test_listdir(self): + expected = self.unicodefn + found = set(os.listdir(self.dir)) + self.assertEqual(found, expected) + # test listdir without arguments + current_directory = os.getcwd() + try: + os.chdir(os.sep) + self.assertEqual(set(os.listdir()), set(os.listdir(os.sep))) + finally: + os.chdir(current_directory) + + def test_open(self): + for fn in self.unicodefn: + f = open(os.path.join(self.dir, fn), 'rb') + f.close() + + @unittest.skipUnless(hasattr(os, 'statvfs'), + "need os.statvfs()") + def test_statvfs(self): + # issue #9645 + for fn in self.unicodefn: + # should not fail with file not found error + fullname = os.path.join(self.dir, fn) + os.statvfs(fullname) + + def test_stat(self): + for fn in self.unicodefn: + os.stat(os.path.join(self.dir, fn)) + +@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") +class Win32KillTests(unittest.TestCase): + def _kill(self, sig): + # Start sys.executable as a subprocess and communicate from the + # subprocess to the parent that the interpreter is ready. When it + # becomes ready, send *sig* via os.kill to the subprocess and check + # that the return code is equal to *sig*. + import ctypes + from ctypes import wintypes + import msvcrt + + # Since we can't access the contents of the process' stdout until the + # process has exited, use PeekNamedPipe to see what's inside stdout + # without waiting. This is done so we can tell that the interpreter + # is started and running at a point where it could handle a signal. + PeekNamedPipe = ctypes.windll.kernel32.PeekNamedPipe + PeekNamedPipe.restype = wintypes.BOOL + PeekNamedPipe.argtypes = (wintypes.HANDLE, # Pipe handle + ctypes.POINTER(ctypes.c_char), # stdout buf + wintypes.DWORD, # Buffer size + ctypes.POINTER(wintypes.DWORD), # bytes read + ctypes.POINTER(wintypes.DWORD), # bytes avail + ctypes.POINTER(wintypes.DWORD)) # bytes left + msg = "running" + proc = subprocess.Popen([sys.executable, "-c", + "import sys;" + "sys.stdout.write('{}');" + "sys.stdout.flush();" + "input()".format(msg)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE) + self.addCleanup(proc.stdout.close) + self.addCleanup(proc.stderr.close) + self.addCleanup(proc.stdin.close) + + count, max = 0, 100 + while count < max and proc.poll() is None: + # Create a string buffer to store the result of stdout from the pipe + buf = ctypes.create_string_buffer(len(msg)) + # Obtain the text currently in proc.stdout + # Bytes read/avail/left are left as NULL and unused + rslt = PeekNamedPipe(msvcrt.get_osfhandle(proc.stdout.fileno()), + buf, ctypes.sizeof(buf), None, None, None) + self.assertNotEqual(rslt, 0, "PeekNamedPipe failed") + if buf.value: + self.assertEqual(msg, buf.value.decode()) + break + time.sleep(0.1) + count += 1 + else: + self.fail("Did not receive communication from the subprocess") + + os.kill(proc.pid, sig) + self.assertEqual(proc.wait(), sig) + + def test_kill_sigterm(self): + # SIGTERM doesn't mean anything special, but make sure it works + self._kill(signal.SIGTERM) + + def test_kill_int(self): + # os.kill on Windows can take an int which gets set as the exit code + self._kill(100) + + def _kill_with_event(self, event, name): + tagname = "test_os_%s" % uuid.uuid1() + m = mmap.mmap(-1, 1, tagname) + m[0] = 0 + # Run a script which has console control handling enabled. + proc = subprocess.Popen([sys.executable, + os.path.join(os.path.dirname(__file__), + "win_console_handler.py"), tagname], + creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) + # Let the interpreter startup before we send signals. See #3137. + count, max = 0, 100 + while count < max and proc.poll() is None: + if m[0] == 1: + break + time.sleep(0.1) + count += 1 + else: + # Forcefully kill the process if we weren't able to signal it. + os.kill(proc.pid, signal.SIGINT) + self.fail("Subprocess didn't finish initialization") + os.kill(proc.pid, event) + # proc.send_signal(event) could also be done here. + # Allow time for the signal to be passed and the process to exit. + time.sleep(0.5) + if not proc.poll(): + # Forcefully kill the process if we weren't able to signal it. + os.kill(proc.pid, signal.SIGINT) + self.fail("subprocess did not stop on {}".format(name)) + + @unittest.skip("subprocesses aren't inheriting Ctrl+C property") + def test_CTRL_C_EVENT(self): + from ctypes import wintypes + import ctypes + + # Make a NULL value by creating a pointer with no argument. + NULL = ctypes.POINTER(ctypes.c_int)() + SetConsoleCtrlHandler = ctypes.windll.kernel32.SetConsoleCtrlHandler + SetConsoleCtrlHandler.argtypes = (ctypes.POINTER(ctypes.c_int), + wintypes.BOOL) + SetConsoleCtrlHandler.restype = wintypes.BOOL + + # Calling this with NULL and FALSE causes the calling process to + # handle Ctrl+C, rather than ignore it. This property is inherited + # by subprocesses. + SetConsoleCtrlHandler(NULL, 0) + + self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") + + def test_CTRL_BREAK_EVENT(self): + self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") + + +@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") +class Win32ListdirTests(unittest.TestCase): + """Test listdir on Windows.""" + + def setUp(self): + self.created_paths = [] + for i in range(2): + dir_name = 'SUB%d' % i + dir_path = os.path.join(support.TESTFN, dir_name) + file_name = 'FILE%d' % i + file_path = os.path.join(support.TESTFN, file_name) + os.makedirs(dir_path) + with open(file_path, 'w') as f: + f.write("I'm %s and proud of it. Blame test_os.\n" % file_path) + self.created_paths.extend([dir_name, file_name]) + self.created_paths.sort() + + def tearDown(self): + shutil.rmtree(support.TESTFN) + + def test_listdir_no_extended_path(self): + """Test when the path is not an "extended" path.""" + # unicode + self.assertEqual( + sorted(os.listdir(support.TESTFN)), + self.created_paths) + + # bytes + self.assertEqual( + sorted(os.listdir(os.fsencode(support.TESTFN))), + [os.fsencode(path) for path in self.created_paths]) + + def test_listdir_extended_path(self): + """Test when the path starts with '\\\\?\\'.""" + # See: http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#maxpath + # unicode + path = '\\\\?\\' + os.path.abspath(support.TESTFN) + self.assertEqual( + sorted(os.listdir(path)), + self.created_paths) + + # bytes + path = b'\\\\?\\' + os.fsencode(os.path.abspath(support.TESTFN)) + self.assertEqual( + sorted(os.listdir(path)), + [os.fsencode(path) for path in self.created_paths]) + + +@unittest.skipUnless(hasattr(os, 'readlink'), 'needs os.readlink()') +class ReadlinkTests(unittest.TestCase): + filelink = 'readlinktest' + filelink_target = os.path.abspath(__file__) + filelinkb = os.fsencode(filelink) + filelinkb_target = os.fsencode(filelink_target) + + def assertPathEqual(self, left, right): + left = os.path.normcase(left) + right = os.path.normcase(right) + if sys.platform == 'win32': + # Bad practice to blindly strip the prefix as it may be required to + # correctly refer to the file, but we're only comparing paths here. + has_prefix = lambda p: p.startswith( + b'\\\\?\\' if isinstance(p, bytes) else '\\\\?\\') + if has_prefix(left): + left = left[4:] + if has_prefix(right): + right = right[4:] + self.assertEqual(left, right) + + def setUp(self): + self.assertTrue(os.path.exists(self.filelink_target)) + self.assertTrue(os.path.exists(self.filelinkb_target)) + self.assertFalse(os.path.exists(self.filelink)) + self.assertFalse(os.path.exists(self.filelinkb)) + + def test_not_symlink(self): + filelink_target = FakePath(self.filelink_target) + self.assertRaises(OSError, os.readlink, self.filelink_target) + self.assertRaises(OSError, os.readlink, filelink_target) + + def test_missing_link(self): + self.assertRaises(FileNotFoundError, os.readlink, 'missing-link') + self.assertRaises(FileNotFoundError, os.readlink, + FakePath('missing-link')) + + @support.skip_unless_symlink + def test_pathlike(self): + os.symlink(self.filelink_target, self.filelink) + self.addCleanup(support.unlink, self.filelink) + filelink = FakePath(self.filelink) + self.assertPathEqual(os.readlink(filelink), self.filelink_target) + + @support.skip_unless_symlink + def test_pathlike_bytes(self): + os.symlink(self.filelinkb_target, self.filelinkb) + self.addCleanup(support.unlink, self.filelinkb) + path = os.readlink(FakePath(self.filelinkb)) + self.assertPathEqual(path, self.filelinkb_target) + self.assertIsInstance(path, bytes) + + @support.skip_unless_symlink + def test_bytes(self): + os.symlink(self.filelinkb_target, self.filelinkb) + self.addCleanup(support.unlink, self.filelinkb) + path = os.readlink(self.filelinkb) + self.assertPathEqual(path, self.filelinkb_target) + self.assertIsInstance(path, bytes) + + +@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") +@support.skip_unless_symlink +class Win32SymlinkTests(unittest.TestCase): + filelink = 'filelinktest' + filelink_target = os.path.abspath(__file__) + dirlink = 'dirlinktest' + dirlink_target = os.path.dirname(filelink_target) + missing_link = 'missing link' + + def setUp(self): + assert os.path.exists(self.dirlink_target) + assert os.path.exists(self.filelink_target) + assert not os.path.exists(self.dirlink) + assert not os.path.exists(self.filelink) + assert not os.path.exists(self.missing_link) + + def tearDown(self): + if os.path.exists(self.filelink): + os.remove(self.filelink) + if os.path.exists(self.dirlink): + os.rmdir(self.dirlink) + if os.path.lexists(self.missing_link): + os.remove(self.missing_link) + + def test_directory_link(self): + os.symlink(self.dirlink_target, self.dirlink) + self.assertTrue(os.path.exists(self.dirlink)) + self.assertTrue(os.path.isdir(self.dirlink)) + self.assertTrue(os.path.islink(self.dirlink)) + self.check_stat(self.dirlink, self.dirlink_target) + + def test_file_link(self): + os.symlink(self.filelink_target, self.filelink) + self.assertTrue(os.path.exists(self.filelink)) + self.assertTrue(os.path.isfile(self.filelink)) + self.assertTrue(os.path.islink(self.filelink)) + self.check_stat(self.filelink, self.filelink_target) + + def _create_missing_dir_link(self): + 'Create a "directory" link to a non-existent target' + linkname = self.missing_link + if os.path.lexists(linkname): + os.remove(linkname) + target = r'c:\\target does not exist.29r3c740' + assert not os.path.exists(target) + target_is_dir = True + os.symlink(target, linkname, target_is_dir) + + def test_remove_directory_link_to_missing_target(self): + self._create_missing_dir_link() + # For compatibility with Unix, os.remove will check the + # directory status and call RemoveDirectory if the symlink + # was created with target_is_dir==True. + os.remove(self.missing_link) + + def test_isdir_on_directory_link_to_missing_target(self): + self._create_missing_dir_link() + self.assertFalse(os.path.isdir(self.missing_link)) + + def test_rmdir_on_directory_link_to_missing_target(self): + self._create_missing_dir_link() + os.rmdir(self.missing_link) + + def check_stat(self, link, target): + self.assertEqual(os.stat(link), os.stat(target)) + self.assertNotEqual(os.lstat(link), os.stat(link)) + + bytes_link = os.fsencode(link) + self.assertEqual(os.stat(bytes_link), os.stat(target)) + self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link)) + + def test_12084(self): + level1 = os.path.abspath(support.TESTFN) + level2 = os.path.join(level1, "level2") + level3 = os.path.join(level2, "level3") + self.addCleanup(support.rmtree, level1) + + os.mkdir(level1) + os.mkdir(level2) + os.mkdir(level3) + + file1 = os.path.abspath(os.path.join(level1, "file1")) + create_file(file1) + + orig_dir = os.getcwd() + try: + os.chdir(level2) + link = os.path.join(level2, "link") + os.symlink(os.path.relpath(file1), "link") + self.assertIn("link", os.listdir(os.getcwd())) + + # Check os.stat calls from the same dir as the link + self.assertEqual(os.stat(file1), os.stat("link")) + + # Check os.stat calls from a dir below the link + os.chdir(level1) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) + + # Check os.stat calls from a dir above the link + os.chdir(level3) + self.assertEqual(os.stat(file1), + os.stat(os.path.relpath(link))) + finally: + os.chdir(orig_dir) + + @unittest.skipUnless(os.path.lexists(r'C:\Users\All Users') + and os.path.exists(r'C:\ProgramData'), + 'Test directories not found') + def test_29248(self): + # os.symlink() calls CreateSymbolicLink, which creates + # the reparse data buffer with the print name stored + # first, so the offset is always 0. CreateSymbolicLink + # stores the "PrintName" DOS path (e.g. "C:\") first, + # with an offset of 0, followed by the "SubstituteName" + # NT path (e.g. "\??\C:\"). The "All Users" link, on + # the other hand, seems to have been created manually + # with an inverted order. + target = os.readlink(r'C:\Users\All Users') + self.assertTrue(os.path.samefile(target, r'C:\ProgramData')) + + def test_buffer_overflow(self): + # Older versions would have a buffer overflow when detecting + # whether a link source was a directory. This test ensures we + # no longer crash, but does not otherwise validate the behavior + segment = 'X' * 27 + path = os.path.join(*[segment] * 10) + test_cases = [ + # overflow with absolute src + ('\\' + path, segment), + # overflow dest with relative src + (segment, path), + # overflow when joining src + (path[:180], path[:180]), + ] + for src, dest in test_cases: + try: + os.symlink(src, dest) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + # Also test with bytes, since that is a separate code path. + try: + os.symlink(os.fsencode(src), os.fsencode(dest)) + except FileNotFoundError: + pass + else: + try: + os.remove(dest) + except OSError: + pass + + def test_appexeclink(self): + root = os.path.expandvars(r'%LOCALAPPDATA%\Microsoft\WindowsApps') + if not os.path.isdir(root): + self.skipTest("test requires a WindowsApps directory") + + aliases = [os.path.join(root, a) + for a in fnmatch.filter(os.listdir(root), '*.exe')] + + for alias in aliases: + if support.verbose: + print() + print("Testing with", alias) + st = os.lstat(alias) + self.assertEqual(st, os.stat(alias)) + self.assertFalse(stat.S_ISLNK(st.st_mode)) + self.assertEqual(st.st_reparse_tag, stat.IO_REPARSE_TAG_APPEXECLINK) + # testing the first one we see is sufficient + break + else: + self.skipTest("test requires an app execution alias") + +@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") +class Win32JunctionTests(unittest.TestCase): + junction = 'junctiontest' + junction_target = os.path.dirname(os.path.abspath(__file__)) + + def setUp(self): + assert os.path.exists(self.junction_target) + assert not os.path.lexists(self.junction) + + def tearDown(self): + if os.path.lexists(self.junction): + os.unlink(self.junction) + + def test_create_junction(self): + _winapi.CreateJunction(self.junction_target, self.junction) + self.assertTrue(os.path.lexists(self.junction)) + self.assertTrue(os.path.exists(self.junction)) + self.assertTrue(os.path.isdir(self.junction)) + self.assertNotEqual(os.stat(self.junction), os.lstat(self.junction)) + self.assertEqual(os.stat(self.junction), os.stat(self.junction_target)) + + # bpo-37834: Junctions are not recognized as links. + self.assertFalse(os.path.islink(self.junction)) + self.assertEqual(os.path.normcase("\\\\?\\" + self.junction_target), + os.path.normcase(os.readlink(self.junction))) + + def test_unlink_removes_junction(self): + _winapi.CreateJunction(self.junction_target, self.junction) + self.assertTrue(os.path.exists(self.junction)) + self.assertTrue(os.path.lexists(self.junction)) + + os.unlink(self.junction) + self.assertFalse(os.path.exists(self.junction)) + +@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") +class Win32NtTests(unittest.TestCase): + def test_getfinalpathname_handles(self): + nt = support.import_module('nt') + ctypes = support.import_module('ctypes') + import ctypes.wintypes + + kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) + kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE + + kernel.GetProcessHandleCount.restype = ctypes.wintypes.BOOL + kernel.GetProcessHandleCount.argtypes = (ctypes.wintypes.HANDLE, + ctypes.wintypes.LPDWORD) + + # This is a pseudo-handle that doesn't need to be closed + hproc = kernel.GetCurrentProcess() + + handle_count = ctypes.wintypes.DWORD() + ok = kernel.GetProcessHandleCount(hproc, ctypes.byref(handle_count)) + self.assertEqual(1, ok) + + before_count = handle_count.value + + # The first two test the error path, __file__ tests the success path + filenames = [ + r'\\?\C:', + r'\\?\NUL', + r'\\?\CONIN', + __file__, + ] + + for _ in range(10): + for name in filenames: + try: + nt._getfinalpathname(name) + except Exception: + # Failure is expected + pass + try: + os.stat(name) + except Exception: + pass + + ok = kernel.GetProcessHandleCount(hproc, ctypes.byref(handle_count)) + self.assertEqual(1, ok) + + handle_delta = handle_count.value - before_count + + self.assertEqual(0, handle_delta) + +@support.skip_unless_symlink +class NonLocalSymlinkTests(unittest.TestCase): + + def setUp(self): + r""" + Create this structure: + + base + \___ some_dir + """ + os.makedirs('base/some_dir') + + def tearDown(self): + shutil.rmtree('base') + + def test_directory_link_nonlocal(self): + """ + The symlink target should resolve relative to the link, not relative + to the current directory. + + Then, link base/some_link -> base/some_dir and ensure that some_link + is resolved as a directory. + + In issue13772, it was discovered that directory detection failed if + the symlink target was not specified relative to the current + directory, which was a defect in the implementation. + """ + src = os.path.join('base', 'some_link') + os.symlink('some_dir', src) + assert os.path.isdir(src) + + +class FSEncodingTests(unittest.TestCase): + def test_nop(self): + self.assertEqual(os.fsencode(b'abc\xff'), b'abc\xff') + self.assertEqual(os.fsdecode('abc\u0141'), 'abc\u0141') + + def test_identity(self): + # assert fsdecode(fsencode(x)) == x + for fn in ('unicode\u0141', 'latin\xe9', 'ascii'): + try: + bytesfn = os.fsencode(fn) + except UnicodeEncodeError: + continue + self.assertEqual(os.fsdecode(bytesfn), fn) + + + +class DeviceEncodingTests(unittest.TestCase): + + def test_bad_fd(self): + # Return None when an fd doesn't actually exist. + self.assertIsNone(os.device_encoding(123456)) + + @unittest.skipUnless(os.isatty(0) and not win32_is_iot() and (sys.platform.startswith('win') or + (hasattr(locale, 'nl_langinfo') and hasattr(locale, 'CODESET'))), + 'test requires a tty and either Windows or nl_langinfo(CODESET)') + def test_device_encoding(self): + encoding = os.device_encoding(0) + self.assertIsNotNone(encoding) + self.assertTrue(codecs.lookup(encoding)) + + +class PidTests(unittest.TestCase): + @unittest.skipUnless(hasattr(os, 'getppid'), "test needs os.getppid") + def test_getppid(self): + p = subprocess.Popen([sys.executable, '-c', + 'import os; print(os.getppid())'], + stdout=subprocess.PIPE) + stdout, _ = p.communicate() + # We are the parent of our subprocess + self.assertEqual(int(stdout), os.getpid()) + + def check_waitpid(self, code, exitcode): + if sys.platform == 'win32': + # On Windows, os.spawnv() simply joins arguments with spaces: + # arguments need to be quoted + args = [f'"{sys.executable}"', '-c', f'"{code}"'] + else: + args = [sys.executable, '-c', code] + pid = os.spawnv(os.P_NOWAIT, sys.executable, args) + + pid2, status = os.waitpid(pid, 0) + if sys.platform == 'win32': + self.assertEqual(status, exitcode << 8) + else: + self.assertTrue(os.WIFEXITED(status), status) + self.assertEqual(os.WEXITSTATUS(status), exitcode) + self.assertEqual(pid2, pid) + + def test_waitpid(self): + self.check_waitpid(code='pass', exitcode=0) + + def test_waitpid_exitcode(self): + exitcode = 23 + code = f'import sys; sys.exit({exitcode})' + self.check_waitpid(code, exitcode=exitcode) + + @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') + def test_waitpid_windows(self): + # bpo-40138: test os.waitpid() with exit code larger than INT_MAX. + STATUS_CONTROL_C_EXIT = 0xC000013A + code = f'import _winapi; _winapi.ExitProcess({STATUS_CONTROL_C_EXIT})' + self.check_waitpid(code, exitcode=STATUS_CONTROL_C_EXIT) + + +class SpawnTests(unittest.TestCase): + def create_args(self, *, with_env=False, use_bytes=False): + self.exitcode = 17 + + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + + if not with_env: + code = 'import sys; sys.exit(%s)' % self.exitcode + else: + self.env = dict(os.environ) + # create an unique key + self.key = str(uuid.uuid4()) + self.env[self.key] = self.key + # read the variable from os.environ to check that it exists + code = ('import sys, os; magic = os.environ[%r]; sys.exit(%s)' + % (self.key, self.exitcode)) + + with open(filename, "w") as fp: + fp.write(code) + + args = [sys.executable, filename] + if use_bytes: + args = [os.fsencode(a) for a in args] + self.env = {os.fsencode(k): os.fsencode(v) + for k, v in self.env.items()} + + return args + + @requires_os_func('spawnl') + def test_spawnl(self): + args = self.create_args() + exitcode = os.spawnl(os.P_WAIT, args[0], *args) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnle') + def test_spawnle(self): + args = self.create_args(with_env=True) + exitcode = os.spawnle(os.P_WAIT, args[0], *args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnlp') + def test_spawnlp(self): + args = self.create_args() + exitcode = os.spawnlp(os.P_WAIT, args[0], *args) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnlpe') + def test_spawnlpe(self): + args = self.create_args(with_env=True) + exitcode = os.spawnlpe(os.P_WAIT, args[0], *args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnv') + def test_spawnv(self): + args = self.create_args() + exitcode = os.spawnv(os.P_WAIT, args[0], args) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnve') + def test_spawnve(self): + args = self.create_args(with_env=True) + exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnvp') + def test_spawnvp(self): + args = self.create_args() + exitcode = os.spawnvp(os.P_WAIT, args[0], args) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnvpe') + def test_spawnvpe(self): + args = self.create_args(with_env=True) + exitcode = os.spawnvpe(os.P_WAIT, args[0], args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnv') + def test_nowait(self): + args = self.create_args() + pid = os.spawnv(os.P_NOWAIT, args[0], args) + result = os.waitpid(pid, 0) + self.assertEqual(result[0], pid) + status = result[1] + if hasattr(os, 'WIFEXITED'): + self.assertTrue(os.WIFEXITED(status)) + self.assertEqual(os.WEXITSTATUS(status), self.exitcode) + else: + self.assertEqual(status, self.exitcode << 8) + + @requires_os_func('spawnve') + def test_spawnve_bytes(self): + # Test bytes handling in parse_arglist and parse_envlist (#28114) + args = self.create_args(with_env=True, use_bytes=True) + exitcode = os.spawnve(os.P_WAIT, args[0], args, self.env) + self.assertEqual(exitcode, self.exitcode) + + @requires_os_func('spawnl') + def test_spawnl_noargs(self): + args = self.create_args() + self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, args[0]) + self.assertRaises(ValueError, os.spawnl, os.P_NOWAIT, args[0], '') + + @requires_os_func('spawnle') + def test_spawnle_noargs(self): + args = self.create_args() + self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, args[0], {}) + self.assertRaises(ValueError, os.spawnle, os.P_NOWAIT, args[0], '', {}) + + @requires_os_func('spawnv') + def test_spawnv_noargs(self): + args = self.create_args() + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ()) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], []) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ('',)) + self.assertRaises(ValueError, os.spawnv, os.P_NOWAIT, args[0], ['']) + + @requires_os_func('spawnve') + def test_spawnve_noargs(self): + args = self.create_args() + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], (), {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [], {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], ('',), {}) + self.assertRaises(ValueError, os.spawnve, os.P_NOWAIT, args[0], [''], {}) + + def _test_invalid_env(self, spawn): + args = [sys.executable, '-c', 'pass'] + + # null character in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT\0VEGETABLE"] = "cabbage" + try: + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + except ValueError: + pass + else: + self.assertEqual(exitcode, 127) + + # null character in the environment variable value + newenv = os.environ.copy() + newenv["FRUIT"] = "orange\0VEGETABLE=cabbage" + try: + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + except ValueError: + pass + else: + self.assertEqual(exitcode, 127) + + # equal character in the environment variable name + newenv = os.environ.copy() + newenv["FRUIT=ORANGE"] = "lemon" + try: + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + except ValueError: + pass + else: + self.assertEqual(exitcode, 127) + + # equal character in the environment variable value + filename = support.TESTFN + self.addCleanup(support.unlink, filename) + with open(filename, "w") as fp: + fp.write('import sys, os\n' + 'if os.getenv("FRUIT") != "orange=lemon":\n' + ' raise AssertionError') + args = [sys.executable, filename] + newenv = os.environ.copy() + newenv["FRUIT"] = "orange=lemon" + exitcode = spawn(os.P_WAIT, args[0], args, newenv) + self.assertEqual(exitcode, 0) + + @requires_os_func('spawnve') + def test_spawnve_invalid_env(self): + self._test_invalid_env(os.spawnve) + + @requires_os_func('spawnvpe') + def test_spawnvpe_invalid_env(self): + self._test_invalid_env(os.spawnvpe) + + +# The introduction of this TestCase caused at least two different errors on +# *nix buildbots. Temporarily skip this to let the buildbots move along. +@unittest.skip("Skip due to platform/environment differences on *NIX buildbots") +@unittest.skipUnless(hasattr(os, 'getlogin'), "test needs os.getlogin") +class LoginTests(unittest.TestCase): + def test_getlogin(self): + user_name = os.getlogin() + self.assertNotEqual(len(user_name), 0) + + +@unittest.skipUnless(hasattr(os, 'getpriority') and hasattr(os, 'setpriority'), + "needs os.getpriority and os.setpriority") +class ProgramPriorityTests(unittest.TestCase): + """Tests for os.getpriority() and os.setpriority().""" + + def test_set_get_priority(self): + + base = os.getpriority(os.PRIO_PROCESS, os.getpid()) + os.setpriority(os.PRIO_PROCESS, os.getpid(), base + 1) + try: + new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) + if base >= 19 and new_prio <= 19: + raise unittest.SkipTest("unable to reliably test setpriority " + "at current nice level of %s" % base) + else: + self.assertEqual(new_prio, base + 1) + finally: + try: + os.setpriority(os.PRIO_PROCESS, os.getpid(), base) + except OSError as err: + if err.errno != errno.EACCES: + raise + + +class SendfileTestServer(asyncore.dispatcher, threading.Thread): + + class Handler(asynchat.async_chat): + + def __init__(self, conn): + asynchat.async_chat.__init__(self, conn) + self.in_buffer = [] + self.accumulate = True + self.closed = False + self.push(b"220 ready\r\n") + + def handle_read(self): + data = self.recv(4096) + if self.accumulate: + self.in_buffer.append(data) + + def get_data(self): + return b''.join(self.in_buffer) + + def handle_close(self): + self.close() + self.closed = True + + def handle_error(self): + raise + + def __init__(self, address): + threading.Thread.__init__(self) + asyncore.dispatcher.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.bind(address) + self.listen(5) + self.host, self.port = self.socket.getsockname()[:2] + self.handler_instance = None + self._active = False + self._active_lock = threading.Lock() + + # --- public API + + @property + def running(self): + return self._active + + def start(self): + assert not self.running + self.__flag = threading.Event() + threading.Thread.start(self) + self.__flag.wait() + + def stop(self): + assert self.running + self._active = False + self.join() + + def wait(self): + # wait for handler connection to be closed, then stop the server + while not getattr(self.handler_instance, "closed", False): + time.sleep(0.001) + self.stop() + + # --- internals + + def run(self): + self._active = True + self.__flag.set() + while self._active and asyncore.socket_map: + self._active_lock.acquire() + asyncore.loop(timeout=0.001, count=1) + self._active_lock.release() + asyncore.close_all() + + def handle_accept(self): + conn, addr = self.accept() + self.handler_instance = self.Handler(conn) + + def handle_connect(self): + self.close() + handle_read = handle_connect + + def writable(self): + return 0 + + def handle_error(self): + raise + + +@unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()") +class TestSendfile(unittest.TestCase): + + DATA = b"12345abcde" * 16 * 1024 # 160 KiB + SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ + not sys.platform.startswith("solaris") and \ + not sys.platform.startswith("sunos") + requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, + 'requires headers and trailers support') + requires_32b = unittest.skipUnless(sys.maxsize < 2**32, + 'test is only meaningful on 32-bit builds') + + @classmethod + def setUpClass(cls): + cls.key = support.threading_setup() + create_file(support.TESTFN, cls.DATA) + + @classmethod + def tearDownClass(cls): + support.threading_cleanup(*cls.key) + support.unlink(support.TESTFN) + + def setUp(self): + self.server = SendfileTestServer((support.HOST, 0)) + self.server.start() + self.client = socket.socket() + self.client.connect((self.server.host, self.server.port)) + self.client.settimeout(1) + # synchronize by waiting for "220 ready" response + self.client.recv(1024) + self.sockno = self.client.fileno() + self.file = open(support.TESTFN, 'rb') + self.fileno = self.file.fileno() + + def tearDown(self): + self.file.close() + self.client.close() + if self.server.running: + self.server.stop() + self.server = None + + def sendfile_wrapper(self, *args, **kwargs): + """A higher level wrapper representing how an application is + supposed to use sendfile(). + """ + while True: + try: + return os.sendfile(*args, **kwargs) + except OSError as err: + if err.errno == errno.ECONNRESET: + # disconnected + raise + elif err.errno in (errno.EAGAIN, errno.EBUSY): + # we have to retry send data + continue + else: + raise + + def test_send_whole_file(self): + # normal send + total_sent = 0 + offset = 0 + nbytes = 4096 + while total_sent < len(self.DATA): + sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) + if sent == 0: + break + offset += sent + total_sent += sent + self.assertTrue(sent <= nbytes) + self.assertEqual(offset, total_sent) + + self.assertEqual(total_sent, len(self.DATA)) + self.client.shutdown(socket.SHUT_RDWR) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(len(data), len(self.DATA)) + self.assertEqual(data, self.DATA) + + def test_send_at_certain_offset(self): + # start sending a file at a certain offset + total_sent = 0 + offset = len(self.DATA) // 2 + must_send = len(self.DATA) - offset + nbytes = 4096 + while total_sent < must_send: + sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) + if sent == 0: + break + offset += sent + total_sent += sent + self.assertTrue(sent <= nbytes) + + self.client.shutdown(socket.SHUT_RDWR) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + expected = self.DATA[len(self.DATA) // 2:] + self.assertEqual(total_sent, len(expected)) + self.assertEqual(len(data), len(expected)) + self.assertEqual(data, expected) + + def test_offset_overflow(self): + # specify an offset > file size + offset = len(self.DATA) + 4096 + try: + sent = os.sendfile(self.sockno, self.fileno, offset, 4096) + except OSError as e: + # Solaris can raise EINVAL if offset >= file length, ignore. + if e.errno != errno.EINVAL: + raise + else: + self.assertEqual(sent, 0) + self.client.shutdown(socket.SHUT_RDWR) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(data, b'') + + def test_invalid_offset(self): + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, -1, 4096) + self.assertEqual(cm.exception.errno, errno.EINVAL) + + def test_keywords(self): + # Keyword arguments should be supported + os.sendfile(out=self.sockno, offset=0, count=4096, + **{'in': self.fileno}) + if self.SUPPORT_HEADERS_TRAILERS: + os.sendfile(self.sockno, self.fileno, offset=0, count=4096, + headers=(), trailers=(), flags=0) + + # --- headers / trailers tests + + @requires_headers_trailers + def test_headers(self): + total_sent = 0 + expected_data = b"x" * 512 + b"y" * 256 + self.DATA[:-1] + sent = os.sendfile(self.sockno, self.fileno, 0, 4096, + headers=[b"x" * 512, b"y" * 256]) + self.assertLessEqual(sent, 512 + 256 + 4096) + total_sent += sent + offset = 4096 + while total_sent < len(expected_data): + nbytes = min(len(expected_data) - total_sent, 4096) + sent = self.sendfile_wrapper(self.sockno, self.fileno, + offset, nbytes) + if sent == 0: + break + self.assertLessEqual(sent, nbytes) + total_sent += sent + offset += sent + + self.assertEqual(total_sent, len(expected_data)) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(hash(data), hash(expected_data)) + + @requires_headers_trailers + def test_trailers(self): + TESTFN2 = support.TESTFN + "2" + file_data = b"abcdef" + + self.addCleanup(support.unlink, TESTFN2) + create_file(TESTFN2, file_data) + + with open(TESTFN2, 'rb') as f: + os.sendfile(self.sockno, f.fileno(), 0, 5, + trailers=[b"123456", b"789"]) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(data, b"abcde123456789") + + @requires_headers_trailers + @requires_32b + def test_headers_overflow_32bits(self): + self.server.handler_instance.accumulate = False + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, 0, 0, + headers=[b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) + + @requires_headers_trailers + @requires_32b + def test_trailers_overflow_32bits(self): + self.server.handler_instance.accumulate = False + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, 0, 0, + trailers=[b"x" * 2**16] * 2**15) + self.assertEqual(cm.exception.errno, errno.EINVAL) + + @requires_headers_trailers + @unittest.skipUnless(hasattr(os, 'SF_NODISKIO'), + 'test needs os.SF_NODISKIO') + def test_flags(self): + try: + os.sendfile(self.sockno, self.fileno, 0, 4096, + flags=os.SF_NODISKIO) + except OSError as err: + if err.errno not in (errno.EBUSY, errno.EAGAIN): + raise + + +def supports_extended_attributes(): + if not hasattr(os, "setxattr"): + return False + + try: + with open(support.TESTFN, "xb", 0) as fp: + try: + os.setxattr(fp.fileno(), b"user.test", b"") + except OSError: + return False + finally: + support.unlink(support.TESTFN) + + return True + + +@unittest.skipUnless(supports_extended_attributes(), + "no non-broken extended attribute support") +# Kernels < 2.6.39 don't respect setxattr flags. +@support.requires_linux_version(2, 6, 39) +class ExtendedAttributeTests(unittest.TestCase): + + def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwargs): + fn = support.TESTFN + self.addCleanup(support.unlink, fn) + create_file(fn) + + with self.assertRaises(OSError) as cm: + getxattr(fn, s("user.test"), **kwargs) + self.assertEqual(cm.exception.errno, errno.ENODATA) + + init_xattr = listxattr(fn) + self.assertIsInstance(init_xattr, list) + + setxattr(fn, s("user.test"), b"", **kwargs) + xattr = set(init_xattr) + xattr.add("user.test") + self.assertEqual(set(listxattr(fn)), xattr) + self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"") + setxattr(fn, s("user.test"), b"hello", os.XATTR_REPLACE, **kwargs) + self.assertEqual(getxattr(fn, b"user.test", **kwargs), b"hello") + + with self.assertRaises(OSError) as cm: + setxattr(fn, s("user.test"), b"bye", os.XATTR_CREATE, **kwargs) + self.assertEqual(cm.exception.errno, errno.EEXIST) + + with self.assertRaises(OSError) as cm: + setxattr(fn, s("user.test2"), b"bye", os.XATTR_REPLACE, **kwargs) + self.assertEqual(cm.exception.errno, errno.ENODATA) + + setxattr(fn, s("user.test2"), b"foo", os.XATTR_CREATE, **kwargs) + xattr.add("user.test2") + self.assertEqual(set(listxattr(fn)), xattr) + removexattr(fn, s("user.test"), **kwargs) + + with self.assertRaises(OSError) as cm: + getxattr(fn, s("user.test"), **kwargs) + self.assertEqual(cm.exception.errno, errno.ENODATA) + + xattr.remove("user.test") + self.assertEqual(set(listxattr(fn)), xattr) + self.assertEqual(getxattr(fn, s("user.test2"), **kwargs), b"foo") + setxattr(fn, s("user.test"), b"a"*1024, **kwargs) + self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*1024) + removexattr(fn, s("user.test"), **kwargs) + many = sorted("user.test{}".format(i) for i in range(100)) + for thing in many: + setxattr(fn, thing, b"x", **kwargs) + self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many)) + + def _check_xattrs(self, *args, **kwargs): + self._check_xattrs_str(str, *args, **kwargs) + support.unlink(support.TESTFN) + + self._check_xattrs_str(os.fsencode, *args, **kwargs) + support.unlink(support.TESTFN) + + def test_simple(self): + self._check_xattrs(os.getxattr, os.setxattr, os.removexattr, + os.listxattr) + + def test_lpath(self): + self._check_xattrs(os.getxattr, os.setxattr, os.removexattr, + os.listxattr, follow_symlinks=False) + + def test_fds(self): + def getxattr(path, *args): + with open(path, "rb") as fp: + return os.getxattr(fp.fileno(), *args) + def setxattr(path, *args): + with open(path, "wb", 0) as fp: + os.setxattr(fp.fileno(), *args) + def removexattr(path, *args): + with open(path, "wb", 0) as fp: + os.removexattr(fp.fileno(), *args) + def listxattr(path, *args): + with open(path, "rb") as fp: + return os.listxattr(fp.fileno(), *args) + self._check_xattrs(getxattr, setxattr, removexattr, listxattr) + + +@unittest.skipUnless(hasattr(os, 'get_terminal_size'), "requires os.get_terminal_size") +class TermsizeTests(unittest.TestCase): + def test_does_not_crash(self): + """Check if get_terminal_size() returns a meaningful value. + + There's no easy portable way to actually check the size of the + terminal, so let's check if it returns something sensible instead. + """ + try: + size = os.get_terminal_size() + except OSError as e: + if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY): + # Under win32 a generic OSError can be thrown if the + # handle cannot be retrieved + self.skipTest("failed to query terminal size") + raise + + self.assertGreaterEqual(size.columns, 0) + self.assertGreaterEqual(size.lines, 0) + + def test_stty_match(self): + """Check if stty returns the same results + + stty actually tests stdin, so get_terminal_size is invoked on + stdin explicitly. If stty succeeded, then get_terminal_size() + should work too. + """ + try: + size = subprocess.check_output(['stty', 'size']).decode().split() + except (FileNotFoundError, subprocess.CalledProcessError, + PermissionError): + self.skipTest("stty invocation failed") + expected = (int(size[1]), int(size[0])) # reversed order + + try: + actual = os.get_terminal_size(sys.__stdin__.fileno()) + except OSError as e: + if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY): + # Under win32 a generic OSError can be thrown if the + # handle cannot be retrieved + self.skipTest("failed to query terminal size") + raise + self.assertEqual(expected, actual) + + +@unittest.skipUnless(hasattr(os, 'memfd_create'), 'requires os.memfd_create') +@support.requires_linux_version(3, 17) +class MemfdCreateTests(unittest.TestCase): + def test_memfd_create(self): + fd = os.memfd_create("Hi", os.MFD_CLOEXEC) + self.assertNotEqual(fd, -1) + self.addCleanup(os.close, fd) + self.assertFalse(os.get_inheritable(fd)) + with open(fd, "wb", closefd=False) as f: + f.write(b'memfd_create') + self.assertEqual(f.tell(), 12) + + fd2 = os.memfd_create("Hi") + self.addCleanup(os.close, fd2) + self.assertFalse(os.get_inheritable(fd2)) + + +class OSErrorTests(unittest.TestCase): + def setUp(self): + class Str(str): + pass + + self.bytes_filenames = [] + self.unicode_filenames = [] + if support.TESTFN_UNENCODABLE is not None: + decoded = support.TESTFN_UNENCODABLE + else: + decoded = support.TESTFN + self.unicode_filenames.append(decoded) + self.unicode_filenames.append(Str(decoded)) + if support.TESTFN_UNDECODABLE is not None: + encoded = support.TESTFN_UNDECODABLE + else: + encoded = os.fsencode(support.TESTFN) + self.bytes_filenames.append(encoded) + self.bytes_filenames.append(bytearray(encoded)) + self.bytes_filenames.append(memoryview(encoded)) + + self.filenames = self.bytes_filenames + self.unicode_filenames + + def test_oserror_filename(self): + funcs = [ + (self.filenames, os.chdir,), + (self.filenames, os.chmod, 0o777), + (self.filenames, os.lstat,), + (self.filenames, os.open, os.O_RDONLY), + (self.filenames, os.rmdir,), + (self.filenames, os.stat,), + (self.filenames, os.unlink,), + ] + if sys.platform == "win32": + funcs.extend(( + (self.bytes_filenames, os.rename, b"dst"), + (self.bytes_filenames, os.replace, b"dst"), + (self.unicode_filenames, os.rename, "dst"), + (self.unicode_filenames, os.replace, "dst"), + (self.unicode_filenames, os.listdir, ), + )) + else: + funcs.extend(( + (self.filenames, os.listdir,), + (self.filenames, os.rename, "dst"), + (self.filenames, os.replace, "dst"), + )) + if hasattr(os, "chown"): + funcs.append((self.filenames, os.chown, 0, 0)) + if hasattr(os, "lchown"): + funcs.append((self.filenames, os.lchown, 0, 0)) + if hasattr(os, "truncate"): + funcs.append((self.filenames, os.truncate, 0)) + if hasattr(os, "chflags"): + funcs.append((self.filenames, os.chflags, 0)) + if hasattr(os, "lchflags"): + funcs.append((self.filenames, os.lchflags, 0)) + if hasattr(os, "chroot"): + funcs.append((self.filenames, os.chroot,)) + if hasattr(os, "link"): + if sys.platform == "win32": + funcs.append((self.bytes_filenames, os.link, b"dst")) + funcs.append((self.unicode_filenames, os.link, "dst")) + else: + funcs.append((self.filenames, os.link, "dst")) + if hasattr(os, "listxattr"): + funcs.extend(( + (self.filenames, os.listxattr,), + (self.filenames, os.getxattr, "user.test"), + (self.filenames, os.setxattr, "user.test", b'user'), + (self.filenames, os.removexattr, "user.test"), + )) + if hasattr(os, "lchmod"): + funcs.append((self.filenames, os.lchmod, 0o777)) + if hasattr(os, "readlink"): + funcs.append((self.filenames, os.readlink,)) + + + for filenames, func, *func_args in funcs: + for name in filenames: + try: + if isinstance(name, (str, bytes)): + func(name, *func_args) + else: + with self.assertWarnsRegex(DeprecationWarning, 'should be'): + func(name, *func_args) + except OSError as err: + self.assertIs(err.filename, name, str(func)) + except UnicodeDecodeError: + pass + else: + self.fail("No exception thrown by {}".format(func)) + +class CPUCountTests(unittest.TestCase): + def test_cpu_count(self): + cpus = os.cpu_count() + if cpus is not None: + self.assertIsInstance(cpus, int) + self.assertGreater(cpus, 0) + else: + self.skipTest("Could not determine the number of CPUs") + + +class FDInheritanceTests(unittest.TestCase): + def test_get_set_inheritable(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + os.set_inheritable(fd, True) + self.assertEqual(os.get_inheritable(fd), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_get_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + # clear FD_CLOEXEC flag + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + flags &= ~fcntl.FD_CLOEXEC + fcntl.fcntl(fd, fcntl.F_SETFD, flags) + + self.assertEqual(os.get_inheritable(fd), True) + + @unittest.skipIf(fcntl is None, "need fcntl") + def test_set_inheritable_cloexec(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + fcntl.FD_CLOEXEC) + + os.set_inheritable(fd, True) + self.assertEqual(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC, + 0) + + def test_open(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_inheritable(fd), False) + + @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()") + def test_pipe(self): + rfd, wfd = os.pipe() + self.addCleanup(os.close, rfd) + self.addCleanup(os.close, wfd) + self.assertEqual(os.get_inheritable(rfd), False) + self.assertEqual(os.get_inheritable(wfd), False) + + def test_dup(self): + fd1 = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd1) + + fd2 = os.dup(fd1) + self.addCleanup(os.close, fd2) + self.assertEqual(os.get_inheritable(fd2), False) + + def test_dup_standard_stream(self): + fd = os.dup(1) + self.addCleanup(os.close, fd) + self.assertGreater(fd, 0) + + @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') + def test_dup_nul(self): + # os.dup() was creating inheritable fds for character files. + fd1 = os.open('NUL', os.O_RDONLY) + self.addCleanup(os.close, fd1) + fd2 = os.dup(fd1) + self.addCleanup(os.close, fd2) + self.assertFalse(os.get_inheritable(fd2)) + + @unittest.skipUnless(hasattr(os, 'dup2'), "need os.dup2()") + def test_dup2(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + + # inheritable by default + fd2 = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd2) + self.assertEqual(os.dup2(fd, fd2), fd2) + self.assertTrue(os.get_inheritable(fd2)) + + # force non-inheritable + fd3 = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd3) + self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3) + self.assertFalse(os.get_inheritable(fd3)) + + @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") + def test_openpty(self): + master_fd, slave_fd = os.openpty() + self.addCleanup(os.close, master_fd) + self.addCleanup(os.close, slave_fd) + self.assertEqual(os.get_inheritable(master_fd), False) + self.assertEqual(os.get_inheritable(slave_fd), False) + + +class PathTConverterTests(unittest.TestCase): + # tuples of (function name, allows fd arguments, additional arguments to + # function, cleanup function) + functions = [ + ('stat', True, (), None), + ('lstat', False, (), None), + ('access', False, (os.F_OK,), None), + ('chflags', False, (0,), None), + ('lchflags', False, (0,), None), + ('open', False, (0,), getattr(os, 'close', None)), + ] + + def test_path_t_converter(self): + str_filename = support.TESTFN + if os.name == 'nt': + bytes_fspath = bytes_filename = None + else: + bytes_filename = support.TESTFN.encode('ascii') + bytes_fspath = FakePath(bytes_filename) + fd = os.open(FakePath(str_filename), os.O_WRONLY|os.O_CREAT) + self.addCleanup(support.unlink, support.TESTFN) + self.addCleanup(os.close, fd) + + int_fspath = FakePath(fd) + str_fspath = FakePath(str_filename) + + for name, allow_fd, extra_args, cleanup_fn in self.functions: + with self.subTest(name=name): + try: + fn = getattr(os, name) + except AttributeError: + continue + + for path in (str_filename, bytes_filename, str_fspath, + bytes_fspath): + if path is None: + continue + with self.subTest(name=name, path=path): + result = fn(path, *extra_args) + if cleanup_fn is not None: + cleanup_fn(result) + + with self.assertRaisesRegex( + TypeError, 'to return str or bytes'): + fn(int_fspath, *extra_args) + + if allow_fd: + result = fn(fd, *extra_args) # should not fail + if cleanup_fn is not None: + cleanup_fn(result) + else: + with self.assertRaisesRegex( + TypeError, + 'os.PathLike'): + fn(fd, *extra_args) + + def test_path_t_converter_and_custom_class(self): + msg = r'__fspath__\(\) to return str or bytes, not %s' + with self.assertRaisesRegex(TypeError, msg % r'int'): + os.stat(FakePath(2)) + with self.assertRaisesRegex(TypeError, msg % r'float'): + os.stat(FakePath(2.34)) + with self.assertRaisesRegex(TypeError, msg % r'object'): + os.stat(FakePath(object())) + + +@unittest.skipUnless(hasattr(os, 'get_blocking'), + 'needs os.get_blocking() and os.set_blocking()') +class BlockingTests(unittest.TestCase): + def test_blocking(self): + fd = os.open(__file__, os.O_RDONLY) + self.addCleanup(os.close, fd) + self.assertEqual(os.get_blocking(fd), True) + + os.set_blocking(fd, False) + self.assertEqual(os.get_blocking(fd), False) + + os.set_blocking(fd, True) + self.assertEqual(os.get_blocking(fd), True) + + + +class ExportsTests(unittest.TestCase): + def test_os_all(self): + self.assertIn('open', os.__all__) + self.assertIn('walk', os.__all__) + + +class TestScandir(unittest.TestCase): + check_no_resource_warning = support.check_no_resource_warning + + def setUp(self): + self.path = os.path.realpath(support.TESTFN) + self.bytes_path = os.fsencode(self.path) + self.addCleanup(support.rmtree, self.path) + os.mkdir(self.path) + + def create_file(self, name="file.txt"): + path = self.bytes_path if isinstance(name, bytes) else self.path + filename = os.path.join(path, name) + create_file(filename, b'python') + return filename + + def get_entries(self, names): + entries = dict((entry.name, entry) + for entry in os.scandir(self.path)) + self.assertEqual(sorted(entries.keys()), names) + return entries + + def assert_stat_equal(self, stat1, stat2, skip_fields): + if skip_fields: + for attr in dir(stat1): + if not attr.startswith("st_"): + continue + if attr in ("st_dev", "st_ino", "st_nlink"): + continue + self.assertEqual(getattr(stat1, attr), + getattr(stat2, attr), + (stat1, stat2, attr)) + else: + self.assertEqual(stat1, stat2) + + def check_entry(self, entry, name, is_dir, is_file, is_symlink): + self.assertIsInstance(entry, os.DirEntry) + self.assertEqual(entry.name, name) + self.assertEqual(entry.path, os.path.join(self.path, name)) + self.assertEqual(entry.inode(), + os.stat(entry.path, follow_symlinks=False).st_ino) + + entry_stat = os.stat(entry.path) + self.assertEqual(entry.is_dir(), + stat.S_ISDIR(entry_stat.st_mode)) + self.assertEqual(entry.is_file(), + stat.S_ISREG(entry_stat.st_mode)) + self.assertEqual(entry.is_symlink(), + os.path.islink(entry.path)) + + entry_lstat = os.stat(entry.path, follow_symlinks=False) + self.assertEqual(entry.is_dir(follow_symlinks=False), + stat.S_ISDIR(entry_lstat.st_mode)) + self.assertEqual(entry.is_file(follow_symlinks=False), + stat.S_ISREG(entry_lstat.st_mode)) + + self.assert_stat_equal(entry.stat(), + entry_stat, + os.name == 'nt' and not is_symlink) + self.assert_stat_equal(entry.stat(follow_symlinks=False), + entry_lstat, + os.name == 'nt') + + def test_attributes(self): + link = hasattr(os, 'link') + symlink = support.can_symlink() + + dirname = os.path.join(self.path, "dir") + os.mkdir(dirname) + filename = self.create_file("file.txt") + if link: + try: + os.link(filename, os.path.join(self.path, "link_file.txt")) + except PermissionError as e: + self.skipTest('os.link(): %s' % e) + if symlink: + os.symlink(dirname, os.path.join(self.path, "symlink_dir"), + target_is_directory=True) + os.symlink(filename, os.path.join(self.path, "symlink_file.txt")) + + names = ['dir', 'file.txt'] + if link: + names.append('link_file.txt') + if symlink: + names.extend(('symlink_dir', 'symlink_file.txt')) + entries = self.get_entries(names) + + entry = entries['dir'] + self.check_entry(entry, 'dir', True, False, False) + + entry = entries['file.txt'] + self.check_entry(entry, 'file.txt', False, True, False) + + if link: + entry = entries['link_file.txt'] + self.check_entry(entry, 'link_file.txt', False, True, False) + + if symlink: + entry = entries['symlink_dir'] + self.check_entry(entry, 'symlink_dir', True, False, True) + + entry = entries['symlink_file.txt'] + self.check_entry(entry, 'symlink_file.txt', False, True, True) + + def get_entry(self, name): + path = self.bytes_path if isinstance(name, bytes) else self.path + entries = list(os.scandir(path)) + self.assertEqual(len(entries), 1) + + entry = entries[0] + self.assertEqual(entry.name, name) + return entry + + def create_file_entry(self, name='file.txt'): + filename = self.create_file(name=name) + return self.get_entry(os.path.basename(filename)) + + def test_current_directory(self): + filename = self.create_file() + old_dir = os.getcwd() + try: + os.chdir(self.path) + + # call scandir() without parameter: it must list the content + # of the current directory + entries = dict((entry.name, entry) for entry in os.scandir()) + self.assertEqual(sorted(entries.keys()), + [os.path.basename(filename)]) + finally: + os.chdir(old_dir) + + def test_repr(self): + entry = self.create_file_entry() + self.assertEqual(repr(entry), "") + + def test_fspath_protocol(self): + entry = self.create_file_entry() + self.assertEqual(os.fspath(entry), os.path.join(self.path, 'file.txt')) + + def test_fspath_protocol_bytes(self): + bytes_filename = os.fsencode('bytesfile.txt') + bytes_entry = self.create_file_entry(name=bytes_filename) + fspath = os.fspath(bytes_entry) + self.assertIsInstance(fspath, bytes) + self.assertEqual(fspath, + os.path.join(os.fsencode(self.path),bytes_filename)) + + def test_removed_dir(self): + path = os.path.join(self.path, 'dir') + + os.mkdir(path) + entry = self.get_entry('dir') + os.rmdir(path) + + # On POSIX, is_dir() result depends if scandir() filled d_type or not + if os.name == 'nt': + self.assertTrue(entry.is_dir()) + self.assertFalse(entry.is_file()) + self.assertFalse(entry.is_symlink()) + if os.name == 'nt': + self.assertRaises(FileNotFoundError, entry.inode) + # don't fail + entry.stat() + entry.stat(follow_symlinks=False) + else: + self.assertGreater(entry.inode(), 0) + self.assertRaises(FileNotFoundError, entry.stat) + self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) + + def test_removed_file(self): + entry = self.create_file_entry() + os.unlink(entry.path) + + self.assertFalse(entry.is_dir()) + # On POSIX, is_dir() result depends if scandir() filled d_type or not + if os.name == 'nt': + self.assertTrue(entry.is_file()) + self.assertFalse(entry.is_symlink()) + if os.name == 'nt': + self.assertRaises(FileNotFoundError, entry.inode) + # don't fail + entry.stat() + entry.stat(follow_symlinks=False) + else: + self.assertGreater(entry.inode(), 0) + self.assertRaises(FileNotFoundError, entry.stat) + self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) + + def test_broken_symlink(self): + if not support.can_symlink(): + return self.skipTest('cannot create symbolic link') + + filename = self.create_file("file.txt") + os.symlink(filename, + os.path.join(self.path, "symlink.txt")) + entries = self.get_entries(['file.txt', 'symlink.txt']) + entry = entries['symlink.txt'] + os.unlink(filename) + + self.assertGreater(entry.inode(), 0) + self.assertFalse(entry.is_dir()) + self.assertFalse(entry.is_file()) # broken symlink returns False + self.assertFalse(entry.is_dir(follow_symlinks=False)) + self.assertFalse(entry.is_file(follow_symlinks=False)) + self.assertTrue(entry.is_symlink()) + self.assertRaises(FileNotFoundError, entry.stat) + # don't fail + entry.stat(follow_symlinks=False) + + def test_bytes(self): + self.create_file("file.txt") + + path_bytes = os.fsencode(self.path) + entries = list(os.scandir(path_bytes)) + self.assertEqual(len(entries), 1, entries) + entry = entries[0] + + self.assertEqual(entry.name, b'file.txt') + self.assertEqual(entry.path, + os.fsencode(os.path.join(self.path, 'file.txt'))) + + def test_bytes_like(self): + self.create_file("file.txt") + + for cls in bytearray, memoryview: + path_bytes = cls(os.fsencode(self.path)) + with self.assertWarns(DeprecationWarning): + entries = list(os.scandir(path_bytes)) + self.assertEqual(len(entries), 1, entries) + entry = entries[0] + + self.assertEqual(entry.name, b'file.txt') + self.assertEqual(entry.path, + os.fsencode(os.path.join(self.path, 'file.txt'))) + self.assertIs(type(entry.name), bytes) + self.assertIs(type(entry.path), bytes) + + @unittest.skipUnless(os.listdir in os.supports_fd, + 'fd support for listdir required for this test.') + def test_fd(self): + self.assertIn(os.scandir, os.supports_fd) + self.create_file('file.txt') + expected_names = ['file.txt'] + if support.can_symlink(): + os.symlink('file.txt', os.path.join(self.path, 'link')) + expected_names.append('link') + + fd = os.open(self.path, os.O_RDONLY) + try: + with os.scandir(fd) as it: + entries = list(it) + names = [entry.name for entry in entries] + self.assertEqual(sorted(names), expected_names) + self.assertEqual(names, os.listdir(fd)) + for entry in entries: + self.assertEqual(entry.path, entry.name) + self.assertEqual(os.fspath(entry), entry.name) + self.assertEqual(entry.is_symlink(), entry.name == 'link') + if os.stat in os.supports_dir_fd: + st = os.stat(entry.name, dir_fd=fd) + self.assertEqual(entry.stat(), st) + st = os.stat(entry.name, dir_fd=fd, follow_symlinks=False) + self.assertEqual(entry.stat(follow_symlinks=False), st) + finally: + os.close(fd) + + def test_empty_path(self): + self.assertRaises(FileNotFoundError, os.scandir, '') + + def test_consume_iterator_twice(self): + self.create_file("file.txt") + iterator = os.scandir(self.path) + + entries = list(iterator) + self.assertEqual(len(entries), 1, entries) + + # check than consuming the iterator twice doesn't raise exception + entries2 = list(iterator) + self.assertEqual(len(entries2), 0, entries2) + + def test_bad_path_type(self): + for obj in [1.234, {}, []]: + self.assertRaises(TypeError, os.scandir, obj) + + def test_close(self): + self.create_file("file.txt") + self.create_file("file2.txt") + iterator = os.scandir(self.path) + next(iterator) + iterator.close() + # multiple closes + iterator.close() + with self.check_no_resource_warning(): + del iterator + + def test_context_manager(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with os.scandir(self.path) as iterator: + next(iterator) + with self.check_no_resource_warning(): + del iterator + + def test_context_manager_close(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with os.scandir(self.path) as iterator: + next(iterator) + iterator.close() + + def test_context_manager_exception(self): + self.create_file("file.txt") + self.create_file("file2.txt") + with self.assertRaises(ZeroDivisionError): + with os.scandir(self.path) as iterator: + next(iterator) + 1/0 + with self.check_no_resource_warning(): + del iterator + + def test_resource_warning(self): + self.create_file("file.txt") + self.create_file("file2.txt") + iterator = os.scandir(self.path) + next(iterator) + with self.assertWarns(ResourceWarning): + del iterator + support.gc_collect() + # exhausted iterator + iterator = os.scandir(self.path) + list(iterator) + with self.check_no_resource_warning(): + del iterator + + +class TestPEP519(unittest.TestCase): + + # Abstracted so it can be overridden to test pure Python implementation + # if a C version is provided. + fspath = staticmethod(os.fspath) + + def test_return_bytes(self): + for b in b'hello', b'goodbye', b'some/path/and/file': + self.assertEqual(b, self.fspath(b)) + + def test_return_string(self): + for s in 'hello', 'goodbye', 'some/path/and/file': + self.assertEqual(s, self.fspath(s)) + + def test_fsencode_fsdecode(self): + for p in "path/like/object", b"path/like/object": + pathlike = FakePath(p) + + self.assertEqual(p, self.fspath(pathlike)) + self.assertEqual(b"path/like/object", os.fsencode(pathlike)) + self.assertEqual("path/like/object", os.fsdecode(pathlike)) + + def test_pathlike(self): + self.assertEqual('#feelthegil', self.fspath(FakePath('#feelthegil'))) + self.assertTrue(issubclass(FakePath, os.PathLike)) + self.assertTrue(isinstance(FakePath('x'), os.PathLike)) + + def test_garbage_in_exception_out(self): + vapor = type('blah', (), {}) + for o in int, type, os, vapor(): + self.assertRaises(TypeError, self.fspath, o) + + def test_argument_required(self): + self.assertRaises(TypeError, self.fspath) + + def test_bad_pathlike(self): + # __fspath__ returns a value other than str or bytes. + self.assertRaises(TypeError, self.fspath, FakePath(42)) + # __fspath__ attribute that is not callable. + c = type('foo', (), {}) + c.__fspath__ = 1 + self.assertRaises(TypeError, self.fspath, c()) + # __fspath__ raises an exception. + self.assertRaises(ZeroDivisionError, self.fspath, + FakePath(ZeroDivisionError())) + + def test_pathlike_subclasshook(self): + # bpo-38878: subclasshook causes subclass checks + # true on abstract implementation. + class A(os.PathLike): + pass + self.assertFalse(issubclass(FakePath, A)) + self.assertTrue(issubclass(FakePath, os.PathLike)) + + +class TimesTests(unittest.TestCase): + def test_times(self): + times = os.times() + self.assertIsInstance(times, os.times_result) + + for field in ('user', 'system', 'children_user', 'children_system', + 'elapsed'): + value = getattr(times, field) + self.assertIsInstance(value, float) + + if os.name == 'nt': + self.assertEqual(times.children_user, 0) + self.assertEqual(times.children_system, 0) + self.assertEqual(times.elapsed, 0) + + +# Only test if the C version is provided, otherwise TestPEP519 already tested +# the pure Python implementation. +if hasattr(os, "_fspath"): + class TestPEP519PurePython(TestPEP519): + + """Explicitly test the pure Python implementation of os.fspath().""" + + fspath = staticmethod(os._fspath) + + +if __name__ == "__main__": + unittest.main() From 46f22981a81c2231b6c2a0ea2a2a94fcdec93581 Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Thu, 6 Aug 2020 20:53:03 +0900 Subject: [PATCH 02/10] Add modules for test_os --- Lib/asynchat.py | 307 +++++++++++++++++++++++ Lib/asyncore.py | 642 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 949 insertions(+) create mode 100644 Lib/asynchat.py create mode 100644 Lib/asyncore.py diff --git a/Lib/asynchat.py b/Lib/asynchat.py new file mode 100644 index 000000000..fc1146adb --- /dev/null +++ b/Lib/asynchat.py @@ -0,0 +1,307 @@ +# -*- Mode: Python; tab-width: 4 -*- +# Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp +# Author: Sam Rushing + +# ====================================================================== +# Copyright 1996 by Sam Rushing +# +# All Rights Reserved +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose and without fee is hereby +# granted, provided that the above copyright notice appear in all +# copies and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of Sam +# Rushing not be used in advertising or publicity pertaining to +# distribution of the software without specific, written prior +# permission. +# +# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN +# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# ====================================================================== + +r"""A class supporting chat-style (command/response) protocols. + +This class adds support for 'chat' style protocols - where one side +sends a 'command', and the other sends a response (examples would be +the common internet protocols - smtp, nntp, ftp, etc..). + +The handle_read() method looks at the input stream for the current +'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n' +for multi-line output), calling self.found_terminator() on its +receipt. + +for example: +Say you build an async nntp client using this class. At the start +of the connection, you'll have self.terminator set to '\r\n', in +order to process the single-line greeting. Just before issuing a +'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST +command will be accumulated (using your own 'collect_incoming_data' +method) up to the terminator, and then control will be returned to +you - by calling your self.found_terminator() method. +""" +import asyncore +from collections import deque + + +class async_chat(asyncore.dispatcher): + """This is an abstract class. You must derive from this class, and add + the two methods collect_incoming_data() and found_terminator()""" + + # these are overridable defaults + + ac_in_buffer_size = 65536 + ac_out_buffer_size = 65536 + + # we don't want to enable the use of encoding by default, because that is a + # sign of an application bug that we don't want to pass silently + + use_encoding = 0 + encoding = 'latin-1' + + def __init__(self, sock=None, map=None): + # for string terminator matching + self.ac_in_buffer = b'' + + # we use a list here rather than io.BytesIO for a few reasons... + # del lst[:] is faster than bio.truncate(0) + # lst = [] is faster than bio.truncate(0) + self.incoming = [] + + # we toss the use of the "simple producer" and replace it with + # a pure deque, which the original fifo was a wrapping of + self.producer_fifo = deque() + asyncore.dispatcher.__init__(self, sock, map) + + def collect_incoming_data(self, data): + raise NotImplementedError("must be implemented in subclass") + + def _collect_incoming_data(self, data): + self.incoming.append(data) + + def _get_data(self): + d = b''.join(self.incoming) + del self.incoming[:] + return d + + def found_terminator(self): + raise NotImplementedError("must be implemented in subclass") + + def set_terminator(self, term): + """Set the input delimiter. + + Can be a fixed string of any length, an integer, or None. + """ + if isinstance(term, str) and self.use_encoding: + term = bytes(term, self.encoding) + elif isinstance(term, int) and term < 0: + raise ValueError('the number of received bytes must be positive') + self.terminator = term + + def get_terminator(self): + return self.terminator + + # grab some more data from the socket, + # throw it to the collector method, + # check for the terminator, + # if found, transition to the next state. + + def handle_read(self): + + try: + data = self.recv(self.ac_in_buffer_size) + except BlockingIOError: + return + except OSError as why: + self.handle_error() + return + + if isinstance(data, str) and self.use_encoding: + data = bytes(str, self.encoding) + self.ac_in_buffer = self.ac_in_buffer + data + + # Continue to search for self.terminator in self.ac_in_buffer, + # while calling self.collect_incoming_data. The while loop + # is necessary because we might read several data+terminator + # combos with a single recv(4096). + + while self.ac_in_buffer: + lb = len(self.ac_in_buffer) + terminator = self.get_terminator() + if not terminator: + # no terminator, collect it all + self.collect_incoming_data(self.ac_in_buffer) + self.ac_in_buffer = b'' + elif isinstance(terminator, int): + # numeric terminator + n = terminator + if lb < n: + self.collect_incoming_data(self.ac_in_buffer) + self.ac_in_buffer = b'' + self.terminator = self.terminator - lb + else: + self.collect_incoming_data(self.ac_in_buffer[:n]) + self.ac_in_buffer = self.ac_in_buffer[n:] + self.terminator = 0 + self.found_terminator() + else: + # 3 cases: + # 1) end of buffer matches terminator exactly: + # collect data, transition + # 2) end of buffer matches some prefix: + # collect data to the prefix + # 3) end of buffer does not match any prefix: + # collect data + terminator_len = len(terminator) + index = self.ac_in_buffer.find(terminator) + if index != -1: + # we found the terminator + if index > 0: + # don't bother reporting the empty string + # (source of subtle bugs) + self.collect_incoming_data(self.ac_in_buffer[:index]) + self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:] + # This does the Right Thing if the terminator + # is changed here. + self.found_terminator() + else: + # check for a prefix of the terminator + index = find_prefix_at_end(self.ac_in_buffer, terminator) + if index: + if index != lb: + # we found a prefix, collect up to the prefix + self.collect_incoming_data(self.ac_in_buffer[:-index]) + self.ac_in_buffer = self.ac_in_buffer[-index:] + break + else: + # no prefix, collect it all + self.collect_incoming_data(self.ac_in_buffer) + self.ac_in_buffer = b'' + + def handle_write(self): + self.initiate_send() + + def handle_close(self): + self.close() + + def push(self, data): + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be byte-ish (%r)', + type(data)) + sabs = self.ac_out_buffer_size + if len(data) > sabs: + for i in range(0, len(data), sabs): + self.producer_fifo.append(data[i:i+sabs]) + else: + self.producer_fifo.append(data) + self.initiate_send() + + def push_with_producer(self, producer): + self.producer_fifo.append(producer) + self.initiate_send() + + def readable(self): + "predicate for inclusion in the readable for select()" + # cannot use the old predicate, it violates the claim of the + # set_terminator method. + + # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size) + return 1 + + def writable(self): + "predicate for inclusion in the writable for select()" + return self.producer_fifo or (not self.connected) + + def close_when_done(self): + "automatically close this channel once the outgoing queue is empty" + self.producer_fifo.append(None) + + def initiate_send(self): + while self.producer_fifo and self.connected: + first = self.producer_fifo[0] + # handle empty string/buffer or None entry + if not first: + del self.producer_fifo[0] + if first is None: + self.handle_close() + return + + # handle classic producer behavior + obs = self.ac_out_buffer_size + try: + data = first[:obs] + except TypeError: + data = first.more() + if data: + self.producer_fifo.appendleft(data) + else: + del self.producer_fifo[0] + continue + + if isinstance(data, str) and self.use_encoding: + data = bytes(data, self.encoding) + + # send the data + try: + num_sent = self.send(data) + except OSError: + self.handle_error() + return + + if num_sent: + if num_sent < len(data) or obs < len(first): + self.producer_fifo[0] = first[num_sent:] + else: + del self.producer_fifo[0] + # we tried to send some actual data + return + + def discard_buffers(self): + # Emergencies only! + self.ac_in_buffer = b'' + del self.incoming[:] + self.producer_fifo.clear() + + +class simple_producer: + + def __init__(self, data, buffer_size=512): + self.data = data + self.buffer_size = buffer_size + + def more(self): + if len(self.data) > self.buffer_size: + result = self.data[:self.buffer_size] + self.data = self.data[self.buffer_size:] + return result + else: + result = self.data + self.data = b'' + return result + + +# Given 'haystack', see if any prefix of 'needle' is at its end. This +# assumes an exact match has already been checked. Return the number of +# characters matched. +# for example: +# f_p_a_e("qwerty\r", "\r\n") => 1 +# f_p_a_e("qwertydkjf", "\r\n") => 0 +# f_p_a_e("qwerty\r\n", "\r\n") => + +# this could maybe be made faster with a computed regex? +# [answer: no; circa Python-2.0, Jan 2001] +# new python: 28961/s +# old python: 18307/s +# re: 12820/s +# regex: 14035/s + +def find_prefix_at_end(haystack, needle): + l = len(needle) - 1 + while l and not haystack.endswith(needle[:l]): + l -= 1 + return l diff --git a/Lib/asyncore.py b/Lib/asyncore.py new file mode 100644 index 000000000..0e92be3ad --- /dev/null +++ b/Lib/asyncore.py @@ -0,0 +1,642 @@ +# -*- Mode: Python -*- +# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp +# Author: Sam Rushing + +# ====================================================================== +# Copyright 1996 by Sam Rushing +# +# All Rights Reserved +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose and without fee is hereby +# granted, provided that the above copyright notice appear in all +# copies and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of Sam +# Rushing not be used in advertising or publicity pertaining to +# distribution of the software without specific, written prior +# permission. +# +# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN +# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# ====================================================================== + +"""Basic infrastructure for asynchronous socket service clients and servers. + +There are only two ways to have a program on a single processor do "more +than one thing at a time". Multi-threaded programming is the simplest and +most popular way to do it, but there is another very different technique, +that lets you have nearly all the advantages of multi-threading, without +actually using multiple threads. it's really only practical if your program +is largely I/O bound. If your program is CPU bound, then pre-emptive +scheduled threads are probably what you really need. Network servers are +rarely CPU-bound, however. + +If your operating system supports the select() system call in its I/O +library (and nearly all do), then you can use it to juggle multiple +communication channels at once; doing other work while your I/O is taking +place in the "background." Although this strategy can seem strange and +complex, especially at first, it is in many ways easier to understand and +control than multi-threaded programming. The module documented here solves +many of the difficult problems for you, making the task of building +sophisticated high-performance network servers and clients a snap. +""" + +import select +import socket +import sys +import time +import warnings + +import os +from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ + ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ + errorcode + +_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, + EBADF}) + +try: + socket_map +except NameError: + socket_map = {} + +def _strerror(err): + try: + return os.strerror(err) + except (ValueError, OverflowError, NameError): + if err in errorcode: + return errorcode[err] + return "Unknown error %s" %err + +class ExitNow(Exception): + pass + +_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit) + +def read(obj): + try: + obj.handle_read_event() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def write(obj): + try: + obj.handle_write_event() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def _exception(obj): + try: + obj.handle_expt_event() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def readwrite(obj, flags): + try: + if flags & select.POLLIN: + obj.handle_read_event() + if flags & select.POLLOUT: + obj.handle_write_event() + if flags & select.POLLPRI: + obj.handle_expt_event() + if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): + obj.handle_close() + except OSError as e: + if e.args[0] not in _DISCONNECTED: + obj.handle_error() + else: + obj.handle_close() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def poll(timeout=0.0, map=None): + if map is None: + map = socket_map + if map: + r = []; w = []; e = [] + for fd, obj in list(map.items()): + is_r = obj.readable() + is_w = obj.writable() + if is_r: + r.append(fd) + # accepting sockets should not be writable + if is_w and not obj.accepting: + w.append(fd) + if is_r or is_w: + e.append(fd) + if [] == r == w == e: + time.sleep(timeout) + return + + r, w, e = select.select(r, w, e, timeout) + + for fd in r: + obj = map.get(fd) + if obj is None: + continue + read(obj) + + for fd in w: + obj = map.get(fd) + if obj is None: + continue + write(obj) + + for fd in e: + obj = map.get(fd) + if obj is None: + continue + _exception(obj) + +def poll2(timeout=0.0, map=None): + # Use the poll() support added to the select module in Python 2.0 + if map is None: + map = socket_map + if timeout is not None: + # timeout is in milliseconds + timeout = int(timeout*1000) + pollster = select.poll() + if map: + for fd, obj in list(map.items()): + flags = 0 + if obj.readable(): + flags |= select.POLLIN | select.POLLPRI + # accepting sockets should not be writable + if obj.writable() and not obj.accepting: + flags |= select.POLLOUT + if flags: + pollster.register(fd, flags) + + r = pollster.poll(timeout) + for fd, flags in r: + obj = map.get(fd) + if obj is None: + continue + readwrite(obj, flags) + +poll3 = poll2 # Alias for backward compatibility + +def loop(timeout=30.0, use_poll=False, map=None, count=None): + if map is None: + map = socket_map + + if use_poll and hasattr(select, 'poll'): + poll_fun = poll2 + else: + poll_fun = poll + + if count is None: + while map: + poll_fun(timeout, map) + + else: + while map and count > 0: + poll_fun(timeout, map) + count = count - 1 + +class dispatcher: + + debug = False + connected = False + accepting = False + connecting = False + closing = False + addr = None + ignore_log_types = frozenset({'warning'}) + + def __init__(self, sock=None, map=None): + if map is None: + self._map = socket_map + else: + self._map = map + + self._fileno = None + + if sock: + # Set to nonblocking just to make sure for cases where we + # get a socket from a blocking source. + sock.setblocking(0) + self.set_socket(sock, map) + self.connected = True + # The constructor no longer requires that the socket + # passed be connected. + try: + self.addr = sock.getpeername() + except OSError as err: + if err.args[0] in (ENOTCONN, EINVAL): + # To handle the case where we got an unconnected + # socket. + self.connected = False + else: + # The socket is broken in some unknown way, alert + # the user and remove it from the map (to prevent + # polling of broken sockets). + self.del_channel(map) + raise + else: + self.socket = None + + def __repr__(self): + status = [self.__class__.__module__+"."+self.__class__.__qualname__] + if self.accepting and self.addr: + status.append('listening') + elif self.connected: + status.append('connected') + if self.addr is not None: + try: + status.append('%s:%d' % self.addr) + except TypeError: + status.append(repr(self.addr)) + return '<%s at %#x>' % (' '.join(status), id(self)) + + def add_channel(self, map=None): + #self.log_info('adding channel %s' % self) + if map is None: + map = self._map + map[self._fileno] = self + + def del_channel(self, map=None): + fd = self._fileno + if map is None: + map = self._map + if fd in map: + #self.log_info('closing channel %d:%s' % (fd, self)) + del map[fd] + self._fileno = None + + def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM): + self.family_and_type = family, type + sock = socket.socket(family, type) + sock.setblocking(0) + self.set_socket(sock) + + def set_socket(self, sock, map=None): + self.socket = sock + self._fileno = sock.fileno() + self.add_channel(map) + + def set_reuse_addr(self): + # try to re-use a server port if possible + try: + self.socket.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, + self.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR) | 1 + ) + except OSError: + pass + + # ================================================== + # predicates for select() + # these are used as filters for the lists of sockets + # to pass to select(). + # ================================================== + + def readable(self): + return True + + def writable(self): + return True + + # ================================================== + # socket object methods. + # ================================================== + + def listen(self, num): + self.accepting = True + if os.name == 'nt' and num > 5: + num = 5 + return self.socket.listen(num) + + def bind(self, addr): + self.addr = addr + return self.socket.bind(addr) + + def connect(self, address): + self.connected = False + self.connecting = True + err = self.socket.connect_ex(address) + if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ + or err == EINVAL and os.name == 'nt': + self.addr = address + return + if err in (0, EISCONN): + self.addr = address + self.handle_connect_event() + else: + raise OSError(err, errorcode[err]) + + def accept(self): + # XXX can return either an address pair or None + try: + conn, addr = self.socket.accept() + except TypeError: + return None + except OSError as why: + if why.args[0] in (EWOULDBLOCK, ECONNABORTED, EAGAIN): + return None + else: + raise + else: + return conn, addr + + def send(self, data): + try: + result = self.socket.send(data) + return result + except OSError as why: + if why.args[0] == EWOULDBLOCK: + return 0 + elif why.args[0] in _DISCONNECTED: + self.handle_close() + return 0 + else: + raise + + def recv(self, buffer_size): + try: + data = self.socket.recv(buffer_size) + if not data: + # a closed connection is indicated by signaling + # a read condition, and having recv() return 0. + self.handle_close() + return b'' + else: + return data + except OSError as why: + # winsock sometimes raises ENOTCONN + if why.args[0] in _DISCONNECTED: + self.handle_close() + return b'' + else: + raise + + def close(self): + self.connected = False + self.accepting = False + self.connecting = False + self.del_channel() + if self.socket is not None: + try: + self.socket.close() + except OSError as why: + if why.args[0] not in (ENOTCONN, EBADF): + raise + + # log and log_info may be overridden to provide more sophisticated + # logging and warning methods. In general, log is for 'hit' logging + # and 'log_info' is for informational, warning and error logging. + + def log(self, message): + sys.stderr.write('log: %s\n' % str(message)) + + def log_info(self, message, type='info'): + if type not in self.ignore_log_types: + print('%s: %s' % (type, message)) + + def handle_read_event(self): + if self.accepting: + # accepting sockets are never connected, they "spawn" new + # sockets that are connected + self.handle_accept() + elif not self.connected: + if self.connecting: + self.handle_connect_event() + self.handle_read() + else: + self.handle_read() + + def handle_connect_event(self): + err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if err != 0: + raise OSError(err, _strerror(err)) + self.handle_connect() + self.connected = True + self.connecting = False + + def handle_write_event(self): + if self.accepting: + # Accepting sockets shouldn't get a write event. + # We will pretend it didn't happen. + return + + if not self.connected: + if self.connecting: + self.handle_connect_event() + self.handle_write() + + def handle_expt_event(self): + # handle_expt_event() is called if there might be an error on the + # socket, or if there is OOB data + # check for the error condition first + err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if err != 0: + # we can get here when select.select() says that there is an + # exceptional condition on the socket + # since there is an error, we'll go ahead and close the socket + # like we would in a subclassed handle_read() that received no + # data + self.handle_close() + else: + self.handle_expt() + + def handle_error(self): + nil, t, v, tbinfo = compact_traceback() + + # sometimes a user repr method will crash. + try: + self_repr = repr(self) + except: + self_repr = '<__repr__(self) failed for object at %0x>' % id(self) + + self.log_info( + 'uncaptured python exception, closing channel %s (%s:%s %s)' % ( + self_repr, + t, + v, + tbinfo + ), + 'error' + ) + self.handle_close() + + def handle_expt(self): + self.log_info('unhandled incoming priority event', 'warning') + + def handle_read(self): + self.log_info('unhandled read event', 'warning') + + def handle_write(self): + self.log_info('unhandled write event', 'warning') + + def handle_connect(self): + self.log_info('unhandled connect event', 'warning') + + def handle_accept(self): + pair = self.accept() + if pair is not None: + self.handle_accepted(*pair) + + def handle_accepted(self, sock, addr): + sock.close() + self.log_info('unhandled accepted event', 'warning') + + def handle_close(self): + self.log_info('unhandled close event', 'warning') + self.close() + +# --------------------------------------------------------------------------- +# adds simple buffered output capability, useful for simple clients. +# [for more sophisticated usage use asynchat.async_chat] +# --------------------------------------------------------------------------- + +class dispatcher_with_send(dispatcher): + + def __init__(self, sock=None, map=None): + dispatcher.__init__(self, sock, map) + self.out_buffer = b'' + + def initiate_send(self): + num_sent = 0 + num_sent = dispatcher.send(self, self.out_buffer[:65536]) + self.out_buffer = self.out_buffer[num_sent:] + + def handle_write(self): + self.initiate_send() + + def writable(self): + return (not self.connected) or len(self.out_buffer) + + def send(self, data): + if self.debug: + self.log_info('sending %s' % repr(data)) + self.out_buffer = self.out_buffer + data + self.initiate_send() + +# --------------------------------------------------------------------------- +# used for debugging. +# --------------------------------------------------------------------------- + +def compact_traceback(): + t, v, tb = sys.exc_info() + tbinfo = [] + if not tb: # Must have a traceback + raise AssertionError("traceback does not exist") + while tb: + tbinfo.append(( + tb.tb_frame.f_code.co_filename, + tb.tb_frame.f_code.co_name, + str(tb.tb_lineno) + )) + tb = tb.tb_next + + # just to be safe + del tb + + file, function, line = tbinfo[-1] + info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo]) + return (file, function, line), t, v, info + +def close_all(map=None, ignore_all=False): + if map is None: + map = socket_map + for x in list(map.values()): + try: + x.close() + except OSError as x: + if x.args[0] == EBADF: + pass + elif not ignore_all: + raise + except _reraised_exceptions: + raise + except: + if not ignore_all: + raise + map.clear() + +# Asynchronous File I/O: +# +# After a little research (reading man pages on various unixen, and +# digging through the linux kernel), I've determined that select() +# isn't meant for doing asynchronous file i/o. +# Heartening, though - reading linux/mm/filemap.c shows that linux +# supports asynchronous read-ahead. So _MOST_ of the time, the data +# will be sitting in memory for us already when we go to read it. +# +# What other OS's (besides NT) support async file i/o? [VMS?] +# +# Regardless, this is useful for pipes, and stdin/stdout... + +if os.name == 'posix': + class file_wrapper: + # Here we override just enough to make a file + # look like a socket for the purposes of asyncore. + # The passed fd is automatically os.dup()'d + + def __init__(self, fd): + self.fd = os.dup(fd) + + def __del__(self): + if self.fd >= 0: + warnings.warn("unclosed file %r" % self, ResourceWarning, + source=self) + self.close() + + def recv(self, *args): + return os.read(self.fd, *args) + + def send(self, *args): + return os.write(self.fd, *args) + + def getsockopt(self, level, optname, buflen=None): + if (level == socket.SOL_SOCKET and + optname == socket.SO_ERROR and + not buflen): + return 0 + raise NotImplementedError("Only asyncore specific behaviour " + "implemented.") + + read = recv + write = send + + def close(self): + if self.fd < 0: + return + fd = self.fd + self.fd = -1 + os.close(fd) + + def fileno(self): + return self.fd + + class file_dispatcher(dispatcher): + + def __init__(self, fd, map=None): + dispatcher.__init__(self, None, map) + self.connected = True + try: + fd = fd.fileno() + except AttributeError: + pass + self.set_file(fd) + # set it to non-blocking mode + os.set_blocking(fd, False) + + def set_file(self, fd): + self.socket = file_wrapper(fd) + self._fileno = self.socket.fileno() + self.add_channel() From 74fe9d30d2995c76d58cac592f8fa604be96aebf Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Thu, 6 Aug 2020 22:47:15 +0900 Subject: [PATCH 03/10] Skip tests that use mmap --- Lib/test/test_os.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 2a4ae1573..e6ea06266 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -12,7 +12,7 @@ import fnmatch import fractions import itertools import locale -import mmap +# import mmap import os import pickle import shutil @@ -2228,6 +2228,7 @@ class Win32KillTests(unittest.TestCase): self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") + @unittest.skip("TODO: RUSTPYTHON") def test_CTRL_BREAK_EVENT(self): self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") From 06176954b168b43216e681f8ce7e8f4f7c061ff8 Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Fri, 7 Aug 2020 02:51:51 +0900 Subject: [PATCH 04/10] Skip tests that raise `invalid mode: x` --- Lib/test/test_os.py | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index e6ea06266..b9a4181b1 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -198,6 +198,7 @@ class FileTests(unittest.TestCase): self.assertEqual(type(s), bytes) self.assertEqual(s, b"spam") + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") @support.cpython_only # Skip the test on 32-bit platforms: the number of bytes must fit in a # Py_ssize_t type @@ -261,6 +262,7 @@ class FileTests(unittest.TestCase): self.fdopen_helper('r') self.fdopen_helper('r', 100) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_replace(self): TESTFN2 = support.TESTFN + ".2" self.addCleanup(support.unlink, support.TESTFN) @@ -443,9 +445,11 @@ class StatAttributeTests(unittest.TestCase): except TypeError: pass + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_stat_attributes(self): self.check_stat_attributes(self.fname) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_stat_attributes_bytes(self): try: fname = self.fname.encode(sys.getfilesystemencoding()) @@ -453,6 +457,7 @@ class StatAttributeTests(unittest.TestCase): self.skipTest("cannot encode %a for the filesystem" % self.fname) self.check_stat_attributes(fname) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_stat_result_pickle(self): result = os.stat(self.fname) for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -648,6 +653,7 @@ class UtimeTests(unittest.TestCase): self.assertEqual(st.st_atime_ns, atime_ns) self.assertEqual(st.st_mtime_ns, mtime_ns) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_utime(self): def set_time(filename, ns): # test the ns keyword parameter @@ -661,6 +667,7 @@ class UtimeTests(unittest.TestCase): # issue, os.utime() rounds towards minus infinity. return (ns * 1e-9) + 0.5e-9 + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_utime_by_indexed(self): # pass times as floating point seconds as the second indexed parameter def set_time(filename, ns): @@ -672,6 +679,7 @@ class UtimeTests(unittest.TestCase): os.utime(filename, (atime, mtime)) self._test_utime(set_time) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_utime_by_times(self): def set_time(filename, ns): atime_ns, mtime_ns = ns @@ -714,6 +722,7 @@ class UtimeTests(unittest.TestCase): os.close(dirfd) self._test_utime(set_time) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_utime_directory(self): def set_time(filename, ns): # test calling os.utime() on a directory @@ -742,12 +751,14 @@ class UtimeTests(unittest.TestCase): self.assertAlmostEqual(st.st_mtime, current, delta=delta, msg=msg) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_utime_current(self): def set_time(filename): # Set to the current time in the new way os.utime(self.fname) self._test_utime_current(set_time) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_utime_current_old(self): def set_time(filename): # Set to the current time in the old explicit way. @@ -767,6 +778,7 @@ class UtimeTests(unittest.TestCase): return buf.value # return None if the filesystem is unknown + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_large_time(self): # Many filesystems are limited to the year 2038. At least, the test # pass with NTFS filesystem. @@ -777,6 +789,7 @@ class UtimeTests(unittest.TestCase): os.utime(self.fname, (large, large)) self.assertEqual(os.stat(self.fname).st_mtime, large) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive with self.assertRaises(ValueError): @@ -1083,6 +1096,7 @@ class WalkTests(unittest.TestCase): os.rmdir(sub21_path) del self.sub2_tree[1][:1] + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'x')") def test_walk_topdown(self): # Walk top-down. all = list(self.walk(self.walk_path)) @@ -1100,6 +1114,7 @@ class WalkTests(unittest.TestCase): self.assertEqual(all[2 + flipped], (self.sub11_path, [], [])) self.assertEqual(all[3 - 2 * flipped], self.sub2_tree) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'x')") def test_walk_prune(self, walk_path=None): if walk_path is None: walk_path = self.walk_path @@ -1119,9 +1134,11 @@ class WalkTests(unittest.TestCase): all[1][1].sort() self.assertEqual(all[1], self.sub2_tree) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'x')") def test_file_like_path(self): self.test_walk_prune(FakePath(self.walk_path)) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'x')") def test_walk_bottom_up(self): # Walk bottom-up. all = list(self.walk(self.walk_path, topdown=False)) @@ -1143,6 +1160,7 @@ class WalkTests(unittest.TestCase): self.assertEqual(all[2 - 2 * flipped], self.sub2_tree) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'x')") def test_walk_symlink(self): if not support.can_symlink(): self.skipTest("need symlink support") @@ -1157,6 +1175,7 @@ class WalkTests(unittest.TestCase): else: self.fail("Didn't follow symlink with followlinks=True") + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'x')") def test_walk_bad_dir(self): # Walk top-down. errors = [] @@ -1178,6 +1197,7 @@ class WalkTests(unittest.TestCase): finally: os.rename(path1new, path1) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'x')") def test_walk_many_open_files(self): depth = 30 base = os.path.join(support.TESTFN, 'deep') @@ -1469,6 +1489,7 @@ class RemoveDirsTests(unittest.TestCase): self.assertFalse(os.path.exists(dira)) self.assertFalse(os.path.exists(support.TESTFN)) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_remove_partial(self): dira = os.path.join(support.TESTFN, 'dira') os.mkdir(dira) @@ -1480,6 +1501,7 @@ class RemoveDirsTests(unittest.TestCase): self.assertTrue(os.path.exists(dira)) self.assertTrue(os.path.exists(support.TESTFN)) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_remove_nothing(self): dira = os.path.join(support.TESTFN, 'dira') os.mkdir(dira) @@ -1625,6 +1647,7 @@ class URandomFDTests(unittest.TestCase): """ rc, out, err = assert_python_ok('-Sc', code) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. @@ -1959,13 +1982,16 @@ class LinkTests(unittest.TestCase): with open(file1, "r") as f1, open(file2, "r") as f2: self.assertTrue(os.path.sameopenfile(f1.fileno(), f2.fileno())) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_link(self): self._test_link(self.file1, self.file2) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_link_bytes(self): self._test_link(bytes(self.file1, sys.getfilesystemencoding()), bytes(self.file2, sys.getfilesystemencoding())) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_unicode_name(self): try: os.fsencode("\xf1") @@ -2228,7 +2254,7 @@ class Win32KillTests(unittest.TestCase): self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") - @unittest.skip("TODO: RUSTPYTHON") + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'x')") def test_CTRL_BREAK_EVENT(self): self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") @@ -3725,6 +3751,7 @@ class TestScandir(unittest.TestCase): entry_lstat, os.name == 'nt') + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_attributes(self): link = hasattr(os, 'link') symlink = support.can_symlink() @@ -3779,6 +3806,7 @@ class TestScandir(unittest.TestCase): filename = self.create_file(name=name) return self.get_entry(os.path.basename(filename)) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_current_directory(self): filename = self.create_file() old_dir = os.getcwd() @@ -3793,14 +3821,17 @@ class TestScandir(unittest.TestCase): finally: os.chdir(old_dir) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_repr(self): entry = self.create_file_entry() self.assertEqual(repr(entry), "") + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_fspath_protocol(self): entry = self.create_file_entry() self.assertEqual(os.fspath(entry), os.path.join(self.path, 'file.txt')) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_fspath_protocol_bytes(self): bytes_filename = os.fsencode('bytesfile.txt') bytes_entry = self.create_file_entry(name=bytes_filename) @@ -3831,6 +3862,7 @@ class TestScandir(unittest.TestCase): self.assertRaises(FileNotFoundError, entry.stat) self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_removed_file(self): entry = self.create_file_entry() os.unlink(entry.path) @@ -3850,6 +3882,7 @@ class TestScandir(unittest.TestCase): self.assertRaises(FileNotFoundError, entry.stat) self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_broken_symlink(self): if not support.can_symlink(): return self.skipTest('cannot create symbolic link') @@ -3871,6 +3904,7 @@ class TestScandir(unittest.TestCase): # don't fail entry.stat(follow_symlinks=False) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_bytes(self): self.create_file("file.txt") @@ -3883,6 +3917,7 @@ class TestScandir(unittest.TestCase): self.assertEqual(entry.path, os.fsencode(os.path.join(self.path, 'file.txt'))) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_bytes_like(self): self.create_file("file.txt") @@ -3931,6 +3966,7 @@ class TestScandir(unittest.TestCase): def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_consume_iterator_twice(self): self.create_file("file.txt") iterator = os.scandir(self.path) @@ -3946,6 +3982,7 @@ class TestScandir(unittest.TestCase): for obj in [1.234, {}, []]: self.assertRaises(TypeError, os.scandir, obj) + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_close(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -3957,6 +3994,7 @@ class TestScandir(unittest.TestCase): with self.check_no_resource_warning(): del iterator + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_context_manager(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -3965,6 +4003,7 @@ class TestScandir(unittest.TestCase): with self.check_no_resource_warning(): del iterator + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_context_manager_close(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -3972,6 +4011,7 @@ class TestScandir(unittest.TestCase): next(iterator) iterator.close() + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_context_manager_exception(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -3982,6 +4022,7 @@ class TestScandir(unittest.TestCase): with self.check_no_resource_warning(): del iterator + @unittest.skip("TODO: RUSTPYTHON (ValueError: invalid mode: 'xb')") def test_resource_warning(self): self.create_file("file.txt") self.create_file("file2.txt") From 1f1e2ee001e9ae0faaa1451659a1c954c63a8db0 Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Fri, 7 Aug 2020 03:07:16 +0900 Subject: [PATCH 05/10] Expect failure for tests raise AttributeError --- Lib/test/test_os.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index b9a4181b1..19c420c67 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -141,6 +141,8 @@ class MiscTests(unittest.TestCase): if support.verbose: print(f"Tested current directory length: {len(cwd)}") + # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'getcwdb') + @unittest.expectedFailure def test_getcwdb(self): cwd = os.getcwdb() self.assertIsInstance(cwd, bytes) @@ -159,6 +161,8 @@ class FileTests(unittest.TestCase): os.close(f) self.assertTrue(os.access(support.TESTFN, os.W_OK)) + # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'dup') + @unittest.expectedFailure def test_closerange(self): first = os.open(support.TESTFN, os.O_CREAT|os.O_RDWR) # We must allocate two consecutive file descriptors, otherwise @@ -1912,6 +1916,8 @@ class TestInvalidFD(unittest.TestCase): def test_fchmod(self): self.check(os.fchmod, 0) + # TODO: RUSTPYTHON (AttributeError: 'OSError' object has no attribute 'errno') + @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'fchown'), 'test needs os.fchown()') def test_fchown(self): self.check(os.fchown, -1, -1) @@ -2678,7 +2684,8 @@ class FSEncodingTests(unittest.TestCase): class DeviceEncodingTests(unittest.TestCase): - + # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'device_encoding') + @unittest.expectedFailure def test_bad_fd(self): # Return None when an fd doesn't actually exist. self.assertIsNone(os.device_encoding(123456)) @@ -2719,9 +2726,13 @@ class PidTests(unittest.TestCase): self.assertEqual(os.WEXITSTATUS(status), exitcode) self.assertEqual(pid2, pid) + # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv') + @unittest.expectedFailure def test_waitpid(self): self.check_waitpid(code='pass', exitcode=0) + # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'spawnv') + @unittest.expectedFailure def test_waitpid_exitcode(self): exitcode = 23 code = f'import sys; sys.exit({exitcode})' @@ -3427,6 +3438,8 @@ class OSErrorTests(unittest.TestCase): self.filenames = self.bytes_filenames + self.unicode_filenames + # TODO: RUSTPYTHON (AttributeError: 'FileNotFoundError' object has no attribute 'filename') + @unittest.expectedFailure def test_oserror_filename(self): funcs = [ (self.filenames, os.chdir,), @@ -3553,6 +3566,8 @@ class FDInheritanceTests(unittest.TestCase): self.assertEqual(os.get_inheritable(rfd), False) self.assertEqual(os.get_inheritable(wfd), False) + # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'dup') + @unittest.expectedFailure def test_dup(self): fd1 = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd1) @@ -3561,6 +3576,8 @@ class FDInheritanceTests(unittest.TestCase): self.addCleanup(os.close, fd2) self.assertEqual(os.get_inheritable(fd2), False) + # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'dup') + @unittest.expectedFailure def test_dup_standard_stream(self): fd = os.dup(1) self.addCleanup(os.close, fd) @@ -4094,6 +4111,8 @@ class TestPEP519(unittest.TestCase): class TimesTests(unittest.TestCase): + # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'times') + @unittest.expectedFailure def test_times(self): times = os.times() self.assertIsInstance(times, os.times_result) From 35667e7a480ffb28de5c448468eada9209524c2f Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Fri, 7 Aug 2020 03:12:26 +0900 Subject: [PATCH 06/10] Expect failure for failed tests --- Lib/test/test_os.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 19c420c67..ebd34cc83 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -1341,6 +1341,8 @@ class MakedirTests(unittest.TestCase): 'dir5', 'dir6') os.makedirs(path) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_mode(self): with support.temp_umask(0o002): base = support.TESTFN @@ -3609,6 +3611,8 @@ class FDInheritanceTests(unittest.TestCase): self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3) self.assertFalse(os.get_inheritable(fd3)) + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") def test_openpty(self): master_fd, slave_fd = os.openpty() @@ -3630,6 +3634,8 @@ class PathTConverterTests(unittest.TestCase): ('open', False, (0,), getattr(os, 'close', None)), ] + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_path_t_converter(self): str_filename = support.TESTFN if os.name == 'nt': @@ -3674,6 +3680,8 @@ class PathTConverterTests(unittest.TestCase): 'os.PathLike'): fn(fd, *extra_args) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_path_t_converter_and_custom_class(self): msg = r'__fspath__\(\) to return str or bytes, not %s' with self.assertRaisesRegex(TypeError, msg % r'int'): @@ -4101,6 +4109,8 @@ class TestPEP519(unittest.TestCase): self.assertRaises(ZeroDivisionError, self.fspath, FakePath(ZeroDivisionError())) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_pathlike_subclasshook(self): # bpo-38878: subclasshook causes subclass checks # true on abstract implementation. From e4770ebdf10e31960e67be3c007dae41e848d9c2 Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Sun, 9 Aug 2020 15:51:02 +0900 Subject: [PATCH 07/10] Expect failrure for tests raise TypeError --- Lib/test/test_os.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index ebd34cc83..743256854 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -222,6 +222,8 @@ class FileTests(unittest.TestCase): # operating system is free to return less bytes than requested. self.assertEqual(data, b'test') + # TODO: RUSTPYTHON (TypeError: a bytes-like object is required, not memoryview) + @unittest.expectedFailure def test_write(self): # os.write() accepts bytes- and buffer-like objects but not strings fd = os.open(support.TESTFN, os.O_CREAT | os.O_WRONLY) @@ -280,11 +282,15 @@ class FileTests(unittest.TestCase): with open(TESTFN2, 'r') as f: self.assertEqual(f.read(), "1") + # TODO: RUSTPYTHON (TypeError: Expected at least 2 arguments (0 given)) + @unittest.expectedFailure def test_open_keywords(self): f = os.open(path=__file__, flags=os.O_RDONLY, mode=0o777, dir_fd=None) os.close(f) + # TODO: RUSTPYTHON (TypeError: Expected at least 2 arguments (0 given)) + @unittest.expectedFailure def test_symlink_keywords(self): symlink = support.get_attribute(os, "symlink") try: @@ -1729,6 +1735,8 @@ def _execvpe_mockup(defpath=None): @unittest.skipUnless(hasattr(os, 'execv'), "need os.execv()") class ExecTests(unittest.TestCase): + # TODO: RUSTPYTHON (TypeError: Expected type , not ) + @unittest.expectedFailure @unittest.skipIf(USING_LINUXTHREADS, "avoid triggering a linuxthreads bug: see issue #4970") def test_execvpe_with_bad_program(self): @@ -1741,6 +1749,8 @@ class ExecTests(unittest.TestCase): self.assertRaises(ValueError, os.execv, 'notepad', ('',)) self.assertRaises(ValueError, os.execv, 'notepad', ['']) + # TODO: RUSTPYTHON (TypeError: Expected type , not ) + @unittest.expectedFailure def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) self.assertRaises(ValueError, os.execvpe, 'notepad', [], {}) @@ -2120,6 +2130,8 @@ class Pep383Tests(unittest.TestCase): def tearDown(self): shutil.rmtree(self.dir) + # TODO: RUSTPYTHON (TypeError: Expected at least 1 arguments (0 given)) + @unittest.expectedFailure def test_listdir(self): expected = self.unicodefn found = set(os.listdir(self.dir)) From 7a3ebfcd95b02865184631e358346c250546b37c Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Sun, 9 Aug 2020 16:15:46 +0900 Subject: [PATCH 08/10] Comment EnvironTests class --- Lib/test/test_os.py | 303 ++++++++++++++++++++++---------------------- 1 file changed, 152 insertions(+), 151 deletions(-) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 743256854..d4d07432b 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -847,184 +847,185 @@ class UtimeTests(unittest.TestCase): from test import mapping_tests -class EnvironTests(mapping_tests.BasicTestMappingProtocol): - """check that os.environ object conform to mapping protocol""" - type2test = None +# TODO: RUSTPYTHON (KeyError: 'surrogateescape') +# class EnvironTests(mapping_tests.BasicTestMappingProtocol): +# """check that os.environ object conform to mapping protocol""" +# type2test = None - def setUp(self): - self.__save = dict(os.environ) - if os.supports_bytes_environ: - self.__saveb = dict(os.environb) - for key, value in self._reference().items(): - os.environ[key] = value +# def setUp(self): +# self.__save = dict(os.environ) +# if os.supports_bytes_environ: +# self.__saveb = dict(os.environb) +# for key, value in self._reference().items(): +# os.environ[key] = value - def tearDown(self): - os.environ.clear() - os.environ.update(self.__save) - if os.supports_bytes_environ: - os.environb.clear() - os.environb.update(self.__saveb) +# def tearDown(self): +# os.environ.clear() +# os.environ.update(self.__save) +# if os.supports_bytes_environ: +# os.environb.clear() +# os.environb.update(self.__saveb) - def _reference(self): - return {"KEY1":"VALUE1", "KEY2":"VALUE2", "KEY3":"VALUE3"} +# def _reference(self): +# return {"KEY1":"VALUE1", "KEY2":"VALUE2", "KEY3":"VALUE3"} - def _empty_mapping(self): - os.environ.clear() - return os.environ +# def _empty_mapping(self): +# os.environ.clear() +# return os.environ - # Bug 1110478 - @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), - 'requires a shell') - def test_update2(self): - os.environ.clear() - os.environ.update(HELLO="World") - with os.popen("%s -c 'echo $HELLO'" % unix_shell) as popen: - value = popen.read().strip() - self.assertEqual(value, "World") +# # Bug 1110478 +# @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), +# 'requires a shell') +# def test_update2(self): +# os.environ.clear() +# os.environ.update(HELLO="World") +# with os.popen("%s -c 'echo $HELLO'" % unix_shell) as popen: +# value = popen.read().strip() +# self.assertEqual(value, "World") - @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), - 'requires a shell') - def test_os_popen_iter(self): - with os.popen("%s -c 'echo \"line1\nline2\nline3\"'" - % unix_shell) as popen: - it = iter(popen) - self.assertEqual(next(it), "line1\n") - self.assertEqual(next(it), "line2\n") - self.assertEqual(next(it), "line3\n") - self.assertRaises(StopIteration, next, it) +# @unittest.skipUnless(unix_shell and os.path.exists(unix_shell), +# 'requires a shell') +# def test_os_popen_iter(self): +# with os.popen("%s -c 'echo \"line1\nline2\nline3\"'" +# % unix_shell) as popen: +# it = iter(popen) +# self.assertEqual(next(it), "line1\n") +# self.assertEqual(next(it), "line2\n") +# self.assertEqual(next(it), "line3\n") +# self.assertRaises(StopIteration, next, it) - # Verify environ keys and values from the OS are of the - # correct str type. - def test_keyvalue_types(self): - for key, val in os.environ.items(): - self.assertEqual(type(key), str) - self.assertEqual(type(val), str) +# # Verify environ keys and values from the OS are of the +# # correct str type. +# def test_keyvalue_types(self): +# for key, val in os.environ.items(): +# self.assertEqual(type(key), str) +# self.assertEqual(type(val), str) - def test_items(self): - for key, value in self._reference().items(): - self.assertEqual(os.environ.get(key), value) +# def test_items(self): +# for key, value in self._reference().items(): +# self.assertEqual(os.environ.get(key), value) - # Issue 7310 - def test___repr__(self): - """Check that the repr() of os.environ looks like environ({...}).""" - env = os.environ - self.assertEqual(repr(env), 'environ({{{}}})'.format(', '.join( - '{!r}: {!r}'.format(key, value) - for key, value in env.items()))) +# # Issue 7310 +# def test___repr__(self): +# """Check that the repr() of os.environ looks like environ({...}).""" +# env = os.environ +# self.assertEqual(repr(env), 'environ({{{}}})'.format(', '.join( +# '{!r}: {!r}'.format(key, value) +# for key, value in env.items()))) - def test_get_exec_path(self): - defpath_list = os.defpath.split(os.pathsep) - test_path = ['/monty', '/python', '', '/flying/circus'] - test_env = {'PATH': os.pathsep.join(test_path)} +# def test_get_exec_path(self): +# defpath_list = os.defpath.split(os.pathsep) +# test_path = ['/monty', '/python', '', '/flying/circus'] +# test_env = {'PATH': os.pathsep.join(test_path)} - saved_environ = os.environ - try: - os.environ = dict(test_env) - # Test that defaulting to os.environ works. - self.assertSequenceEqual(test_path, os.get_exec_path()) - self.assertSequenceEqual(test_path, os.get_exec_path(env=None)) - finally: - os.environ = saved_environ +# saved_environ = os.environ +# try: +# os.environ = dict(test_env) +# # Test that defaulting to os.environ works. +# self.assertSequenceEqual(test_path, os.get_exec_path()) +# self.assertSequenceEqual(test_path, os.get_exec_path(env=None)) +# finally: +# os.environ = saved_environ - # No PATH environment variable - self.assertSequenceEqual(defpath_list, os.get_exec_path({})) - # Empty PATH environment variable - self.assertSequenceEqual(('',), os.get_exec_path({'PATH':''})) - # Supplied PATH environment variable - self.assertSequenceEqual(test_path, os.get_exec_path(test_env)) +# # No PATH environment variable +# self.assertSequenceEqual(defpath_list, os.get_exec_path({})) +# # Empty PATH environment variable +# self.assertSequenceEqual(('',), os.get_exec_path({'PATH':''})) +# # Supplied PATH environment variable +# self.assertSequenceEqual(test_path, os.get_exec_path(test_env)) - if os.supports_bytes_environ: - # env cannot contain 'PATH' and b'PATH' keys - try: - # ignore BytesWarning warning - with warnings.catch_warnings(record=True): - mixed_env = {'PATH': '1', b'PATH': b'2'} - except BytesWarning: - # mixed_env cannot be created with python -bb - pass - else: - self.assertRaises(ValueError, os.get_exec_path, mixed_env) +# if os.supports_bytes_environ: +# # env cannot contain 'PATH' and b'PATH' keys +# try: +# # ignore BytesWarning warning +# with warnings.catch_warnings(record=True): +# mixed_env = {'PATH': '1', b'PATH': b'2'} +# except BytesWarning: +# # mixed_env cannot be created with python -bb +# pass +# else: +# self.assertRaises(ValueError, os.get_exec_path, mixed_env) - # bytes key and/or value - self.assertSequenceEqual(os.get_exec_path({b'PATH': b'abc'}), - ['abc']) - self.assertSequenceEqual(os.get_exec_path({b'PATH': 'abc'}), - ['abc']) - self.assertSequenceEqual(os.get_exec_path({'PATH': b'abc'}), - ['abc']) +# # bytes key and/or value +# self.assertSequenceEqual(os.get_exec_path({b'PATH': b'abc'}), +# ['abc']) +# self.assertSequenceEqual(os.get_exec_path({b'PATH': 'abc'}), +# ['abc']) +# self.assertSequenceEqual(os.get_exec_path({'PATH': b'abc'}), +# ['abc']) - @unittest.skipUnless(os.supports_bytes_environ, - "os.environb required for this test.") - def test_environb(self): - # os.environ -> os.environb - value = 'euro\u20ac' - try: - value_bytes = value.encode(sys.getfilesystemencoding(), - 'surrogateescape') - except UnicodeEncodeError: - msg = "U+20AC character is not encodable to %s" % ( - sys.getfilesystemencoding(),) - self.skipTest(msg) - os.environ['unicode'] = value - self.assertEqual(os.environ['unicode'], value) - self.assertEqual(os.environb[b'unicode'], value_bytes) +# @unittest.skipUnless(os.supports_bytes_environ, +# "os.environb required for this test.") +# def test_environb(self): +# # os.environ -> os.environb +# value = 'euro\u20ac' +# try: +# value_bytes = value.encode(sys.getfilesystemencoding(), +# 'surrogateescape') +# except UnicodeEncodeError: +# msg = "U+20AC character is not encodable to %s" % ( +# sys.getfilesystemencoding(),) +# self.skipTest(msg) +# os.environ['unicode'] = value +# self.assertEqual(os.environ['unicode'], value) +# self.assertEqual(os.environb[b'unicode'], value_bytes) - # os.environb -> os.environ - value = b'\xff' - os.environb[b'bytes'] = value - self.assertEqual(os.environb[b'bytes'], value) - value_str = value.decode(sys.getfilesystemencoding(), 'surrogateescape') - self.assertEqual(os.environ['bytes'], value_str) +# # os.environb -> os.environ +# value = b'\xff' +# os.environb[b'bytes'] = value +# self.assertEqual(os.environb[b'bytes'], value) +# value_str = value.decode(sys.getfilesystemencoding(), 'surrogateescape') +# self.assertEqual(os.environ['bytes'], value_str) - # On OS X < 10.6, unsetenv() doesn't return a value (bpo-13415). - @support.requires_mac_ver(10, 6) - def test_unset_error(self): - if sys.platform == "win32": - # an environment variable is limited to 32,767 characters - key = 'x' * 50000 - self.assertRaises(ValueError, os.environ.__delitem__, key) - else: - # "=" is not allowed in a variable name - key = 'key=' - self.assertRaises(OSError, os.environ.__delitem__, key) +# # On OS X < 10.6, unsetenv() doesn't return a value (bpo-13415). +# @support.requires_mac_ver(10, 6) +# def test_unset_error(self): +# if sys.platform == "win32": +# # an environment variable is limited to 32,767 characters +# key = 'x' * 50000 +# self.assertRaises(ValueError, os.environ.__delitem__, key) +# else: +# # "=" is not allowed in a variable name +# key = 'key=' +# self.assertRaises(OSError, os.environ.__delitem__, key) - def test_key_type(self): - missing = 'missingkey' - self.assertNotIn(missing, os.environ) +# def test_key_type(self): +# missing = 'missingkey' +# self.assertNotIn(missing, os.environ) - with self.assertRaises(KeyError) as cm: - os.environ[missing] - self.assertIs(cm.exception.args[0], missing) - self.assertTrue(cm.exception.__suppress_context__) +# with self.assertRaises(KeyError) as cm: +# os.environ[missing] +# self.assertIs(cm.exception.args[0], missing) +# self.assertTrue(cm.exception.__suppress_context__) - with self.assertRaises(KeyError) as cm: - del os.environ[missing] - self.assertIs(cm.exception.args[0], missing) - self.assertTrue(cm.exception.__suppress_context__) +# with self.assertRaises(KeyError) as cm: +# del os.environ[missing] +# self.assertIs(cm.exception.args[0], missing) +# self.assertTrue(cm.exception.__suppress_context__) - def _test_environ_iteration(self, collection): - iterator = iter(collection) - new_key = "__new_key__" +# def _test_environ_iteration(self, collection): +# iterator = iter(collection) +# new_key = "__new_key__" - next(iterator) # start iteration over os.environ.items +# next(iterator) # start iteration over os.environ.items - # add a new key in os.environ mapping - os.environ[new_key] = "test_environ_iteration" +# # add a new key in os.environ mapping +# os.environ[new_key] = "test_environ_iteration" - try: - next(iterator) # force iteration over modified mapping - self.assertEqual(os.environ[new_key], "test_environ_iteration") - finally: - del os.environ[new_key] +# try: +# next(iterator) # force iteration over modified mapping +# self.assertEqual(os.environ[new_key], "test_environ_iteration") +# finally: +# del os.environ[new_key] - def test_iter_error_when_changing_os_environ(self): - self._test_environ_iteration(os.environ) +# def test_iter_error_when_changing_os_environ(self): +# self._test_environ_iteration(os.environ) - def test_iter_error_when_changing_os_environ_items(self): - self._test_environ_iteration(os.environ.items()) +# def test_iter_error_when_changing_os_environ_items(self): +# self._test_environ_iteration(os.environ.items()) - def test_iter_error_when_changing_os_environ_values(self): - self._test_environ_iteration(os.environ.values()) +# def test_iter_error_when_changing_os_environ_values(self): +# self._test_environ_iteration(os.environ.values()) class WalkTests(unittest.TestCase): From 70e4ee92bd5eca073094f420295bc5c0909b6a04 Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Sun, 9 Aug 2020 16:29:17 +0900 Subject: [PATCH 09/10] Expect failure for tests raise various errors --- Lib/test/test_os.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index d4d07432b..881231e31 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -192,6 +192,8 @@ class FileTests(unittest.TestCase): new = sys.getrefcount(path) self.assertEqual(old, new) + # TODO: RUSTPYTHON (OSError: Bad file descriptor (os error 9)) + @unittest.expectedFailure def test_read(self): with open(support.TESTFN, "w+b") as fobj: fobj.write(b"spam") @@ -1646,6 +1648,8 @@ class URandomFDTests(unittest.TestCase): """ assert_python_ok('-c', code) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_urandom_fd_closed(self): # Issue #21207: urandom() should reopen its fd to /dev/urandom if # closed. @@ -1806,11 +1810,15 @@ class ExecTests(unittest.TestCase): self.assertSequenceEqual(calls[0], ('execve', native_fullpath, (arguments, env_path))) + # TODO: RUSTPYTHON (NameError: name 'orig_execve' is not defined) + @unittest.expectedFailure def test_internal_execvpe_str(self): self._test_internal_execvpe(str) if os.name != "nt": self._test_internal_execvpe(bytes) + # TODO: RUSTPYTHON (AttributeError: module 'os' has no attribute 'execve') + @unittest.expectedFailure def test_execve_invalid_env(self): args = [sys.executable, '-c', 'pass'] @@ -2064,6 +2072,8 @@ class PosixUidGidTests(unittest.TestCase): self.assertRaises(OverflowError, os.setreuid, self.UID_OVERFLOW, 0) self.assertRaises(OverflowError, os.setreuid, 0, self.UID_OVERFLOW) + # TODO: RUSTPYTHON (subprocess.CalledProcessError) + @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()') def test_setreuid_neg1(self): # Needs to accept -1. We run this in a subprocess to avoid @@ -3878,6 +3888,8 @@ class TestScandir(unittest.TestCase): self.assertEqual(fspath, os.path.join(os.fsencode(self.path),bytes_filename)) + # TODO: RUSTPYTHON (FileNotFoundError: No such file or directory (os error 2)) + @unittest.expectedFailure def test_removed_dir(self): path = os.path.join(self.path, 'dir') From 10b5c2f078b63152a821a95e255b532090ca215c Mon Sep 17 00:00:00 2001 From: Yi Hyunjoon Date: Sun, 9 Aug 2020 17:36:36 +0900 Subject: [PATCH 10/10] Patch for ubuntu-latest CI --- Lib/test/test_os.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 881231e31..1d1431f09 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -2091,6 +2091,8 @@ class PosixUidGidTests(unittest.TestCase): self.assertRaises(OverflowError, os.setregid, self.GID_OVERFLOW, 0) self.assertRaises(OverflowError, os.setregid, 0, self.GID_OVERFLOW) + # TODO: RUSTPYTHON (subprocess.CalledProcessError) + @unittest.expectedFailure @unittest.skipUnless(hasattr(os, 'setregid'), 'test needs os.setregid()') def test_setregid_neg1(self): # Needs to accept -1. We run this in a subprocess to avoid