diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 7bdf415db655f32..0a4a02c45b533bd 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1448,7 +1448,7 @@ or `the MSDN `_ on Windo Return a pair of file descriptors ``(r, w)`` usable for reading and writing, respectively. - .. availability:: Unix, not WASI, not macOS, not iOS. + .. availability:: Unix, macOS >= 27.0, not WASI, not iOS. .. versionadded:: 3.3 diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 660847ae3fe3c85..893154246ae4d81 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -963,7 +963,7 @@ The :mod:`!test.support` module defines the following functions: .. currentmodule:: test.support.isolation -.. decorator:: runInSubprocess() +.. decorator:: runInSubprocess(*, options=(), env=None, timeout=None) Decorator that runs the decorated test in a fresh interpreter subprocess, in isolation, so that it does not share global or interpreter state with the @@ -997,6 +997,19 @@ The :mod:`!test.support` module defines the following functions: :func:`~test.support.bigmemtest` and the like behave consistently in both processes. + *options* is a sequence of interpreter command line options + to run the subprocess with, + and *env* is a mapping of environment variables to set in it, + on top of the inherited environment. + A value of ``None`` in *env* unsets the variable. + Note that :option:`-E` and :option:`-I` make the subprocess ignore + the ``PYTHON*`` environment variables, including :envvar:`PYTHONPATH`. + + *timeout* is the number of seconds to wait for the subprocess; + the test is reported as an error if it does not complete in time. + By default there is no timeout, + and a hung test is left to the timeout of the test runner. + The test is skipped on platforms without subprocess support. diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index bd73ddb95cd07f7..bc2fb742cb9cfca 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -10,7 +10,7 @@ __all__ = ["version", "bootstrap"] -_PIP_VERSION = "26.2" +_PIP_VERSION = "26.2.1" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora diff --git a/Lib/ensurepip/_bundled/pip-26.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-26.2.1-py3-none-any.whl similarity index 94% rename from Lib/ensurepip/_bundled/pip-26.2-py3-none-any.whl rename to Lib/ensurepip/_bundled/pip-26.2.1-py3-none-any.whl index e0cc1e2f8dd971c..bc442f6b99306b3 100644 Binary files a/Lib/ensurepip/_bundled/pip-26.2-py3-none-any.whl and b/Lib/ensurepip/_bundled/pip-26.2.1-py3-none-any.whl differ diff --git a/Lib/test/_isolated_sample.py b/Lib/test/_isolated_sample.py index c89f7145e7328d3..5853b654fc28cb6 100644 --- a/Lib/test/_isolated_sample.py +++ b/Lib/test/_isolated_sample.py @@ -7,6 +7,7 @@ import atexit import os +import sys import time import unittest from test.support import isolation @@ -141,3 +142,39 @@ def test_pass(self): def test_dies(self): _die_at_exit() + + +@isolation.runInSubprocess(options=['-X', 'dev', '-W', 'error::BytesWarning']) +class OptionsSample(unittest.TestCase): + + def test_options_applied(self): + self.assertTrue(sys.flags.dev_mode) + self.assertIn('error::BytesWarning', sys.warnoptions) + + +class EnvSample(unittest.TestCase): + + @isolation.runInSubprocess(env={'_PYTHON_ISOLATION_PROBE': 'set-by-test'}) + def test_env_set(self): + self.assertEqual(os.environ.get('_PYTHON_ISOLATION_PROBE'), 'set-by-test') + + @isolation.runInSubprocess(env={'_PYTHON_ISOLATION_PROBE': None}) + def test_env_unset(self): + self.assertNotIn('_PYTHON_ISOLATION_PROBE', os.environ) + + @isolation.runInSubprocess() + def test_env_inherited(self): + # Without env= the subprocess inherits the parent environment as it is. + self.assertEqual(os.environ.get('_PYTHON_ISOLATION_PROBE'), 'set-by-parent') + + +# TimeoutSample hangs this long, so that the timeout always fires first. +TIMEOUT_HANG = 60.0 +TIMEOUT = 0.5 + + +class TimeoutSample(unittest.TestCase): + + @isolation.runInSubprocess(timeout=TIMEOUT) + def test_hang(self): + time.sleep(TIMEOUT_HANG) diff --git a/Lib/test/clinic.test.c b/Lib/test/clinic.test.c index 171570588e7a2b5..146f57a2a11342a 100644 --- a/Lib/test/clinic.test.c +++ b/Lib/test/clinic.test.c @@ -5769,6 +5769,146 @@ Test___init___impl(TestObj *self, PyObject *a, int group_right_1, /*[clinic end generated code: output=2bbb8ea60e8f57a6 input=10f5d0f1e8e466ef]*/ +/*[clinic input] +group_and_optional_parameter + [ + a: object + b: object + ] + c: object = None + / +The optional parameter can be omitted with or without the group. +[clinic start generated code]*/ + +PyDoc_STRVAR(group_and_optional_parameter__doc__, +"group_and_optional_parameter([a, b,] c=None)\n" +"The optional parameter can be omitted with or without the group."); + +#define GROUP_AND_OPTIONAL_PARAMETER_METHODDEF \ + {"group_and_optional_parameter", (PyCFunction)group_and_optional_parameter, METH_VARARGS, group_and_optional_parameter__doc__}, + +static PyObject * +group_and_optional_parameter_impl(PyObject *module, int group_left_1, + PyObject *a, PyObject *b, PyObject *c); + +static PyObject * +group_and_optional_parameter(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int group_left_1 = 0; + PyObject *a = NULL; + PyObject *b = NULL; + PyObject *c = Py_None; + + switch (PyTuple_GET_SIZE(args)) { + case 0: + case 1: + if (!PyArg_ParseTuple(args, "|O:group_and_optional_parameter", &c)) { + goto exit; + } + break; + case 2: + case 3: + if (!PyArg_ParseTuple(args, "OO|O:group_and_optional_parameter", &a, &b, &c)) { + goto exit; + } + group_left_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "group_and_optional_parameter requires 0 to 3 arguments"); + goto exit; + } + return_value = group_and_optional_parameter_impl(module, group_left_1, a, b, c); + +exit: + return return_value; +} + +static PyObject * +group_and_optional_parameter_impl(PyObject *module, int group_left_1, + PyObject *a, PyObject *b, PyObject *c) +/*[clinic end generated code: output=3faea69eafd5bbbe input=7f0fbb6124f5a972]*/ + + +/*[clinic input] +two_groups_on_the_same_level + [ + a: object + b: object + ] + [ + c: object + ] + d: object + / +Groups on the same level are independent of each other. +[clinic start generated code]*/ + +PyDoc_STRVAR(two_groups_on_the_same_level__doc__, +"two_groups_on_the_same_level([a, b,] [c,] d)\n" +"Groups on the same level are independent of each other."); + +#define TWO_GROUPS_ON_THE_SAME_LEVEL_METHODDEF \ + {"two_groups_on_the_same_level", (PyCFunction)two_groups_on_the_same_level, METH_VARARGS, two_groups_on_the_same_level__doc__}, + +static PyObject * +two_groups_on_the_same_level_impl(PyObject *module, int group_left_1, + PyObject *a, PyObject *b, int group_left_2, + PyObject *c, PyObject *d); + +static PyObject * +two_groups_on_the_same_level(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int group_left_1 = 0; + PyObject *a = NULL; + PyObject *b = NULL; + int group_left_2 = 0; + PyObject *c = NULL; + PyObject *d; + + switch (PyTuple_GET_SIZE(args)) { + case 1: + if (!PyArg_ParseTuple(args, "O:two_groups_on_the_same_level", &d)) { + goto exit; + } + break; + case 2: + if (!PyArg_ParseTuple(args, "OO:two_groups_on_the_same_level", &c, &d)) { + goto exit; + } + group_left_2 = 1; + break; + case 3: + if (!PyArg_ParseTuple(args, "OOO:two_groups_on_the_same_level", &a, &b, &d)) { + goto exit; + } + group_left_1 = 1; + break; + case 4: + if (!PyArg_ParseTuple(args, "OOOO:two_groups_on_the_same_level", &a, &b, &c, &d)) { + goto exit; + } + group_left_1 = 1; + group_left_2 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "two_groups_on_the_same_level requires 1 to 4 arguments"); + goto exit; + } + return_value = two_groups_on_the_same_level_impl(module, group_left_1, a, b, group_left_2, c, d); + +exit: + return return_value; +} + +static PyObject * +two_groups_on_the_same_level_impl(PyObject *module, int group_left_1, + PyObject *a, PyObject *b, int group_left_2, + PyObject *c, PyObject *d) +/*[clinic end generated code: output=508a61ee582da21e input=1b45d9b675b32d1a]*/ + + /*[clinic input] Test._pyarg_parsestackandkeywords cls: defining_class diff --git a/Lib/test/support/isolation.py b/Lib/test/support/isolation.py index bc2189329c03997..bb4fa6b003cc20c 100644 --- a/Lib/test/support/isolation.py +++ b/Lib/test/support/isolation.py @@ -78,7 +78,11 @@ def _decode(data): def _remote(detail): # Wrap the subprocess traceback the way concurrent.futures does, so it is - # clearly delimited when shown as the cause. + # clearly delimited when shown as the cause. Return None if the subprocess + # said nothing (a hung one usually does not), so that "raise ... from None" + # suppresses an empty cause. + if not detail: + return None return _RemoteTraceback(f'\n"""\n{detail}"""') @@ -90,7 +94,21 @@ def _check_subprocess_support(): raise unittest.SkipTest('requires subprocess support') -def _run_in_subprocess(module, qualname): +def _child_environ(env): + # Start from the inherited environment, so that *env* only has to name what + # the test changes. + if not env: + return None + environ = dict(os.environ) + for name, value in env.items(): + if value is None: + environ.pop(name, None) + else: + environ[name] = value + return environ + + +def _run_in_subprocess(module, qualname, options, env, timeout): """Run module.qualname (a test method or class) in a fresh subprocess. Return ``(payload, output, returncode)``, where *payload* is the decoded @@ -104,13 +122,22 @@ def _run_in_subprocess(module, qualname): os.close(fd) try: # Pass the config on the command line, not in the environment, so that - # the test cannot pass it on to the processes it spawns itself. Use - # marshal, not json: it is built in, so the child imports nothing that - # the test would not see in a normal test run. - cmd = [sys.executable, '-m', 'test.support.subprocess_runner', + # the test cannot pass it on to the processes it spawns itself, and so + # that it survives the -E and -I options. Use marshal, not json: it is + # built in, so the child imports nothing that the test would not see in + # a normal test run. + cmd = [sys.executable, *options, '-m', 'test.support.subprocess_runner', module, qualname, result_path, marshal.dumps(_child_config()).hex()] - proc = subprocess.run(cmd, capture_output=True) + try: + proc = subprocess.run(cmd, capture_output=True, + env=_child_environ(env), timeout=timeout) + except subprocess.TimeoutExpired as exc: + # Report the hang rather than leaving the test runner stuck. + output = _decode(exc.stdout) + _decode(exc.stderr) + raise _SubprocessTestError( + f'test did not complete in a subprocess ' + f'within {timeout} seconds') from _remote(output) try: with open(result_path, 'rb') as f: payload = marshal.load(f) @@ -173,7 +200,7 @@ def _check_returncode(returncode, output, what): raise exc from _remote(output) -def _isolate_method(func): +def _isolate_method(func, options, env, timeout): @functools.wraps(func) def wrapper(self, /, *args, **kwargs): if runningInSubprocess: @@ -183,7 +210,8 @@ def wrapper(self, /, *args, **kwargs): cls = type(self) qualname = f'{cls.__qualname__}.{func.__name__}' payload, output, returncode = _run_in_subprocess(cls.__module__, - qualname) + qualname, options, + env, timeout) if payload is None: exc = _SubprocessTestError( f'test did not complete in a subprocess (exit code {returncode})') @@ -196,7 +224,7 @@ def wrapper(self, /, *args, **kwargs): return wrapper -def _isolate_class(cls): +def _isolate_class(cls, options, env, timeout): # Unwrap to the plain functions so the replacements can call them with the # runtime cls; a bound classmethod would freeze the decoration-time class # and a subclass would run the fixtures bound to the base class. @@ -217,7 +245,8 @@ def setUpClass(cls): # Run the whole class in a single subprocess and stash the outcomes # for the test methods to replay. payload, output, returncode = _run_in_subprocess(cls.__module__, - cls.__qualname__) + cls.__qualname__, + options, env, timeout) if payload is None: exc = _SubprocessTestError( f'class did not complete in a subprocess (exit code {returncode})') @@ -283,7 +312,7 @@ def _addDuration(self, result, elapsed): return cls -def runInSubprocess(): +def runInSubprocess(*, options=(), env=None, timeout=None): """Decorator to run a test method or class in a fresh subprocess. The decorated test runs in a separate, fresh Python process, so it does not @@ -293,6 +322,16 @@ def runInSubprocess(): once there; when a method is decorated, only that method runs in a subprocess. Decorated methods must take no extra arguments. + *options* is a sequence of interpreter command line options for the + subprocess, and *env* is a mapping of environment variables to set in it, + on top of the inherited environment; a value of ``None`` unsets a variable. + Note that ``-E`` and ``-I`` make the subprocess ignore the ``PYTHON*`` + variables, including ``PYTHONPATH``. + + *timeout* is the number of seconds to wait for the subprocess; the test is + reported as an error if it does not complete in time. By default there is + no timeout, and a hung test is left to the timeout of the test runner. + A failure, error or skip of the whole test is reported for the test, and individual subtests (:meth:`~unittest.TestCase.subTest`) that fail or are skipped are reported individually. The original subprocess traceback is @@ -304,6 +343,6 @@ def runInSubprocess(): """ def decorator(obj): if isinstance(obj, type) and issubclass(obj, unittest.TestCase): - return _isolate_class(obj) - return _isolate_method(obj) + return _isolate_class(obj, options, env, timeout) + return _isolate_method(obj, options, env, timeout) return decorator diff --git a/Lib/test/test_clinic.py b/Lib/test/test_clinic.py index 93c284e58764f46..cb4507dcac2336d 100644 --- a/Lib/test/test_clinic.py +++ b/Lib/test/test_clinic.py @@ -9,6 +9,7 @@ from test.support.os_helper import TESTFN, unlink, rmtree from textwrap import dedent from unittest import TestCase +import difflib import inspect import os.path import re @@ -330,6 +331,24 @@ def __init__(self): """ self.expect_failure(block, err, lineno=8) + def test_ambiguous_group_and_optional_parameters(self): + err = ("Function 'my_test_func' has an ambiguous group configuration: " + "a call with 2 argument(s) can be parsed in more than one way.") + block = """ + /*[clinic input] + my_test_func + + [ + a: object + b: object + ] + c: object = None + d: object = None + / + [clinic start generated code]*/ + """ + self.expect_failure(block, err) + def test_star_after_vararg(self): err = "'my_test_func' uses '*' more than once." block = """ @@ -817,7 +836,7 @@ def _test(self, l, m, r, output): self.assertEqual(output, computed) def test_range(self): - self._test([['start']], ['stop'], [['step']], + self._test([[['start']]], ['stop'], [[['step']]], ( ('stop',), ('start', 'stop',), @@ -825,7 +844,7 @@ def test_range(self): )) def test_add_window(self): - self._test([['x', 'y']], ['ch'], [['attr']], + self._test([[['x', 'y']]], ['ch'], [[['attr']]], ( ('ch',), ('ch', 'attr'), @@ -834,7 +853,8 @@ def test_add_window(self): )) def test_ludicrous(self): - self._test([['a1', 'a2', 'a3'], ['b1', 'b2']], ['c1'], [['d1', 'd2'], ['e1', 'e2', 'e3']], + self._test([[['a1', 'a2', 'a3'], ['b1', 'b2']]], ['c1'], + [[['d1', 'd2'], ['e1', 'e2', 'e3']]], ( ('c1',), ('b1', 'b2', 'c1'), @@ -845,7 +865,7 @@ def test_ludicrous(self): )) def test_right_only(self): - self._test([], [], [['a'],['b'],['c']], + self._test([], [], [[['a'],['b'],['c']]], ( (), ('a',), @@ -853,9 +873,28 @@ def test_right_only(self): ('a', 'b', 'c') )) + def test_chgat(self): + # Two independent groups on the left. + self._test([[['y', 'x']], [['n']]], ['attr'], [], + ( + ('attr',), + ('n', 'attr'), + ('y', 'x', 'attr'), + ('y', 'x', 'n', 'attr'), + )) + + def test_independent_groups_on_the_right(self): + self._test([], ['a'], [[['b']], [['c', 'd']]], + ( + ('a',), + ('a', 'b'), + ('a', 'c', 'd'), + ('a', 'b', 'c', 'd'), + )) + def test_have_left_options_but_required_is_empty(self): def fn(): - permute_optional_groups(['a'], [], []) + permute_optional_groups([[['a']]], [], []) self.assertRaises(ValueError, fn) @@ -1696,41 +1735,74 @@ def test_nested_groups(self): Attributes for the character. """) - def test_disallowed_grouping__two_top_groups_on_left(self): - err = ( - "Function 'two_top_groups_on_left' has an unsupported group " - "configuration. (Unexpected state 2.b)" - ) - block = """ - module foo - foo.two_top_groups_on_left + def test_two_top_groups_on_left(self): + function = self.parse_function(""" + module curses + curses.chgat [ - group1 : int + y: int + Y-coordinate. + x: int + X-coordinate. ] [ - group2 : int + num: int + Number of characters. ] - param: int - """ - self.expect_failure(block, err, lineno=5) + attr: long + Attributes for the characters. + / + """) + dataset = ( + ('y', -1), ('x', -1), + ('num', -2), + ('attr', 0), + ) + for name, group in dataset: + with self.subTest(name=name, group=group): + p = function.parameters[name] + self.assertEqual(p.group, group) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + self.checkDocstring(function, """ + chgat([y, x,] [num,] attr) - def test_disallowed_grouping__two_top_groups_on_right(self): - block = """ + + y + Y-coordinate. + x + X-coordinate. + num + Number of characters. + attr + Attributes for the characters. + """) + + def test_two_top_groups_on_right(self): + function = self.parse_function(""" module foo foo.two_top_groups_on_right param: int [ - group1 : int + group1: int ] [ - group2 : int + group2: int ] - """ - err = ( - "Function 'two_top_groups_on_right' has an unsupported group " - "configuration. (Unexpected state 6.b)" + / + """) + dataset = ( + ('param', 0), + ('group1', 1), + ('group2', 2), ) - self.expect_failure(block, err) + for name, group in dataset: + with self.subTest(name=name, group=group): + p = function.parameters[name] + self.assertEqual(p.group, group) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + self.checkDocstring(function, """ + two_top_groups_on_right(param, [group1,] [group2]) + """) def test_disallowed_grouping__parameter_after_group_on_right(self): block = """ @@ -3042,6 +3114,148 @@ def test_cli_force(self): generated = f.read() self.assertEndsWith(generated, checksum) + DRY_RUN_CODE = dedent(""" + /*[clinic input] + func + a: int + / + + Docstring. + [clinic start generated code]*/ + """) + + def make_dry_run_file(self, tmp_dir): + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write(self.DRY_RUN_CODE) + return fn + + @staticmethod + def dest_file(fn): + # The default destination for the generated code. Its path is + # built from the "{dirname}/clinic/{basename}.h" template, so it + # always uses forward slashes, even on Windows. + dirname, basename = os.path.split(fn) + return f"{dirname}/clinic/{basename}.h" + + def check_unchanged(self, tmp_dir, fn, pre_mtime): + # Neither the source file nor the destination file + # nor its directory is created or modified. + with open(fn, encoding="utf-8") as f: + self.assertEqual(f.read(), self.DRY_RUN_CODE) + self.assertEqual(os.stat(fn).st_mtime_ns, pre_mtime) + self.assertEqual(os.listdir(tmp_dir), ["test.c"]) + + def test_cli_dry_run(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_dry_run_file(tmp_dir) + pre_mtime = os.stat(fn).st_mtime_ns + out = self.expect_success("--dry-run", fn) + self.assertEqual(out.splitlines(), [ + f"would create {self.dest_file(fn)}", + f"would update {fn}", + ]) + self.check_unchanged(tmp_dir, fn, pre_mtime) + + def test_cli_dry_run_no_change(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_dry_run_file(tmp_dir) + self.expect_success(fn) + self.assertEqual(self.expect_success("--dry-run", fn), "") + self.assertEqual(self.expect_success("--diff", fn), "") + + def test_cli_dry_run_no_clinic_block(self): + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write("int x;\n") + self.assertEqual(self.expect_success("--dry-run", fn), "") + + def test_cli_dry_run_output(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_dry_run_file(tmp_dir) + out_fn = os.path.join(tmp_dir, "output.c") + out = self.expect_success("--dry-run", "-o", out_fn, fn) + self.assertIn(f"would create {out_fn}", out) + self.assertNotIn(f"would update {fn}", out) + self.assertFalse(os.path.exists(out_fn)) + + def test_cli_dry_run_make(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_dry_run_file(tmp_dir) + pre_mtime = os.stat(fn).st_mtime_ns + out = self.expect_success("--dry-run", "--make", "--srcdir", tmp_dir) + self.assertIn(f"would update {fn}", out) + self.check_unchanged(tmp_dir, fn, pre_mtime) + + def test_cli_dry_run_verbose(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_dry_run_file(tmp_dir) + out, err, code = self.run_clinic("-v", "--dry-run", fn) + self.assertEqual(code, 0) + # The progress goes to stderr, so that the standard output + # contains only the report. + self.assertEqual(err.splitlines(), [fn]) + self.assertEqual(out.splitlines(), [ + f"would create {self.dest_file(fn)}", + f"would update {fn}", + ]) + + def test_cli_dry_run_checksum_mismatch(self): + invalid_input = dedent(""" + /*[clinic input] + output preset block + module test + test.fn + a: int + [clinic start generated code]*/ + /*[clinic end generated code: output=bogus input=bogus]*/ + """) + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write(invalid_input) + pre_mtime = os.stat(fn).st_mtime_ns + # The dry run does not disable the checksum verification. + _, err = self.expect_failure("--dry-run", fn) + self.assertIn("Checksum mismatch!", err) + # With -f the change is reported, but still not written. + out = self.expect_success("--dry-run", "-f", fn) + self.assertIn(f"would update {fn}", out) + with open(fn, encoding="utf-8") as f: + self.assertEqual(f.read(), invalid_input) + self.assertEqual(os.stat(fn).st_mtime_ns, pre_mtime) + + def test_cli_diff(self): + with os_helper.temp_dir() as tmp_dir: + fn = self.make_dry_run_file(tmp_dir) + pre_mtime = os.stat(fn).st_mtime_ns + out = self.expect_success("--diff", fn) + self.check_unchanged(tmp_dir, fn, pre_mtime) + + # A new file is created by the patch. + dest_fn = self.dest_file(fn) + self.assertStartsWith(out, f"--- /dev/null\n+++ {dest_fn}\n@@ -0,0 +1,") + self.assertIn(f"--- {fn}\n+++ {fn}\n", out) + self.assertIn("+/*[clinic end generated code:", out) + + # The patch is what clinic would have written. + self.expect_success(fn) + with open(fn, encoding="utf-8") as f: + new_contents = f.read() + expected = "".join(difflib.unified_diff( + self.DRY_RUN_CODE.splitlines(keepends=True), + new_contents.splitlines(keepends=True), + fromfile=fn, tofile=fn)) + self.assertEndsWith(out, expected) + + def test_cli_fail_converters_and_dry_run(self): + for opt in "--dry-run", "--diff": + with self.subTest(opt=opt): + _, err = self.expect_failure("--converters", opt) + msg = "can't use --dry-run or --diff with --converters" + self.assertIn(msg, err) + def test_cli_make(self): c_code = dedent(""" /*[clinic input] @@ -3203,10 +3417,66 @@ def test_cli_converters(self): with self.subTest(converter=converter): self.assertStartsWith(line, converter) - def test_cli_fail_converters_and_filename(self): - _, err = self.expect_failure("--converters", "test.c") - msg = "can't specify --converters and a filename at the same time" - self.assertIn(msg, err) + def test_cli_converters_file(self): + code = dedent(""" + /*[python input] + class my_type_converter(CConverter): + type = 'my_type' + converter = 'my_type_converter' + + def converter_init(self, *, strict=False): + pass + + class my_result_return_converter(CReturnConverter): + type = 'my_result' + [python start generated code]*/ + """) + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write(code) + out = self.expect_success("--converters", fn) + self.assertIn("Converters:\n my_type(strict=False)\n", out) + self.assertIn("Return converters:\n my_result()\n", out) + # Only the converters defined in the file are listed. + self.assertNotIn("Legacy converters:", out) + self.assertNotIn("bool(", out) + # Listing the converters does not write anything. + with open(fn, encoding="utf-8") as f: + self.assertEqual(f.read(), code) + self.assertEqual(os.listdir(tmp_dir), ["test.c"]) + + def test_cli_converters_make(self): + code = dedent(""" + /*[python input] + class my_type_converter(CConverter): + type = 'my_type' + converter = 'my_type_converter' + [python start generated code]*/ + """) + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write(code) + out = self.expect_success("--converters", "--make", + "--srcdir", tmp_dir) + self.assertIn("Converters:\n my_type()\n", out) + with open(fn, encoding="utf-8") as f: + self.assertEqual(f.read(), code) + + def test_cli_converters_no_converters(self): + with os_helper.temp_dir() as tmp_dir: + fn = os.path.join(tmp_dir, "test.c") + with open(fn, "w", encoding="utf-8") as f: + f.write("/*[clinic input]\n[clinic start generated code]*/\n") + self.assertEqual(self.expect_success("--converters", fn), "") + + def test_cli_fail_directory(self): + with os_helper.temp_dir() as tmp_dir: + subdir = os.path.join(tmp_dir, "test.c") + os.mkdir(subdir) + _, err = self.expect_failure(subdir) + self.assertIn(f"Can't read file {subdir!r}: it is a directory", err) def test_cli_fail_no_filename(self): _, err = self.expect_failure() @@ -3865,6 +4135,47 @@ def test_varpos_kwonly_req_opt(self): self.assertEqual(fn(1, a=2, b=3), ((1,), 2, 3, False)) self.assertEqual(fn(1, a=2, b=3, c=4), ((1,), 2, 3, 4)) + def test_group_and_opt(self): + # fn([a, b,] c=None) + fn = ac_tester.group_and_opt + self.assertEqual(fn(), (False, None, None, None)) + self.assertEqual(fn(1), (False, None, None, 1)) + self.assertEqual(fn(1, 2), (True, 1, 2, None)) + self.assertEqual(fn(1, 2, 3), (True, 1, 2, 3)) + self.assertRaises(TypeError, fn, 1, 2, 3, 4) + self.assertRaises(TypeError, fn, c=1) + + def test_group_and_two_opt(self): + # fn([a, b, c,] d=None, e=None) + fn = ac_tester.group_and_two_opt + self.assertEqual(fn(), (False, None, None, None, None, None)) + self.assertEqual(fn(1), (False, None, None, None, 1, None)) + self.assertEqual(fn(1, 2), (False, None, None, None, 1, 2)) + self.assertEqual(fn(1, 2, 3), (True, 1, 2, 3, None, None)) + self.assertEqual(fn(1, 2, 3, 4), (True, 1, 2, 3, 4, None)) + self.assertEqual(fn(1, 2, 3, 4, 5), (True, 1, 2, 3, 4, 5)) + self.assertRaises(TypeError, fn, 1, 2, 3, 4, 5, 6) + + def test_two_groups_on_left(self): + # fn([a, b,] [c,] d) + fn = ac_tester.two_groups_on_left + self.assertRaises(TypeError, fn) + self.assertEqual(fn(1), (False, None, None, False, None, 1)) + self.assertEqual(fn(1, 2), (False, None, None, True, 1, 2)) + self.assertEqual(fn(1, 2, 3), (True, 1, 2, False, None, 3)) + self.assertEqual(fn(1, 2, 3, 4), (True, 1, 2, True, 3, 4)) + self.assertRaises(TypeError, fn, 1, 2, 3, 4, 5) + + def test_two_groups_on_right(self): + # fn(a, [b,] [c, d]) + fn = ac_tester.two_groups_on_right + self.assertRaises(TypeError, fn) + self.assertEqual(fn(1), (1, False, None, False, None, None)) + self.assertEqual(fn(1, 2), (1, True, 2, False, None, None)) + self.assertEqual(fn(1, 2, 3), (1, False, None, True, 2, 3)) + self.assertEqual(fn(1, 2, 3, 4), (1, True, 2, True, 3, 4)) + self.assertRaises(TypeError, fn, 1, 2, 3, 4, 5) + def test_gh_32092_oob(self): ac_tester.gh_32092_oob(1, 2, 3, 4, kw1=5, kw2=6) @@ -4467,21 +4778,21 @@ def test_permute_optional_groups(self): "expected": ((),), } noleft1 = { - "left": (), "required": ("b",), "right": ("c",), + "left": (), "required": ("b",), "right": (("c",),), "expected": ( ("b",), ("b", "c"), ), } noleft2 = { - "left": (), "required": ("b", "c",), "right": ("d",), + "left": (), "required": ("b", "c",), "right": (("d",),), "expected": ( ("b", "c"), ("b", "c", "d"), ), } noleft3 = { - "left": (), "required": ("b", "c",), "right": ("d", "e"), + "left": (), "required": ("b", "c",), "right": (("d", "e"),), "expected": ( ("b", "c"), ("b", "c", "d"), @@ -4489,21 +4800,21 @@ def test_permute_optional_groups(self): ), } noright1 = { - "left": ("a",), "required": ("b",), "right": (), + "left": (("a",),), "required": ("b",), "right": (), "expected": ( ("b",), ("a", "b"), ), } noright2 = { - "left": ("a",), "required": ("b", "c"), "right": (), + "left": (("a",),), "required": ("b", "c"), "right": (), "expected": ( ("b", "c"), ("a", "b", "c"), ), } noright3 = { - "left": ("a", "b"), "required": ("c",), "right": (), + "left": (("a", "b"),), "required": ("c",), "right": (), "expected": ( ("c",), ("b", "c"), @@ -4511,7 +4822,7 @@ def test_permute_optional_groups(self): ), } leftandright1 = { - "left": ("a",), "required": ("b",), "right": ("c",), + "left": (("a",),), "required": ("b",), "right": (("c",),), "expected": ( ("b",), ("a", "b"), # Prefer left. @@ -4519,7 +4830,7 @@ def test_permute_optional_groups(self): ), } leftandright2 = { - "left": ("a", "b"), "required": ("c", "d"), "right": ("e", "f"), + "left": (("a", "b"),), "required": ("c", "d"), "right": (("e", "f"),), "expected": ( ("c", "d"), ("b", "c", "d"), # Prefer left. @@ -4528,11 +4839,28 @@ def test_permute_optional_groups(self): ("a", "b", "c", "d", "e", "f"), ), } + independentleft = { + "left": (("a",), ("b",)), "required": ("c",), "right": (), + "expected": ( + ("c",), + ("b", "c"), + ("a", "b", "c"), + ), + } + independentright = { + "left": (), "required": ("a",), "right": (("b",), ("c",)), + "expected": ( + ("a",), + ("a", "b"), + ("a", "b", "c"), + ), + } dataset = ( empty, noleft1, noleft2, noleft3, noright1, noright2, noright3, leftandright1, leftandright2, + independentleft, independentright, ) for params in dataset: with self.subTest(**params): diff --git a/Lib/test/test_os/test_posix.py b/Lib/test/test_os/test_posix.py index 41a730708974c25..8743b0bf0bc4939 100644 --- a/Lib/test/test_os/test_posix.py +++ b/Lib/test/test_os/test_posix.py @@ -2388,6 +2388,22 @@ def test_pwritev(self): self.assertNotHasAttr(os, "pwritev") self.assertNotHasAttr(os, "preadv") + def test_pipe2(self): + self._verify_available("HAVE_PIPE2") + if self.mac_ver >= (27, 0): + self.assertHasAttr(os, "pipe2") + else: + self.assertNotHasAttr(os, "pipe2") + + def test_dup3(self): + self._verify_available("HAVE_DUP3") + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + # Must not crash even when dup3 unavailable at runtime. + # os.dup2 returns fd2 (here w); do not double-close. + os.dup2(r, w, inheritable=False) + def test_stat(self): self._verify_available("HAVE_FSTATAT") if self.mac_ver >= (10, 10): diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 2317077b30ac388..7c59bb38aaee9ae 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -1205,6 +1205,32 @@ def test_class_subprocess_dying_after_the_tests_is_reported(self): self.assertIn('tearDownClass', str(result.errors[0][0])) self.assertIn(f'exited with code {EXIT_CODE}', result.errors[0][1]) + @support.requires_subprocess() + def test_options_passed_to_subprocess(self): + result = self._run('OptionsSample') + self.assertEqual(result.testsRun, 1) + self.assertEqual(result.failures, []) + self.assertEqual(result.errors, []) + + @support.requires_subprocess() + def test_env_passed_to_subprocess(self): + # The samples check the variable, so set it here to let them tell + # env= from the inherited environment. + with os_helper.EnvironmentVarGuard() as env: + env['_PYTHON_ISOLATION_PROBE'] = 'set-by-parent' + result = self._run('EnvSample') + self.assertEqual(result.testsRun, 3) + self.assertEqual(result.failures, []) + self.assertEqual(result.errors, []) + + @support.requires_subprocess() + def test_timeout_reported_as_error(self): + from test._isolated_sample import TIMEOUT + result = self._run('TimeoutSample') + self.assertEqual(result.testsRun, 1) + self.assertEqual(len(result.errors), 1) + self.assertIn(f'within {TIMEOUT} seconds', result.errors[0][1]) + def test_skipped_without_subprocess_support(self): # On a platform without subprocess support the test is skipped in the # parent, before any subprocess is spawned. diff --git a/Misc/NEWS.d/next/Library/2026-08-04-14-14-31.gh-issue-153711.PBpc1g.rst b/Misc/NEWS.d/next/Library/2026-08-04-14-14-31.gh-issue-153711.PBpc1g.rst new file mode 100644 index 000000000000000..9552dde73918c43 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-04-14-14-31.gh-issue-153711.PBpc1g.rst @@ -0,0 +1,4 @@ +On macOS, add run-time checks around the syscalls :manpage:`pipe2 (2)` and +:manpage:`dup3 (2)`, in addition to the existing build-time checks. This +means that Python built on macOS 27 (where these calls are available) can +run on macOS 26 (where they aren't). diff --git a/Misc/NEWS.d/next/Library/2026-08-04-19-03-20.gh-issue-155063.gN_mCU.rst b/Misc/NEWS.d/next/Library/2026-08-04-19-03-20.gh-issue-155063.gN_mCU.rst new file mode 100644 index 000000000000000..fdb03c907a581ca --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-04-19-03-20.gh-issue-155063.gN_mCU.rst @@ -0,0 +1 @@ +Bump the version of pip bundled in ensurepip to version 26.2.1 diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-08-04-17-12-33.gh-issue-64502.Qv7mLp.rst b/Misc/NEWS.d/next/Tools-Demos/2026-08-04-17-12-33.gh-issue-64502.Qv7mLp.rst new file mode 100644 index 000000000000000..da9647d1fdd369b --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-08-04-17-12-33.gh-issue-64502.Qv7mLp.rst @@ -0,0 +1,3 @@ +Fix Argument Clinic support of parameters with a default value used together +with optional groups. +Such parameters were always required in the generated parsing code. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-08-04-19-05-41.gh-issue-64502.Rb2wKt.rst b/Misc/NEWS.d/next/Tools-Demos/2026-08-04-19-05-41.gh-issue-64502.Rb2wKt.rst new file mode 100644 index 000000000000000..d3e75435df91824 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-08-04-19-05-41.gh-issue-64502.Rb2wKt.rst @@ -0,0 +1,3 @@ +Argument Clinic now supports several optional groups on the same nesting +level, like in ``[y, x,] [n,] attr``. +Such groups can be omitted independently of each other. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-08-05-10-24-17.gh-issue-155207.Wv3nKq.rst b/Misc/NEWS.d/next/Tools-Demos/2026-08-05-10-24-17.gh-issue-155207.Wv3nKq.rst new file mode 100644 index 000000000000000..392ab42bffa726e --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-08-05-10-24-17.gh-issue-155207.Wv3nKq.rst @@ -0,0 +1,3 @@ +Argument Clinic now supports the ``--dry-run`` and ``--diff`` options. +They list the files which would be changed, or write a unified diff of the +changes to the standard output, without modifying any file. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-08-05-12-40-08.gh-issue-155212.Lx8vTm.rst b/Misc/NEWS.d/next/Tools-Demos/2026-08-05-12-40-08.gh-issue-155212.Lx8vTm.rst new file mode 100644 index 000000000000000..a4b52430436f3dd --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-08-05-12-40-08.gh-issue-155212.Lx8vTm.rst @@ -0,0 +1,4 @@ +The ``--converters`` option of Argument Clinic now accepts file names and +can be used with ``--make``. +It prints the converters and return converters which the specified files +define, instead of the built-in ones. diff --git a/Misc/NEWS.d/next/Tools-Demos/2026-08-05-14-02-55.gh-issue-155218.Nq4xZv.rst b/Misc/NEWS.d/next/Tools-Demos/2026-08-05-14-02-55.gh-issue-155218.Nq4xZv.rst new file mode 100644 index 000000000000000..909efc558693c0a --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2026-08-05-14-02-55.gh-issue-155218.Nq4xZv.rst @@ -0,0 +1,2 @@ +Fix Argument Clinic generating the flags of the optional groups in +different order on 32-bit and 64-bit platforms. diff --git a/Modules/_testclinic.c b/Modules/_testclinic.c index 66a375589ba38e5..c53bf4a08753586 100644 --- a/Modules/_testclinic.c +++ b/Modules/_testclinic.c @@ -1237,6 +1237,104 @@ posonly_poskw_varpos_array_impl(PyObject *module, PyObject *a, PyObject *b, } +/*[clinic input] +group_and_opt + + [ + a: object + b: object + ] + c: object = None + / + +[clinic start generated code]*/ + +static PyObject * +group_and_opt_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, PyObject *c) +/*[clinic end generated code: output=23413ec545526111 input=8a84d8f44bc8bd0b]*/ +{ + return pack_arguments_newref(4, group_left_1 ? Py_True : Py_False, + a, b, c); +} + + +/*[clinic input] +two_groups_on_left + + [ + a: object + b: object + ] + [ + c: object + ] + d: object + / + +[clinic start generated code]*/ + +static PyObject * +two_groups_on_left_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, int group_left_2, PyObject *c, + PyObject *d) +/*[clinic end generated code: output=3a43d6542864e3d8 input=79fc792669696ac7]*/ +{ + return pack_arguments_newref(6, group_left_1 ? Py_True : Py_False, a, b, + group_left_2 ? Py_True : Py_False, c, d); +} + + +/*[clinic input] +two_groups_on_right + + a: object + [ + b: object + ] + [ + c: object + d: object + ] + / + +[clinic start generated code]*/ + +static PyObject * +two_groups_on_right_impl(PyObject *module, PyObject *a, int group_right_1, + PyObject *b, int group_right_2, PyObject *c, + PyObject *d) +/*[clinic end generated code: output=045f60f127c6e448 input=96895285f29bb501]*/ +{ + return pack_arguments_newref(6, a, group_right_1 ? Py_True : Py_False, b, + group_right_2 ? Py_True : Py_False, c, d); +} + + +/*[clinic input] +group_and_two_opt + + [ + a: object + b: object + c: object + ] + d: object = None + e: object = None + / + +[clinic start generated code]*/ + +static PyObject * +group_and_two_opt_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, PyObject *e) +/*[clinic end generated code: output=1427c4b3c35f24ff input=cdda98eec1e365ea]*/ +{ + return pack_arguments_newref(6, group_left_1 ? Py_True : Py_False, + a, b, c, d, e); +} + + /*[clinic input] gh_32092_oob @@ -2455,6 +2553,10 @@ static PyMethodDef tester_methods[] = { POSONLY_VARPOS_ARRAY_METHODDEF POSONLY_REQ_OPT_VARPOS_ARRAY_METHODDEF POSONLY_POSKW_VARPOS_ARRAY_METHODDEF + GROUP_AND_OPT_METHODDEF + GROUP_AND_TWO_OPT_METHODDEF + TWO_GROUPS_ON_LEFT_METHODDEF + TWO_GROUPS_ON_RIGHT_METHODDEF GH_32092_OOB_METHODDEF GH_32092_KW_PASS_METHODDEF diff --git a/Modules/clinic/_cursesmodule.c.h b/Modules/clinic/_cursesmodule.c.h index dfd589ba45089e7..58087416796f8fc 100644 --- a/Modules/clinic/_cursesmodule.c.h +++ b/Modules/clinic/_cursesmodule.c.h @@ -247,8 +247,8 @@ _curses_window_addch(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "iiOO&:addch", &y, &x, &ch, attr_converter, &attr)) { goto exit; } - group_right_1 = 1; group_left_1 = 1; + group_right_1 = 1; break; default: PyErr_SetString(PyExc_TypeError, "_curses.window.addch requires 1 to 4 arguments"); @@ -319,8 +319,8 @@ _curses_window_addstr(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "iiOO&:addstr", &y, &x, &str, attr_converter, &attr)) { goto exit; } - group_right_1 = 1; group_left_1 = 1; + group_right_1 = 1; break; default: PyErr_SetString(PyExc_TypeError, "_curses.window.addstr requires 1 to 4 arguments"); @@ -394,8 +394,8 @@ _curses_window_addnstr(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "iiOiO&:addnstr", &y, &x, &str, &n, attr_converter, &attr)) { goto exit; } - group_right_1 = 1; group_left_1 = 1; + group_right_1 = 1; break; default: PyErr_SetString(PyExc_TypeError, "_curses.window.addnstr requires 2 to 5 arguments"); @@ -1451,8 +1451,8 @@ _curses_window_hline(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "iiOiO&:hline", &y, &x, &ch, &n, attr_converter, &attr)) { goto exit; } - group_right_1 = 1; group_left_1 = 1; + group_right_1 = 1; break; default: PyErr_SetString(PyExc_TypeError, "_curses.window.hline requires 2 to 5 arguments"); @@ -1521,8 +1521,8 @@ _curses_window_insch(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "iiOO&:insch", &y, &x, &ch, attr_converter, &attr)) { goto exit; } - group_right_1 = 1; group_left_1 = 1; + group_right_1 = 1; break; default: PyErr_SetString(PyExc_TypeError, "_curses.window.insch requires 1 to 4 arguments"); @@ -1640,8 +1640,8 @@ _curses_window_insstr(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "iiOO&:insstr", &y, &x, &str, attr_converter, &attr)) { goto exit; } - group_right_1 = 1; group_left_1 = 1; + group_right_1 = 1; break; default: PyErr_SetString(PyExc_TypeError, "_curses.window.insstr requires 1 to 4 arguments"); @@ -1717,8 +1717,8 @@ _curses_window_insnstr(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "iiOiO&:insnstr", &y, &x, &str, &n, attr_converter, &attr)) { goto exit; } - group_right_1 = 1; group_left_1 = 1; + group_right_1 = 1; break; default: PyErr_SetString(PyExc_TypeError, "_curses.window.insnstr requires 2 to 5 arguments"); @@ -2328,8 +2328,8 @@ _curses_window_vline(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "iiOiO&:vline", &y, &x, &ch, &n, attr_converter, &attr)) { goto exit; } - group_right_1 = 1; group_left_1 = 1; + group_right_1 = 1; break; default: PyErr_SetString(PyExc_TypeError, "_curses.window.vline requires 2 to 5 arguments"); @@ -6234,4 +6234,4 @@ _curses_has_extended_color_support(PyObject *module, PyObject *Py_UNUSED(ignored #ifndef _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #define _CURSES_ASSUME_DEFAULT_COLORS_METHODDEF #endif /* !defined(_CURSES_ASSUME_DEFAULT_COLORS_METHODDEF) */ -/*[clinic end generated code: output=cb5525c88ae5c440 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c1f85ec415c303bf input=a9049054013a1b77]*/ diff --git a/Modules/clinic/_testclinic.c.h b/Modules/clinic/_testclinic.c.h index 05615c1fdd81b9c..3fe32d704f0140f 100644 --- a/Modules/clinic/_testclinic.c.h +++ b/Modules/clinic/_testclinic.c.h @@ -3477,6 +3477,210 @@ posonly_poskw_varpos_array(PyObject *module, PyObject *const *args, Py_ssize_t n return return_value; } +PyDoc_STRVAR(group_and_opt__doc__, +"group_and_opt([a, b,] c=None)"); + +#define GROUP_AND_OPT_METHODDEF \ + {"group_and_opt", (PyCFunction)group_and_opt, METH_VARARGS, group_and_opt__doc__}, + +static PyObject * +group_and_opt_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, PyObject *c); + +static PyObject * +group_and_opt(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int group_left_1 = 0; + PyObject *a = NULL; + PyObject *b = NULL; + PyObject *c = Py_None; + + switch (PyTuple_GET_SIZE(args)) { + case 0: + case 1: + if (!PyArg_ParseTuple(args, "|O:group_and_opt", &c)) { + goto exit; + } + break; + case 2: + case 3: + if (!PyArg_ParseTuple(args, "OO|O:group_and_opt", &a, &b, &c)) { + goto exit; + } + group_left_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "group_and_opt requires 0 to 3 arguments"); + goto exit; + } + return_value = group_and_opt_impl(module, group_left_1, a, b, c); + +exit: + return return_value; +} + +PyDoc_STRVAR(two_groups_on_left__doc__, +"two_groups_on_left([a, b,] [c,] d)"); + +#define TWO_GROUPS_ON_LEFT_METHODDEF \ + {"two_groups_on_left", (PyCFunction)two_groups_on_left, METH_VARARGS, two_groups_on_left__doc__}, + +static PyObject * +two_groups_on_left_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, int group_left_2, PyObject *c, + PyObject *d); + +static PyObject * +two_groups_on_left(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int group_left_1 = 0; + PyObject *a = NULL; + PyObject *b = NULL; + int group_left_2 = 0; + PyObject *c = NULL; + PyObject *d; + + switch (PyTuple_GET_SIZE(args)) { + case 1: + if (!PyArg_ParseTuple(args, "O:two_groups_on_left", &d)) { + goto exit; + } + break; + case 2: + if (!PyArg_ParseTuple(args, "OO:two_groups_on_left", &c, &d)) { + goto exit; + } + group_left_2 = 1; + break; + case 3: + if (!PyArg_ParseTuple(args, "OOO:two_groups_on_left", &a, &b, &d)) { + goto exit; + } + group_left_1 = 1; + break; + case 4: + if (!PyArg_ParseTuple(args, "OOOO:two_groups_on_left", &a, &b, &c, &d)) { + goto exit; + } + group_left_1 = 1; + group_left_2 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "two_groups_on_left requires 1 to 4 arguments"); + goto exit; + } + return_value = two_groups_on_left_impl(module, group_left_1, a, b, group_left_2, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(two_groups_on_right__doc__, +"two_groups_on_right(a, [b,] [c, d])"); + +#define TWO_GROUPS_ON_RIGHT_METHODDEF \ + {"two_groups_on_right", (PyCFunction)two_groups_on_right, METH_VARARGS, two_groups_on_right__doc__}, + +static PyObject * +two_groups_on_right_impl(PyObject *module, PyObject *a, int group_right_1, + PyObject *b, int group_right_2, PyObject *c, + PyObject *d); + +static PyObject * +two_groups_on_right(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + PyObject *a; + int group_right_1 = 0; + PyObject *b = NULL; + int group_right_2 = 0; + PyObject *c = NULL; + PyObject *d = NULL; + + switch (PyTuple_GET_SIZE(args)) { + case 1: + if (!PyArg_ParseTuple(args, "O:two_groups_on_right", &a)) { + goto exit; + } + break; + case 2: + if (!PyArg_ParseTuple(args, "OO:two_groups_on_right", &a, &b)) { + goto exit; + } + group_right_1 = 1; + break; + case 3: + if (!PyArg_ParseTuple(args, "OOO:two_groups_on_right", &a, &c, &d)) { + goto exit; + } + group_right_2 = 1; + break; + case 4: + if (!PyArg_ParseTuple(args, "OOOO:two_groups_on_right", &a, &b, &c, &d)) { + goto exit; + } + group_right_1 = 1; + group_right_2 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "two_groups_on_right requires 1 to 4 arguments"); + goto exit; + } + return_value = two_groups_on_right_impl(module, a, group_right_1, b, group_right_2, c, d); + +exit: + return return_value; +} + +PyDoc_STRVAR(group_and_two_opt__doc__, +"group_and_two_opt([a, b, c,] d=None, e=None)"); + +#define GROUP_AND_TWO_OPT_METHODDEF \ + {"group_and_two_opt", (PyCFunction)group_and_two_opt, METH_VARARGS, group_and_two_opt__doc__}, + +static PyObject * +group_and_two_opt_impl(PyObject *module, int group_left_1, PyObject *a, + PyObject *b, PyObject *c, PyObject *d, PyObject *e); + +static PyObject * +group_and_two_opt(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + int group_left_1 = 0; + PyObject *a = NULL; + PyObject *b = NULL; + PyObject *c = NULL; + PyObject *d = Py_None; + PyObject *e = Py_None; + + switch (PyTuple_GET_SIZE(args)) { + case 0: + case 1: + case 2: + if (!PyArg_ParseTuple(args, "|OO:group_and_two_opt", &d, &e)) { + goto exit; + } + break; + case 3: + case 4: + case 5: + if (!PyArg_ParseTuple(args, "OOO|OO:group_and_two_opt", &a, &b, &c, &d, &e)) { + goto exit; + } + group_left_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "group_and_two_opt requires 0 to 5 arguments"); + goto exit; + } + return_value = group_and_two_opt_impl(module, group_left_1, a, b, c, d, e); + +exit: + return return_value; +} + PyDoc_STRVAR(gh_32092_oob__doc__, "gh_32092_oob($module, /, pos1, pos2, *varargs, kw1=None, kw2=None)\n" "--\n" @@ -4600,4 +4804,4 @@ _testclinic_TestClass_posonly_poskw_varpos_array_no_fastcall(PyObject *type, PyO exit: return return_value; } -/*[clinic end generated code: output=9971dbbc5f62b8d2 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d9d4091b2f2ed359 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c34e3fc5eb600df..db65d5862440655 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -504,6 +504,8 @@ static const unsigned int _Py_STATX_KNOWN = (STATX_BASIC_STATS | STATX_BTIME # define HAVE_MKFIFOAT_RUNTIME __builtin_available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) # define HAVE_MKNODAT_RUNTIME __builtin_available(macOS 13.0, iOS 16.0, tvOS 16.0, watchOS 9.0, *) # define HAVE_PTSNAME_R_RUNTIME __builtin_available(macOS 10.13.4, iOS 11.3, tvOS 11.3, watchOS 4.3, *) +# define HAVE_DUP3_RUNTIME __builtin_available(macOS 27.0, *) +# define HAVE_PIPE2_RUNTIME __builtin_available(macOS 27.0, *) # define HAVE_POSIX_SPAWN_SETSID_RUNTIME __builtin_available(macOS 10.15, *) @@ -589,6 +591,14 @@ static const unsigned int _Py_STATX_KNOWN = (STATX_BASIC_STATS | STATX_BTIME # define HAVE_PTSNAME_R_RUNTIME (ptsname_r != NULL) # endif +# ifdef HAVE_DUP3 +# define HAVE_DUP3_RUNTIME (dup3 != NULL) +# endif + +# ifdef HAVE_PIPE2 +# define HAVE_PIPE2_RUNTIME (pipe2 != NULL) +# endif + #endif #ifdef HAVE_FUTIMESAT @@ -619,6 +629,8 @@ static const unsigned int _Py_STATX_KNOWN = (STATX_BASIC_STATS | STATX_BTIME # define HAVE_MKFIFOAT_RUNTIME 1 # define HAVE_MKNODAT_RUNTIME 1 # define HAVE_PTSNAME_R_RUNTIME 1 +# define HAVE_DUP3_RUNTIME 1 +# define HAVE_PIPE2_RUNTIME 1 #endif @@ -11866,11 +11878,16 @@ os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) /*[clinic end generated code: output=bc059d34a73404d1 input=c3cddda8922b038d]*/ { int res = 0; -#if defined(HAVE_DUP3) && \ - !(defined(HAVE_FCNTL_H) && defined(F_DUP2FD_CLOEXEC)) - /* dup3() is available on Linux 2.6.27+ and glibc 2.9 */ - static int dup3_works = -1; -#endif + + /* dup3() is available on Linux 2.6.27+ and glibc 2.9 and macOS 27.0; + * it needs runtime detection for the case of running on older kernels. + * Values: -1: unknown; 0: doesn't work; 1: works + * For thread safety, use a process-global with one read & one store, + * both relaxed. (It's fine if two threads race and do the detection + * simultaneously; they should get the same result.) + */ + static int dup3_works_atomic = -1; + (void) dup3_works_atomic; // unused on some platforms /* dup2() can fail with EINTR if the target FD is already open, because it * then has to be closed. See os_close_impl() for why we don't handle EINTR @@ -11909,18 +11926,27 @@ os_dup2_impl(PyObject *module, int fd, int fd2, int inheritable) #else #ifdef HAVE_DUP3 + int dup3_works = FT_ATOMIC_LOAD_INT_RELAXED(dup3_works_atomic); if (!inheritable && dup3_works != 0) { - Py_BEGIN_ALLOW_THREADS - res = dup3(fd, fd2, O_CLOEXEC); - Py_END_ALLOW_THREADS - if (res < 0) { - if (dup3_works == -1) - dup3_works = (errno != ENOSYS); - if (dup3_works) { - posix_error(); - return -1; + if (HAVE_DUP3_RUNTIME) { + Py_BEGIN_ALLOW_THREADS + res = dup3(fd, fd2, O_CLOEXEC); + Py_END_ALLOW_THREADS + if (res < 0) { + if (dup3_works == -1) { + dup3_works = (errno != ENOSYS); + FT_ATOMIC_STORE_INT_RELAXED(dup3_works_atomic, dup3_works); + } + if (dup3_works) { + posix_error(); + return -1; + } } } + else { + dup3_works = 0; + FT_ATOMIC_STORE_INT_RELAXED(dup3_works_atomic, dup3_works); + } } if (inheritable || dup3_works == 0) @@ -12761,7 +12787,13 @@ os_pipe_impl(PyObject *module) SECURITY_ATTRIBUTES attr; BOOL ok; #else - int res; + int res = -1; + + /* pipe2() is available on some newer linux/glibc & macOS; + * use the same runtime detection as for dup3 above. + */ + static int pipe2_works_atomic = -1; + (void) pipe2_works_atomic; // unused on some platforms #endif #ifdef MS_WINDOWS @@ -12787,11 +12819,30 @@ os_pipe_impl(PyObject *module) #else #ifdef HAVE_PIPE2 - Py_BEGIN_ALLOW_THREADS - res = pipe2(fds, O_CLOEXEC); - Py_END_ALLOW_THREADS + int pipe2_works = FT_ATOMIC_LOAD_INT_RELAXED(pipe2_works_atomic); + if (pipe2_works != 0) { + if (HAVE_PIPE2_RUNTIME) { + Py_BEGIN_ALLOW_THREADS + res = pipe2(fds, O_CLOEXEC); + Py_END_ALLOW_THREADS + if (pipe2_works == -1) { + if (res != 0 && errno == ENOSYS) { + pipe2_works = 0; + } + else { + // pipe2 is present but this call failed + pipe2_works = 1; + } + FT_ATOMIC_STORE_INT_RELAXED(pipe2_works_atomic, pipe2_works); + } + } + else { + pipe2_works = 0; + FT_ATOMIC_STORE_INT_RELAXED(pipe2_works_atomic, pipe2_works); + } + } - if (res != 0 && errno == ENOSYS) + if (pipe2_works == 0) { #endif Py_BEGIN_ALLOW_THREADS @@ -12814,8 +12865,9 @@ os_pipe_impl(PyObject *module) } #endif - if (res != 0) + if (res != 0) { return PyErr_SetFromErrno(PyExc_OSError); + } #endif /* !MS_WINDOWS */ return Py_BuildValue("(ii)", fds[0], fds[1]); } @@ -12845,9 +12897,17 @@ os_pipe2_impl(PyObject *module, int flags) int fds[2]; int res; - res = pipe2(fds, flags); - if (res != 0) + if (HAVE_PIPE2_RUNTIME) { + res = pipe2(fds, flags); + } + else { + res = -1; + errno = ENOSYS; + } + if (res != 0) { return posix_error(); + } + return Py_BuildValue("(ii)", fds[0], fds[1]); } #endif /* HAVE_PIPE2 */ @@ -18844,6 +18904,22 @@ posixmodule_exec(PyObject *m) } #endif +#if HAVE_PIPE2 + if (HAVE_PIPE2_RUNTIME) { + // Do nothing. (`__builtin_available` doesn't allow `!`; see + // "using negations" in a comment above.) + } + else { + PyObject* dct = PyModule_GetDict(m); + if (dct == NULL) { + return -1; + } + if (PyDict_PopString(dct, "pipe2", NULL) < 0) { + return -1; + } + } +#endif + /* Initialize environ dictionary */ if (PyModule_Add(m, "environ", convertenviron()) != 0) { return -1; diff --git a/Tools/c-analyzer/cpython/_parser.py b/Tools/c-analyzer/cpython/_parser.py index 2875f45cb8d3756..3d755765b967097 100644 --- a/Tools/c-analyzer/cpython/_parser.py +++ b/Tools/c-analyzer/cpython/_parser.py @@ -318,6 +318,7 @@ def format_tsv_lines(lines): _abs('Modules/_remote_debugging/debug_offsets_validation.h'): (25_000, 1000), _abs('Modules/_remote_debugging/*.h'): (20_000, 1000), _abs('Modules/_testcapimodule.c'): (20_000, 400), + _abs('Modules/_testclinic.c'): (20_000, 400), _abs('Modules/expat/expat.h'): (10_000, 400), _abs('Objects/stringlib/unicode_format.h'): (10_000, 400), _abs('Objects/typeobject.c'): (380_000, 13_000), diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index ef314625d507d61..4c143164650a2ba 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -19,9 +19,9 @@ Python/bootstrap_hash.c py_getrandom getrandom_works - Python/bootstrap_hash.c py_getentropy getentropy_works - Python/fileutils.c - _Py_open_cloexec_works - Python/fileutils.c set_inheritable ioctl_works - -# (set lazily, *after* first init) -# XXX Is this thread-safe? -Modules/posixmodule.c os_dup2_impl dup3_works - +# (set lazily, atomically, *after* first init) +Modules/posixmodule.c os_dup2_impl dup3_works_atomic - +Modules/posixmodule.c os_pipe_impl pipe2_works_atomic - ## guards around resource init Python/thread_pthread.h PyThread__init_thread lib_initialized - diff --git a/Tools/clinic/libclinic/__init__.py b/Tools/clinic/libclinic/__init__.py index 5ee165d0c138a85..ff8f44be0b774ff 100644 --- a/Tools/clinic/libclinic/__init__.py +++ b/Tools/clinic/libclinic/__init__.py @@ -26,6 +26,8 @@ is_legal_py_identifier, ) from .utils import ( + FileChange, + FileWriter, FormatCounterFormatter, NULL, NullType, @@ -33,6 +35,7 @@ VersionTuple, compute_checksum, create_regex, + read_file, unknown, unspecified, write_file, @@ -66,6 +69,8 @@ "is_legal_py_identifier", # Utility functions + "FileChange", + "FileWriter", "FormatCounterFormatter", "NULL", "NullType", @@ -73,6 +78,7 @@ "VersionTuple", "compute_checksum", "create_regex", + "read_file", "unknown", "unspecified", "write_file", diff --git a/Tools/clinic/libclinic/app.py b/Tools/clinic/libclinic/app.py index 9e8cec5320f8772..d8de3687a35ce64 100644 --- a/Tools/clinic/libclinic/app.py +++ b/Tools/clinic/libclinic/app.py @@ -87,6 +87,7 @@ def __init__( filename: str, limited_capi: bool, verify: bool = True, + writer: libclinic.FileWriter | None = None, ) -> None: # maps strings to Parser objects. # (instantiated from the "parsers" global.) @@ -95,6 +96,7 @@ def __init__( if printer: fail("Custom printers are broken right now") self.printer = printer or BlockPrinter(language) + self.writer = writer or libclinic.FileWriter() self.verify = verify self.limited_capi = limited_capi self.filename = filename @@ -213,7 +215,7 @@ def parse(self, input: str) -> str: try: dirname = os.path.dirname(destination.filename) try: - os.makedirs(dirname) + self.writer.makedirs(dirname) except FileExistsError: if not os.path.isdir(dirname): fail(f"Can't write to destination " @@ -234,8 +236,8 @@ def parse(self, input: str) -> str: printer_2 = BlockPrinter(self.language) printer_2.print_block(block, header_includes=includes) - libclinic.write_file(destination.filename, - printer_2.f.getvalue()) + self.writer.write(destination.filename, + printer_2.f.getvalue()) continue return printer.f.getvalue() diff --git a/Tools/clinic/libclinic/clanguage.py b/Tools/clinic/libclinic/clanguage.py index 7f02c7790f015aa..1581a19a4fd78ab 100644 --- a/Tools/clinic/libclinic/clanguage.py +++ b/Tools/clinic/libclinic/clanguage.py @@ -1,6 +1,5 @@ from __future__ import annotations import itertools -import sys import textwrap from typing import TYPE_CHECKING, Literal, Final from operator import attrgetter @@ -12,7 +11,7 @@ from libclinic.codegen import CRenderData, TemplateDict, CodeGen from libclinic.language import Language from libclinic.function import ( - Module, Class, Function, Parameter, + Module, Class, Function, Parameter, ParamTuple, permute_optional_groups, GETTER, SETTER, METHOD_INIT) from libclinic.converters import self_converter @@ -21,6 +20,20 @@ from libclinic.app import Clinic +def count_required(subset: ParamTuple) -> int: + """Return the number of arguments which cannot be omitted. + + A parameter in an optional group is passed together with its group, + so only trailing parameters with a default value can be omitted. + """ + count = len(subset) + for p in reversed(subset): + if p.group or not p.is_optional(): + break + count -= 1 + return count + + def c_id(name: str) -> str: if len(name) == 1 and ord(name) < 256: if name.isalnum(): @@ -275,44 +288,67 @@ def render_option_group_parsing( # What if the number of arguments leads us to an ambiguous result? # Clinic prefers groups on the left. So in the above example, # five arguments would map to B+C, not C+D. + # + # A nested group can only be omitted together with the group + # containing it, but groups on the same level, like G and H in + # + # [ G1 G2 ] [ H1 ] I1 I2 + # + # can be omitted independently of each other. out = [] parameters = list(f.parameters.values()) if isinstance(parameters[0].converter, self_converter): del parameters[0] + # Groups are collected into chains of nested groups. A group which + # is not nested in the preceding one starts a new chain. group: list[Parameter] | None = None - left = [] - right = [] + left: list[list[list[Parameter]]] = [] + right: list[list[list[Parameter]]] = [] required: list[Parameter] = [] last: int | Literal[Sentinels.unspecified] = unspecified + last_depth = 0 for p in parameters: group_id = p.group if group_id != last: last = group_id group = [] - if group_id < 0: - left.append(group) - elif group_id == 0: + if group_id == 0: group = required else: - right.append(group) + chains = left if group_id < 0 else right + nested = ((p.group_depth < last_depth) if group_id < 0 + else (p.group_depth > last_depth)) + if chains and nested: + chains[-1].append(group) + else: + chains.append([group]) + last_depth = p.group_depth assert group is not None group.append(p) - count_min = sys.maxsize - count_max = -1 + # Map the number of arguments to the subset which accepts it. + subsets: dict[int, ParamTuple] = {} + for subset in permute_optional_groups(left, required, right): + for count in range(count_required(subset), len(subset) + 1): + if count in subsets: + fail(f"Function {f.full_name!r} has an ambiguous group " + f"configuration: a call with {count} argument(s) " + f"can be parsed in more than one way.") + subsets[count] = subset if limited_capi: nargs = 'PyTuple_Size(args)' else: nargs = 'PyTuple_GET_SIZE(args)' out.append(f"switch ({nargs}) {{\n") - for subset in permute_optional_groups(left, required, right): - count = len(subset) - count_min = min(count_min, count) - count_max = max(count_max, count) + for count, subset in sorted(subsets.items()): + if count < len(subset): + # The omitted parameters are parsed by the following case. + out.append(f" case {count}:\n") + continue if count == 0: out.append(""" case 0: @@ -320,18 +356,24 @@ def render_option_group_parsing( """) continue - group_ids = {p.group for p in subset} # eliminate duplicates + # A set would eliminate duplicates too, but the iteration + # order of small negative integers depends on the platform. + group_ids = dict.fromkeys(p.group for p in subset) d: dict[str, str | int] = {} d['count'] = count d['name'] = f.name - d['format_units'] = "".join(p.converter.format_unit for p in subset) + format_units = [p.converter.format_unit for p in subset] + n_required = count_required(subset) + if n_required < count: + format_units.insert(n_required, '|') + d['format_units'] = "".join(format_units) parse_arguments: list[str] = [] for p in subset: p.converter.parse_argument(parse_arguments) d['parse_arguments'] = ", ".join(parse_arguments) - group_ids.discard(0) + group_ids.pop(0, None) lines = "\n".join([ self.group_to_variable_name(g) + " = 1;" for g in group_ids @@ -351,7 +393,7 @@ def render_option_group_parsing( out.append(" default:\n") s = ' PyErr_SetString(PyExc_TypeError, "{} requires {} to {} arguments");\n' - out.append(s.format(f.full_name, count_min, count_max)) + out.append(s.format(f.full_name, min(subsets), max(subsets))) out.append(' goto exit;\n') out.append("}") diff --git a/Tools/clinic/libclinic/cli.py b/Tools/clinic/libclinic/cli.py index f36c6d04efd3835..c66084cf3144826 100644 --- a/Tools/clinic/libclinic/cli.py +++ b/Tools/clinic/libclinic/cli.py @@ -1,11 +1,12 @@ from __future__ import annotations import argparse +import difflib import inspect import os import re import sys -from collections.abc import Callable +from collections.abc import Callable, Iterable, Iterator, Mapping from typing import NoReturn @@ -52,9 +53,12 @@ def parse_file( limited_capi: bool, output: str | None = None, verify: bool = True, + writer: libclinic.FileWriter | None = None, ) -> None: if not output: output = filename + if writer is None: + writer = libclinic.FileWriter() extension = os.path.splitext(filename)[1][1:] if not extension: @@ -65,6 +69,9 @@ def parse_file( except KeyError: raise ClinicError(f"Can't identify file type for file {filename!r}") + if os.path.isdir(filename): + raise ClinicError(f"Can't read file {filename!r}: it is a directory") + with open(filename, encoding="utf-8") as f: raw = f.read() @@ -80,10 +87,11 @@ def parse_file( clinic = Clinic(language, verify=verify, filename=filename, - limited_capi=limited_capi) + limited_capi=limited_capi, + writer=writer) cooked = clinic.parse(raw) - libclinic.write_file(output, cooked) + writer.write(output, cooked) def create_cli() -> argparse.ArgumentParser: @@ -102,9 +110,17 @@ def create_cli() -> argparse.ArgumentParser: help="redirect file output to OUTPUT") cmdline.add_argument("-v", "--verbose", action='store_true', help="enable verbose mode") + cmdline.add_argument("--dry-run", action='store_true', + help=("don't write any file, only list the files " + "which would be changed")) + cmdline.add_argument("--diff", action='store_true', + help=("don't write any file, write a unified diff " + "of the changes to the standard output")) cmdline.add_argument("--converters", action='store_true', help=("print a list of all supported converters " - "and return converters")) + "and return converters; if files are " + "specified, print only the converters " + "which they define")) cmdline.add_argument("--make", action='store_true', help="walk --srcdir to run over all relevant files") cmdline.add_argument("--srcdir", type=str, default=os.curdir, @@ -119,104 +135,161 @@ def create_cli() -> argparse.ArgumentParser: return cmdline -def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: - if ns.converters: - if ns.filename: - parser.error( - "can't specify --converters and a filename at the same time" - ) - AnyConverterType = ConverterType | ReturnConverterType - converter_list: list[tuple[str, AnyConverterType]] = [] - return_converter_list: list[tuple[str, AnyConverterType]] = [] - - for name, converter in converters.items(): - converter_list.append(( - name, - converter, - )) - for name, return_converter in return_converters.items(): - return_converter_list.append(( - name, - return_converter - )) +def print_diff(change: libclinic.FileChange) -> None: + if change.old_contents is None: + fromfile = "/dev/null" + old_lines: list[str] = [] + else: + fromfile = change.filename + old_lines = change.old_contents.splitlines(keepends=True) + sys.stdout.writelines(difflib.unified_diff( + old_lines, + change.new_contents.splitlines(keepends=True), + fromfile=fromfile, + tofile=change.filename, + )) + + +def report_changes(writer: libclinic.FileWriter, *, diff: bool) -> None: + for change in sorted(writer.changes, key=lambda change: change.filename): + if diff: + print_diff(change) + else: + action = "create" if change.old_contents is None else "update" + print(f"would {action} {change.filename}") - print() +AnyConverterType = ConverterType | ReturnConverterType + + +def defined_in_files( + registry: Mapping[str, AnyConverterType], + builtin: Mapping[str, AnyConverterType], +) -> dict[str, AnyConverterType]: + """Return the converters which the parsed files define or redefine.""" + return {name: cls for name, cls in registry.items() + if builtin.get(name) is not cls} + + +def print_converter_list( + title: str, + attribute: str, + registry: Mapping[str, AnyConverterType], +) -> None: + print(title + ":") + for name, cls in sorted(registry.items(), key=lambda item: item[0].lower()): + callable = getattr(cls, attribute, None) + if not callable: + continue + signature = inspect.signature(callable) + parameters = [] + for parameter_name, parameter in signature.parameters.items(): + if parameter.kind == inspect.Parameter.KEYWORD_ONLY: + if parameter.default != inspect.Parameter.empty: + s = f'{parameter_name}={parameter.default!r}' + else: + s = parameter_name + parameters.append(s) + print(' {}({})'.format(name, ', '.join(parameters))) + print() + + +def print_converters( + converters: Mapping[str, AnyConverterType], + legacy_converters: Mapping[str, AnyConverterType], + return_converters: Mapping[str, AnyConverterType], +) -> None: + if not (converters or legacy_converters or return_converters): + return + print() + if legacy_converters: print("Legacy converters:") legacy = sorted(legacy_converters) - print(' ' + ' '.join(c for c in legacy if c[0].isupper())) - print(' ' + ' '.join(c for c in legacy if c[0].islower())) + # A converter defined in a file can use any string, even a C + # expression, as its format unit, not only a letter. + groups = ([c for c in legacy if c[0].isupper()], + [c for c in legacy if c[0].islower()], + [c for c in legacy if not c[0].isalpha()]) + for group in groups: + if group: + print(' ' + ' '.join(group)) print() + if converters: + print_converter_list("Converters", 'converter_init', converters) + if return_converters: + print_converter_list("Return converters", 'return_converter_init', + return_converters) + print("All converters also accept (c_default=None, py_default=None, annotation=None).") + print("All return converters also accept (py_default=None).") + + +def walk_srcdir(srcdir: str, exclude: list[str] | None) -> Iterator[str]: + """Yield the C files in the source directory tree.""" + if exclude: + excludes = [os.path.normpath(os.path.join(srcdir, f)) for f in exclude] + else: + excludes = [] + for root, dirs, files in os.walk(srcdir): + for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): + if rcs_dir in dirs: + dirs.remove(rcs_dir) + for filename in files: + # handle .c, .cpp and .h files + if not filename.endswith(('.c', '.cpp', '.h')): + continue + path = os.path.normpath(os.path.join(root, filename)) + if path in excludes: + continue + yield path - for title, attribute, ids in ( - ("Converters", 'converter_init', converter_list), - ("Return converters", 'return_converter_init', return_converter_list), - ): - print(title + ":") - - ids.sort(key=lambda item: item[0].lower()) - longest = -1 - for name, _ in ids: - longest = max(longest, len(name)) - - for name, cls in ids: - callable = getattr(cls, attribute, None) - if not callable: - continue - signature = inspect.signature(callable) - parameters = [] - for parameter_name, parameter in signature.parameters.items(): - if parameter.kind == inspect.Parameter.KEYWORD_ONLY: - if parameter.default != inspect.Parameter.empty: - s = f'{parameter_name}={parameter.default!r}' - else: - s = parameter_name - parameters.append(s) - print(' {}({})'.format(name, ', '.join(parameters))) - print() - print("All converters also accept (c_default=None, py_default=None, annotation=None).") - print("All return converters also accept (py_default=None).") - return +def run_clinic(parser: argparse.ArgumentParser, ns: argparse.Namespace) -> None: + dry_run = ns.dry_run or ns.diff + # The report is written to the standard output, so the progress + # is written to the standard error stream to not mix them. + verbose_file = sys.stderr if dry_run else sys.stdout + + filenames: Iterable[str] if ns.make: if ns.output or ns.filename: parser.error("can't use -o or filenames with --make") if not ns.srcdir: parser.error("--srcdir must not be empty with --make") - if ns.exclude: - excludes = [os.path.join(ns.srcdir, f) for f in ns.exclude] - excludes = [os.path.normpath(f) for f in excludes] - else: - excludes = [] - for root, dirs, files in os.walk(ns.srcdir): - for rcs_dir in ('.svn', '.git', '.hg', 'build', 'externals'): - if rcs_dir in dirs: - dirs.remove(rcs_dir) - for filename in files: - # handle .c, .cpp and .h files - if not filename.endswith(('.c', '.cpp', '.h')): - continue - path = os.path.join(root, filename) - path = os.path.normpath(path) - if path in excludes: - continue - if ns.verbose: - print(path) - parse_file(path, - verify=not ns.force, limited_capi=ns.limited_capi) - return - - if not ns.filename: - parser.error("no input files") - - if ns.output and len(ns.filename) > 1: - parser.error("can't use -o with multiple filenames") + filenames = walk_srcdir(ns.srcdir, ns.exclude) + else: + if not ns.filename and not ns.converters: + parser.error("no input files") + if ns.output and len(ns.filename) > 1: + parser.error("can't use -o with multiple filenames") + filenames = ns.filename - for filename in ns.filename: + if ns.converters: + if dry_run: + parser.error("can't use --dry-run or --diff with --converters") + if not ns.make and not ns.filename: + print_converters(converters, legacy_converters, return_converters) + return + # Converters defined in a file are added to the same registries + # as the built-in ones, so remember the latter to tell them apart. + builtin_converters = dict(converters) + builtin_legacy_converters = dict(legacy_converters) + builtin_return_converters = dict(return_converters) + + writer = libclinic.FileWriter(dry_run=dry_run or ns.converters) + for filename in filenames: if ns.verbose: - print(filename) + print(filename, file=verbose_file) parse_file(filename, output=ns.output, - verify=not ns.force, limited_capi=ns.limited_capi) + verify=not ns.force, limited_capi=ns.limited_capi, + writer=writer) + + if ns.converters: + print_converters( + defined_in_files(converters, builtin_converters), + defined_in_files(legacy_converters, builtin_legacy_converters), + defined_in_files(return_converters, builtin_return_converters)) + else: + report_changes(writer, diff=ns.diff) def main(argv: list[str] | None = None) -> NoReturn: diff --git a/Tools/clinic/libclinic/dsl_parser.py b/Tools/clinic/libclinic/dsl_parser.py index 90e2e0d3d9c928c..4dcbc815cc6f25b 100644 --- a/Tools/clinic/libclinic/dsl_parser.py +++ b/Tools/clinic/libclinic/dsl_parser.py @@ -112,8 +112,8 @@ class ParamState(enum.IntEnum): """Parameter parsing state. - [ [ a, b, ] c, ] d, e, f=3, [ g, h, [ i ] ] <- line - 01 2 3 4 5 6 <- state transitions + [ [ a, b, ] c, ] [ d, ] e, f=3, [ g, h, [ i ] ] [ j ] <- line + 01 2 3 12 3 4 5 6 5 6 <- state transitions """ # Before we've seen anything. # Legal transitions: to LEFT_SQUARE_BEFORE or REQUIRED @@ -251,7 +251,8 @@ class DSLParser: positional_only: bool deprecated_positional: VersionTuple | None deprecated_keyword: VersionTuple | None - group: int + group_stack: list[int] + group_count: int parameter_state: ParamState indent: IndentStack kind: FunctionKind @@ -291,7 +292,8 @@ def reset(self) -> None: self.positional_only = False self.deprecated_positional = None self.deprecated_keyword = None - self.group = 0 + self.group_stack = [] + self.group_count = 0 self.parameter_state: ParamState = ParamState.START self.indent = IndentStack() self.kind = CALLABLE @@ -829,6 +831,7 @@ def to_required(self) -> None: assert self.function is not None for p in self.function.parameters.values(): p.group = -p.group + self.group_count = 0 def state_parameter(self, line: str) -> None: assert isinstance(self.function, Function) @@ -888,7 +891,7 @@ def parse_parameter(self, line: str) -> None: case ParamState.LEFT_SQUARE_BEFORE: self.parameter_state = ParamState.GROUP_BEFORE case ParamState.GROUP_BEFORE: - if not self.group: + if not self.group_stack: self.to_required() case ParamState.GROUP_AFTER | ParamState.OPTIONAL: pass @@ -1082,7 +1085,7 @@ def bad_node(self, node: ast.AST) -> None: if isinstance(converter, self_converter): if len(self.function.parameters) == 1: - if self.group: + if self.group_stack: fail("A 'self' parameter cannot be in an optional group.") assert self.parameter_state is ParamState.REQUIRED assert value is unspecified @@ -1096,7 +1099,7 @@ def bad_node(self, node: ast.AST) -> None: if isinstance(converter, defining_class_converter): _lp = len(self.function.parameters) if _lp == 1: - if self.group: + if self.group_stack: fail("A 'defining_class' parameter cannot be in an optional group.") if self.function.cls is None: fail("A 'defining_class' parameter cannot be defined at module level.") @@ -1110,7 +1113,9 @@ def bad_node(self, node: ast.AST) -> None: p = Parameter(parameter_name, kind, function=self.function, - converter=converter, default=value, group=self.group, + converter=converter, default=value, + group=self.group_stack[-1] if self.group_stack else 0, + group_depth=len(self.group_stack), deprecated_positional=self.deprecated_positional) names = [k.name for k in self.function.parameters.values()] @@ -1189,26 +1194,34 @@ def parse_star(self, function: Function, version: VersionTuple | None) -> None: def parse_opening_square_bracket(self, function: Function) -> None: """Parse opening parameter group symbol '['.""" + # A group can only be nested in a group which does not contain + # parameters yet, but two groups on the same nesting level can + # follow each other. match self.parameter_state: case ParamState.START | ParamState.LEFT_SQUARE_BEFORE: self.parameter_state = ParamState.LEFT_SQUARE_BEFORE + case ParamState.GROUP_BEFORE if not self.group_stack: + self.parameter_state = ParamState.LEFT_SQUARE_BEFORE case ParamState.REQUIRED | ParamState.GROUP_AFTER: self.parameter_state = ParamState.GROUP_AFTER + case ParamState.RIGHT_SQUARE_AFTER if not self.group_stack: + self.parameter_state = ParamState.GROUP_AFTER case st: fail(f"Function {function.name!r} " f"has an unsupported group configuration. " f"(Unexpected state {st}.b)") - self.group += 1 + self.group_count += 1 + self.group_stack.append(self.group_count) function.docstring_only = True def parse_closing_square_bracket(self, function: Function) -> None: """Parse closing parameter group symbol ']'.""" - if not self.group: + if not self.group_stack: fail(f"Function {function.name!r} has a ']' without a matching '['.") - if not any(p.group == self.group for p in function.parameters.values()): + group = self.group_stack.pop() + if not any(p.group == group for p in function.parameters.values()): fail(f"Function {function.name!r} has an empty group. " "All groups must contain at least one parameter.") - self.group -= 1 match self.parameter_state: case ParamState.LEFT_SQUARE_BEFORE | ParamState.GROUP_BEFORE: self.parameter_state = ParamState.GROUP_BEFORE @@ -1268,7 +1281,7 @@ def parse_slash(self, function: Function, version: VersionTuple | None) -> None: ParamState.RIGHT_SQUARE_AFTER, ParamState.GROUP_BEFORE, } - if (self.parameter_state not in allowed) or self.group: + if (self.parameter_state not in allowed) or self.group_stack: fail(f"Function {function.name!r} has an unsupported group configuration. " f"(Unexpected state {self.parameter_state}.d)") # fixup preceding parameters @@ -1329,7 +1342,7 @@ def state_parameter_docstring(self, line: str) -> None: def state_function_docstring(self, line: str) -> None: assert self.function is not None - if self.group: + if self.group_stack: fail(f"Function {self.function.name!r} has a ']' without a matching '['.") if not self.valid_line(line): @@ -1364,16 +1377,25 @@ def format_docstring_signature( else: assert positional_only if positional_only: - p.right_bracket_count = abs(p.group) + p.right_bracket_count = p.group_depth else: # don't put any right brackets around non-positional-only parameters, ever. p.right_bracket_count = 0 right_bracket_count = 0 + last_group = 0 - def fix_right_bracket_count(desired: int) -> str: - nonlocal right_bracket_count + def fix_right_bracket_count(desired: int, group: int = 0) -> str: + nonlocal right_bracket_count, last_group s = '' + if (group != last_group and right_bracket_count and + ((desired >= right_bracket_count) if group < 0 else + (desired <= right_bracket_count))): + # The group is not nested in the previous group, + # close the brackets of the latter first. + s += ']' * right_bracket_count + right_bracket_count = 0 + last_group = group while right_bracket_count < desired: s += '[' right_bracket_count += 1 @@ -1441,7 +1463,8 @@ def add_parameter(text: str) -> None: added_star = True add_parameter('*,') - p_lines = [fix_right_bracket_count(p.right_bracket_count)] + p_lines = [fix_right_bracket_count(p.right_bracket_count, + p.group)] if isinstance(p.converter, self_converter): # annotate first parameter as being a "self". diff --git a/Tools/clinic/libclinic/function.py b/Tools/clinic/libclinic/function.py index 1c643caea98e3b5..325633eb010608f 100644 --- a/Tools/clinic/libclinic/function.py +++ b/Tools/clinic/libclinic/function.py @@ -205,7 +205,11 @@ class Parameter: converter: CConverter annotation: object = inspect.Parameter.empty docstring: str = '' + # Identifier of the optional group containing the parameter (0 if none). + # It is negative for groups before the required parameters. group: int = 0 + # Nesting level of that group (0 if none). + group_depth: int = 0 # (`None` signifies that there is no deprecation) deprecated_positional: VersionTuple | None = None deprecated_keyword: VersionTuple | None = None @@ -301,15 +305,18 @@ def permute_right_option_groups( def permute_optional_groups( - left: Sequence[Iterable[Parameter]], + left: Sequence[Sequence[Iterable[Parameter]]], required: Iterable[Parameter], - right: Sequence[Iterable[Parameter]] + right: Sequence[Sequence[Iterable[Parameter]]] ) -> tuple[ParamTuple, ...]: """ Generator function that computes the set of acceptable argument lists for the provided iterables of argument groups. (Actually it generates a tuple of tuples.) + "left" and "right" are sequences of chains of nested groups. + Groups of different chains are independent of each other. + Algorithm: prefer left options over right options. If required is empty, left must also be empty. @@ -319,10 +326,21 @@ def permute_optional_groups( if left: raise ValueError("required is empty but left is not") + left_options: list[ParamTuple] = [()] + for chain in left: + left_options = [option + t + for option in left_options + for t in permute_left_option_groups(chain)] + right_options: list[ParamTuple] = [()] + for chain in reversed(right): + right_options = [t + option + for option in right_options + for t in permute_right_option_groups(chain)] + accumulator: list[ParamTuple] = [] counts = set() - for r in permute_right_option_groups(right): - for l in permute_left_option_groups(left): + for r in right_options: + for l in left_options: t = l + required + r if len(t) in counts: continue diff --git a/Tools/clinic/libclinic/utils.py b/Tools/clinic/libclinic/utils.py index 3df64f270dd074a..8fc8748f0f9ae10 100644 --- a/Tools/clinic/libclinic/utils.py +++ b/Tools/clinic/libclinic/utils.py @@ -1,4 +1,5 @@ import collections +import dataclasses as dc import enum import hashlib import os @@ -7,17 +8,20 @@ from typing import Literal, Final -def write_file(filename: str, new_contents: str) -> None: - """Write new content to file, iff the content changed.""" +def read_file(filename: str) -> str | None: + """Return the content of the file, or None if it does not exist.""" try: with open(filename, encoding="utf-8") as fp: - old_contents = fp.read() - - if old_contents == new_contents: - # no change: avoid modifying the file modification time - return + return fp.read() except FileNotFoundError: - pass + return None + + +def write_file(filename: str, new_contents: str) -> None: + """Write new content to file, iff the content changed.""" + if read_file(filename) == new_contents: + # no change: avoid modifying the file modification time + return # Atomic write using a temporary file and os.replace() filename_new = f"{filename}.new" with open(filename_new, "w", encoding="utf-8") as fp: @@ -29,6 +33,42 @@ def write_file(filename: str, new_contents: str) -> None: raise +@dc.dataclass(slots=True, frozen=True) +class FileChange: + filename: str + # None if the file does not exist yet. + old_contents: str | None + new_contents: str + + +@dc.dataclass(slots=True) +class FileWriter: + """Write the generated files. + + In the dry run mode no file is written, the changes are only recorded. + """ + + dry_run: bool = False + changes: list[FileChange] = dc.field(default_factory=list) + + def makedirs(self, dirname: str) -> None: + if not self.dry_run: + os.makedirs(dirname) + elif os.path.exists(dirname): + # Create nothing, but fail as os.makedirs() does, so that + # the caller can report an existing non-directory. + raise FileExistsError(dirname) + + def write(self, filename: str, new_contents: str) -> None: + if not self.dry_run: + write_file(filename, new_contents) + return + old_contents = read_file(filename) + if old_contents != new_contents: + self.changes.append( + FileChange(filename, old_contents, new_contents)) + + def compute_checksum(input_: str, length: int | None = None) -> str: checksum = hashlib.sha1(input_.encode("utf-8")).hexdigest() if length: