From 2cd5ce024cadfbe70c460f21ae94683ff497bd30 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 15 Mar 2021 16:55:49 +0100 Subject: [PATCH 01/93] Initial commit Add a Barrier object to asyncio synchronized primitives --- Lib/asyncio/locks.py | 173 +++++++++++++- Lib/test/test_asyncio/test_locks.py | 356 +++++++++++++++++++++++++++- 2 files changed, 527 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index a7453fb1c77287..e5fcf9648de5d9 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -1,6 +1,6 @@ """Synchronization primitives.""" -__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore') +__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') import collections @@ -418,3 +418,174 @@ def release(self): if self._value >= self._bound_value: raise ValueError('BoundedSemaphore released too many times') super().release() + + +# A barrier class. Inspired in part by the pthread_barrier_* api and +# the CyclicBarrier class from Java. See +# http://sourceware.org/pthreads-win32/manual/pthread_barrier_init.html and +# http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/ +# CyclicBarrier.html +# for information. +# We maintain two main states, 'filling' and 'draining' enabling the barrier +# to be cyclic. Tasks are not allowed into it until it has fully drained +# since the previous cycle. In addition, a 'resetting' state exists which is +# similar to 'draining' except that tasks leave with a BrokenBarrierError, +# and a 'broken' state in which all tasks get the exception. + +class Barrier(mixins._LoopBoundMixin): + """Asynchronous equivalent to threading.Barrier + + Implements a Barrier. + Useful for synchronizing a fixed number of tasks at known synchronization + points. Tasks block on 'wait()' and are simultaneously awoken once they + have all made that call. + """ + + def __init__(self, parties, action=None, *, loop=mixins._marker): + """Create a barrier, initialised to 'parties' tasks. + 'action' is a callable which, when supplied, will be called by one of + the tasks after they have all entered the barrier and just prior to + releasing them all. + """ + super().__init__(loop=loop) + if parties < 1: + raise ValueError('parties must be > 0') + + self._waiting = Event() # used notify all waiting tasks + self._blocking = Event() # used block tasks while wainting tasks are draining or broken + self._action = action + self._parties = parties + self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken + self._count = 0 # count waiting tasks + + def __repr__(self): + res = super().__repr__() + _wait = 'set' if self._waiting.is_set() else 'unset' + _block = 'set' if self._blocking.is_set() else 'unset' + extra = f'{_wait}, count:{self._count}/{self._parties}, {_block}, state:{self._state}' + return f'<{res[1:-1]} [{extra}]>' + + async def wait(self): + """Wait for the barrier. + When the specified number of tasks have started waiting, they are all + simultaneously awoken. If an 'action' was provided for the barrier, one + of the tasks will have executed that callback prior to returning. + Returns an individual index number from 0 to 'parties-1'. + """ + await self._block() # Block while the barrier drains or resets. + index = self._count + self._count += 1 + try: + if index + 1 == self._parties: + # We release the barrier + self._release() + else: + # We wait until someone releases us + await self._wait() + return index + finally: + self._count -= 1 + # Wake up any tasks waiting for barrier to drain. + self._exit() + + # Block until the barrier is ready for us, or raise an exception + # if it is broken. + async def _block (self): + if self._state in (-1, 1): + # It is draining or resetting, wait until done + await self._blocking.wait() + + #see if the barrier is in a broken state + if self._state < 0: + raise BrokenBarrierError + assert self._state == 0, repr(self) + + # Optionally run the 'action' and release the tasks waiting + # in the barrier. + def _release(self): + try: + if self._action: + self._action() + # enter draining state + self._state = 1 + self._blocking.clear() + self._waiting.set() + except: + #an exception during the _action handler. Break and reraise + self._state = -2 + self._blocking.clear() + self._waiting.set() + raise + + # Wait in the barrier until we are released. Raise an exception + # if the barrier is reset or broken. + async def _wait(self): + await self._waiting.wait() + # no timeout so + if self._state < 0: + raise BrokenBarrierError + assert self._state == 1, repr(self) + + # If we are the last tasks to exit the barrier, signal any tasks + # waiting for the barrier to drain. + def _exit(self): + if self._count == 0: + if self._state == 1: + self._state = 0 + elif self._state == -1: + self._state = 0 + self._waiting.clear() + self._blocking.set() + + # async def reset(self): + def reset(self): + """Reset the barrier to the initial state. + Any tasks currently waiting will get the BrokenBarrier exception + raised. + """ + if self._count > 0: + if self._state in (0, 1): + #reset the barrier, waking up tasks + self._state = -1 + elif self._state == -2: + #was broken, set it to reset state + #which clears when the last tasks exits + self._state = -1 + self._waiting.set() + self._blocking.clear() + else: + self._state = 0 + + + # async def abort(self): + def abort(self): + """Place the barrier into a 'broken' state. + Useful in case of error. Any currently waiting tasks and tasks + attempting to 'wait()' will have BrokenBarrierError raised. + """ + self._state = -2 + self._waiting.set() + self._blocking.clear() + + @property + def parties(self): + """Return the number of tasks required to trip the barrier.""" + return self._parties + + @property + def n_waiting(self): + """Return the number of tasks currently waiting at the barrier.""" + # We don't need synchronization here since this is an ephemeral result + # anyway. It returns the correct value in the steady state. + if self._state == 0: + return self._count + return 0 + + @property + def broken(self): + """Return True if the barrier is in a broken state.""" + return self._state == -2 + +# exception raised by the Barrier class +class BrokenBarrierError(RuntimeError): + pass \ No newline at end of file diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 6194cd06176dac..824535a18cb168 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -6,11 +6,13 @@ import asyncio from test.test_asyncio import utils as test_utils +import functools STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' - r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' + r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?(, count:\d+/\d+)?(, state:^-?\d)?' + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier object r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -68,6 +70,16 @@ def test_lock_doesnt_accept_loop_parameter(self): ): cls(loop=self.loop) + # Barrier object has a positional paramater + # so check alone + cls = asyncio.Barrier + with self.assertRaisesRegex( + TypeError, + rf'As of 3.10, the \*loop\* parameter was removed from ' + rf'{cls.__name__}\(\) since it is no longer necessary' + ): + cls(1, loop=self.loop) + def test_lock_by_with_statement(self): loop = asyncio.new_event_loop() # don't use TestLoop quirks self.set_event_loop(loop) @@ -928,5 +940,347 @@ def test_release_no_waiters(self): self.assertFalse(sem.locked()) +class BarrierTests(test_utils.TestCase): + """ + Tests for Barrier objects. + """ + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.N = 5 + + async def run_coros(self, n, coro): + tasks = [self.loop.create_task(coro()) for _ in range(n)] + res = await asyncio.gather(*tasks) + return res, tasks + + def test_repr(self): + async def coro(): + try: + i = await barrier.wait() + except: + ... + else: + return await asyncio.sleep(1, True) + + self.N = 999 + barrier = asyncio.Barrier(self.N) + self.assertTrue("[unset," in repr(barrier)) + self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(repr(barrier).endswith('unset, state:0]>')) + + nb_waits = 3 + tasks = [] + for _ in range(nb_waits): + tasks.append(self.loop.create_task(coro())) + test_utils.run_briefly(self.loop) + self.assertTrue("[unset," in repr(barrier)) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith('unset, state:0]>')) + + barrier.reset() + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue("[set," in repr(barrier)) + self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith('unset, state:-1]>')) + test_utils.run_briefly(self.loop) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue("[unset," in repr(barrier)) + self.assertTrue(f"count:0/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith('set, state:0]>')) + test_utils.run_briefly(self.loop) + + barrier.abort() + test_utils.run_briefly(self.loop) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(f"0/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith(':-2]>')) + + def test_init(self): + self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) + self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) + + def test_wait(self): + async def coro(result): + r = await barrier.wait() + result.append(r) + + barrier = asyncio.Barrier(self.N) + results = [] + _ = [self.loop.create_task(coro(results)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + self.assertEqual(sum(results), sum(range(self.N))) + self.assertFalse(barrier.broken) + + def test_wait_task_multi(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) + self.assertEqual(barrier.n_waiting, 0) + self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 2) + self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_wait_task(self): + self.N = 1 + barrier = asyncio.Barrier(self.N) + r1 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + r2 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + self.assertEqual(r1, r2) + self.assertFalse(barrier.broken) + + def test_wait_step_by_step(self): + async def coro(result, value): + await barrier.wait() + result.append(value) + + results = [] + barrier = asyncio.Barrier(self.N) + value = 1 + tasks = [] + for n in range(self.N-1): + tasks.append(self.loop.create_task(coro(results, value))) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, n+1) + + tasks.append(self.loop.create_task(coro(results, value))) + test_utils.run_briefly(self.loop) + self.assertEqual(results, [value]* self.N) + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(all(t.done() for t in tasks)) + self.assertTrue(all(t.result() is None for t in tasks)) + self.assertFalse(barrier.broken) + + def test_wait_multi_return(self): + results1 = [] + results2 = [] + results3 = [] + async def coro(): + await barrier.wait() + results1.append(True) + i = await barrier.wait() + results2.append(True) + results3.append(i) + return i + + barrier = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertEqual(len(results1), self.N) + self.assertTrue(all(results1)) + self.assertEqual(len(results2), self.N) + self.assertTrue(all(results2)) + self.assertEqual(sum(res), sum(results3)) + + def test_barrier(self, multipass=1, nn=5): + results = [[] for _ in range(multipass)] + async def coro(result, value): + ret = await barrier.wait() + result.append(value) + return ret + + barrier = asyncio.Barrier(nn) + for i in range(multipass): + _ = [self.loop.create_task(coro(results[i], value)) for value in range(nn)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(results[i]), nn) + self.assertEqual(sum(results[i]), sum(range(nn))) + if i > 0: + self.assertEqual(sum(results[i]), sum(results[i-1])) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) + + def test_barrier_multipass(self): + self.test_barrier(10) + + def test_action_callback(self): + async def coro(result, value): + ret = await barrier.wait() + result.append(value) + return ret + + result = [] + result1 = [] + value = 1 + barrier = asyncio.Barrier(1, action=lambda: result1.append(True)) + _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(result1), self.N) + self.assertTrue(all(result1)) + self.assertEqual(len(result), self.N) + self.assertEqual(sum(result), self.N*value) + self.assertFalse(barrier.broken) + + def test_action_callback_n(self): + async def coro(result, value): + ret = await barrier.wait() + result.append(value) + return ret + + result = [] + result1 = [] + value = 1 + barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) + _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(result1), 1) + self.assertTrue(result1[0]) + self.assertEqual(len(result), self.N) + self.assertEqual(sum(result), value*self.N) + self.assertFalse(barrier.broken) + + def test_action_callback_error(self): + results1 = [] + results2 = [] + results3 = [] + def raise_except(e): + raise e + + async def coro(e): + try: + ret = await barrier.wait() + except e: + results1.append(False) + except: + results2.append(True) + else: + results3.append(None) + + barrier = asyncio.Barrier(self.N, lambda: raise_except(ZeroDivisionError)) + _ = [self.loop.create_task(coro(ZeroDivisionError)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + self.assertEqual(len(results1), 1) + self.assertFalse(results1[0]) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(all(results2)) + self.assertEqual(len(results3), 0) + self.assertTrue(barrier.broken) + + def test_reset(self): + results1 = [] + results2 = [] + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + results1.append(True) + finally: + await barrier.wait() + results2.append(True) + + async def coro2(): + await barrier.wait() + results2.append(True) + + + barrier = asyncio.Barrier(self.N) + _ = [self.loop.create_task(coro()) for _ in range(self.N-1)] + test_utils.run_briefly(self.loop) + barrier.reset() + test_utils.run_briefly(self.loop) + self.loop.create_task(coro2()) + test_utils.run_briefly(self.loop) + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), self.N-1) + self.assertEqual(len(results2), self.N) + + def test_reset_multi_wait(self): + results1 = [] + results2 = [] + results3 = [] + alls = [] + async def coro(): + i = await barrier1.wait() + alls.append(i) + if len(alls) == self.N: # i == self.N//2: + barrier.reset() + else: + try: + await barrier.wait() + results1.append(True) + except Exception as e: + results2.append(True) + + # Now, pass the barrier again + await barrier1.wait() + results3.append(True) + + barrier = asyncio.Barrier(self.N) + barrier1 = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + def test_abort(self): + results1 = [] + results2 = [] + async def coro(): + try: + i = await barrier.wait() + if i == self.N//2: + raise RuntimeError + await barrier.wait() + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + + barrier = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertTrue(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + + def test_abort_and_reset(self): + results1 = [] + results2 = [] + results3 = [] + async def coro(): + try: + i = await barrier.wait() + if i == self.N//2: + raise RuntimeError + await barrier.wait() + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + i = await barrier2.wait() + if i == self.N//2: + barrier.reset() + await barrier2.wait() + await barrier.wait() + results3.append(True) + + barrier = asyncio.Barrier(self.N) + barrier2 = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + if __name__ == '__main__': unittest.main() From 7dfa8ef7379549df7e19031704e3cb78f876c69b Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 15 Mar 2021 21:04:37 +0100 Subject: [PATCH 02/93] make patchcheck Change after run make patchcheck --- Lib/asyncio/locks.py | 18 ++++++++-------- Lib/test/test_asyncio/test_locks.py | 32 ++++++++++++++--------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index e5fcf9648de5d9..4bc69100ed225d 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -433,7 +433,7 @@ def release(self): # and a 'broken' state in which all tasks get the exception. class Barrier(mixins._LoopBoundMixin): - """Asynchronous equivalent to threading.Barrier + """Asynchronous equivalent to threading.Barrier Implements a Barrier. Useful for synchronizing a fixed number of tasks at known synchronization @@ -445,18 +445,18 @@ def __init__(self, parties, action=None, *, loop=mixins._marker): """Create a barrier, initialised to 'parties' tasks. 'action' is a callable which, when supplied, will be called by one of the tasks after they have all entered the barrier and just prior to - releasing them all. + releasing them all. """ super().__init__(loop=loop) if parties < 1: raise ValueError('parties must be > 0') self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block tasks while wainting tasks are draining or broken + self._blocking = Event() # used block tasks while wainting tasks are draining or broken self._action = action self._parties = parties self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken - self._count = 0 # count waiting tasks + self._count = 0 # count waiting tasks def __repr__(self): res = super().__repr__() @@ -494,7 +494,7 @@ async def _block (self): if self._state in (-1, 1): # It is draining or resetting, wait until done await self._blocking.wait() - + #see if the barrier is in a broken state if self._state < 0: raise BrokenBarrierError @@ -515,7 +515,7 @@ def _release(self): self._state = -2 self._blocking.clear() self._waiting.set() - raise + raise # Wait in the barrier until we are released. Raise an exception # if the barrier is reset or broken. @@ -529,7 +529,7 @@ async def _wait(self): # If we are the last tasks to exit the barrier, signal any tasks # waiting for the barrier to drain. def _exit(self): - if self._count == 0: + if self._count == 0: if self._state == 1: self._state = 0 elif self._state == -1: @@ -543,7 +543,7 @@ def reset(self): Any tasks currently waiting will get the BrokenBarrier exception raised. """ - if self._count > 0: + if self._count > 0: if self._state in (0, 1): #reset the barrier, waking up tasks self._state = -1 @@ -588,4 +588,4 @@ def broken(self): # exception raised by the Barrier class class BrokenBarrierError(RuntimeError): - pass \ No newline at end of file + pass diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 824535a18cb168..42ef44db2d3758 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -70,8 +70,8 @@ def test_lock_doesnt_accept_loop_parameter(self): ): cls(loop=self.loop) - # Barrier object has a positional paramater - # so check alone + # Barrier object has a positional paramater + # so check alone cls = asyncio.Barrier with self.assertRaisesRegex( TypeError, @@ -947,7 +947,7 @@ class BarrierTests(test_utils.TestCase): def setUp(self): super().setUp() self.loop = self.new_test_loop() - self.N = 5 + self.N = 5 async def run_coros(self, n, coro): tasks = [self.loop.create_task(coro()) for _ in range(n)] @@ -969,8 +969,8 @@ async def coro(): self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue(repr(barrier).endswith('unset, state:0]>')) - - nb_waits = 3 + + nb_waits = 3 tasks = [] for _ in range(nb_waits): tasks.append(self.loop.create_task(coro())) @@ -1002,16 +1002,16 @@ def test_init(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - def test_wait(self): + def test_wait_more(self): async def coro(result): r = await barrier.wait() result.append(r) - + barrier = asyncio.Barrier(self.N) results = [] _ = [self.loop.create_task(coro(results)) for _ in range(self.N)] test_utils.run_briefly(self.loop) - self.assertEqual(sum(results), sum(range(self.N))) + self.assertEqual(sum(results), sum(range(self.N))) self.assertFalse(barrier.broken) def test_wait_task_multi(self): @@ -1029,7 +1029,7 @@ def test_wait_task_multi(self): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_wait_task(self): + def test_wait_task_one(self): self.N = 1 barrier = asyncio.Barrier(self.N) r1 = self.loop.run_until_complete(barrier.wait()) @@ -1115,7 +1115,7 @@ async def coro(result, value): value = 1 barrier = asyncio.Barrier(1, action=lambda: result1.append(True)) _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] - test_utils.run_briefly(self.loop) + test_utils.run_briefly(self.loop) self.assertEqual(len(result1), self.N) self.assertTrue(all(result1)) @@ -1134,7 +1134,7 @@ async def coro(result, value): value = 1 barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] - test_utils.run_briefly(self.loop) + test_utils.run_briefly(self.loop) self.assertEqual(len(result1), 1) self.assertTrue(result1[0]) @@ -1173,9 +1173,9 @@ def test_reset(self): results1 = [] results2 = [] async def coro(): - try: + try: await barrier.wait() - except asyncio.BrokenBarrierError: + except asyncio.BrokenBarrierError: results1.append(True) finally: await barrier.wait() @@ -1217,7 +1217,7 @@ async def coro(): # Now, pass the barrier again await barrier1.wait() results3.append(True) - + barrier = asyncio.Barrier(self.N) barrier1 = asyncio.Barrier(self.N) res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) @@ -1235,7 +1235,7 @@ async def coro(): if i == self.N//2: raise RuntimeError await barrier.wait() - results1.append(True) + results1.append(True) except asyncio.BrokenBarrierError: results2.append(True) except RuntimeError: @@ -1272,7 +1272,7 @@ async def coro(): await barrier2.wait() await barrier.wait() results3.append(True) - + barrier = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) From f19ea24d2c401920d5abc368e2673ed28016bd89 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 15 Mar 2021 21:06:51 +0100 Subject: [PATCH 03/93] make patchcheck run make patchcheck --- Doc/c-api/exceptions.rst.bak | 1054 +++++++++++++ Doc/c-api/memory.rst.bak | 642 ++++++++ Doc/faq/design.rst.bak | 759 +++++++++ Doc/howto/descriptor.rst.bak | 1575 +++++++++++++++++++ Doc/library/_thread.rst.bak | 215 +++ Doc/library/ast.rst.bak | 1993 ++++++++++++++++++++++++ Doc/library/asyncio-api-index.rst.bak | 221 +++ Doc/library/collections.rst.bak | 1278 +++++++++++++++ Doc/library/dataclasses.rst.bak | 595 +++++++ Doc/library/gc.rst.bak | 322 ++++ Doc/library/importlib.metadata.rst.bak | 262 ++++ Doc/library/logging.rst.bak | 1350 ++++++++++++++++ Doc/library/readline.rst.bak | 361 +++++ Doc/library/sqlite3.rst.bak | 1094 +++++++++++++ Doc/library/statistics.rst.bak | 879 +++++++++++ Doc/library/tempfile.rst.bak | 374 +++++ Doc/library/venv.rst.bak | 496 ++++++ Doc/library/xml.sax.handler.rst.bak | 463 ++++++ Doc/library/zipimport.rst.bak | 209 +++ Doc/whatsnew/3.10.rst.bak | 1315 ++++++++++++++++ 20 files changed, 15457 insertions(+) create mode 100644 Doc/c-api/exceptions.rst.bak create mode 100644 Doc/c-api/memory.rst.bak create mode 100644 Doc/faq/design.rst.bak create mode 100644 Doc/howto/descriptor.rst.bak create mode 100644 Doc/library/_thread.rst.bak create mode 100644 Doc/library/ast.rst.bak create mode 100644 Doc/library/asyncio-api-index.rst.bak create mode 100644 Doc/library/collections.rst.bak create mode 100644 Doc/library/dataclasses.rst.bak create mode 100644 Doc/library/gc.rst.bak create mode 100644 Doc/library/importlib.metadata.rst.bak create mode 100644 Doc/library/logging.rst.bak create mode 100644 Doc/library/readline.rst.bak create mode 100644 Doc/library/sqlite3.rst.bak create mode 100644 Doc/library/statistics.rst.bak create mode 100644 Doc/library/tempfile.rst.bak create mode 100644 Doc/library/venv.rst.bak create mode 100644 Doc/library/xml.sax.handler.rst.bak create mode 100644 Doc/library/zipimport.rst.bak create mode 100644 Doc/whatsnew/3.10.rst.bak diff --git a/Doc/c-api/exceptions.rst.bak b/Doc/c-api/exceptions.rst.bak new file mode 100644 index 00000000000000..4e99a0167a632d --- /dev/null +++ b/Doc/c-api/exceptions.rst.bak @@ -0,0 +1,1054 @@ +.. highlight:: c + + +.. _exceptionhandling: + +****************** +Exception Handling +****************** + +The functions described in this chapter will let you handle and raise Python +exceptions. It is important to understand some of the basics of Python +exception handling. It works somewhat like the POSIX :c:data:`errno` variable: +there is a global indicator (per thread) of the last error that occurred. Most +C API functions don't clear this on success, but will set it to indicate the +cause of the error on failure. Most C API functions also return an error +indicator, usually ``NULL`` if they are supposed to return a pointer, or ``-1`` +if they return an integer (exception: the :c:func:`PyArg_\*` functions +return ``1`` for success and ``0`` for failure). + +Concretely, the error indicator consists of three object pointers: the +exception's type, the exception's value, and the traceback object. Any +of those pointers can be ``NULL`` if non-set (although some combinations are +forbidden, for example you can't have a non-``NULL`` traceback if the exception +type is ``NULL``). + +When a function must fail because some function it called failed, it generally +doesn't set the error indicator; the function it called already set it. It is +responsible for either handling the error and clearing the exception or +returning after cleaning up any resources it holds (such as object references or +memory allocations); it should *not* continue normally if it is not prepared to +handle the error. If returning due to an error, it is important to indicate to +the caller that an error has been set. If the error is not handled or carefully +propagated, additional calls into the Python/C API may not behave as intended +and may fail in mysterious ways. + +.. note:: + The error indicator is **not** the result of :func:`sys.exc_info()`. + The former corresponds to an exception that is not yet caught (and is + therefore still propagating), while the latter returns an exception after + it is caught (and has therefore stopped propagating). + + +Printing and clearing +===================== + + +.. c:function:: void PyErr_Clear() + + Clear the error indicator. If the error indicator is not set, there is no + effect. + + +.. c:function:: void PyErr_PrintEx(int set_sys_last_vars) + + Print a standard traceback to ``sys.stderr`` and clear the error indicator. + **Unless** the error is a ``SystemExit``, in that case no traceback is + printed and the Python process will exit with the error code specified by + the ``SystemExit`` instance. + + Call this function **only** when the error indicator is set. Otherwise it + will cause a fatal error! + + If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, + :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the + type, value and traceback of the printed exception, respectively. + + +.. c:function:: void PyErr_Print() + + Alias for ``PyErr_PrintEx(1)``. + + +.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) + + Call :func:`sys.unraisablehook` using the current exception and *obj* + argument. + + This utility function prints a warning message to ``sys.stderr`` when an + exception has been set but it is impossible for the interpreter to actually + raise the exception. It is used, for example, when an exception occurs in an + :meth:`__del__` method. + + The function is called with a single argument *obj* that identifies the context + in which the unraisable exception occurred. If possible, + the repr of *obj* will be printed in the warning message. + + An exception must be set when calling this function. + + +Raising exceptions +================== + +These functions help you set the current thread's error indicator. +For convenience, some of these functions will always return a +``NULL`` pointer for use in a ``return`` statement. + + +.. c:function:: void PyErr_SetString(PyObject *type, const char *message) + + This is the most common way to set the error indicator. The first argument + specifies the exception type; it is normally one of the standard exceptions, + e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + The second argument is an error message; it is decoded from ``'utf-8``'. + + +.. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) + + This function is similar to :c:func:`PyErr_SetString` but lets you specify an + arbitrary Python object for the "value" of the exception. + + +.. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) + + This function sets the error indicator and returns ``NULL``. *exception* + should be a Python exception class. The *format* and subsequent + parameters help format the error message; they have the same meaning and + values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded + string. + + +.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs) + + Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather + than a variable number of arguments. + + .. versionadded:: 3.5 + + +.. c:function:: void PyErr_SetNone(PyObject *type) + + This is a shorthand for ``PyErr_SetObject(type, Py_None)``. + + +.. c:function:: int PyErr_BadArgument() + + This is a shorthand for ``PyErr_SetString(PyExc_TypeError, message)``, where + *message* indicates that a built-in operation was invoked with an illegal + argument. It is mostly for internal use. + + +.. c:function:: PyObject* PyErr_NoMemory() + + This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns ``NULL`` + so an object allocation function can write ``return PyErr_NoMemory();`` when it + runs out of memory. + + +.. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) + + .. index:: single: strerror() + + This is a convenience function to raise an exception when a C library function + has returned an error and set the C variable :c:data:`errno`. It constructs a + tuple object whose first item is the integer :c:data:`errno` value and whose + second item is the corresponding error message (gotten from :c:func:`strerror`), + and then calls ``PyErr_SetObject(type, object)``. On Unix, when the + :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, + this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, + leaves it set to that. The function always returns ``NULL``, so a wrapper + function around a system call can write ``return PyErr_SetFromErrno(type);`` + when the system call returns an error. + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) + + Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if + *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as + a third parameter. In the case of :exc:`OSError` exception, + this is used to define the :attr:`filename` attribute of the + exception instance. + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) + + Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second + filename object, for raising errors when a function that takes two filenames + fails. + + .. versionadded:: 3.4 + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) + + Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename + is given as a C string. *filename* is decoded from the :term:`filesystem + encoding and error handler`. + + +.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) + + This is a convenience function to raise :exc:`WindowsError`. If called with + *ierr* of :c:data:`0`, the error code returned by a call to :c:func:`GetLastError` + is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve + the Windows description of error code given by *ierr* or :c:func:`GetLastError`, + then it constructs a tuple object whose first item is the *ierr* value and whose + second item is the corresponding error message (gotten from + :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, + object)``. This function always returns ``NULL``. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr) + + Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter + specifying the exception type to be raised. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the + filename is given as a C string. *filename* is decoded from the filesystem + encoding (:func:`os.fsdecode`). + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an + additional parameter specifying the exception type to be raised. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) + + Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, + but accepts a second filename object. + + .. availability:: Windows. + + .. versionadded:: 3.4 + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional + parameter specifying the exception type to be raised. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path) + + This is a convenience function to raise :exc:`ImportError`. *msg* will be + set as the exception's message string. *name* and *path*, both of which can + be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name`` + and ``path`` attributes. + + .. versionadded:: 3.3 + + +.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) + + Set file, line, and offset information for the current exception. If the + current exception is not a :exc:`SyntaxError`, then it sets additional + attributes, which make the exception printing subsystem think the exception + is a :exc:`SyntaxError`. + + .. versionadded:: 3.4 + + +.. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) + + Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string + decoded from the :term:`filesystem encoding and error handler`. + + .. versionadded:: 3.2 + + +.. c:function:: void PyErr_SyntaxLocation(const char *filename, int lineno) + + Like :c:func:`PyErr_SyntaxLocationEx`, but the col_offset parameter is + omitted. + + +.. c:function:: void PyErr_BadInternalCall() + + This is a shorthand for ``PyErr_SetString(PyExc_SystemError, message)``, + where *message* indicates that an internal operation (e.g. a Python/C API + function) was invoked with an illegal argument. It is mostly for internal + use. + + +Issuing warnings +================ + +Use these functions to issue warnings from C code. They mirror similar +functions exported by the Python :mod:`warnings` module. They normally +print a warning message to *sys.stderr*; however, it is +also possible that the user has specified that warnings are to be turned into +errors, and in that case they will raise an exception. It is also possible that +the functions raise an exception because of a problem with the warning machinery. +The return value is ``0`` if no exception is raised, or ``-1`` if an exception +is raised. (It is not possible to determine whether a warning message is +actually printed, nor what the reason is for the exception; this is +intentional.) If an exception is raised, the caller should do its normal +exception handling (for example, :c:func:`Py_DECREF` owned references and return +an error value). + +.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) + + Issue a warning message. The *category* argument is a warning category (see + below) or ``NULL``; the *message* argument is a UTF-8 encoded string. *stack_level* is a + positive number giving a number of stack frames; the warning will be issued from + the currently executing line of code in that stack frame. A *stack_level* of 1 + is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, + and so forth. + + Warning categories must be subclasses of :c:data:`PyExc_Warning`; + :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; + the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard + Python warning categories are available as global variables whose names are + enumerated at :ref:`standardwarningcategories`. + + For information about warning control, see the documentation for the + :mod:`warnings` module and the :option:`-W` option in the command line + documentation. There is no C API for warning control. + +.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) + + Much like :c:func:`PyErr_SetImportError` but this function allows for + specifying a subclass of :exc:`ImportError` to raise. + + .. versionadded:: 3.6 + + +.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) + + Issue a warning message with explicit control over all warning attributes. This + is a straightforward wrapper around the Python function + :func:`warnings.warn_explicit`, see there for more information. The *module* + and *registry* arguments may be set to ``NULL`` to get the default effect + described there. + + .. versionadded:: 3.4 + + +.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) + + Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and + *module* are UTF-8 encoded strings, and *filename* is decoded from the + :term:`filesystem encoding and error handler`. + + +.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnEx`, but use + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + +.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnFormat`, but *category* is + :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. + + .. versionadded:: 3.6 + + +Querying the error indicator +============================ + +.. c:function:: PyObject* PyErr_Occurred() + + Test whether the error indicator is set. If set, return the exception *type* + (the first argument to the last call to one of the :c:func:`PyErr_Set\*` + functions or to :c:func:`PyErr_Restore`). If not set, return ``NULL``. You do not + own a reference to the return value, so you do not need to :c:func:`Py_DECREF` + it. + + The caller must hold the GIL. + + .. note:: + + Do not compare the return value to a specific exception; use + :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could + easily fail since the exception may be an instance instead of a class, in the + case of a class exception, or it may be a subclass of the expected exception.) + + +.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) + + Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This + should only be called when an exception is actually set; a memory access + violation will occur if no exception has been raised. + + +.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) + + Return true if the *given* exception matches the exception type in *exc*. If + *exc* is a class object, this also returns true when *given* is an instance + of a subclass. If *exc* is a tuple, all exception types in the tuple (and + recursively in subtuples) are searched for a match. + + +.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) + + Retrieve the error indicator into three variables whose addresses are passed. + If the error indicator is not set, set all three variables to ``NULL``. If it is + set, it will be cleared and you own a reference to each object retrieved. The + value and traceback object may be ``NULL`` even when the type object is not. + + .. note:: + + This function is normally only used by code that needs to catch exceptions or + by code that needs to save and restore the error indicator temporarily, e.g.:: + + { + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + + /* ... code that might produce other errors ... */ + + PyErr_Restore(type, value, traceback); + } + + +.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) + + Set the error indicator from the three objects. If the error indicator is + already set, it is cleared first. If the objects are ``NULL``, the error + indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or + traceback. The exception type should be a class. Do not pass an invalid + exception type or value. (Violating these rules will cause subtle problems + later.) This call takes away a reference to each object: you must own a + reference to each object before the call and after the call you no longer own + these references. (If you don't understand this, don't use this function. I + warned you.) + + .. note:: + + This function is normally only used by code that needs to save and restore the + error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current + error indicator. + + +.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) + + Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below + can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance. + + .. note:: + + This function *does not* implicitly set the ``__traceback__`` + attribute on the exception value. If setting the traceback + appropriately is desired, the following additional snippet is needed:: + + if (tb != NULL) { + PyException_SetTraceback(val, tb); + } + + +.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) + + Retrieve the exception info, as known from ``sys.exc_info()``. This refers + to an exception that was *already caught*, not to an exception that was + freshly raised. Returns new references for the three objects, any of which + may be ``NULL``. Does not modify the exception info state. + + .. note:: + + This function is not normally used by code that wants to handle exceptions. + Rather, it can be used when code needs to save and restore the exception + state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the + exception state. + + .. versionadded:: 3.3 + + +.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback) + + Set the exception info, as known from ``sys.exc_info()``. This refers + to an exception that was *already caught*, not to an exception that was + freshly raised. This function steals the references of the arguments. + To clear the exception state, pass ``NULL`` for all three arguments. + For general rules about the three arguments, see :c:func:`PyErr_Restore`. + + .. note:: + + This function is not normally used by code that wants to handle exceptions. + Rather, it can be used when code needs to save and restore the exception + state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception + state. + + .. versionadded:: 3.3 + + +Signal Handling +=============== + + +.. c:function:: int PyErr_CheckSignals() + + .. index:: + module: signal + single: SIGINT + single: KeyboardInterrupt (built-in exception) + + This function interacts with Python's signal handling. It checks whether a + signal has been sent to the processes and if so, invokes the corresponding + signal handler. If the :mod:`signal` module is supported, this can invoke a + signal handler written in Python. In all cases, the default effect for + :const:`SIGINT` is to raise the :exc:`KeyboardInterrupt` exception. If an + exception is raised the error indicator is set and the function returns ``-1``; + otherwise the function returns ``0``. The error indicator may or may not be + cleared if it was previously set. + + +.. c:function:: void PyErr_SetInterrupt() + + .. index:: + single: SIGINT + single: KeyboardInterrupt (built-in exception) + + Simulate the effect of a :const:`SIGINT` signal arriving. The next time + :c:func:`PyErr_CheckSignals` is called, the Python signal handler for + :const:`SIGINT` will be called. + + If :const:`SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. + +.. c:function:: int PySignal_SetWakeupFd(int fd) + + This utility function specifies a file descriptor to which the signal number + is written as a single byte whenever a signal is received. *fd* must be + non-blocking. It returns the previous such file descriptor. + + The value ``-1`` disables the feature; this is the initial state. + This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any + error checking. *fd* should be a valid file descriptor. The function should + only be called from the main thread. + + .. versionchanged:: 3.5 + On Windows, the function now also supports socket handles. + + +Exception Classes +================= + +.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict) + + This utility function creates and returns a new exception class. The *name* + argument must be the name of the new exception, a C string of the form + ``module.classname``. The *base* and *dict* arguments are normally ``NULL``. + This creates a class object derived from :exc:`Exception` (accessible in C as + :c:data:`PyExc_Exception`). + + The :attr:`__module__` attribute of the new class is set to the first part (up + to the last dot) of the *name* argument, and the class name is set to the last + part (after the last dot). The *base* argument can be used to specify alternate + base classes; it can either be only one class or a tuple of classes. The *dict* + argument can be used to specify a dictionary of class variables and methods. + + +.. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) + + Same as :c:func:`PyErr_NewException`, except that the new exception class can + easily be given a docstring: If *doc* is non-``NULL``, it will be used as the + docstring for the exception class. + + .. versionadded:: 3.2 + + +Exception Objects +================= + +.. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) + + Return the traceback associated with the exception as a new reference, as + accessible from Python through :attr:`__traceback__`. If there is no + traceback associated, this returns ``NULL``. + + +.. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) + + Set the traceback associated with the exception to *tb*. Use ``Py_None`` to + clear it. + + +.. c:function:: PyObject* PyException_GetContext(PyObject *ex) + + Return the context (another exception instance during whose handling *ex* was + raised) associated with the exception as a new reference, as accessible from + Python through :attr:`__context__`. If there is no context associated, this + returns ``NULL``. + + +.. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) + + Set the context associated with the exception to *ctx*. Use ``NULL`` to clear + it. There is no type check to make sure that *ctx* is an exception instance. + This steals a reference to *ctx*. + + +.. c:function:: PyObject* PyException_GetCause(PyObject *ex) + + Return the cause (either an exception instance, or :const:`None`, + set by ``raise ... from ...``) associated with the exception as a new + reference, as accessible from Python through :attr:`__cause__`. + + +.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) + + Set the cause associated with the exception to *cause*. Use ``NULL`` to clear + it. There is no type check to make sure that *cause* is either an exception + instance or :const:`None`. This steals a reference to *cause*. + + :attr:`__suppress_context__` is implicitly set to ``True`` by this function. + + +.. _unicodeexceptions: + +Unicode Exception Objects +========================= + +The following functions are used to create and modify Unicode exceptions from C. + +.. c:function:: PyObject* PyUnicodeDecodeError_Create(const char *encoding, const char *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeDecodeError` object with the attributes *encoding*, + *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are + UTF-8 encoded strings. + +.. c:function:: PyObject* PyUnicodeEncodeError_Create(const char *encoding, const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeEncodeError` object with the attributes *encoding*, + *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are + UTF-8 encoded strings. + + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", ...)``. + +.. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeTranslateError` object with the attributes *object*, + *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. + + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeTranslateError, "Onns", ...)``. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) + + Return the *encoding* attribute of the given exception object. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetObject(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetObject(PyObject *exc) + PyObject* PyUnicodeTranslateError_GetObject(PyObject *exc) + + Return the *object* attribute of the given exception object. + +.. c:function:: int PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) + int PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) + int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) + + Get the *start* attribute of the given exception object and place it into + *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on + failure. + +.. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) + int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) + int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) + + Set the *start* attribute of the given exception object to *start*. Return + ``0`` on success, ``-1`` on failure. + +.. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) + int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) + int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) + + Get the *end* attribute of the given exception object and place it into + *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on + failure. + +.. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) + int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) + int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) + + Set the *end* attribute of the given exception object to *end*. Return ``0`` + on success, ``-1`` on failure. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) + PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) + + Return the *reason* attribute of the given exception object. + +.. c:function:: int PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) + int PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) + int PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) + + Set the *reason* attribute of the given exception object to *reason*. Return + ``0`` on success, ``-1`` on failure. + + +.. _recursion: + +Recursion Control +================= + +These two functions provide a way to perform safe recursive calls at the C +level, both in the core and in extension modules. They are needed if the +recursive code does not necessarily invoke Python code (which tracks its +recursion depth automatically). +They are also not needed for *tp_call* implementations +because the :ref:`call protocol ` takes care of recursion handling. + +.. c:function:: int Py_EnterRecursiveCall(const char *where) + + Marks a point where a recursive C-level call is about to be performed. + + If :const:`USE_STACKCHECK` is defined, this function checks if the OS + stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it + sets a :exc:`MemoryError` and returns a nonzero value. + + The function then checks if the recursion limit is reached. If this is the + case, a :exc:`RecursionError` is set and a nonzero value is returned. + Otherwise, zero is returned. + + *where* should be a UTF-8 encoded string such as ``" in instance check"`` to + be concatenated to the :exc:`RecursionError` message caused by the recursion + depth limit. + + .. versionchanged:: 3.9 + This function is now also available in the limited API. + +.. c:function:: void Py_LeaveRecursiveCall(void) + + Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each + *successful* invocation of :c:func:`Py_EnterRecursiveCall`. + + .. versionchanged:: 3.9 + This function is now also available in the limited API. + +Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires +special recursion handling. In addition to protecting the stack, +:c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The +following two functions facilitate this functionality. Effectively, +these are the C equivalent to :func:`reprlib.recursive_repr`. + +.. c:function:: int Py_ReprEnter(PyObject *object) + + Called at the beginning of the :c:member:`~PyTypeObject.tp_repr` implementation to + detect cycles. + + If the object has already been processed, the function returns a + positive integer. In that case the :c:member:`~PyTypeObject.tp_repr` implementation + should return a string object indicating a cycle. As examples, + :class:`dict` objects return ``{...}`` and :class:`list` objects + return ``[...]``. + + The function will return a negative integer if the recursion limit + is reached. In that case the :c:member:`~PyTypeObject.tp_repr` implementation should + typically return ``NULL``. + + Otherwise, the function returns zero and the :c:member:`~PyTypeObject.tp_repr` + implementation can continue normally. + +.. c:function:: void Py_ReprLeave(PyObject *object) + + Ends a :c:func:`Py_ReprEnter`. Must be called once for each + invocation of :c:func:`Py_ReprEnter` that returns zero. + + +.. _standardexceptions: + +Standard Exceptions +=================== + +All standard Python exceptions are available as global variables whose names are +``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_BufferError + single: PyExc_ChildProcessError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_GeneratorExit + single: PyExc_ImportError + single: PyExc_IndentationError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_LookupError + single: PyExc_MemoryError + single: PyExc_ModuleNotFoundError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_RecursionError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_StopAsyncIteration + single: PyExc_StopIteration + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TabError + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_UnboundLocalError + single: PyExc_UnicodeDecodeError + single: PyExc_UnicodeEncodeError + single: PyExc_UnicodeError + single: PyExc_UnicodeTranslateError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + ++-----------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++=========================================+=================================+==========+ +| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NameError` | :exc:`NameError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TabError` | :exc:`TabError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | ++-----------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.3 + :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, + :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`, + :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`, + :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`, + :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`, + :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`, + :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` + and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. + +.. versionadded:: 3.5 + :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. + +.. versionadded:: 3.6 + :c:data:`PyExc_ModuleNotFoundError`. + +These are compatibility aliases to :c:data:`PyExc_OSError`: + +.. index:: + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + ++-------------------------------------+----------+ +| C Name | Notes | ++=====================================+==========+ +| :c:data:`PyExc_EnvironmentError` | | ++-------------------------------------+----------+ +| :c:data:`PyExc_IOError` | | ++-------------------------------------+----------+ +| :c:data:`PyExc_WindowsError` | \(3) | ++-------------------------------------+----------+ + +.. versionchanged:: 3.3 + These aliases used to be separate exception types. + +Notes: + +(1) + This is a base class for other standard exceptions. + +(2) + Only defined on Windows; protect code that uses this by testing that the + preprocessor macro ``MS_WINDOWS`` is defined. + +.. _standardwarningcategories: + +Standard Warning Categories +=========================== + +All standard Python warning categories are available as global variables whose +names are ``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_Warning + single: PyExc_BytesWarning + single: PyExc_DeprecationWarning + single: PyExc_FutureWarning + single: PyExc_ImportWarning + single: PyExc_PendingDeprecationWarning + single: PyExc_ResourceWarning + single: PyExc_RuntimeWarning + single: PyExc_SyntaxWarning + single: PyExc_UnicodeWarning + single: PyExc_UserWarning + ++------------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++==========================================+=================================+==========+ +| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | ++------------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.2 + :c:data:`PyExc_ResourceWarning`. + +Notes: + +(1) + This is a base class for other standard warning categories. diff --git a/Doc/c-api/memory.rst.bak b/Doc/c-api/memory.rst.bak new file mode 100644 index 00000000000000..588668ee853c65 --- /dev/null +++ b/Doc/c-api/memory.rst.bak @@ -0,0 +1,642 @@ +.. highlight:: c + + +.. _memory: + +***************** +Memory Management +***************** + +.. sectionauthor:: Vladimir Marangozov + + + +.. _memoryoverview: + +Overview +======== + +Memory management in Python involves a private heap containing all Python +objects and data structures. The management of this private heap is ensured +internally by the *Python memory manager*. The Python memory manager has +different components which deal with various dynamic storage management aspects, +like sharing, segmentation, preallocation or caching. + +At the lowest level, a raw memory allocator ensures that there is enough room in +the private heap for storing all Python-related data by interacting with the +memory manager of the operating system. On top of the raw memory allocator, +several object-specific allocators operate on the same heap and implement +distinct memory management policies adapted to the peculiarities of every object +type. For example, integer objects are managed differently within the heap than +strings, tuples or dictionaries because integers imply different storage +requirements and speed/space tradeoffs. The Python memory manager thus delegates +some of the work to the object-specific allocators, but ensures that the latter +operate within the bounds of the private heap. + +It is important to understand that the management of the Python heap is +performed by the interpreter itself and that the user has no control over it, +even if they regularly manipulate object pointers to memory blocks inside that +heap. The allocation of heap space for Python objects and other internal +buffers is performed on demand by the Python memory manager through the Python/C +API functions listed in this document. + +.. index:: + single: malloc() + single: calloc() + single: realloc() + single: free() + +To avoid memory corruption, extension writers should never try to operate on +Python objects with the functions exported by the C library: :c:func:`malloc`, +:c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. This will result in mixed +calls between the C allocator and the Python memory manager with fatal +consequences, because they implement different algorithms and operate on +different heaps. However, one may safely allocate and release memory blocks +with the C library allocator for individual purposes, as shown in the following +example:: + + PyObject *res; + char *buf = (char *) malloc(BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + ...Do some I/O operation involving buf... + res = PyBytes_FromString(buf); + free(buf); /* malloc'ed */ + return res; + +In this example, the memory request for the I/O buffer is handled by the C +library allocator. The Python memory manager is involved only in the allocation +of the bytes object returned as a result. + +In most situations, however, it is recommended to allocate memory from the +Python heap specifically because the latter is under control of the Python +memory manager. For example, this is required when the interpreter is extended +with new object types written in C. Another reason for using the Python heap is +the desire to *inform* the Python memory manager about the memory needs of the +extension module. Even when the requested memory is used exclusively for +internal, highly-specific purposes, delegating all memory requests to the Python +memory manager causes the interpreter to have a more accurate image of its +memory footprint as a whole. Consequently, under certain circumstances, the +Python memory manager may or may not trigger appropriate actions, like garbage +collection, memory compaction or other preventive procedures. Note that by using +the C library allocator as shown in the previous example, the allocated memory +for the I/O buffer escapes completely the Python memory manager. + +.. seealso:: + + The :envvar:`PYTHONMALLOC` environment variable can be used to configure + the memory allocators used by Python. + + The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print + statistics of the :ref:`pymalloc memory allocator ` every time a + new pymalloc object arena is created, and on shutdown. + +Allocator Domains +================= + +All allocating functions belong to one of three different "domains" (see also +:c:type:`PyMemAllocatorDomain`). These domains represent different allocation +strategies and are optimized for different purposes. The specific details on +how every domain allocates memory or what internal functions each domain calls +is considered an implementation detail, but for debugging purposes a simplified +table can be found at :ref:`here `. There is no hard +requirement to use the memory returned by the allocation functions belonging to +a given domain for only the purposes hinted by that domain (although this is the +recommended practice). For example, one could use the memory returned by +:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned +by :c:func:`PyObject_Malloc` for allocating memory for buffers. + +The three allocation domains are: + +* Raw domain: intended for allocating memory for general-purpose memory + buffers where the allocation *must* go to the system allocator or where the + allocator can operate without the :term:`GIL`. The memory is requested directly + to the system. + +* "Mem" domain: intended for allocating memory for Python buffers and + general-purpose memory buffers where the allocation must be performed with + the :term:`GIL` held. The memory is taken from the Python private heap. + +* Object domain: intended for allocating memory belonging to Python objects. The + memory is taken from the Python private heap. + +When freeing memory previously allocated by the allocating functions belonging to a +given domain,the matching specific deallocating functions must be used. For example, +:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. + +Raw Memory Interface +==================== + +The following function sets are wrappers to the system allocator. These +functions are thread-safe, the :term:`GIL ` does not +need to be held. + +The :ref:`default raw memory allocator ` uses +the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` +and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting +zero bytes. + +.. versionadded:: 3.4 + +.. c:function:: void* PyMem_RawMalloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. + + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as + if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-``NULL`` pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been + called instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will + be unchanged to the minimum of the old and the new sizes. + + If *p* is ``NULL``, the call is equivalent to ``PyMem_RawMalloc(n)``; else if + *n* is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-``NULL``. + + Unless *p* is ``NULL``, it must have been returned by a previous call to + :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or + :c:func:`PyMem_RawCalloc`. + + If the request fails, :c:func:`PyMem_RawRealloc` returns ``NULL`` and *p* + remains a valid pointer to the previous memory area. + + +.. c:function:: void PyMem_RawFree(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or + :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been + called before, undefined behavior occurs. + + If *p* is ``NULL``, no operation is performed. + + +.. _memoryinterface: + +Memory Interface +================ + +The following function sets, modeled after the ANSI C standard, but specifying +behavior when requesting zero bytes, are available for allocating and releasing +memory from the Python heap. + +The :ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `. + +.. warning:: + + The :term:`GIL ` must be held when using these + functions. + +.. versionchanged:: 3.6 + + The default allocator is now pymalloc instead of system :c:func:`malloc`. + +.. c:function:: void* PyMem_Malloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. + + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as + if ``PyMem_Malloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-``NULL`` pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called + instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyMem_Realloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will be + unchanged to the minimum of the old and the new sizes. + + If *p* is ``NULL``, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* + is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-``NULL``. + + Unless *p* is ``NULL``, it must have been returned by a previous call to + :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. + + If the request fails, :c:func:`PyMem_Realloc` returns ``NULL`` and *p* remains + a valid pointer to the previous memory area. + + +.. c:function:: void PyMem_Free(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or + :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called + before, undefined behavior occurs. + + If *p* is ``NULL``, no operation is performed. + +The following type-oriented macros are provided for convenience. Note that +*TYPE* refers to any C type. + + +.. c:function:: TYPE* PyMem_New(TYPE, size_t n) + + Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of + memory. Returns a pointer cast to :c:type:`TYPE*`. The memory will not have + been initialized in any way. + + +.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) + + Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * + sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE*`. On return, + *p* will be a pointer to the new memory area, or ``NULL`` in the event of + failure. + + This is a C preprocessor macro; *p* is always reassigned. Save the original + value of *p* to avoid losing memory when handling errors. + + +.. c:function:: void PyMem_Del(void *p) + + Same as :c:func:`PyMem_Free`. + +In addition, the following macro sets are provided for calling the Python memory +allocator directly, without involving the C API functions listed above. However, +note that their use does not preserve binary compatibility across Python +versions and is therefore deprecated in extension modules. + +* ``PyMem_MALLOC(size)`` +* ``PyMem_NEW(type, size)`` +* ``PyMem_REALLOC(ptr, size)`` +* ``PyMem_RESIZE(ptr, type, size)`` +* ``PyMem_FREE(ptr)`` +* ``PyMem_DEL(ptr)`` + + +Object allocators +================= + +The following function sets, modeled after the ANSI C standard, but specifying +behavior when requesting zero bytes, are available for allocating and releasing +memory from the Python heap. + +.. note:: + There is no guarantee that the memory returned by these allocators can be + succesfully casted to a Python object when intercepting the allocating + functions in this domain by the methods described in + the :ref:`Customize Memory Allocators ` section. + +The :ref:`default object allocator ` uses the +:ref:`pymalloc memory allocator `. + +.. warning:: + + The :term:`GIL ` must be held when using these + functions. + +.. c:function:: void* PyObject_Malloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. + + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as + if ``PyObject_Malloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-``NULL`` pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called + instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyObject_Realloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will be + unchanged to the minimum of the old and the new sizes. + + If *p* is ``NULL``, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* + is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-``NULL``. + + Unless *p* is ``NULL``, it must have been returned by a previous call to + :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. + + If the request fails, :c:func:`PyObject_Realloc` returns ``NULL`` and *p* remains + a valid pointer to the previous memory area. + + +.. c:function:: void PyObject_Free(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or + :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called + before, undefined behavior occurs. + + If *p* is ``NULL``, no operation is performed. + + +.. _default-memory-allocators: + +Default Memory Allocators +========================= + +Default memory allocators: + +=============================== ==================== ================== ===================== ==================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=============================== ==================== ================== ===================== ==================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +=============================== ==================== ================== ===================== ==================== + +Legend: + +* Name: value for :envvar:`PYTHONMALLOC` environment variable +* ``malloc``: system allocators from the standard C library, C functions: + :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free` +* ``pymalloc``: :ref:`pymalloc memory allocator ` +* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` + +.. _customize-memory-allocators: + +Customize Memory Allocators +=========================== + +.. versionadded:: 3.4 + +.. c:type:: PyMemAllocatorEx + + Structure used to describe a memory block allocator. The structure has + four fields: + + +----------------------------------------------------------+---------------------------------------+ + | Field | Meaning | + +==========================================================+=======================================+ + | ``void *ctx`` | user context passed as first argument | + +----------------------------------------------------------+---------------------------------------+ + | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | + +----------------------------------------------------------+---------------------------------------+ + | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized | + | | with zeros | + +----------------------------------------------------------+---------------------------------------+ + | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | + +----------------------------------------------------------+---------------------------------------+ + | ``void free(void *ctx, void *ptr)`` | free a memory block | + +----------------------------------------------------------+---------------------------------------+ + + .. versionchanged:: 3.5 + The :c:type:`PyMemAllocator` structure was renamed to + :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. + + +.. c:type:: PyMemAllocatorDomain + + Enum used to identify an allocator domain. Domains: + + .. c:macro:: PYMEM_DOMAIN_RAW + + Functions: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + + .. c:macro:: PYMEM_DOMAIN_MEM + + Functions: + + * :c:func:`PyMem_Malloc`, + * :c:func:`PyMem_Realloc` + * :c:func:`PyMem_Calloc` + * :c:func:`PyMem_Free` + + .. c:macro:: PYMEM_DOMAIN_OBJ + + Functions: + + * :c:func:`PyObject_Malloc` + * :c:func:`PyObject_Realloc` + * :c:func:`PyObject_Calloc` + * :c:func:`PyObject_Free` + +.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) + + Get the memory block allocator of the specified domain. + + +.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) + + Set the memory block allocator of the specified domain. + + The new allocator must return a distinct non-``NULL`` pointer when requesting + zero bytes. + + For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be + thread-safe: the :term:`GIL ` is not held when the + allocator is called. + + If the new allocator is not a hook (does not call the previous allocator), + the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the + debug hooks on top on the new allocator. + + +.. c:function:: void PyMem_SetupDebugHooks(void) + + Setup hooks to detect bugs in the Python memory allocator functions. + + Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``), + freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks + are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``). + + Runtime checks: + + - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer + allocated by :c:func:`PyMem_Malloc` + - Detect write before the start of the buffer (buffer underflow) + - Detect write after the end of the buffer (buffer overflow) + - Check that the :term:`GIL ` is held when + allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: + :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: + :c:func:`PyMem_Malloc`) domains are called + + On error, the debug hooks use the :mod:`tracemalloc` module to get the + traceback where a memory block was allocated. The traceback is only + displayed if :mod:`tracemalloc` is tracing Python memory allocations and the + memory block was traced. + + These hooks are :ref:`installed by default ` if + Python is compiled in debug + mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install + debug hooks on a Python compiled in release mode. + + .. versionchanged:: 3.6 + This function now also works on Python compiled in release mode. + On error, the debug hooks now use :mod:`tracemalloc` to get the traceback + where a memory block was allocated. The debug hooks now also check + if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and + :c:data:`PYMEM_DOMAIN_MEM` domains are called. + + .. versionchanged:: 3.8 + Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and + ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` + and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` + and ``free()``. + + +.. _pymalloc: + +The pymalloc allocator +====================== + +Python has a *pymalloc* allocator optimized for small objects (smaller or equal +to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" +with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and +:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. + +*pymalloc* is the :ref:`default allocator ` of the +:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and +:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. + +The arena allocator uses the following functions: + +* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, +* :c:func:`mmap` and :c:func:`munmap` if available, +* :c:func:`malloc` and :c:func:`free` otherwise. + +Customize pymalloc Arena Allocator +---------------------------------- + +.. versionadded:: 3.4 + +.. c:type:: PyObjectArenaAllocator + + Structure used to describe an arena allocator. The structure has + three fields: + + +--------------------------------------------------+---------------------------------------+ + | Field | Meaning | + +==================================================+=======================================+ + | ``void *ctx`` | user context passed as first argument | + +--------------------------------------------------+---------------------------------------+ + | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | + +--------------------------------------------------+---------------------------------------+ + | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | + +--------------------------------------------------+---------------------------------------+ + +.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) + + Get the arena allocator. + +.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) + + Set the arena allocator. + + +tracemalloc C API +================= + +.. versionadded:: 3.7 + +.. c:function:: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) + + Track an allocated memory block in the :mod:`tracemalloc` module. + + Return ``0`` on success, return ``-1`` on error (failed to allocate memory to + store the trace). Return ``-2`` if tracemalloc is disabled. + + If memory block is already tracked, update the existing trace. + +.. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) + + Untrack an allocated memory block in the :mod:`tracemalloc` module. + Do nothing if the block was not tracked. + + Return ``-2`` if tracemalloc is disabled, otherwise return ``0``. + + +.. _memoryexamples: + +Examples +======== + +Here is the example from section :ref:`memoryoverview`, rewritten so that the +I/O buffer is allocated from the Python heap by using the first function set:: + + PyObject *res; + char *buf = (char *) PyMem_Malloc(BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + /* ...Do some I/O operation involving buf... */ + res = PyBytes_FromString(buf); + PyMem_Free(buf); /* allocated with PyMem_Malloc */ + return res; + +The same code using the type-oriented function set:: + + PyObject *res; + char *buf = PyMem_New(char, BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + /* ...Do some I/O operation involving buf... */ + res = PyBytes_FromString(buf); + PyMem_Del(buf); /* allocated with PyMem_New */ + return res; + +Note that in the two examples above, the buffer is always manipulated via +functions belonging to the same set. Indeed, it is required to use the same +memory API family for a given memory block, so that the risk of mixing different +allocators is reduced to a minimum. The following code sequence contains two +errors, one of which is labeled as *fatal* because it mixes two different +allocators operating on different heaps. :: + + char *buf1 = PyMem_New(char, BUFSIZ); + char *buf2 = (char *) malloc(BUFSIZ); + char *buf3 = (char *) PyMem_Malloc(BUFSIZ); + ... + PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ + free(buf2); /* Right -- allocated via malloc() */ + free(buf1); /* Fatal -- should be PyMem_Del() */ + +In addition to the functions aimed at handling raw memory blocks from the Python +heap, objects in Python are allocated and released with :c:func:`PyObject_New`, +:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`. + +These will be explained in the next chapter on defining and implementing new +object types in C. diff --git a/Doc/faq/design.rst.bak b/Doc/faq/design.rst.bak new file mode 100644 index 00000000000000..7fe1c6d58f58a1 --- /dev/null +++ b/Doc/faq/design.rst.bak @@ -0,0 +1,759 @@ +====================== +Design and History FAQ +====================== + +.. only:: html + + .. contents:: + + +Why does Python use indentation for grouping of statements? +----------------------------------------------------------- + +Guido van Rossum believes that using indentation for grouping is extremely +elegant and contributes a lot to the clarity of the average Python program. +Most people learn to love this feature after a while. + +Since there are no begin/end brackets there cannot be a disagreement between +grouping perceived by the parser and the human reader. Occasionally C +programmers will encounter a fragment of code like this:: + + if (x <= y) + x++; + y--; + z++; + +Only the ``x++`` statement is executed if the condition is true, but the +indentation leads many to believe otherwise. Even experienced C programmers will +sometimes stare at it a long time wondering as to why ``y`` is being decremented even +for ``x > y``. + +Because there are no begin/end brackets, Python is much less prone to +coding-style conflicts. In C there are many different ways to place the braces. +After becoming used to reading and writing code using a particular style, +it is normal to feel somewhat uneasy when reading (or being required to write) +in a different one. + + +Many coding styles place begin/end brackets on a line by themselves. This makes +programs considerably longer and wastes valuable screen space, making it harder +to get a good overview of a program. Ideally, a function should fit on one +screen (say, 20--30 lines). 20 lines of Python can do a lot more work than 20 +lines of C. This is not solely due to the lack of begin/end brackets -- the +lack of declarations and the high-level data types are also responsible -- but +the indentation-based syntax certainly helps. + + +Why am I getting strange results with simple arithmetic operations? +------------------------------------------------------------------- + +See the next question. + + +Why are floating-point calculations so inaccurate? +-------------------------------------------------- + +Users are often surprised by results like this:: + + >>> 1.2 - 1.0 + 0.19999999999999996 + +and think it is a bug in Python. It's not. This has little to do with Python, +and much more to do with how the underlying platform handles floating-point +numbers. + +The :class:`float` type in CPython uses a C ``double`` for storage. A +:class:`float` object's value is stored in binary floating-point with a fixed +precision (typically 53 bits) and Python uses C operations, which in turn rely +on the hardware implementation in the processor, to perform floating-point +operations. This means that as far as floating-point operations are concerned, +Python behaves like many popular languages including C and Java. + +Many numbers that can be written easily in decimal notation cannot be expressed +exactly in binary floating-point. For example, after:: + + >>> x = 1.2 + +the value stored for ``x`` is a (very good) approximation to the decimal value +``1.2``, but is not exactly equal to it. On a typical machine, the actual +stored value is:: + + 1.0011001100110011001100110011001100110011001100110011 (binary) + +which is exactly:: + + 1.1999999999999999555910790149937383830547332763671875 (decimal) + +The typical precision of 53 bits provides Python floats with 15--16 +decimal digits of accuracy. + +For a fuller explanation, please see the :ref:`floating point arithmetic +` chapter in the Python tutorial. + + +Why are Python strings immutable? +--------------------------------- + +There are several advantages. + +One is performance: knowing that a string is immutable means we can allocate +space for it at creation time, and the storage requirements are fixed and +unchanging. This is also one of the reasons for the distinction between tuples +and lists. + +Another advantage is that strings in Python are considered as "elemental" as +numbers. No amount of activity will change the value 8 to anything else, and in +Python, no amount of activity will change the string "eight" to anything else. + + +.. _why-self: + +Why must 'self' be used explicitly in method definitions and calls? +------------------------------------------------------------------- + +The idea was borrowed from Modula-3. It turns out to be very useful, for a +variety of reasons. + +First, it's more obvious that you are using a method or instance attribute +instead of a local variable. Reading ``self.x`` or ``self.meth()`` makes it +absolutely clear that an instance variable or method is used even if you don't +know the class definition by heart. In C++, you can sort of tell by the lack of +a local variable declaration (assuming globals are rare or easily recognizable) +-- but in Python, there are no local variable declarations, so you'd have to +look up the class definition to be sure. Some C++ and Java coding standards +call for instance attributes to have an ``m_`` prefix, so this explicitness is +still useful in those languages, too. + +Second, it means that no special syntax is necessary if you want to explicitly +reference or call the method from a particular class. In C++, if you want to +use a method from a base class which is overridden in a derived class, you have +to use the ``::`` operator -- in Python you can write +``baseclass.methodname(self, )``. This is particularly useful +for :meth:`__init__` methods, and in general in cases where a derived class +method wants to extend the base class method of the same name and thus has to +call the base class method somehow. + +Finally, for instance variables it solves a syntactic problem with assignment: +since local variables in Python are (by definition!) those variables to which a +value is assigned in a function body (and that aren't explicitly declared +global), there has to be some way to tell the interpreter that an assignment was +meant to assign to an instance variable instead of to a local variable, and it +should preferably be syntactic (for efficiency reasons). C++ does this through +declarations, but Python doesn't have declarations and it would be a pity having +to introduce them just for this purpose. Using the explicit ``self.var`` solves +this nicely. Similarly, for using instance variables, having to write +``self.var`` means that references to unqualified names inside a method don't +have to search the instance's directories. To put it another way, local +variables and instance variables live in two different namespaces, and you need +to tell Python which namespace to use. + + +.. _why-can-t-i-use-an-assignment-in-an-expression: + +Why can't I use an assignment in an expression? +----------------------------------------------- + +Starting in Python 3.8, you can! + +Assignment expressions using the walrus operator `:=` assign a variable in an +expression:: + + while chunk := fp.read(200): + print(chunk) + +See :pep:`572` for more information. + + + +Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))? +---------------------------------------------------------------------------------------------------------------- + +As Guido said: + + (a) For some operations, prefix notation just reads better than + postfix -- prefix (and infix!) operations have a long tradition in + mathematics which likes notations where the visuals help the + mathematician thinking about a problem. Compare the easy with which we + rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of + doing the same thing using a raw OO notation. + + (b) When I read code that says len(x) I *know* that it is asking for + the length of something. This tells me two things: the result is an + integer, and the argument is some kind of container. To the contrary, + when I read x.len(), I have to already know that x is some kind of + container implementing an interface or inheriting from a class that + has a standard len(). Witness the confusion we occasionally have when + a class that is not implementing a mapping has a get() or keys() + method, or something that isn't a file has a write() method. + + -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html + + +Why is join() a string method instead of a list or tuple method? +---------------------------------------------------------------- + +Strings became much more like other standard types starting in Python 1.6, when +methods were added which give the same functionality that has always been +available using the functions of the string module. Most of these new methods +have been widely accepted, but the one which appears to make some programmers +feel uncomfortable is:: + + ", ".join(['1', '2', '4', '8', '16']) + +which gives the result:: + + "1, 2, 4, 8, 16" + +There are two common arguments against this usage. + +The first runs along the lines of: "It looks really ugly using a method of a +string literal (string constant)", to which the answer is that it might, but a +string literal is just a fixed value. If the methods are to be allowed on names +bound to strings there is no logical reason to make them unavailable on +literals. + +The second objection is typically cast as: "I am really telling a sequence to +join its members together with a string constant". Sadly, you aren't. For some +reason there seems to be much less difficulty with having :meth:`~str.split` as +a string method, since in that case it is easy to see that :: + + "1, 2, 4, 8, 16".split(", ") + +is an instruction to a string literal to return the substrings delimited by the +given separator (or, by default, arbitrary runs of white space). + +:meth:`~str.join` is a string method because in using it you are telling the +separator string to iterate over a sequence of strings and insert itself between +adjacent elements. This method can be used with any argument which obeys the +rules for sequence objects, including any new classes you might define yourself. +Similar methods exist for bytes and bytearray objects. + + +How fast are exceptions? +------------------------ + +A try/except block is extremely efficient if no exceptions are raised. Actually +catching an exception is expensive. In versions of Python prior to 2.0 it was +common to use this idiom:: + + try: + value = mydict[key] + except KeyError: + mydict[key] = getvalue(key) + value = mydict[key] + +This only made sense when you expected the dict to have the key almost all the +time. If that wasn't the case, you coded it like this:: + + if key in mydict: + value = mydict[key] + else: + value = mydict[key] = getvalue(key) + +For this specific case, you could also use ``value = dict.setdefault(key, +getvalue(key))``, but only if the ``getvalue()`` call is cheap enough because it +is evaluated in all cases. + + +Why isn't there a switch or case statement in Python? +----------------------------------------------------- + +You can do this easily enough with a sequence of ``if... elif... elif... else``. +For literal values, or constants within a namespace, you can also use a +``match ... case`` statement. + +For cases where you need to choose from a very large number of possibilities, +you can create a dictionary mapping case values to functions to call. For +example:: + + def function_1(...): + ... + + functions = {'a': function_1, + 'b': function_2, + 'c': self.method_1, ...} + + func = functions[value] + func() + +For calling methods on objects, you can simplify yet further by using the +:func:`getattr` built-in to retrieve methods with a particular name:: + + def visit_a(self, ...): + ... + ... + + def dispatch(self, value): + method_name = 'visit_' + str(value) + method = getattr(self, method_name) + method() + +It's suggested that you use a prefix for the method names, such as ``visit_`` in +this example. Without such a prefix, if values are coming from an untrusted +source, an attacker would be able to call any method on your object. + + +Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? +-------------------------------------------------------------------------------------------------------- + +Answer 1: Unfortunately, the interpreter pushes at least one C stack frame for +each Python stack frame. Also, extensions can call back into Python at almost +random moments. Therefore, a complete threads implementation requires thread +support for C. + +Answer 2: Fortunately, there is `Stackless Python `_, +which has a completely redesigned interpreter loop that avoids the C stack. + + +Why can't lambda expressions contain statements? +------------------------------------------------ + +Python lambda expressions cannot contain statements because Python's syntactic +framework can't handle statements nested inside expressions. However, in +Python, this is not a serious problem. Unlike lambda forms in other languages, +where they add functionality, Python lambdas are only a shorthand notation if +you're too lazy to define a function. + +Functions are already first class objects in Python, and can be declared in a +local scope. Therefore the only advantage of using a lambda instead of a +locally-defined function is that you don't need to invent a name for the +function -- but that's just a local variable to which the function object (which +is exactly the same type of object that a lambda expression yields) is assigned! + + +Can Python be compiled to machine code, C or some other language? +----------------------------------------------------------------- + +`Cython `_ compiles a modified version of Python with +optional annotations into C extensions. `Nuitka `_ is +an up-and-coming compiler of Python into C++ code, aiming to support the full +Python language. For compiling to Java you can consider +`VOC `_. + + +How does Python manage memory? +------------------------------ + +The details of Python memory management depend on the implementation. The +standard implementation of Python, :term:`CPython`, uses reference counting to +detect inaccessible objects, and another mechanism to collect reference cycles, +periodically executing a cycle detection algorithm which looks for inaccessible +cycles and deletes the objects involved. The :mod:`gc` module provides functions +to perform a garbage collection, obtain debugging statistics, and tune the +collector's parameters. + +Other implementations (such as `Jython `_ or +`PyPy `_), however, can rely on a different mechanism +such as a full-blown garbage collector. This difference can cause some +subtle porting problems if your Python code depends on the behavior of the +reference counting implementation. + +In some Python implementations, the following code (which is fine in CPython) +will probably run out of file descriptors:: + + for file in very_long_list_of_files: + f = open(file) + c = f.read(1) + +Indeed, using CPython's reference counting and destructor scheme, each new +assignment to *f* closes the previous file. With a traditional GC, however, +those file objects will only get collected (and closed) at varying and possibly +long intervals. + +If you want to write code that will work with any Python implementation, +you should explicitly close the file or use the :keyword:`with` statement; +this will work regardless of memory management scheme:: + + for file in very_long_list_of_files: + with open(file) as f: + c = f.read(1) + + +Why doesn't CPython use a more traditional garbage collection scheme? +--------------------------------------------------------------------- + +For one thing, this is not a C standard feature and hence it's not portable. +(Yes, we know about the Boehm GC library. It has bits of assembler code for +*most* common platforms, not for all of them, and although it is mostly +transparent, it isn't completely transparent; patches are required to get +Python to work with it.) + +Traditional GC also becomes a problem when Python is embedded into other +applications. While in a standalone Python it's fine to replace the standard +malloc() and free() with versions provided by the GC library, an application +embedding Python may want to have its *own* substitute for malloc() and free(), +and may not want Python's. Right now, CPython works with anything that +implements malloc() and free() properly. + + +Why isn't all memory freed when CPython exits? +---------------------------------------------- + +Objects referenced from the global namespaces of Python modules are not always +deallocated when Python exits. This may happen if there are circular +references. There are also certain bits of memory that are allocated by the C +library that are impossible to free (e.g. a tool like Purify will complain about +these). Python is, however, aggressive about cleaning up memory on exit and +does try to destroy every single object. + +If you want to force Python to delete certain things on deallocation use the +:mod:`atexit` module to run a function that will force those deletions. + + +Why are there separate tuple and list data types? +------------------------------------------------- + +Lists and tuples, while similar in many respects, are generally used in +fundamentally different ways. Tuples can be thought of as being similar to +Pascal records or C structs; they're small collections of related data which may +be of different types which are operated on as a group. For example, a +Cartesian coordinate is appropriately represented as a tuple of two or three +numbers. + +Lists, on the other hand, are more like arrays in other languages. They tend to +hold a varying number of objects all of which have the same type and which are +operated on one-by-one. For example, ``os.listdir('.')`` returns a list of +strings representing the files in the current directory. Functions which +operate on this output would generally not break if you added another file or +two to the directory. + +Tuples are immutable, meaning that once a tuple has been created, you can't +replace any of its elements with a new value. Lists are mutable, meaning that +you can always change a list's elements. Only immutable elements can be used as +dictionary keys, and hence only tuples and not lists can be used as keys. + + +How are lists implemented in CPython? +------------------------------------- + +CPython's lists are really variable-length arrays, not Lisp-style linked lists. +The implementation uses a contiguous array of references to other objects, and +keeps a pointer to this array and the array's length in a list head structure. + +This makes indexing a list ``a[i]`` an operation whose cost is independent of +the size of the list or the value of the index. + +When items are appended or inserted, the array of references is resized. Some +cleverness is applied to improve the performance of appending items repeatedly; +when the array must be grown, some extra space is allocated so the next few +times don't require an actual resize. + + +How are dictionaries implemented in CPython? +-------------------------------------------- + +CPython's dictionaries are implemented as resizable hash tables. Compared to +B-trees, this gives better performance for lookup (the most common operation by +far) under most circumstances, and the implementation is simpler. + +Dictionaries work by computing a hash code for each key stored in the dictionary +using the :func:`hash` built-in function. The hash code varies widely depending +on the key and a per-process seed; for example, "Python" could hash to +-539294296 while "python", a string that differs by a single bit, could hash +to 1142331976. The hash code is then used to calculate a location in an +internal array where the value will be stored. Assuming that you're storing +keys that all have different hash values, this means that dictionaries take +constant time -- O(1), in Big-O notation -- to retrieve a key. + + +Why must dictionary keys be immutable? +-------------------------------------- + +The hash table implementation of dictionaries uses a hash value calculated from +the key value to find the key. If the key were a mutable object, its value +could change, and thus its hash could also change. But since whoever changes +the key object can't tell that it was being used as a dictionary key, it can't +move the entry around in the dictionary. Then, when you try to look up the same +object in the dictionary it won't be found because its hash value is different. +If you tried to look up the old value it wouldn't be found either, because the +value of the object found in that hash bin would be different. + +If you want a dictionary indexed with a list, simply convert the list to a tuple +first; the function ``tuple(L)`` creates a tuple with the same entries as the +list ``L``. Tuples are immutable and can therefore be used as dictionary keys. + +Some unacceptable solutions that have been proposed: + +- Hash lists by their address (object ID). This doesn't work because if you + construct a new list with the same value it won't be found; e.g.:: + + mydict = {[1, 2]: '12'} + print(mydict[[1, 2]]) + + would raise a :exc:`KeyError` exception because the id of the ``[1, 2]`` used in the + second line differs from that in the first line. In other words, dictionary + keys should be compared using ``==``, not using :keyword:`is`. + +- Make a copy when using a list as a key. This doesn't work because the list, + being a mutable object, could contain a reference to itself, and then the + copying code would run into an infinite loop. + +- Allow lists as keys but tell the user not to modify them. This would allow a + class of hard-to-track bugs in programs when you forgot or modified a list by + accident. It also invalidates an important invariant of dictionaries: every + value in ``d.keys()`` is usable as a key of the dictionary. + +- Mark lists as read-only once they are used as a dictionary key. The problem + is that it's not just the top-level object that could change its value; you + could use a tuple containing a list as a key. Entering anything as a key into + a dictionary would require marking all objects reachable from there as + read-only -- and again, self-referential objects could cause an infinite loop. + +There is a trick to get around this if you need to, but use it at your own risk: +You can wrap a mutable structure inside a class instance which has both a +:meth:`__eq__` and a :meth:`__hash__` method. You must then make sure that the +hash value for all such wrapper objects that reside in a dictionary (or other +hash based structure), remain fixed while the object is in the dictionary (or +other structure). :: + + class ListWrapper: + def __init__(self, the_list): + self.the_list = the_list + + def __eq__(self, other): + return self.the_list == other.the_list + + def __hash__(self): + l = self.the_list + result = 98767 - len(l)*555 + for i, el in enumerate(l): + try: + result = result + (hash(el) % 9999999) * 1001 + i + except Exception: + result = (result % 7777777) + i * 333 + return result + +Note that the hash computation is complicated by the possibility that some +members of the list may be unhashable and also by the possibility of arithmetic +overflow. + +Furthermore it must always be the case that if ``o1 == o2`` (ie ``o1.__eq__(o2) +is True``) then ``hash(o1) == hash(o2)`` (ie, ``o1.__hash__() == o2.__hash__()``), +regardless of whether the object is in a dictionary or not. If you fail to meet +these restrictions dictionaries and other hash based structures will misbehave. + +In the case of ListWrapper, whenever the wrapper object is in a dictionary the +wrapped list must not change to avoid anomalies. Don't do this unless you are +prepared to think hard about the requirements and the consequences of not +meeting them correctly. Consider yourself warned. + + +Why doesn't list.sort() return the sorted list? +----------------------------------------------- + +In situations where performance matters, making a copy of the list just to sort +it would be wasteful. Therefore, :meth:`list.sort` sorts the list in place. In +order to remind you of that fact, it does not return the sorted list. This way, +you won't be fooled into accidentally overwriting a list when you need a sorted +copy but also need to keep the unsorted version around. + +If you want to return a new list, use the built-in :func:`sorted` function +instead. This function creates a new list from a provided iterable, sorts +it and returns it. For example, here's how to iterate over the keys of a +dictionary in sorted order:: + + for key in sorted(mydict): + ... # do whatever with mydict[key]... + + +How do you specify and enforce an interface spec in Python? +----------------------------------------------------------- + +An interface specification for a module as provided by languages such as C++ and +Java describes the prototypes for the methods and functions of the module. Many +feel that compile-time enforcement of interface specifications helps in the +construction of large programs. + +Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes +(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check +whether an instance or a class implements a particular ABC. The +:mod:`collections.abc` module defines a set of useful ABCs such as +:class:`~collections.abc.Iterable`, :class:`~collections.abc.Container`, and +:class:`~collections.abc.MutableMapping`. + +For Python, many of the advantages of interface specifications can be obtained +by an appropriate test discipline for components. + +A good test suite for a module can both provide a regression test and serve as a +module interface specification and a set of examples. Many Python modules can +be run as a script to provide a simple "self test." Even modules which use +complex external interfaces can often be tested in isolation using trivial +"stub" emulations of the external interface. The :mod:`doctest` and +:mod:`unittest` modules or third-party test frameworks can be used to construct +exhaustive test suites that exercise every line of code in a module. + +An appropriate testing discipline can help build large complex applications in +Python as well as having interface specifications would. In fact, it can be +better because an interface specification cannot test certain properties of a +program. For example, the :meth:`append` method is expected to add new elements +to the end of some internal list; an interface specification cannot test that +your :meth:`append` implementation will actually do this correctly, but it's +trivial to check this property in a test suite. + +Writing test suites is very helpful, and you might want to design your code to +make it easily tested. One increasingly popular technique, test-driven +development, calls for writing parts of the test suite first, before you write +any of the actual code. Of course Python allows you to be sloppy and not write +test cases at all. + + +Why is there no goto? +--------------------- + +You can use exceptions to provide a "structured goto" that even works across +function calls. Many feel that exceptions can conveniently emulate all +reasonable uses of the "go" or "goto" constructs of C, Fortran, and other +languages. For example:: + + class label(Exception): pass # declare a label + + try: + ... + if condition: raise label() # goto label + ... + except label: # where to goto + pass + ... + +This doesn't allow you to jump into the middle of a loop, but that's usually +considered an abuse of goto anyway. Use sparingly. + + +Why can't raw strings (r-strings) end with a backslash? +------------------------------------------------------- + +More precisely, they can't end with an odd number of backslashes: the unpaired +backslash at the end escapes the closing quote character, leaving an +unterminated string. + +Raw strings were designed to ease creating input for processors (chiefly regular +expression engines) that want to do their own backslash escape processing. Such +processors consider an unmatched trailing backslash to be an error anyway, so +raw strings disallow that. In return, they allow you to pass on the string +quote character by escaping it with a backslash. These rules work well when +r-strings are used for their intended purpose. + +If you're trying to build Windows pathnames, note that all Windows system calls +accept forward slashes too:: + + f = open("/mydir/file.txt") # works fine! + +If you're trying to build a pathname for a DOS command, try e.g. one of :: + + dir = r"\this\is\my\dos\dir" "\\" + dir = r"\this\is\my\dos\dir\ "[:-1] + dir = "\\this\\is\\my\\dos\\dir\\" + + +Why doesn't Python have a "with" statement for attribute assignments? +--------------------------------------------------------------------- + +Python has a 'with' statement that wraps the execution of a block, calling code +on the entrance and exit from the block. Some languages have a construct that +looks like this:: + + with obj: + a = 1 # equivalent to obj.a = 1 + total = total + 1 # obj.total = obj.total + 1 + +In Python, such a construct would be ambiguous. + +Other languages, such as Object Pascal, Delphi, and C++, use static types, so +it's possible to know, in an unambiguous way, what member is being assigned +to. This is the main point of static typing -- the compiler *always* knows the +scope of every variable at compile time. + +Python uses dynamic types. It is impossible to know in advance which attribute +will be referenced at runtime. Member attributes may be added or removed from +objects on the fly. This makes it impossible to know, from a simple reading, +what attribute is being referenced: a local one, a global one, or a member +attribute? + +For instance, take the following incomplete snippet:: + + def foo(a): + with a: + print(x) + +The snippet assumes that "a" must have a member attribute called "x". However, +there is nothing in Python that tells the interpreter this. What should happen +if "a" is, let us say, an integer? If there is a global variable named "x", +will it be used inside the with block? As you see, the dynamic nature of Python +makes such choices much harder. + +The primary benefit of "with" and similar language features (reduction of code +volume) can, however, easily be achieved in Python by assignment. Instead of:: + + function(args).mydict[index][index].a = 21 + function(args).mydict[index][index].b = 42 + function(args).mydict[index][index].c = 63 + +write this:: + + ref = function(args).mydict[index][index] + ref.a = 21 + ref.b = 42 + ref.c = 63 + +This also has the side-effect of increasing execution speed because name +bindings are resolved at run-time in Python, and the second version only needs +to perform the resolution once. + + +Why are colons required for the if/while/def/class statements? +-------------------------------------------------------------- + +The colon is required primarily to enhance readability (one of the results of +the experimental ABC language). Consider this:: + + if a == b + print(a) + +versus :: + + if a == b: + print(a) + +Notice how the second one is slightly easier to read. Notice further how a +colon sets off the example in this FAQ answer; it's a standard usage in English. + +Another minor reason is that the colon makes it easier for editors with syntax +highlighting; they can look for colons to decide when indentation needs to be +increased instead of having to do a more elaborate parsing of the program text. + + +Why does Python allow commas at the end of lists and tuples? +------------------------------------------------------------ + +Python lets you add a trailing comma at the end of lists, tuples, and +dictionaries:: + + [1, 2, 3,] + ('a', 'b', 'c',) + d = { + "A": [1, 5], + "B": [6, 7], # last trailing comma is optional but good style + } + + +There are several reasons to allow this. + +When you have a literal value for a list, tuple, or dictionary spread across +multiple lines, it's easier to add more elements because you don't have to +remember to add a comma to the previous line. The lines can also be reordered +without creating a syntax error. + +Accidentally omitting the comma can lead to errors that are hard to diagnose. +For example:: + + x = [ + "fee", + "fie" + "foo", + "fum" + ] + +This list looks like it has four elements, but it actually contains three: +"fee", "fiefoo" and "fum". Always adding the comma avoids this source of error. + +Allowing the trailing comma may also make programmatic code generation easier. diff --git a/Doc/howto/descriptor.rst.bak b/Doc/howto/descriptor.rst.bak new file mode 100644 index 00000000000000..94a8b4e6b40b96 --- /dev/null +++ b/Doc/howto/descriptor.rst.bak @@ -0,0 +1,1575 @@ +.. _descriptorhowto: + +====================== +Descriptor HowTo Guide +====================== + +:Author: Raymond Hettinger +:Contact: + +.. Contents:: + + +:term:`Descriptors ` let objects customize attribute lookup, +storage, and deletion. + +This guide has four major sections: + +1) The "primer" gives a basic overview, moving gently from simple examples, + adding one feature at a time. Start here if you're new to descriptors. + +2) The second section shows a complete, practical descriptor example. If you + already know the basics, start there. + +3) The third section provides a more technical tutorial that goes into the + detailed mechanics of how descriptors work. Most people don't need this + level of detail. + +4) The last section has pure Python equivalents for built-in descriptors that + are written in C. Read this if you're curious about how functions turn + into bound methods or about the implementation of common tools like + :func:`classmethod`, :func:`staticmethod`, :func:`property`, and + :term:`__slots__`. + + +Primer +^^^^^^ + +In this primer, we start with the most basic possible example and then we'll +add new capabilities one by one. + + +Simple example: A descriptor that returns a constant +---------------------------------------------------- + +The :class:`Ten` class is a descriptor whose :meth:`__get__` method always +returns the constant ``10``: + +.. testcode:: + + class Ten: + def __get__(self, obj, objtype=None): + return 10 + +To use the descriptor, it must be stored as a class variable in another class: + +.. testcode:: + + class A: + x = 5 # Regular class attribute + y = Ten() # Descriptor instance + +An interactive session shows the difference between normal attribute lookup +and descriptor lookup: + +.. doctest:: + + >>> a = A() # Make an instance of class A + >>> a.x # Normal attribute lookup + 5 + >>> a.y # Descriptor lookup + 10 + +In the ``a.x`` attribute lookup, the dot operator finds ``'x': 5`` +in the class dictionary. In the ``a.y`` lookup, the dot operator +finds a descriptor instance, recognized by its ``__get__`` method. +Calling that method returns ``10``. + +Note that the value ``10`` is not stored in either the class dictionary or the +instance dictionary. Instead, the value ``10`` is computed on demand. + +This example shows how a simple descriptor works, but it isn't very useful. +For retrieving constants, normal attribute lookup would be better. + +In the next section, we'll create something more useful, a dynamic lookup. + + +Dynamic lookups +--------------- + +Interesting descriptors typically run computations instead of returning +constants: + +.. testcode:: + + import os + + class DirectorySize: + + def __get__(self, obj, objtype=None): + return len(os.listdir(obj.dirname)) + + class Directory: + + size = DirectorySize() # Descriptor instance + + def __init__(self, dirname): + self.dirname = dirname # Regular instance attribute + +An interactive session shows that the lookup is dynamic — it computes +different, updated answers each time:: + + >>> s = Directory('songs') + >>> g = Directory('games') + >>> s.size # The songs directory has twenty files + 20 + >>> g.size # The games directory has three files + 3 + >>> open('games/newfile').close() # Add a fourth file to the directory + >>> g.size # File count is automatically updated + 4 + +Besides showing how descriptors can run computations, this example also +reveals the purpose of the parameters to :meth:`__get__`. The *self* +parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is +either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that +lets the :meth:`__get__` method learn the target directory. The *objtype* +parameter is the class *Directory*. + + +Managed attributes +------------------ + +A popular use for descriptors is managing access to instance data. The +descriptor is assigned to a public attribute in the class dictionary while the +actual data is stored as a private attribute in the instance dictionary. The +descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when +the public attribute is accessed. + +In the following example, *age* is the public attribute and *_age* is the +private attribute. When the public attribute is accessed, the descriptor logs +the lookup or update: + +.. testcode:: + + import logging + + logging.basicConfig(level=logging.INFO) + + class LoggedAgeAccess: + + def __get__(self, obj, objtype=None): + value = obj._age + logging.info('Accessing %r giving %r', 'age', value) + return value + + def __set__(self, obj, value): + logging.info('Updating %r to %r', 'age', value) + obj._age = value + + class Person: + + age = LoggedAgeAccess() # Descriptor instance + + def __init__(self, name, age): + self.name = name # Regular instance attribute + self.age = age # Calls __set__() + + def birthday(self): + self.age += 1 # Calls both __get__() and __set__() + + +An interactive session shows that all access to the managed attribute *age* is +logged, but that the regular attribute *name* is not logged: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: + + >>> mary = Person('Mary M', 30) # The initial age update is logged + INFO:root:Updating 'age' to 30 + >>> dave = Person('David D', 40) + INFO:root:Updating 'age' to 40 + + >>> vars(mary) # The actual data is in a private attribute + {'name': 'Mary M', '_age': 30} + >>> vars(dave) + {'name': 'David D', '_age': 40} + + >>> mary.age # Access the data and log the lookup + INFO:root:Accessing 'age' giving 30 + 30 + >>> mary.birthday() # Updates are logged as well + INFO:root:Accessing 'age' giving 30 + INFO:root:Updating 'age' to 31 + + >>> dave.name # Regular attribute lookup isn't logged + 'David D' + >>> dave.age # Only the managed attribute is logged + INFO:root:Accessing 'age' giving 40 + 40 + +One major issue with this example is that the private name *_age* is hardwired in +the *LoggedAgeAccess* class. That means that each instance can only have one +logged attribute and that its name is unchangeable. In the next example, +we'll fix that problem. + + +Customized names +---------------- + +When a class uses descriptors, it can inform each descriptor about which +variable name was used. + +In this example, the :class:`Person` class has two descriptor instances, +*name* and *age*. When the :class:`Person` class is defined, it makes a +callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can +be recorded, giving each descriptor its own *public_name* and *private_name*: + +.. testcode:: + + import logging + + logging.basicConfig(level=logging.INFO) + + class LoggedAccess: + + def __set_name__(self, owner, name): + self.public_name = name + self.private_name = '_' + name + + def __get__(self, obj, objtype=None): + value = getattr(obj, self.private_name) + logging.info('Accessing %r giving %r', self.public_name, value) + return value + + def __set__(self, obj, value): + logging.info('Updating %r to %r', self.public_name, value) + setattr(obj, self.private_name, value) + + class Person: + + name = LoggedAccess() # First descriptor instance + age = LoggedAccess() # Second descriptor instance + + def __init__(self, name, age): + self.name = name # Calls the first descriptor + self.age = age # Calls the second descriptor + + def birthday(self): + self.age += 1 + +An interactive session shows that the :class:`Person` class has called +:meth:`__set_name__` so that the field names would be recorded. Here +we call :func:`vars` to look up the descriptor without triggering it: + +.. doctest:: + + >>> vars(vars(Person)['name']) + {'public_name': 'name', 'private_name': '_name'} + >>> vars(vars(Person)['age']) + {'public_name': 'age', 'private_name': '_age'} + +The new class now logs access to both *name* and *age*: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: + + >>> pete = Person('Peter P', 10) + INFO:root:Updating 'name' to 'Peter P' + INFO:root:Updating 'age' to 10 + >>> kate = Person('Catherine C', 20) + INFO:root:Updating 'name' to 'Catherine C' + INFO:root:Updating 'age' to 20 + +The two *Person* instances contain only the private names:: + + >>> vars(pete) + {'_name': 'Peter P', '_age': 10} + >>> vars(kate) + {'_name': 'Catherine C', '_age': 20} + + +Closing thoughts +---------------- + +A :term:`descriptor` is what we call any object that defines :meth:`__get__`, +:meth:`__set__`, or :meth:`__delete__`. + +Optionally, descriptors can have a :meth:`__set_name__` method. This is only +used in cases where a descriptor needs to know either the class where it was +created or the name of class variable it was assigned to. (This method, if +present, is called even if the class is not a descriptor.) + +Descriptors get invoked by the dot operator during attribute lookup. If a +descriptor is accessed indirectly with ``vars(some_class)[descriptor_name]``, +the descriptor instance is returned without invoking it. + +Descriptors only work when used as class variables. When put in instances, +they have no effect. + +The main motivation for descriptors is to provide a hook allowing objects +stored in class variables to control what happens during attribute lookup. + +Traditionally, the calling class controls what happens during lookup. +Descriptors invert that relationship and allow the data being looked-up to +have a say in the matter. + +Descriptors are used throughout the language. It is how functions turn into +bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, +:func:`property`, and :func:`functools.cached_property` are all implemented as +descriptors. + + +Complete Practical Example +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In this example, we create a practical and powerful tool for locating +notoriously hard to find data corruption bugs. + + +Validator class +--------------- + +A validator is a descriptor for managed attribute access. Prior to storing +any data, it verifies that the new value meets various type and range +restrictions. If those restrictions aren't met, it raises an exception to +prevent data corruption at its source. + +This :class:`Validator` class is both an :term:`abstract base class` and a +managed attribute descriptor: + +.. testcode:: + + from abc import ABC, abstractmethod + + class Validator(ABC): + + def __set_name__(self, owner, name): + self.private_name = '_' + name + + def __get__(self, obj, objtype=None): + return getattr(obj, self.private_name) + + def __set__(self, obj, value): + self.validate(value) + setattr(obj, self.private_name, value) + + @abstractmethod + def validate(self, value): + pass + +Custom validators need to inherit from :class:`Validator` and must supply a +:meth:`validate` method to test various restrictions as needed. + + +Custom validators +----------------- + +Here are three practical data validation utilities: + +1) :class:`OneOf` verifies that a value is one of a restricted set of options. + +2) :class:`Number` verifies that a value is either an :class:`int` or + :class:`float`. Optionally, it verifies that a value is between a given + minimum or maximum. + +3) :class:`String` verifies that a value is a :class:`str`. Optionally, it + validates a given minimum or maximum length. It can validate a + user-defined `predicate + `_ as well. + +.. testcode:: + + class OneOf(Validator): + + def __init__(self, *options): + self.options = set(options) + + def validate(self, value): + if value not in self.options: + raise ValueError(f'Expected {value!r} to be one of {self.options!r}') + + class Number(Validator): + + def __init__(self, minvalue=None, maxvalue=None): + self.minvalue = minvalue + self.maxvalue = maxvalue + + def validate(self, value): + if not isinstance(value, (int, float)): + raise TypeError(f'Expected {value!r} to be an int or float') + if self.minvalue is not None and value < self.minvalue: + raise ValueError( + f'Expected {value!r} to be at least {self.minvalue!r}' + ) + if self.maxvalue is not None and value > self.maxvalue: + raise ValueError( + f'Expected {value!r} to be no more than {self.maxvalue!r}' + ) + + class String(Validator): + + def __init__(self, minsize=None, maxsize=None, predicate=None): + self.minsize = minsize + self.maxsize = maxsize + self.predicate = predicate + + def validate(self, value): + if not isinstance(value, str): + raise TypeError(f'Expected {value!r} to be an str') + if self.minsize is not None and len(value) < self.minsize: + raise ValueError( + f'Expected {value!r} to be no smaller than {self.minsize!r}' + ) + if self.maxsize is not None and len(value) > self.maxsize: + raise ValueError( + f'Expected {value!r} to be no bigger than {self.maxsize!r}' + ) + if self.predicate is not None and not self.predicate(value): + raise ValueError( + f'Expected {self.predicate} to be true for {value!r}' + ) + + +Practical application +--------------------- + +Here's how the data validators can be used in a real class: + +.. testcode:: + + class Component: + + name = String(minsize=3, maxsize=10, predicate=str.isupper) + kind = OneOf('wood', 'metal', 'plastic') + quantity = Number(minvalue=0) + + def __init__(self, name, kind, quantity): + self.name = name + self.kind = kind + self.quantity = quantity + +The descriptors prevent invalid instances from being created: + +.. doctest:: + + >>> Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase + Traceback (most recent call last): + ... + ValueError: Expected to be true for 'Widget' + + >>> Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled + Traceback (most recent call last): + ... + ValueError: Expected 'metle' to be one of {'metal', 'plastic', 'wood'} + + >>> Component('WIDGET', 'metal', -5) # Blocked: -5 is negative + Traceback (most recent call last): + ... + ValueError: Expected -5 to be at least 0 + >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number + Traceback (most recent call last): + ... + TypeError: Expected 'V' to be an int or float + + >>> c = Component('WIDGET', 'metal', 5) # Allowed: The inputs are valid + + +Technical Tutorial +^^^^^^^^^^^^^^^^^^ + +What follows is a more technical tutorial for the mechanics and details of how +descriptors work. + + +Abstract +-------- + +Defines descriptors, summarizes the protocol, and shows how descriptors are +called. Provides an example showing how object relational mappings work. + +Learning about descriptors not only provides access to a larger toolset, it +creates a deeper understanding of how Python works. + + +Definition and introduction +--------------------------- + +In general, a descriptor is an attribute value that has one of the methods in +the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, +and :meth:`__delete__`. If any of those methods are defined for an +attribute, it is said to be a :term:`descriptor`. + +The default behavior for attribute access is to get, set, or delete the +attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain +starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and +continuing through the method resolution order of ``type(a)``. If the +looked-up value is an object defining one of the descriptor methods, then Python +may override the default behavior and invoke the descriptor method instead. +Where this occurs in the precedence chain depends on which descriptor methods +were defined. + +Descriptors are a powerful, general purpose protocol. They are the mechanism +behind properties, methods, static methods, class methods, and +:func:`super()`. They are used throughout Python itself. Descriptors +simplify the underlying C code and offer a flexible set of new tools for +everyday Python programs. + + +Descriptor protocol +------------------- + +``descr.__get__(self, obj, type=None) -> value`` + +``descr.__set__(self, obj, value) -> None`` + +``descr.__delete__(self, obj) -> None`` + +That is all there is to it. Define any of these methods and an object is +considered a descriptor and can override default behavior upon being looked up +as an attribute. + +If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered +a data descriptor. Descriptors that only define :meth:`__get__` are called +non-data descriptors (they are often used for methods but other uses are +possible). + +Data and non-data descriptors differ in how overrides are calculated with +respect to entries in an instance's dictionary. If an instance's dictionary +has an entry with the same name as a data descriptor, the data descriptor +takes precedence. If an instance's dictionary has an entry with the same +name as a non-data descriptor, the dictionary entry takes precedence. + +To make a read-only data descriptor, define both :meth:`__get__` and +:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when +called. Defining the :meth:`__set__` method with an exception raising +placeholder is enough to make it a data descriptor. + + +Overview of descriptor invocation +--------------------------------- + +A descriptor can be called directly with ``desc.__get__(obj)`` or +``desc.__get__(None, cls)``. + +But it is more common for a descriptor to be invoked automatically from +attribute access. + +The expression ``obj.x`` looks up the attribute ``x`` in the chain of +namespaces for ``obj``. If the search finds a descriptor outside of the +instance ``__dict__``, its :meth:`__get__` method is invoked according to the +precedence rules listed below. + +The details of invocation depend on whether ``obj`` is an object, class, or +instance of super. + + +Invocation from an instance +--------------------------- + +Instance lookup scans through a chain of namespaces giving data descriptors +the highest priority, followed by instance variables, then non-data +descriptors, then class variables, and lastly :meth:`__getattr__` if it is +provided. + +If a descriptor is found for ``a.x``, then it is invoked with: +``desc.__get__(a, type(a))``. + +The logic for a dotted lookup is in :meth:`object.__getattribute__`. Here is +a pure Python equivalent: + +.. testcode:: + + def object_getattribute(obj, name): + "Emulate PyObject_GenericGetAttr() in Objects/object.c" + null = object() + objtype = type(obj) + cls_var = getattr(objtype, name, null) + descr_get = getattr(type(cls_var), '__get__', null) + if descr_get is not null: + if (hasattr(type(cls_var), '__set__') + or hasattr(type(cls_var), '__delete__')): + return descr_get(cls_var, obj, objtype) # data descriptor + if hasattr(obj, '__dict__') and name in vars(obj): + return vars(obj)[name] # instance variable + if descr_get is not null: + return descr_get(cls_var, obj, objtype) # non-data descriptor + if cls_var is not null: + return cls_var # class variable + raise AttributeError(name) + + +.. testcode:: + :hide: + + # Test the fidelity of object_getattribute() by comparing it with the + # normal object.__getattribute__(). The former will be accessed by + # square brackets and the latter by the dot operator. + + class Object: + + def __getitem__(obj, name): + try: + return object_getattribute(obj, name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + + class DualOperator(Object): + + x = 10 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + @property + def p3(self): + return 3 * self.x + + def m5(self, y): + return 5 * y + + def m7(self, y): + return 7 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + class DualOperatorWithSlots: + + __getitem__ = Object.__getitem__ + + __slots__ = ['z'] + + x = 15 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + def m5(self, y): + return 5 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + +.. doctest:: + :hide: + + >>> a = DualOperator(11) + >>> vars(a).update(p3 = '_p3', m7 = '_m7') + >>> a.x == a['x'] == 10 + True + >>> a.z == a['z'] == 11 + True + >>> a.p2 == a['p2'] == 20 + True + >>> a.p3 == a['p3'] == 30 + True + >>> a.m5(100) == a.m5(100) == 500 + True + >>> a.m7 == a['m7'] == '_m7' + True + >>> a.g == a['g'] == ('getattr_hook', a, 'g') + True + + >>> b = DualOperatorWithSlots(22) + >>> b.x == b['x'] == 15 + True + >>> b.z == b['z'] == 22 + True + >>> b.p2 == b['p2'] == 30 + True + >>> b.m5(200) == b['m5'](200) == 1000 + True + >>> b.g == b['g'] == ('getattr_hook', b, 'g') + True + + +Interestingly, attribute lookup doesn't call :meth:`object.__getattribute__` +directly. Instead, both the dot operator and the :func:`getattr` function +perform attribute lookup by way of a helper function: + +.. testcode:: + + def getattr_hook(obj, name): + "Emulate slot_tp_getattr_hook() in Objects/typeobject.c" + try: + return obj.__getattribute__(name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + +So if :meth:`__getattr__` exists, it is called whenever :meth:`__getattribute__` +raises :exc:`AttributeError` (either directly or in one of the descriptor calls). + +Also, if a user calls :meth:`object.__getattribute__` directly, the +:meth:`__getattr__` hook is bypassed entirely. + + +Invocation from a class +----------------------- + +The logic for a dotted lookup such as ``A.x`` is in +:meth:`type.__getattribute__`. The steps are similar to those for +:meth:`object.__getattribute__` but the instance dictionary lookup is replaced +by a search through the class's :term:`method resolution order`. + +If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. + +The full C implementation can be found in :c:func:`type_getattro()` and +:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`. + + +Invocation from super +--------------------- + +The logic for super's dotted lookup is in the :meth:`__getattribute__` method for +object returned by :class:`super()`. + +A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` +for the base class ``B`` immediately following ``A`` and then returns +``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned +unchanged. + +The full C implementation can be found in :c:func:`super_getattro()` in +:source:`Objects/typeobject.c`. A pure Python equivalent can be found in +`Guido's Tutorial +`_. + + +Summary of invocation logic +--------------------------- + +The mechanism for descriptors is embedded in the :meth:`__getattribute__()` +methods for :class:`object`, :class:`type`, and :func:`super`. + +The important points to remember are: + +* Descriptors are invoked by the :meth:`__getattribute__` method. + +* Classes inherit this machinery from :class:`object`, :class:`type`, or + :func:`super`. + +* Overriding :meth:`__getattribute__` prevents automatic descriptor calls + because all the descriptor logic is in that method. + +* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make + different calls to :meth:`__get__`. The first includes the instance and may + include the class. The second puts in ``None`` for the instance and always + includes the class. + +* Data descriptors always override instance dictionaries. + +* Non-data descriptors may be overridden by instance dictionaries. + + +Automatic name notification +--------------------------- + +Sometimes it is desirable for a descriptor to know what class variable name it +was assigned to. When a new class is created, the :class:`type` metaclass +scans the dictionary of the new class. If any of the entries are descriptors +and if they define :meth:`__set_name__`, that method is called with two +arguments. The *owner* is the class where the descriptor is used, and the +*name* is the class variable the descriptor was assigned to. + +The implementation details are in :c:func:`type_new()` and +:c:func:`set_names()` in :source:`Objects/typeobject.c`. + +Since the update logic is in :meth:`type.__new__`, notifications only take +place at the time of class creation. If descriptors are added to the class +afterwards, :meth:`__set_name__` will need to be called manually. + + +ORM example +----------- + +The following code is simplified skeleton showing how data descriptors could +be used to implement an `object relational mapping +`_. + +The essential idea is that the data is stored in an external database. The +Python instances only hold keys to the database's tables. Descriptors take +care of lookups or updates: + +.. testcode:: + + class Field: + + def __set_name__(self, owner, name): + self.fetch = f'SELECT {name} FROM {owner.table} WHERE {owner.key}=?;' + self.store = f'UPDATE {owner.table} SET {name}=? WHERE {owner.key}=?;' + + def __get__(self, obj, objtype=None): + return conn.execute(self.fetch, [obj.key]).fetchone()[0] + + def __set__(self, obj, value): + conn.execute(self.store, [value, obj.key]) + conn.commit() + +We can use the :class:`Field` class to define `models +`_ that describe the schema for +each table in a database: + +.. testcode:: + + class Movie: + table = 'Movies' # Table name + key = 'title' # Primary key + director = Field() + year = Field() + + def __init__(self, key): + self.key = key + + class Song: + table = 'Music' + key = 'title' + artist = Field() + year = Field() + genre = Field() + + def __init__(self, key): + self.key = key + +To use the models, first connect to the database:: + + >>> import sqlite3 + >>> conn = sqlite3.connect('entertainment.db') + +An interactive session shows how data is retrieved from the database and how +it can be updated: + +.. testsetup:: + + song_data = [ + ('Country Roads', 'John Denver', 1972), + ('Me and Bobby McGee', 'Janice Joplin', 1971), + ('Coal Miners Daughter', 'Loretta Lynn', 1970), + ] + + movie_data = [ + ('Star Wars', 'George Lucas', 1977), + ('Jaws', 'Steven Spielberg', 1975), + ('Aliens', 'James Cameron', 1986), + ] + + import sqlite3 + + conn = sqlite3.connect(':memory:') + conn.execute('CREATE TABLE Music (title text, artist text, year integer);') + conn.execute('CREATE INDEX MusicNdx ON Music (title);') + conn.executemany('INSERT INTO Music VALUES (?, ?, ?);', song_data) + conn.execute('CREATE TABLE Movies (title text, director text, year integer);') + conn.execute('CREATE INDEX MovieNdx ON Music (title);') + conn.executemany('INSERT INTO Movies VALUES (?, ?, ?);', movie_data) + conn.commit() + +.. doctest:: + + >>> Movie('Star Wars').director + 'George Lucas' + >>> jaws = Movie('Jaws') + >>> f'Released in {jaws.year} by {jaws.director}' + 'Released in 1975 by Steven Spielberg' + + >>> Song('Country Roads').artist + 'John Denver' + + >>> Movie('Star Wars').director = 'J.J. Abrams' + >>> Movie('Star Wars').director + 'J.J. Abrams' + + +Pure Python Equivalents +^^^^^^^^^^^^^^^^^^^^^^^ + +The descriptor protocol is simple and offers exciting possibilities. Several +use cases are so common that they have been prepackaged into built-in tools. +Properties, bound methods, static methods, class methods, and \_\_slots\_\_ are +all based on the descriptor protocol. + + +Properties +---------- + +Calling :func:`property` is a succinct way of building a data descriptor that +triggers a function call upon access to an attribute. Its signature is:: + + property(fget=None, fset=None, fdel=None, doc=None) -> property + +The documentation shows a typical use to define a managed attribute ``x``: + +.. testcode:: + + class C: + def getx(self): return self.__x + def setx(self, value): self.__x = value + def delx(self): del self.__x + x = property(getx, setx, delx, "I'm the 'x' property.") + +To see how :func:`property` is implemented in terms of the descriptor protocol, +here is a pure Python equivalent: + +.. testcode:: + + class Property: + "Emulate PyProperty_Type() in Objects/descrobject.c" + + def __init__(self, fget=None, fset=None, fdel=None, doc=None): + self.fget = fget + self.fset = fset + self.fdel = fdel + if doc is None and fget is not None: + doc = fget.__doc__ + self.__doc__ = doc + self._name = '' + + def __set_name__(self, owner, name): + self._name = name + + def __get__(self, obj, objtype=None): + if obj is None: + return self + if self.fget is None: + raise AttributeError(f'unreadable attribute {self._name}') + return self.fget(obj) + + def __set__(self, obj, value): + if self.fset is None: + raise AttributeError(f"can't set attribute {self._name}") + self.fset(obj, value) + + def __delete__(self, obj): + if self.fdel is None: + raise AttributeError(f"can't delete attribute {self._name}") + self.fdel(obj) + + def getter(self, fget): + prop = type(self)(fget, self.fset, self.fdel, self.__doc__) + prop._name = self._name + return prop + + def setter(self, fset): + prop = type(self)(self.fget, fset, self.fdel, self.__doc__) + prop._name = self._name + return prop + + def deleter(self, fdel): + prop = type(self)(self.fget, self.fset, fdel, self.__doc__) + prop._name = self._name + return prop + +.. testcode:: + :hide: + + # Verify the Property() emulation + + class CC: + def getx(self): + return self.__x + def setx(self, value): + self.__x = value + def delx(self): + del self.__x + x = Property(getx, setx, delx, "I'm the 'x' property.") + + # Now do it again but use the decorator style + + class CCC: + @Property + def x(self): + return self.__x + @x.setter + def x(self, value): + self.__x = value + @x.deleter + def x(self): + del self.__x + + +.. doctest:: + :hide: + + >>> cc = CC() + >>> hasattr(cc, 'x') + False + >>> cc.x = 33 + >>> cc.x + 33 + >>> del cc.x + >>> hasattr(cc, 'x') + False + + >>> ccc = CCC() + >>> hasattr(ccc, 'x') + False + >>> ccc.x = 333 + >>> ccc.x == 333 + True + >>> del ccc.x + >>> hasattr(ccc, 'x') + False + +The :func:`property` builtin helps whenever a user interface has granted +attribute access and then subsequent changes require the intervention of a +method. + +For instance, a spreadsheet class may grant access to a cell value through +``Cell('b10').value``. Subsequent improvements to the program require the cell +to be recalculated on every access; however, the programmer does not want to +affect existing client code accessing the attribute directly. The solution is +to wrap access to the value attribute in a property data descriptor: + +.. testcode:: + + class Cell: + ... + + @property + def value(self): + "Recalculate the cell before returning value" + self.recalc() + return self._value + +Either the built-in :func:`property` or our :func:`Property` equivalent would +work in this example. + + +Functions and methods +--------------------- + +Python's object oriented features are built upon a function based environment. +Using non-data descriptors, the two are merged seamlessly. + +Functions stored in class dictionaries get turned into methods when invoked. +Methods only differ from regular functions in that the object instance is +prepended to the other arguments. By convention, the instance is called +*self* but could be called *this* or any other variable name. + +Methods can be created manually with :class:`types.MethodType` which is +roughly equivalent to: + +.. testcode:: + + class MethodType: + "Emulate Py_MethodType in Objects/classobject.c" + + def __init__(self, func, obj): + self.__func__ = func + self.__self__ = obj + + def __call__(self, *args, **kwargs): + func = self.__func__ + obj = self.__self__ + return func(obj, *args, **kwargs) + +To support automatic creation of methods, functions include the +:meth:`__get__` method for binding methods during attribute access. This +means that functions are non-data descriptors that return bound methods +during dotted lookup from an instance. Here's how it works: + +.. testcode:: + + class Function: + ... + + def __get__(self, obj, objtype=None): + "Simulate func_descr_get() in Objects/funcobject.c" + if obj is None: + return self + return MethodType(self, obj) + +Running the following class in the interpreter shows how the function +descriptor works in practice: + +.. testcode:: + + class D: + def f(self, x): + return x + +The function has a :term:`qualified name` attribute to support introspection: + +.. doctest:: + + >>> D.f.__qualname__ + 'D.f' + +Accessing the function through the class dictionary does not invoke +:meth:`__get__`. Instead, it just returns the underlying function object:: + + >>> D.__dict__['f'] + + +Dotted access from a class calls :meth:`__get__` which just returns the +underlying function unchanged:: + + >>> D.f + + +The interesting behavior occurs during dotted access from an instance. The +dotted lookup calls :meth:`__get__` which returns a bound method object:: + + >>> d = D() + >>> d.f + > + +Internally, the bound method stores the underlying function and the bound +instance:: + + >>> d.f.__func__ + + + >>> d.f.__self__ + <__main__.D object at 0x1012e1f98> + +If you have ever wondered where *self* comes from in regular methods or where +*cls* comes from in class methods, this is it! + + +Static methods +-------------- + +Non-data descriptors provide a simple mechanism for variations on the usual +patterns of binding functions into methods. + +To recap, functions have a :meth:`__get__` method so that they can be converted +to a method when accessed as attributes. The non-data descriptor transforms an +``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` +becomes ``f(*args)``. + +This chart summarizes the binding and its two most useful variants: + + +-----------------+----------------------+------------------+ + | Transformation | Called from an | Called from a | + | | object | class | + +=================+======================+==================+ + | function | f(obj, \*args) | f(\*args) | + +-----------------+----------------------+------------------+ + | staticmethod | f(\*args) | f(\*args) | + +-----------------+----------------------+------------------+ + | classmethod | f(type(obj), \*args) | f(cls, \*args) | + +-----------------+----------------------+------------------+ + +Static methods return the underlying function without changes. Calling either +``c.f`` or ``C.f`` is the equivalent of a direct lookup into +``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a +result, the function becomes identically accessible from either an object or a +class. + +Good candidates for static methods are methods that do not reference the +``self`` variable. + +For instance, a statistics package may include a container class for +experimental data. The class provides normal methods for computing the average, +mean, median, and other descriptive statistics that depend on the data. However, +there may be useful functions which are conceptually related but do not depend +on the data. For instance, ``erf(x)`` is handy conversion routine that comes up +in statistical work but does not directly depend on a particular dataset. +It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or +``Sample.erf(1.5) --> .9332``. + +Since static methods return the underlying function with no changes, the +example calls are unexciting: + +.. testcode:: + + class E: + @staticmethod + def f(x): + print(x) + +.. doctest:: + + >>> E.f(3) + 3 + >>> E().f(3) + 3 + +Using the non-data descriptor protocol, a pure Python version of +:func:`staticmethod` would look like this: + +.. doctest:: + + class StaticMethod: + "Emulate PyStaticMethod_Type() in Objects/funcobject.c" + + def __init__(self, f): + self.f = f + + def __get__(self, obj, objtype=None): + return self.f + + +Class methods +------------- + +Unlike static methods, class methods prepend the class reference to the +argument list before calling the function. This format is the same +for whether the caller is an object or a class: + +.. testcode:: + + class F: + @classmethod + def f(cls, x): + return cls.__name__, x + +.. doctest:: + + >>> F.f(3) + ('F', 3) + >>> F().f(3) + ('F', 3) + +This behavior is useful whenever the method only needs to have a class +reference and does not rely on data stored in a specific instance. One use for +class methods is to create alternate class constructors. For example, the +classmethod :func:`dict.fromkeys` creates a new dictionary from a list of +keys. The pure Python equivalent is: + +.. testcode:: + + class Dict(dict): + @classmethod + def fromkeys(cls, iterable, value=None): + "Emulate dict_fromkeys() in Objects/dictobject.c" + d = cls() + for key in iterable: + d[key] = value + return d + +Now a new dictionary of unique keys can be constructed like this: + +.. doctest:: + + >>> d = Dict.fromkeys('abracadabra') + >>> type(d) is Dict + True + >>> d + {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} + +Using the non-data descriptor protocol, a pure Python version of +:func:`classmethod` would look like this: + +.. testcode:: + + class ClassMethod: + "Emulate PyClassMethod_Type() in Objects/funcobject.c" + + def __init__(self, f): + self.f = f + + def __get__(self, obj, cls=None): + if cls is None: + cls = type(obj) + if hasattr(obj, '__get__'): + return self.f.__get__(cls) + return MethodType(self.f, cls) + +.. testcode:: + :hide: + + # Verify the emulation works + class T: + @ClassMethod + def cm(cls, x, y): + return (cls, x, y) + +.. doctest:: + :hide: + + >>> T.cm(11, 22) + (, 11, 22) + + # Also call it from an instance + >>> t = T() + >>> t.cm(11, 22) + (, 11, 22) + +The code path for ``hasattr(obj, '__get__')`` was added in Python 3.9 and +makes it possible for :func:`classmethod` to support chained decorators. +For example, a classmethod and property could be chained together: + +.. testcode:: + + class G: + @classmethod + @property + def __doc__(cls): + return f'A doc for {cls.__name__!r}' + +.. doctest:: + + >>> G.__doc__ + "A doc for 'G'" + + +Member objects and __slots__ +---------------------------- + +When a class defines ``__slots__``, it replaces instance dictionaries with a +fixed-length array of slot values. From a user point of view that has +several effects: + +1. Provides immediate detection of bugs due to misspelled attribute +assignments. Only attribute names specified in ``__slots__`` are allowed: + +.. testcode:: + + class Vehicle: + __slots__ = ('id_number', 'make', 'model') + +.. doctest:: + + >>> auto = Vehicle() + >>> auto.id_nubmer = 'VYE483814LQEX' + Traceback (most recent call last): + ... + AttributeError: 'Vehicle' object has no attribute 'id_nubmer' + +2. Helps create immutable objects where descriptors manage access to private +attributes stored in ``__slots__``: + +.. testcode:: + + class Immutable: + + __slots__ = ('_dept', '_name') # Replace the instance dictionary + + def __init__(self, dept, name): + self._dept = dept # Store to private attribute + self._name = name # Store to private attribute + + @property # Read-only descriptor + def dept(self): + return self._dept + + @property + def name(self): # Read-only descriptor + return self._name + +.. doctest:: + + >>> mark = Immutable('Botany', 'Mark Watney') + >>> mark.dept + 'Botany' + >>> mark.dept = 'Space Pirate' + Traceback (most recent call last): + ... + AttributeError: can't set attribute + >>> mark.location = 'Mars' + Traceback (most recent call last): + ... + AttributeError: 'Immutable' object has no attribute 'location' + +3. Saves memory. On a 64-bit Linux build, an instance with two attributes +takes 48 bytes with ``__slots__`` and 152 bytes without. This `flyweight +design pattern `_ likely only +matters when a large number of instances are going to be created. + +4. Improves speed. Reading instance variables is 35% faster with +``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). + +5. Blocks tools like :func:`functools.cached_property` which require an +instance dictionary to function correctly: + +.. testcode:: + + from functools import cached_property + + class CP: + __slots__ = () # Eliminates the instance dict + + @cached_property # Requires an instance dict + def pi(self): + return 4 * sum((-1.0)**n / (2.0*n + 1.0) + for n in reversed(range(100_000))) + +.. doctest:: + + >>> CP().pi + Traceback (most recent call last): + ... + TypeError: No '__dict__' attribute on 'CP' instance to cache 'pi' property. + +It is not possible to create an exact drop-in pure Python version of +``__slots__`` because it requires direct access to C structures and control +over object memory allocation. However, we can build a mostly faithful +simulation where the actual C structure for slots is emulated by a private +``_slotvalues`` list. Reads and writes to that private structure are managed +by member descriptors: + +.. testcode:: + + null = object() + + class Member: + + def __init__(self, name, clsname, offset): + 'Emulate PyMemberDef in Include/structmember.h' + # Also see descr_new() in Objects/descrobject.c + self.name = name + self.clsname = clsname + self.offset = offset + + def __get__(self, obj, objtype=None): + 'Emulate member_get() in Objects/descrobject.c' + # Also see PyMember_GetOne() in Python/structmember.c + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + return value + + def __set__(self, obj, value): + 'Emulate member_set() in Objects/descrobject.c' + obj._slotvalues[self.offset] = value + + def __delete__(self, obj): + 'Emulate member_delete() in Objects/descrobject.c' + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + obj._slotvalues[self.offset] = null + + def __repr__(self): + 'Emulate member_repr() in Objects/descrobject.c' + return f'' + +The :meth:`type.__new__` method takes care of adding member objects to class +variables: + +.. testcode:: + + class Type(type): + 'Simulate how the type metaclass adds member objects for slots' + + def __new__(mcls, clsname, bases, mapping): + 'Emuluate type_new() in Objects/typeobject.c' + # type_new() calls PyTypeReady() which calls add_methods() + slot_names = mapping.get('slot_names', []) + for offset, name in enumerate(slot_names): + mapping[name] = Member(name, clsname, offset) + return type.__new__(mcls, clsname, bases, mapping) + +The :meth:`object.__new__` method takes care of creating instances that have +slots instead of an instance dictionary. Here is a rough simulation in pure +Python: + +.. testcode:: + + class Object: + 'Simulate how object.__new__() allocates memory for __slots__' + + def __new__(cls, *args): + 'Emulate object_new() in Objects/typeobject.c' + inst = super().__new__(cls) + if hasattr(cls, 'slot_names'): + empty_slots = [null] * len(cls.slot_names) + object.__setattr__(inst, '_slotvalues', empty_slots) + return inst + + def __setattr__(self, name, value): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__setattr__(name, value) + + def __delattr__(self, name): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__delattr__(name) + +To use the simulation in a real class, just inherit from :class:`Object` and +set the :term:`metaclass` to :class:`Type`: + +.. testcode:: + + class H(Object, metaclass=Type): + 'Instance variables stored in slots' + + slot_names = ['x', 'y'] + + def __init__(self, x, y): + self.x = x + self.y = y + +At this point, the metaclass has loaded member objects for *x* and *y*:: + + >>> from pprint import pp + >>> pp(dict(vars(H))) + {'__module__': '__main__', + '__doc__': 'Instance variables stored in slots', + 'slot_names': ['x', 'y'], + '__init__': , + 'x': , + 'y': } + +.. doctest:: + :hide: + + # We test this separately because the preceding section is not + # doctestable due to the hex memory address for the __init__ function + >>> isinstance(vars(H)['x'], Member) + True + >>> isinstance(vars(H)['y'], Member) + True + +When instances are created, they have a ``slot_values`` list where the +attributes are stored: + +.. doctest:: + + >>> h = H(10, 20) + >>> vars(h) + {'_slotvalues': [10, 20]} + >>> h.x = 55 + >>> vars(h) + {'_slotvalues': [55, 20]} + +Misspelled or unassigned attributes will raise an exception: + +.. doctest:: + + >>> h.xz + Traceback (most recent call last): + ... + AttributeError: 'H' object has no attribute 'xz' + +.. doctest:: + :hide: + + # Examples for deleted attributes are not shown because this section + # is already a bit lengthy. We still test that code here. + >>> del h.x + >>> hasattr(h, 'x') + False + + # Also test the code for uninitialized slots + >>> class HU(Object, metaclass=Type): + ... slot_names = ['x', 'y'] + ... + >>> hu = HU() + >>> hasattr(hu, 'x') + False + >>> hasattr(hu, 'y') + False diff --git a/Doc/library/_thread.rst.bak b/Doc/library/_thread.rst.bak new file mode 100644 index 00000000000000..bd653ab32bb9c4 --- /dev/null +++ b/Doc/library/_thread.rst.bak @@ -0,0 +1,215 @@ +:mod:`_thread` --- Low-level threading API +========================================== + +.. module:: _thread + :synopsis: Low-level threading API. + +.. index:: + single: light-weight processes + single: processes, light-weight + single: binary semaphores + single: semaphores, binary + +-------------- + +This module provides low-level primitives for working with multiple threads +(also called :dfn:`light-weight processes` or :dfn:`tasks`) --- multiple threads of +control sharing their global data space. For synchronization, simple locks +(also called :dfn:`mutexes` or :dfn:`binary semaphores`) are provided. +The :mod:`threading` module provides an easier to use and higher-level +threading API built on top of this module. + +.. index:: + single: pthreads + pair: threads; POSIX + +.. versionchanged:: 3.7 + This module used to be optional, it is now always available. + +This module defines the following constants and functions: + +.. exception:: error + + Raised on thread-specific errors. + + .. versionchanged:: 3.3 + This is now a synonym of the built-in :exc:`RuntimeError`. + + +.. data:: LockType + + This is the type of lock objects. + + +.. function:: start_new_thread(function, args[, kwargs]) + + Start a new thread and return its identifier. The thread executes the + function *function* with the argument list *args* (which must be a tuple). + The optional *kwargs* argument specifies a dictionary of keyword arguments. + + When the function returns, the thread silently exits. + + When the function terminates with an unhandled exception, + :func:`sys.unraisablehook` is called to handle the exception. The *object* + attribute of the hook argument is *function*. By default, a stack trace is + printed and then the thread exits (but other threads continue to run). + + When the function raises a :exc:`SystemExit` exception, it is silently + ignored. + + .. versionchanged:: 3.8 + :func:`sys.unraisablehook` is now used to handle unhandled exceptions. + + +.. function:: interrupt_main() + + Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main + thread. A thread can use this function to interrupt the main thread. + + If :data:`signal.SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. + + +.. function:: exit() + + Raise the :exc:`SystemExit` exception. When not caught, this will cause the + thread to exit silently. + +.. + function:: exit_prog(status) + + Exit all threads and report the value of the integer argument + *status* as the exit status of the entire program. + **Caveat:** code in pending :keyword:`finally` clauses, in this thread + or in other threads, is not executed. + + +.. function:: allocate_lock() + + Return a new lock object. Methods of locks are described below. The lock is + initially unlocked. + + +.. function:: get_ident() + + Return the 'thread identifier' of the current thread. This is a nonzero + integer. Its value has no direct meaning; it is intended as a magic cookie to + be used e.g. to index a dictionary of thread-specific data. Thread identifiers + may be recycled when a thread exits and another thread is created. + + +.. function:: get_native_id() + + Return the native integral Thread ID of the current thread assigned by the kernel. + This is a non-negative integer. + Its value may be used to uniquely identify this particular thread system-wide + (until the thread terminates, after which the value may be recycled by the OS). + + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. + + .. versionadded:: 3.8 + + +.. function:: stack_size([size]) + + Return the thread stack size used when creating new threads. The optional + *size* argument specifies the stack size to be used for subsequently created + threads, and must be 0 (use platform or configured default) or a positive + integer value of at least 32,768 (32 KiB). If *size* is not specified, + 0 is used. If changing the thread stack size is + unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is + invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB + is currently the minimum supported stack size value to guarantee sufficient + stack space for the interpreter itself. Note that some platforms may have + particular restrictions on values for the stack size, such as requiring a + minimum stack size > 32 KiB or requiring allocation in multiples of the system + memory page size - platform documentation should be referred to for more + information (4 KiB pages are common; using multiples of 4096 for the stack size is + the suggested approach in the absence of more specific information). + + .. availability:: Windows, systems with POSIX threads. + + +.. data:: TIMEOUT_MAX + + The maximum value allowed for the *timeout* parameter of + :meth:`Lock.acquire`. Specifying a timeout greater than this value will + raise an :exc:`OverflowError`. + + .. versionadded:: 3.2 + + +Lock objects have the following methods: + + +.. method:: lock.acquire(waitflag=1, timeout=-1) + + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). + + If the integer *waitflag* argument is present, the action depends on its + value: if it is zero, the lock is only acquired if it can be acquired + immediately without waiting, while if it is nonzero, the lock is acquired + unconditionally as above. + + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *waitflag* is zero. + + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. + + .. versionchanged:: 3.2 + The *timeout* parameter is new. + + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. + + +.. method:: lock.release() + + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. + + +.. method:: lock.locked() + + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. + +In addition to these methods, lock objects can also be used via the +:keyword:`with` statement, e.g.:: + + import _thread + + a_lock = _thread.allocate_lock() + + with a_lock: + print("a_lock is locked while this executes") + +**Caveats:** + + .. index:: module: signal + +* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` + exception will be received by an arbitrary thread. (When the :mod:`signal` + module is available, interrupts always go to the main thread.) + +* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is + equivalent to calling :func:`_thread.exit`. + +* It is not possible to interrupt the :meth:`acquire` method on a lock --- the + :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. + +* When the main thread exits, it is system defined whether the other threads + survive. On most systems, they are killed without executing + :keyword:`try` ... :keyword:`finally` clauses or executing object + destructors. + +* When the main thread exits, it does not do any of its usual cleanup (except + that :keyword:`try` ... :keyword:`finally` clauses are honored), and the + standard I/O files are not flushed. + diff --git a/Doc/library/ast.rst.bak b/Doc/library/ast.rst.bak new file mode 100644 index 00000000000000..b5e1be82691157 --- /dev/null +++ b/Doc/library/ast.rst.bak @@ -0,0 +1,1993 @@ +:mod:`ast` --- Abstract Syntax Trees +==================================== + +.. module:: ast + :synopsis: Abstract Syntax Tree classes and manipulation. + +.. sectionauthor:: Martin v. Löwis +.. sectionauthor:: Georg Brandl + +.. testsetup:: + + import ast + +**Source code:** :source:`Lib/ast.py` + +-------------- + +The :mod:`ast` module helps Python applications to process trees of the Python +abstract syntax grammar. The abstract syntax itself might change with each +Python release; this module helps to find out programmatically what the current +grammar looks like. + +An abstract syntax tree can be generated by passing :data:`ast.PyCF_ONLY_AST` as +a flag to the :func:`compile` built-in function, or using the :func:`parse` +helper provided in this module. The result will be a tree of objects whose +classes all inherit from :class:`ast.AST`. An abstract syntax tree can be +compiled into a Python code object using the built-in :func:`compile` function. + + +.. _abstract-grammar: + +Abstract Grammar +---------------- + +The abstract grammar is currently defined as follows: + +.. literalinclude:: ../../Parser/Python.asdl + :language: asdl + + +Node classes +------------ + +.. class:: AST + + This is the base of all AST node classes. The actual node classes are + derived from the :file:`Parser/Python.asdl` file, which is reproduced + :ref:`below `. They are defined in the :mod:`_ast` C + module and re-exported in :mod:`ast`. + + There is one class defined for each left-hand side symbol in the abstract + grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, + there is one class defined for each constructor on the right-hand side; these + classes inherit from the classes for the left-hand side trees. For example, + :class:`ast.BinOp` inherits from :class:`ast.expr`. For production rules + with alternatives (aka "sums"), the left-hand side class is abstract: only + instances of specific constructor nodes are ever created. + + .. index:: single: ? (question mark); in AST grammar + .. index:: single: * (asterisk); in AST grammar + + .. attribute:: _fields + + Each concrete class has an attribute :attr:`_fields` which gives the names + of all child nodes. + + Each instance of a concrete class has one attribute for each child node, + of the type as defined in the grammar. For example, :class:`ast.BinOp` + instances have an attribute :attr:`left` of type :class:`ast.expr`. + + If these attributes are marked as optional in the grammar (using a + question mark), the value might be ``None``. If the attributes can have + zero-or-more values (marked with an asterisk), the values are represented + as Python lists. All possible attributes must be present and have valid + values when compiling an AST with :func:`compile`. + + .. attribute:: lineno + col_offset + end_lineno + end_col_offset + + Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have + :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and + :attr:`end_col_offset` attributes. The :attr:`lineno` and :attr:`end_lineno` + are the first and last line numbers of source text span (1-indexed so the + first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset` + are the corresponding UTF-8 byte offsets of the first and last tokens that + generated the node. The UTF-8 offset is recorded because the parser uses + UTF-8 internally. + + Note that the end positions are not required by the compiler and are + therefore optional. The end offset is *after* the last symbol, for example + one can get the source segment of a one-line expression node using + ``source_line[node.col_offset : node.end_col_offset]``. + + The constructor of a class :class:`ast.T` parses its arguments as follows: + + * If there are positional arguments, there must be as many as there are items + in :attr:`T._fields`; they will be assigned as attributes of these names. + * If there are keyword arguments, they will set the attributes of the same + names to the given values. + + For example, to create and populate an :class:`ast.UnaryOp` node, you could + use :: + + node = ast.UnaryOp() + node.op = ast.USub() + node.operand = ast.Constant() + node.operand.value = 5 + node.operand.lineno = 0 + node.operand.col_offset = 0 + node.lineno = 0 + node.col_offset = 0 + + or the more compact :: + + node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), + lineno=0, col_offset=0) + +.. versionchanged:: 3.8 + + Class :class:`ast.Constant` is now used for all constants. + +.. versionchanged:: 3.9 + + Simple indices are represented by their value, extended slices are + represented as tuples. + +.. deprecated:: 3.8 + + Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, + :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, + but they will be removed in future Python releases. In the meantime, + instantiating them will return an instance of a different class. + +.. deprecated:: 3.9 + + Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still + available, but they will be removed in future Python releases. + In the meantime, instantiating them will return an instance of + a different class. + +.. note:: + The descriptions of the specific node classes displayed here + were initially adapted from the fantastic `Green Tree + Snakes `__ project and + all its contributors. + +Literals +^^^^^^^^ + +.. class:: Constant(value) + + A constant value. The ``value`` attribute of the ``Constant`` literal contains the + Python object it represents. The values represented can be simple types + such as a number, string or ``None``, but also immutable container types + (tuples and frozensets) if all of their elements are constant. + + .. doctest:: + + >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) + Expression( + body=Constant(value=123)) + + +.. class:: FormattedValue(value, conversion, format_spec) + + Node representing a single formatting field in an f-string. If the string + contains a single formatting field and nothing else the node can be + isolated otherwise it appears in :class:`JoinedStr`. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + * ``conversion`` is an integer: + + * -1: no formatting + * 115: ``!s`` string formatting + * 114: ``!r`` repr formatting + * 97: ``!a`` ascii formatting + + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + + +.. class:: JoinedStr(values) + + An f-string, comprising a series of :class:`FormattedValue` and :class:`Constant` + nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('f"sin({a}) is {sin(a):.3}"', mode='eval'), indent=4)) + Expression( + body=JoinedStr( + values=[ + Constant(value='sin('), + FormattedValue( + value=Name(id='a', ctx=Load()), + conversion=-1), + Constant(value=') is '), + FormattedValue( + value=Call( + func=Name(id='sin', ctx=Load()), + args=[ + Name(id='a', ctx=Load())], + keywords=[]), + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='.3')]))])) + + +.. class:: List(elts, ctx) + Tuple(elts, ctx) + + A list or tuple. ``elts`` holds a list of nodes representing the elements. + ``ctx`` is :class:`Store` if the container is an assignment target (i.e. + ``(x,y)=something``), and :class:`Load` otherwise. + + .. doctest:: + + >>> print(ast.dump(ast.parse('[1, 2, 3]', mode='eval'), indent=4)) + Expression( + body=List( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + ctx=Load())) + >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) + Expression( + body=Tuple( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + ctx=Load())) + + +.. class:: Set(elts) + + A set. ``elts`` holds a list of nodes representing the set's elements. + + .. doctest:: + + >>> print(ast.dump(ast.parse('{1, 2, 3}', mode='eval'), indent=4)) + Expression( + body=Set( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)])) + + +.. class:: Dict(keys, values) + + A dictionary. ``keys`` and ``values`` hold lists of nodes representing the + keys and the values respectively, in matching order (what would be returned + when calling :code:`dictionary.keys()` and :code:`dictionary.values()`). + + When doing dictionary unpacking using dictionary literals the expression to be + expanded goes in the ``values`` list, with a ``None`` at the corresponding + position in ``keys``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('{"a":1, **d}', mode='eval'), indent=4)) + Expression( + body=Dict( + keys=[ + Constant(value='a'), + None], + values=[ + Constant(value=1), + Name(id='d', ctx=Load())])) + + +Variables +^^^^^^^^^ + +.. class:: Name(id, ctx) + + A variable name. ``id`` holds the name as a string, and ``ctx`` is one of + the following types. + + +.. class:: Load() + Store() + Del() + + Variable references can be used to load the value of a variable, to assign + a new value to it, or to delete it. Variable references are given a context + to distinguish these cases. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a'), indent=4)) + Module( + body=[ + Expr( + value=Name(id='a', ctx=Load()))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a = 1'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Name(id='a', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('del a'), indent=4)) + Module( + body=[ + Delete( + targets=[ + Name(id='a', ctx=Del())])], + type_ignores=[]) + + +.. class:: Starred(value, ctx) + + A ``*var`` variable reference. ``value`` holds the variable, typically a + :class:`Name` node. This type must be used when building a :class:`Call` + node with ``*args``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a, *b = it'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Tuple( + elts=[ + Name(id='a', ctx=Store()), + Starred( + value=Name(id='b', ctx=Store()), + ctx=Store())], + ctx=Store())], + value=Name(id='it', ctx=Load()))], + type_ignores=[]) + + +Expressions +^^^^^^^^^^^ + +.. class:: Expr(value) + + When an expression, such as a function call, appears as a statement by itself + with its return value not used or stored, it is wrapped in this container. + ``value`` holds one of the other nodes in this section, a :class:`Constant`, a + :class:`Name`, a :class:`Lambda`, a :class:`Yield` or :class:`YieldFrom` node. + + .. doctest:: + + >>> print(ast.dump(ast.parse('-a'), indent=4)) + Module( + body=[ + Expr( + value=UnaryOp( + op=USub(), + operand=Name(id='a', ctx=Load())))], + type_ignores=[]) + + +.. class:: UnaryOp(op, operand) + + A unary operation. ``op`` is the operator, and ``operand`` any expression + node. + + +.. class:: UAdd + USub + Not + Invert + + Unary operator tokens. :class:`Not` is the ``not`` keyword, :class:`Invert` + is the ``~`` operator. + + .. doctest:: + + >>> print(ast.dump(ast.parse('not x', mode='eval'), indent=4)) + Expression( + body=UnaryOp( + op=Not(), + operand=Name(id='x', ctx=Load()))) + + +.. class:: BinOp(left, op, right) + + A binary operation (like addition or division). ``op`` is the operator, and + ``left`` and ``right`` are any expression nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) + Expression( + body=BinOp( + left=Name(id='x', ctx=Load()), + op=Add(), + right=Name(id='y', ctx=Load()))) + + +.. class:: Add + Sub + Mult + Div + FloorDiv + Mod + Pow + LShift + RShift + BitOr + BitXor + BitAnd + MatMult + + Binary operator tokens. + + +.. class:: BoolOp(op, values) + + A boolean operation, 'or' or 'and'. ``op`` is :class:`Or` or :class:`And`. + ``values`` are the values involved. Consecutive operations with the same + operator, such as ``a or b or c``, are collapsed into one node with several + values. + + This doesn't include ``not``, which is a :class:`UnaryOp`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x or y', mode='eval'), indent=4)) + Expression( + body=BoolOp( + op=Or(), + values=[ + Name(id='x', ctx=Load()), + Name(id='y', ctx=Load())])) + + +.. class:: And + Or + + Boolean operator tokens. + + +.. class:: Compare(left, ops, comparators) + + A comparison of two or more values. ``left`` is the first value in the + comparison, ``ops`` the list of operators, and ``comparators`` the list + of values after the first element in the comparison. + + .. doctest:: + + >>> print(ast.dump(ast.parse('1 <= a < 10', mode='eval'), indent=4)) + Expression( + body=Compare( + left=Constant(value=1), + ops=[ + LtE(), + Lt()], + comparators=[ + Name(id='a', ctx=Load()), + Constant(value=10)])) + + +.. class:: Eq + NotEq + Lt + LtE + Gt + GtE + Is + IsNot + In + NotIn + + Comparison operator tokens. + + +.. class:: Call(func, args, keywords, starargs, kwargs) + + A function call. ``func`` is the function, which will often be a + :class:`Name` or :class:`Attribute` object. Of the arguments: + + * ``args`` holds a list of the arguments passed by position. + * ``keywords`` holds a list of :class:`keyword` objects representing + arguments passed by keyword. + + When creating a ``Call`` node, ``args`` and ``keywords`` are required, but + they can be empty lists. ``starargs`` and ``kwargs`` are optional. + + .. doctest:: + + >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) + Expression( + body=Call( + func=Name(id='func', ctx=Load()), + args=[ + Name(id='a', ctx=Load()), + Starred( + value=Name(id='d', ctx=Load()), + ctx=Load())], + keywords=[ + keyword( + arg='b', + value=Name(id='c', ctx=Load())), + keyword( + value=Name(id='e', ctx=Load()))])) + + +.. class:: keyword(arg, value) + + A keyword argument to a function call or class definition. ``arg`` is a raw + string of the parameter name, ``value`` is a node to pass in. + + +.. class:: IfExp(test, body, orelse) + + An expression such as ``a if b else c``. Each field holds a single node, so + in the following example, all three are :class:`Name` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) + Expression( + body=IfExp( + test=Name(id='b', ctx=Load()), + body=Name(id='a', ctx=Load()), + orelse=Name(id='c', ctx=Load()))) + + +.. class:: Attribute(value, attr, ctx) + + Attribute access, e.g. ``d.keys``. ``value`` is a node, typically a + :class:`Name`. ``attr`` is a bare string giving the name of the attribute, + and ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` according to how + the attribute is acted on. + + .. doctest:: + + >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) + Expression( + body=Attribute( + value=Name(id='snake', ctx=Load()), + attr='colour', + ctx=Load())) + + +.. class:: NamedExpr(target, value) + + A named expression. This AST node is produced by the assignment expressions + operator (also known as the walrus operator). As opposed to the :class:`Assign` + node in which the first argument can be multiple nodes, in this case both + ``target`` and ``value`` must be single nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('(x := 4)', mode='eval'), indent=4)) + Expression( + body=NamedExpr( + target=Name(id='x', ctx=Store()), + value=Constant(value=4))) + + +Subscripting +~~~~~~~~~~~~ + +.. class:: Subscript(value, slice, ctx) + + A subscript, such as ``l[1]``. ``value`` is the subscripted object + (usually sequence or mapping). ``slice`` is an index, slice or key. + It can be a :class:`Tuple` and contain a :class:`Slice`. + ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` + according to the action performed with the subscript. + + .. doctest:: + + >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) + Expression( + body=Subscript( + value=Name(id='l', ctx=Load()), + slice=Tuple( + elts=[ + Slice( + lower=Constant(value=1), + upper=Constant(value=2)), + Constant(value=3)], + ctx=Load()), + ctx=Load())) + + +.. class:: Slice(lower, upper, step) + + Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``). + Can occur only inside the *slice* field of :class:`Subscript`, either + directly or as an element of :class:`Tuple`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) + Expression( + body=Subscript( + value=Name(id='l', ctx=Load()), + slice=Slice( + lower=Constant(value=1), + upper=Constant(value=2)), + ctx=Load())) + + +Comprehensions +~~~~~~~~~~~~~~ + +.. class:: ListComp(elt, generators) + SetComp(elt, generators) + GeneratorExp(elt, generators) + DictComp(key, value, generators) + + List and set comprehensions, generator expressions, and dictionary + comprehensions. ``elt`` (or ``key`` and ``value``) is a single node + representing the part that will be evaluated for each item. + + ``generators`` is a list of :class:`comprehension` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) + Expression( + body=ListComp( + elt=Name(id='x', ctx=Load()), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) + Expression( + body=DictComp( + key=Name(id='x', ctx=Load()), + value=BinOp( + left=Name(id='x', ctx=Load()), + op=Pow(), + right=Constant(value=2)), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) + Expression( + body=SetComp( + elt=Name(id='x', ctx=Load()), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + + +.. class:: comprehension(target, iter, ifs, is_async) + + One ``for`` clause in a comprehension. ``target`` is the reference to use for + each element - typically a :class:`Name` or :class:`Tuple` node. ``iter`` + is the object to iterate over. ``ifs`` is a list of test expressions: each + ``for`` clause can have multiple ``ifs``. + + ``is_async`` indicates a comprehension is asynchronous (using an + ``async for`` instead of ``for``). The value is an integer (0 or 1). + + .. doctest:: + + >>> print(ast.dump(ast.parse('[ord(c) for line in file for c in line]', mode='eval'), + ... indent=4)) # Multiple comprehensions in one. + Expression( + body=ListComp( + elt=Call( + func=Name(id='ord', ctx=Load()), + args=[ + Name(id='c', ctx=Load())], + keywords=[]), + generators=[ + comprehension( + target=Name(id='line', ctx=Store()), + iter=Name(id='file', ctx=Load()), + ifs=[], + is_async=0), + comprehension( + target=Name(id='c', ctx=Store()), + iter=Name(id='line', ctx=Load()), + ifs=[], + is_async=0)])) + + >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), + ... indent=4)) # generator comprehension + Expression( + body=GeneratorExp( + elt=BinOp( + left=Name(id='n', ctx=Load()), + op=Pow(), + right=Constant(value=2)), + generators=[ + comprehension( + target=Name(id='n', ctx=Store()), + iter=Name(id='it', ctx=Load()), + ifs=[ + Compare( + left=Name(id='n', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=5)]), + Compare( + left=Name(id='n', ctx=Load()), + ops=[ + Lt()], + comparators=[ + Constant(value=10)])], + is_async=0)])) + + >>> print(ast.dump(ast.parse('[i async for i in soc]', mode='eval'), + ... indent=4)) # Async comprehension + Expression( + body=ListComp( + elt=Name(id='i', ctx=Load()), + generators=[ + comprehension( + target=Name(id='i', ctx=Store()), + iter=Name(id='soc', ctx=Load()), + ifs=[], + is_async=1)])) + +Statements +^^^^^^^^^^ + +.. class:: Assign(targets, value, type_comment) + + An assignment. ``targets`` is a list of nodes, and ``value`` is a single node. + + Multiple nodes in ``targets`` represents assigning the same value to each. + Unpacking is represented by putting a :class:`Tuple` or :class:`List` + within ``targets``. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a = b = 1'), indent=4)) # Multiple assignment + Module( + body=[ + Assign( + targets=[ + Name(id='a', ctx=Store()), + Name(id='b', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking + Module( + body=[ + Assign( + targets=[ + Tuple( + elts=[ + Name(id='a', ctx=Store()), + Name(id='b', ctx=Store())], + ctx=Store())], + value=Name(id='c', ctx=Load()))], + type_ignores=[]) + + +.. class:: AnnAssign(target, annotation, value, simple) + + An assignment with a type annotation. ``target`` is a single node and can + be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. + ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` + node. ``value`` is a single optional node. ``simple`` is a boolean integer + set to True for a :class:`Name` node in ``target`` that do not appear in + between parenthesis and are hence pure names and not expressions. + + .. doctest:: + + >>> print(ast.dump(ast.parse('c: int'), indent=4)) + Module( + body=[ + AnnAssign( + target=Name(id='c', ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=1)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis + Module( + body=[ + AnnAssign( + target=Name(id='a', ctx=Store()), + annotation=Name(id='int', ctx=Load()), + value=Constant(value=1), + simple=0)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation + Module( + body=[ + AnnAssign( + target=Attribute( + value=Name(id='a', ctx=Load()), + attr='b', + ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=0)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation + Module( + body=[ + AnnAssign( + target=Subscript( + value=Name(id='a', ctx=Load()), + slice=Constant(value=1), + ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=0)], + type_ignores=[]) + + +.. class:: AugAssign(target, op, value) + + Augmented assignment, such as ``a += 1``. In the following example, + ``target`` is a :class:`Name` node for ``x`` (with the :class:`Store` + context), ``op`` is :class:`Add`, and ``value`` is a :class:`Constant` with + value for 1. + + The ``target`` attribute connot be of class :class:`Tuple` or :class:`List`, + unlike the targets of :class:`Assign`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x += 2'), indent=4)) + Module( + body=[ + AugAssign( + target=Name(id='x', ctx=Store()), + op=Add(), + value=Constant(value=2))], + type_ignores=[]) + + +.. class:: Raise(exc, cause) + + A ``raise`` statement. ``exc`` is the exception object to be raised, normally a + :class:`Call` or :class:`Name`, or ``None`` for a standalone ``raise``. + ``cause`` is the optional part for ``y`` in ``raise x from y``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('raise x from y'), indent=4)) + Module( + body=[ + Raise( + exc=Name(id='x', ctx=Load()), + cause=Name(id='y', ctx=Load()))], + type_ignores=[]) + + +.. class:: Assert(test, msg) + + An assertion. ``test`` holds the condition, such as a :class:`Compare` node. + ``msg`` holds the failure message. + + .. doctest:: + + >>> print(ast.dump(ast.parse('assert x,y'), indent=4)) + Module( + body=[ + Assert( + test=Name(id='x', ctx=Load()), + msg=Name(id='y', ctx=Load()))], + type_ignores=[]) + + +.. class:: Delete(targets) + + Represents a ``del`` statement. ``targets`` is a list of nodes, such as + :class:`Name`, :class:`Attribute` or :class:`Subscript` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('del x,y,z'), indent=4)) + Module( + body=[ + Delete( + targets=[ + Name(id='x', ctx=Del()), + Name(id='y', ctx=Del()), + Name(id='z', ctx=Del())])], + type_ignores=[]) + + +.. class:: Pass() + + A ``pass`` statement. + + .. doctest:: + + >>> print(ast.dump(ast.parse('pass'), indent=4)) + Module( + body=[ + Pass()], + type_ignores=[]) + + +Other statements which are only applicable inside functions or loops are +described in other sections. + +Imports +~~~~~~~ + +.. class:: Import(names) + + An import statement. ``names`` is a list of :class:`alias` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('import x,y,z'), indent=4)) + Module( + body=[ + Import( + names=[ + alias(name='x'), + alias(name='y'), + alias(name='z')])], + type_ignores=[]) + + +.. class:: ImportFrom(module, names, level) + + Represents ``from x import y``. ``module`` is a raw string of the 'from' name, + without any leading dots, or ``None`` for statements such as ``from . import foo``. + ``level`` is an integer holding the level of the relative import (0 means + absolute import). + + .. doctest:: + + >>> print(ast.dump(ast.parse('from y import x,y,z'), indent=4)) + Module( + body=[ + ImportFrom( + module='y', + names=[ + alias(name='x'), + alias(name='y'), + alias(name='z')], + level=0)], + type_ignores=[]) + + +.. class:: alias(name, asname) + + Both parameters are raw strings of the names. ``asname`` can be ``None`` if + the regular name is to be used. + + .. doctest:: + + >>> print(ast.dump(ast.parse('from ..foo.bar import a as b, c'), indent=4)) + Module( + body=[ + ImportFrom( + module='foo.bar', + names=[ + alias(name='a', asname='b'), + alias(name='c')], + level=2)], + type_ignores=[]) + +Control flow +^^^^^^^^^^^^ + +.. note:: + Optional clauses such as ``else`` are stored as an empty list if they're + not present. + +.. class:: If(test, body, orelse) + + An ``if`` statement. ``test`` holds a single node, such as a :class:`Compare` + node. ``body`` and ``orelse`` each hold a list of nodes. + + ``elif`` clauses don't have a special representation in the AST, but rather + appear as extra :class:`If` nodes within the ``orelse`` section of the + previous one. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... if x: + ... ... + ... elif y: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + If( + test=Name(id='x', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + If( + test=Name(id='y', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +.. class:: For(target, iter, body, orelse, type_comment) + + A ``for`` loop. ``target`` holds the variable(s) the loop assigns to, as a + single :class:`Name`, :class:`Tuple` or :class:`List` node. ``iter`` holds + the item to be looped over, again as a single node. ``body`` and ``orelse`` + contain lists of nodes to execute. Those in ``orelse`` are executed if the + loop finishes normally, rather than via a ``break`` statement. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... for x in y: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + For( + target=Name(id='x', ctx=Store()), + iter=Name(id='y', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: While(test, body, orelse) + + A ``while`` loop. ``test`` holds the condition, such as a :class:`Compare` + node. + + .. doctest:: + + >> print(ast.dump(ast.parse(""" + ... while x: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + While( + test=Name(id='x', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: Break + Continue + + The ``break`` and ``continue`` statements. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... for a in b: + ... if a > 5: + ... break + ... else: + ... continue + ... + ... """), indent=4)) + Module( + body=[ + For( + target=Name(id='a', ctx=Store()), + iter=Name(id='b', ctx=Load()), + body=[ + If( + test=Compare( + left=Name(id='a', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=5)]), + body=[ + Break()], + orelse=[ + Continue()])], + orelse=[])], + type_ignores=[]) + + +.. class:: Try(body, handlers, orelse, finalbody) + + ``try`` blocks. All attributes are list of nodes to execute, except for + ``handlers``, which is a list of :class:`ExceptHandler` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... try: + ... ... + ... except Exception: + ... ... + ... except OtherException as e: + ... ... + ... else: + ... ... + ... finally: + ... ... + ... """), indent=4)) + Module( + body=[ + Try( + body=[ + Expr( + value=Constant(value=Ellipsis))], + handlers=[ + ExceptHandler( + type=Name(id='Exception', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + ExceptHandler( + type=Name(id='OtherException', ctx=Load()), + name='e', + body=[ + Expr( + value=Constant(value=Ellipsis))])], + orelse=[ + Expr( + value=Constant(value=Ellipsis))], + finalbody=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: ExceptHandler(type, name, body) + + A single ``except`` clause. ``type`` is the exception type it will match, + typically a :class:`Name` node (or ``None`` for a catch-all ``except:`` clause). + ``name`` is a raw string for the name to hold the exception, or ``None`` if + the clause doesn't have ``as foo``. ``body`` is a list of nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... try: + ... a + 1 + ... except TypeError: + ... pass + ... """), indent=4)) + Module( + body=[ + Try( + body=[ + Expr( + value=BinOp( + left=Name(id='a', ctx=Load()), + op=Add(), + right=Constant(value=1)))], + handlers=[ + ExceptHandler( + type=Name(id='TypeError', ctx=Load()), + body=[ + Pass()])], + orelse=[], + finalbody=[])], + type_ignores=[]) + + +.. class:: With(items, body, type_comment) + + A ``with`` block. ``items`` is a list of :class:`withitem` nodes representing + the context managers, and ``body`` is the indented block inside the context. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + +.. class:: withitem(context_expr, optional_vars) + + A single context manager in a ``with`` block. ``context_expr`` is the context + manager, often a :class:`Call` node. ``optional_vars`` is a :class:`Name`, + :class:`Tuple` or :class:`List` for the ``as foo`` part, or ``None`` if that + isn't used. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... with a as b, c as d: + ... something(b, d) + ... """), indent=4)) + Module( + body=[ + With( + items=[ + withitem( + context_expr=Name(id='a', ctx=Load()), + optional_vars=Name(id='b', ctx=Store())), + withitem( + context_expr=Name(id='c', ctx=Load()), + optional_vars=Name(id='d', ctx=Store()))], + body=[ + Expr( + value=Call( + func=Name(id='something', ctx=Load()), + args=[ + Name(id='b', ctx=Load()), + Name(id='d', ctx=Load())], + keywords=[]))])], + type_ignores=[]) + + +.. class:: Match(subject, cases) + + A ``match`` statement. ``subject`` holds the subject of the match (the object + that is being matched against the cases) and ``cases`` contains an iterable of + :class:`match_case` nodes with the different cases. + + +.. class:: match_case(context_expr, optional_vars) + + A single case pattern in a ``match`` statement. ``pattern`` contains the + match pattern that will be used to match the subject against. Notice that + the meaning of the :class:`AST` nodes in this attribute have a different + meaning than in other places, as they represent patterns to match against. + The ``guard`` attribute contains an expression that will be evaluated if + the pattern matches the subject. If the pattern matches and the ``guard`` condition + is truthy, the body of the case shall be executed. ``body`` contains a list + of nodes to execute if the guard is truthy. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] if x>0: + ... ... + ... case tuple(): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=List( + elts=[ + Name(id='x', ctx=Store())], + ctx=Load()), + guard=Compare( + left=Name(id='x', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=0)]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=Call( + func=Name(id='tuple', ctx=Load()), + args=[], + keywords=[]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchAs(pattern, name) + + A match "as-pattern". The as-pattern matches whatever pattern is on its + left-hand side, but also binds the value to a name. ``pattern`` contains + the match pattern that will be used to match the subject agsinst. The ``name`` + attribute contains the name that will be binded if the pattern is successful. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] as y: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchAs( + pattern=List( + elts=[ + Name(id='x', ctx=Store())], + ctx=Load()), + name='y'), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +.. class:: MatchOr(patterns) + + A match "or-pattern". An or-pattern matches each of its subpatterns in turn + to the subject, until one succeeds. The or-pattern is then deemed to + succeed. If none of the subpatterns succeed the or-pattern fails. The + ``patterns`` attribute contains a list of match patterns nodes that will be + matched against the subject. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] | (y): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchOr( + patterns=[ + List( + elts=[ + Name(id='x', ctx=Store())], + ctx=Load()), + Name(id='y', ctx=Store())]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +Function and class definitions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment) + + A function definition. + + * ``name`` is a raw string of the function name. + * ``args`` is a :class:`arguments` node. + * ``body`` is the list of nodes inside the function. + * ``decorator_list`` is the list of decorators to be applied, stored outermost + first (i.e. the first in the list will be applied last). + * ``returns`` is the return annotation. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + +.. class:: Lambda(args, body) + + ``lambda`` is a minimal function definition that can be used inside an + expression. Unlike :class:`FunctionDef`, ``body`` holds a single node. + + .. doctest:: + + >>> print(ast.dump(ast.parse('lambda x,y: ...'), indent=4)) + Module( + body=[ + Expr( + value=Lambda( + args=arguments( + posonlyargs=[], + args=[ + arg(arg='x'), + arg(arg='y')], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=Constant(value=Ellipsis)))], + type_ignores=[]) + + +.. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) + + The arguments for a function. + + * ``posonlyargs``, ``args`` and ``kwonlyargs`` are lists of :class:`arg` nodes. + * ``vararg`` and ``kwarg`` are single :class:`arg` nodes, referring to the + ``*args, **kwargs`` parameters. + * ``kw_defaults`` is a list of default values for keyword-only arguments. If + one is ``None``, the corresponding argument is required. + * ``defaults`` is a list of default values for arguments that can be passed + positionally. If there are fewer defaults, they correspond to the last n + arguments. + + +.. class:: arg(arg, annotation, type_comment) + + A single argument in a list. ``arg`` is a raw string of the argument + name, ``annotation`` is its annotation, such as a :class:`Str` or + :class:`Name` node. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... @decorator1 + ... @decorator2 + ... def f(a: 'annotation', b=1, c=2, *d, e, f=3, **g) -> 'return annotation': + ... pass + ... """), indent=4)) + Module( + body=[ + FunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[ + arg( + arg='a', + annotation=Constant(value='annotation')), + arg(arg='b'), + arg(arg='c')], + vararg=arg(arg='d'), + kwonlyargs=[ + arg(arg='e'), + arg(arg='f')], + kw_defaults=[ + None, + Constant(value=3)], + kwarg=arg(arg='g'), + defaults=[ + Constant(value=1), + Constant(value=2)]), + body=[ + Pass()], + decorator_list=[ + Name(id='decorator1', ctx=Load()), + Name(id='decorator2', ctx=Load())], + returns=Constant(value='return annotation'))], + type_ignores=[]) + + +.. class:: Return(value) + + A ``return`` statement. + + .. doctest:: + + >>> print(ast.dump(ast.parse('return 4'), indent=4)) + Module( + body=[ + Return( + value=Constant(value=4))], + type_ignores=[]) + + +.. class:: Yield(value) + YieldFrom(value) + + A ``yield`` or ``yield from`` expression. Because these are expressions, they + must be wrapped in a :class:`Expr` node if the value sent back is not used. + + .. doctest:: + + >>> print(ast.dump(ast.parse('yield x'), indent=4)) + Module( + body=[ + Expr( + value=Yield( + value=Name(id='x', ctx=Load())))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('yield from x'), indent=4)) + Module( + body=[ + Expr( + value=YieldFrom( + value=Name(id='x', ctx=Load())))], + type_ignores=[]) + + +.. class:: Global(names) + Nonlocal(names) + + ``global`` and ``nonlocal`` statements. ``names`` is a list of raw strings. + + .. doctest:: + + >>> print(ast.dump(ast.parse('global x,y,z'), indent=4)) + Module( + body=[ + Global( + names=[ + 'x', + 'y', + 'z'])], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) + Module( + body=[ + Nonlocal( + names=[ + 'x', + 'y', + 'z'])], + type_ignores=[]) + + +.. class:: ClassDef(name, bases, keywords, starargs, kwargs, body, decorator_list) + + A class definition. + + * ``name`` is a raw string for the class name + * ``bases`` is a list of nodes for explicitly specified base classes. + * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. + Other keywords will be passed to the metaclass, as per `PEP-3115 + `_. + * ``starargs`` and ``kwargs`` are each a single node, as in a function call. + starargs will be expanded to join the list of base classes, and kwargs will + be passed to the metaclass. + * ``body`` is a list of nodes representing the code within the class + definition. + * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... @decorator1 + ... @decorator2 + ... class Foo(base1, base2, metaclass=meta): + ... pass + ... """), indent=4)) + Module( + body=[ + ClassDef( + name='Foo', + bases=[ + Name(id='base1', ctx=Load()), + Name(id='base2', ctx=Load())], + keywords=[ + keyword( + arg='metaclass', + value=Name(id='meta', ctx=Load()))], + body=[ + Pass()], + decorator_list=[ + Name(id='decorator1', ctx=Load()), + Name(id='decorator2', ctx=Load())])], + type_ignores=[]) + +Async and await +^^^^^^^^^^^^^^^ + +.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment) + + An ``async def`` function definition. Has the same fields as + :class:`FunctionDef`. + + +.. class:: Await(value) + + An ``await`` expression. ``value`` is what it waits for. + Only valid in the body of an :class:`AsyncFunctionDef`. + +.. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... async def f(): + ... await other_func() + ... """), indent=4)) + Module( + body=[ + AsyncFunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=[ + Expr( + value=Await( + value=Call( + func=Name(id='other_func', ctx=Load()), + args=[], + keywords=[])))], + decorator_list=[])], + type_ignores=[]) + + +.. class:: AsyncFor(target, iter, body, orelse, type_comment) + AsyncWith(items, body, type_comment) + + ``async for`` loops and ``async with`` context managers. They have the same + fields as :class:`For` and :class:`With`, respectively. Only valid in the + body of an :class:`AsyncFunctionDef`. + +.. note:: + When a string is parsed by :func:`ast.parse`, operator nodes (subclasses + of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, + :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree + will be singletons. Changes to one will be reflected in all other + occurrences of the same value (e.g. :class:`ast.Add`). + + +:mod:`ast` Helpers +------------------ + +Apart from the node classes, the :mod:`ast` module defines these utility functions +and classes for traversing abstract syntax trees: + +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) + + Parse the source into an AST node. Equivalent to ``compile(source, + filename, mode, ast.PyCF_ONLY_AST)``. + + If ``type_comments=True`` is given, the parser is modified to check + and return type comments as specified by :pep:`484` and :pep:`526`. + This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the + flags passed to :func:`compile()`. This will report syntax errors + for misplaced type comments. Without this flag, type comments will + be ignored, and the ``type_comment`` field on selected AST nodes + will always be ``None``. In addition, the locations of ``# type: + ignore`` comments will be returned as the ``type_ignores`` + attribute of :class:`Module` (otherwise it is always an empty list). + + In addition, if ``mode`` is ``'func_type'``, the input syntax is + modified to correspond to :pep:`484` "signature type comments", + e.g. ``(str, int) -> List[str]``. + + Also, setting ``feature_version`` to a tuple ``(major, minor)`` + will attempt to parse using that Python version's grammar. + Currently ``major`` must equal to ``3``. For example, setting + ``feature_version=(3, 4)`` will allow the use of ``async`` and + ``await`` as variable names. The lowest supported version is + ``(3, 4)``; the highest is ``sys.version_info[0:2]``. + + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + + .. versionchanged:: 3.8 + Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. + + +.. function:: unparse(ast_obj) + + Unparse an :class:`ast.AST` object and generate a string with code + that would produce an equivalent :class:`ast.AST` object if parsed + back with :func:`ast.parse`. + + .. warning:: + The produced code string will not necessarily be equal to the original + code that generated the :class:`ast.AST` object (without any compiler + optimizations, such as constant tuples/frozensets). + + .. warning:: + Trying to unparse a highly complex expression would result with + :exc:`RecursionError`. + + .. versionadded:: 3.9 + + +.. function:: literal_eval(node_or_string) + + Safely evaluate an expression node or a string containing a Python literal or + container display. The string or node provided may only consist of the + following Python literal structures: strings, bytes, numbers, tuples, lists, + dicts, sets, booleans, ``None`` and ``Ellipsis``. + + This can be used for safely evaluating strings containing Python values from + untrusted sources without the need to parse the values oneself. It is not + capable of evaluating arbitrarily complex expressions, for example involving + operators or indexing. + + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + + It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, + :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed + input. + + .. versionchanged:: 3.2 + Now allows bytes and set literals. + + .. versionchanged:: 3.9 + Now supports creating empty sets with ``'set()'``. + + .. versionchanged:: 3.10 + For string inputs, leading spaces and tabs are now stripped. + + +.. function:: get_docstring(node, clean=True) + + Return the docstring of the given *node* (which must be a + :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`, + or :class:`Module` node), or ``None`` if it has no docstring. + If *clean* is true, clean up the docstring's indentation with + :func:`inspect.cleandoc`. + + .. versionchanged:: 3.5 + :class:`AsyncFunctionDef` is now supported. + + +.. function:: get_source_segment(source, node, *, padded=False) + + Get source code segment of the *source* that generated *node*. + If some location information (:attr:`lineno`, :attr:`end_lineno`, + :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. + + If *padded* is ``True``, the first line of a multi-line statement will + be padded with spaces to match its original position. + + .. versionadded:: 3.8 + + +.. function:: fix_missing_locations(node) + + When you compile a node tree with :func:`compile`, the compiler expects + :attr:`lineno` and :attr:`col_offset` attributes for every node that supports + them. This is rather tedious to fill in for generated nodes, so this helper + adds these attributes recursively where not already set, by setting them to + the values of the parent node. It works recursively starting at *node*. + + +.. function:: increment_lineno(node, n=1) + + Increment the line number and end line number of each node in the tree + starting at *node* by *n*. This is useful to "move code" to a different + location in a file. + + +.. function:: copy_location(new_node, old_node) + + Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, + and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, + and return *new_node*. + + +.. function:: iter_fields(node) + + Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` + that is present on *node*. + + +.. function:: iter_child_nodes(node) + + Yield all direct child nodes of *node*, that is, all fields that are nodes + and all items of fields that are lists of nodes. + + +.. function:: walk(node) + + Recursively yield all descendant nodes in the tree starting at *node* + (including *node* itself), in no specified order. This is useful if you only + want to modify nodes in place and don't care about the context. + + +.. class:: NodeVisitor() + + A node visitor base class that walks the abstract syntax tree and calls a + visitor function for every node found. This function may return a value + which is forwarded by the :meth:`visit` method. + + This class is meant to be subclassed, with the subclass adding visitor + methods. + + .. method:: visit(node) + + Visit a node. The default implementation calls the method called + :samp:`self.visit_{classname}` where *classname* is the name of the node + class, or :meth:`generic_visit` if that method doesn't exist. + + .. method:: generic_visit(node) + + This visitor calls :meth:`visit` on all children of the node. + + Note that child nodes of nodes that have a custom visitor method won't be + visited unless the visitor calls :meth:`generic_visit` or visits them + itself. + + Don't use the :class:`NodeVisitor` if you want to apply changes to nodes + during traversal. For this a special visitor exists + (:class:`NodeTransformer`) that allows modifications. + + .. deprecated:: 3.8 + + Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, + :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated + now and will not be called in future Python versions. Add the + :meth:`visit_Constant` method to handle all constant nodes. + + +.. class:: NodeTransformer() + + A :class:`NodeVisitor` subclass that walks the abstract syntax tree and + allows modification of nodes. + + The :class:`NodeTransformer` will walk the AST and use the return value of + the visitor methods to replace or remove the old node. If the return value + of the visitor method is ``None``, the node will be removed from its + location, otherwise it is replaced with the return value. The return value + may be the original node in which case no replacement takes place. + + Here is an example transformer that rewrites all occurrences of name lookups + (``foo``) to ``data['foo']``:: + + class RewriteName(NodeTransformer): + + def visit_Name(self, node): + return Subscript( + value=Name(id='data', ctx=Load()), + slice=Constant(value=node.id), + ctx=node.ctx + ) + + Keep in mind that if the node you're operating on has child nodes you must + either transform the child nodes yourself or call the :meth:`generic_visit` + method for the node first. + + For nodes that were part of a collection of statements (that applies to all + statement nodes), the visitor may also return a list of nodes rather than + just a single node. + + If :class:`NodeTransformer` introduces new nodes (that weren't part of + original tree) without giving them location information (such as + :attr:`lineno`), :func:`fix_missing_locations` should be called with + the new sub-tree to recalculate the location information:: + + tree = ast.parse('foo', mode='eval') + new_tree = fix_missing_locations(RewriteName().visit(tree)) + + Usually you use the transformer like this:: + + node = YourTransformer().visit(node) + + +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) + + Return a formatted dump of the tree in *node*. This is mainly useful for + debugging purposes. If *annotate_fields* is true (by default), + the returned string will show the names and the values for fields. + If *annotate_fields* is false, the result string will be more compact by + omitting unambiguous field names. Attributes such as line + numbers and column offsets are not dumped by default. If this is wanted, + *include_attributes* can be set to true. + + If *indent* is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. An indent level + of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) + selects the single line representation. Using a positive integer indent + indents that many spaces per level. If *indent* is a string (such as ``"\t"``), + that string is used to indent each level. + + .. versionchanged:: 3.9 + Added the *indent* option. + + +.. _ast-compiler-flags: + +Compiler Flags +-------------- + +The following flags may be passed to :func:`compile` in order to change +effects on the compilation of a program: + +.. data:: PyCF_ALLOW_TOP_LEVEL_AWAIT + + Enables support for top-level ``await``, ``async for``, ``async with`` + and async comprehensions. + + .. versionadded:: 3.8 + +.. data:: PyCF_ONLY_AST + + Generates and returns an abstract syntax tree instead of returning a + compiled code object. + +.. data:: PyCF_TYPE_COMMENTS + + Enables support for :pep:`484` and :pep:`526` style type comments + (``# type: ``, ``# type: ignore ``). + + .. versionadded:: 3.8 + + +.. _ast-cli: + +Command-Line Usage +------------------ + +.. versionadded:: 3.9 + +The :mod:`ast` module can be executed as a script from the command line. +It is as simple as: + +.. code-block:: sh + + python -m ast [-m ] [-a] [infile] + +The following options are accepted: + +.. program:: ast + +.. cmdoption:: -h, --help + + Show the help message and exit. + +.. cmdoption:: -m + --mode + + Specify what kind of code must be compiled, like the *mode* argument + in :func:`parse`. + +.. cmdoption:: --no-type-comments + + Don't parse type comments. + +.. cmdoption:: -a, --include-attributes + + Include attributes such as line numbers and column offsets. + +.. cmdoption:: -i + --indent + + Indentation of nodes in AST (number of spaces). + +If :file:`infile` is specified its contents are parsed to AST and dumped +to stdout. Otherwise, the content is read from stdin. + + +.. seealso:: + + `Green Tree Snakes `_, an external + documentation resource, has good details on working with Python ASTs. + + `ASTTokens `_ + annotates Python ASTs with the positions of tokens and text in the source + code that generated them. This is helpful for tools that make source code + transformations. + + `leoAst.py `_ unifies the + token-based and parse-tree-based views of python programs by inserting + two-way links between tokens and ast nodes. + + `LibCST `_ parses code as a Concrete Syntax + Tree that looks like an ast tree and keeps all formatting details. It's + useful for building automated refactoring (codemod) applications and + linters. + + `Parso `_ is a Python parser that supports + error recovery and round-trip parsing for different Python versions (in + multiple Python versions). Parso is also able to list multiple syntax errors + in your python file. diff --git a/Doc/library/asyncio-api-index.rst.bak b/Doc/library/asyncio-api-index.rst.bak new file mode 100644 index 00000000000000..047e5bbc58ccad --- /dev/null +++ b/Doc/library/asyncio-api-index.rst.bak @@ -0,0 +1,221 @@ +.. currentmodule:: asyncio + + +==================== +High-level API Index +==================== + +This page lists all high-level async/await enabled asyncio APIs. + + +Tasks +===== + +Utilities to run asyncio programs, create Tasks, and +await on multiple things with timeouts. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :func:`run` + - Create event loop, run a coroutine, close the loop. + + * - :func:`create_task` + - Start an asyncio Task. + + * - ``await`` :func:`sleep` + - Sleep for a number of seconds. + + * - ``await`` :func:`gather` + - Schedule and wait for things concurrently. + + * - ``await`` :func:`wait_for` + - Run with a timeout. + + * - ``await`` :func:`shield` + - Shield from cancellation. + + * - ``await`` :func:`wait` + - Monitor for completion. + + * - :func:`current_task` + - Return the current Task. + + * - :func:`all_tasks` + - Return all tasks for an event loop. + + * - :class:`Task` + - Task object. + + * - :func:`to_thread` + - Asychronously run a function in a separate OS thread. + + * - :func:`run_coroutine_threadsafe` + - Schedule a coroutine from another OS thread. + + * - ``for in`` :func:`as_completed` + - Monitor for completion with a ``for`` loop. + + +.. rubric:: Examples + +* :ref:`Using asyncio.gather() to run things in parallel + `. + +* :ref:`Using asyncio.wait_for() to enforce a timeout + `. + +* :ref:`Cancellation `. + +* :ref:`Using asyncio.sleep() `. + +* See also the main :ref:`Tasks documentation page `. + + +Queues +====== + +Queues should be used to distribute work amongst multiple asyncio Tasks, +implement connection pools, and pub/sub patterns. + + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :class:`Queue` + - A FIFO queue. + + * - :class:`PriorityQueue` + - A priority queue. + + * - :class:`LifoQueue` + - A LIFO queue. + + +.. rubric:: Examples + +* :ref:`Using asyncio.Queue to distribute workload between several + Tasks `. + +* See also the :ref:`Queues documentation page `. + + +Subprocesses +============ + +Utilities to spawn subprocesses and run shell commands. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :func:`create_subprocess_exec` + - Create a subprocess. + + * - ``await`` :func:`create_subprocess_shell` + - Run a shell command. + + +.. rubric:: Examples + +* :ref:`Executing a shell command `. + +* See also the :ref:`subprocess APIs ` + documentation. + + +Streams +======= + +High-level APIs to work with network IO. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :func:`open_connection` + - Establish a TCP connection. + + * - ``await`` :func:`open_unix_connection` + - Establish a Unix socket connection. + + * - ``await`` :func:`start_server` + - Start a TCP server. + + * - ``await`` :func:`start_unix_server` + - Start a Unix socket server. + + * - :class:`StreamReader` + - High-level async/await object to receive network data. + + * - :class:`StreamWriter` + - High-level async/await object to send network data. + + +.. rubric:: Examples + +* :ref:`Example TCP client `. + +* See also the :ref:`streams APIs ` + documentation. + + +Synchronization +=============== + +Threading-like synchronization primitives that can be used in Tasks. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :class:`Lock` + - A mutex lock. + + * - :class:`Event` + - An event object. + + * - :class:`Condition` + - A condition object. + + * - :class:`Semaphore` + - A semaphore. + + * - :class:`BoundedSemaphore` + - A bounded semaphore. + + +.. rubric:: Examples + +* :ref:`Using asyncio.Event `. + +* See also the documentation of asyncio + :ref:`synchronization primitives `. + + +Exceptions +========== + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + + * - :exc:`asyncio.TimeoutError` + - Raised on timeout by functions like :func:`wait_for`. + Keep in mind that ``asyncio.TimeoutError`` is **unrelated** + to the built-in :exc:`TimeoutError` exception. + + * - :exc:`asyncio.CancelledError` + - Raised when a Task is cancelled. See also :meth:`Task.cancel`. + + +.. rubric:: Examples + +* :ref:`Handling CancelledError to run code on cancellation request + `. + +* See also the full list of + :ref:`asyncio-specific exceptions `. diff --git a/Doc/library/collections.rst.bak b/Doc/library/collections.rst.bak new file mode 100644 index 00000000000000..aa0acfaae45a5f --- /dev/null +++ b/Doc/library/collections.rst.bak @@ -0,0 +1,1278 @@ +:mod:`collections` --- Container datatypes +========================================== + +.. module:: collections + :synopsis: Container datatypes + +.. moduleauthor:: Raymond Hettinger +.. sectionauthor:: Raymond Hettinger + +**Source code:** :source:`Lib/collections/__init__.py` + +.. testsetup:: * + + from collections import * + import itertools + __name__ = '' + +-------------- + +This module implements specialized container datatypes providing alternatives to +Python's general purpose built-in containers, :class:`dict`, :class:`list`, +:class:`set`, and :class:`tuple`. + +===================== ==================================================================== +:func:`namedtuple` factory function for creating tuple subclasses with named fields +:class:`deque` list-like container with fast appends and pops on either end +:class:`ChainMap` dict-like class for creating a single view of multiple mappings +:class:`Counter` dict subclass for counting hashable objects +:class:`OrderedDict` dict subclass that remembers the order entries were added +:class:`defaultdict` dict subclass that calls a factory function to supply missing values +:class:`UserDict` wrapper around dictionary objects for easier dict subclassing +:class:`UserList` wrapper around list objects for easier list subclassing +:class:`UserString` wrapper around string objects for easier string subclassing +===================== ==================================================================== + + +:class:`ChainMap` objects +------------------------- + +.. versionadded:: 3.3 + +A :class:`ChainMap` class is provided for quickly linking a number of mappings +so they can be treated as a single unit. It is often much faster than creating +a new dictionary and running multiple :meth:`~dict.update` calls. + +The class can be used to simulate nested scopes and is useful in templating. + +.. class:: ChainMap(*maps) + + A :class:`ChainMap` groups multiple dicts or other mappings together to + create a single, updateable view. If no *maps* are specified, a single empty + dictionary is provided so that a new chain always has at least one mapping. + + The underlying mappings are stored in a list. That list is public and can + be accessed or updated using the *maps* attribute. There is no other state. + + Lookups search the underlying mappings successively until a key is found. In + contrast, writes, updates, and deletions only operate on the first mapping. + + A :class:`ChainMap` incorporates the underlying mappings by reference. So, if + one of the underlying mappings gets updated, those changes will be reflected + in :class:`ChainMap`. + + All of the usual dictionary methods are supported. In addition, there is a + *maps* attribute, a method for creating new subcontexts, and a property for + accessing all but the first mapping: + + .. attribute:: maps + + A user updateable list of mappings. The list is ordered from + first-searched to last-searched. It is the only stored state and can + be modified to change which mappings are searched. The list should + always contain at least one mapping. + + .. method:: new_child(m=None) + + Returns a new :class:`ChainMap` containing a new map followed by + all of the maps in the current instance. If ``m`` is specified, + it becomes the new map at the front of the list of mappings; if not + specified, an empty dict is used, so that a call to ``d.new_child()`` + is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for + creating subcontexts that can be updated without altering values in any + of the parent mappings. + + .. versionchanged:: 3.4 + The optional ``m`` parameter was added. + + .. attribute:: parents + + Property returning a new :class:`ChainMap` containing all of the maps in + the current instance except the first one. This is useful for skipping + the first map in the search. Use cases are similar to those for the + :keyword:`nonlocal` keyword used in :term:`nested scopes `. The use cases also parallel those for the built-in + :func:`super` function. A reference to ``d.parents`` is equivalent to: + ``ChainMap(*d.maps[1:])``. + + Note, the iteration order of a :class:`ChainMap()` is determined by + scanning the mappings last to first:: + + >>> baseline = {'music': 'bach', 'art': 'rembrandt'} + >>> adjustments = {'art': 'van gogh', 'opera': 'carmen'} + >>> list(ChainMap(adjustments, baseline)) + ['music', 'art', 'opera'] + + This gives the same ordering as a series of :meth:`dict.update` calls + starting with the last mapping:: + + >>> combined = baseline.copy() + >>> combined.update(adjustments) + >>> list(combined) + ['music', 'art', 'opera'] + + .. versionchanged:: 3.9 + Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. + +.. seealso:: + + * The `MultiContext class + `_ + in the Enthought `CodeTools package + `_ has options to support + writing to any mapping in the chain. + + * Django's `Context class + `_ + for templating is a read-only chain of mappings. It also features + pushing and popping of contexts similar to the + :meth:`~collections.ChainMap.new_child` method and the + :attr:`~collections.ChainMap.parents` property. + + * The `Nested Contexts recipe + `_ has options to control + whether writes and other mutations apply only to the first mapping or to + any mapping in the chain. + + * A `greatly simplified read-only version of Chainmap + `_. + + +:class:`ChainMap` Examples and Recipes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This section shows various approaches to working with chained maps. + + +Example of simulating Python's internal lookup chain:: + + import builtins + pylookup = ChainMap(locals(), globals(), vars(builtins)) + +Example of letting user specified command-line arguments take precedence over +environment variables which in turn take precedence over default values:: + + import os, argparse + + defaults = {'color': 'red', 'user': 'guest'} + + parser = argparse.ArgumentParser() + parser.add_argument('-u', '--user') + parser.add_argument('-c', '--color') + namespace = parser.parse_args() + command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} + + combined = ChainMap(command_line_args, os.environ, defaults) + print(combined['color']) + print(combined['user']) + +Example patterns for using the :class:`ChainMap` class to simulate nested +contexts:: + + c = ChainMap() # Create root context + d = c.new_child() # Create nested child context + e = c.new_child() # Child of c, independent from d + e.maps[0] # Current context dictionary -- like Python's locals() + e.maps[-1] # Root context -- like Python's globals() + e.parents # Enclosing context chain -- like Python's nonlocals + + d['x'] = 1 # Set value in current context + d['x'] # Get first key in the chain of contexts + del d['x'] # Delete from current context + list(d) # All nested values + k in d # Check all nested values + len(d) # Number of nested values + d.items() # All nested items + dict(d) # Flatten into a regular dictionary + +The :class:`ChainMap` class only makes updates (writes and deletions) to the +first mapping in the chain while lookups will search the full chain. However, +if deep writes and deletions are desired, it is easy to make a subclass that +updates keys found deeper in the chain:: + + class DeepChainMap(ChainMap): + 'Variant of ChainMap that allows direct updates to inner scopes' + + def __setitem__(self, key, value): + for mapping in self.maps: + if key in mapping: + mapping[key] = value + return + self.maps[0][key] = value + + def __delitem__(self, key): + for mapping in self.maps: + if key in mapping: + del mapping[key] + return + raise KeyError(key) + + >>> d = DeepChainMap({'zebra': 'black'}, {'elephant': 'blue'}, {'lion': 'yellow'}) + >>> d['lion'] = 'orange' # update an existing key two levels down + >>> d['snake'] = 'red' # new keys get added to the topmost dict + >>> del d['elephant'] # remove an existing key one level down + >>> d # display result + DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) + + +:class:`Counter` objects +------------------------ + +A counter tool is provided to support convenient and rapid tallies. +For example:: + + >>> # Tally occurrences of words in a list + >>> cnt = Counter() + >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: + ... cnt[word] += 1 + >>> cnt + Counter({'blue': 3, 'red': 2, 'green': 1}) + + >>> # Find the ten most common words in Hamlet + >>> import re + >>> words = re.findall(r'\w+', open('hamlet.txt').read().lower()) + >>> Counter(words).most_common(10) + [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), + ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] + +.. class:: Counter([iterable-or-mapping]) + + A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. + It is a collection where elements are stored as dictionary keys + and their counts are stored as dictionary values. Counts are allowed to be + any integer value including zero or negative counts. The :class:`Counter` + class is similar to bags or multisets in other languages. + + Elements are counted from an *iterable* or initialized from another + *mapping* (or counter): + + >>> c = Counter() # a new, empty counter + >>> c = Counter('gallahad') # a new counter from an iterable + >>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping + >>> c = Counter(cats=4, dogs=8) # a new counter from keyword args + + Counter objects have a dictionary interface except that they return a zero + count for missing items instead of raising a :exc:`KeyError`: + + >>> c = Counter(['eggs', 'ham']) + >>> c['bacon'] # count of a missing element is zero + 0 + + Setting a count to zero does not remove an element from a counter. + Use ``del`` to remove it entirely: + + >>> c['sausage'] = 0 # counter entry with a zero count + >>> del c['sausage'] # del actually removes the entry + + .. versionadded:: 3.1 + + .. versionchanged:: 3.7 As a :class:`dict` subclass, :class:`Counter` + Inherited the capability to remember insertion order. Math operations + on *Counter* objects also preserve order. Results are ordered + according to when an element is first encountered in the left operand + and then by the order encountered in the right operand. + + Counter objects support three methods beyond those available for all + dictionaries: + + .. method:: elements() + + Return an iterator over elements repeating each as many times as its + count. Elements are returned in the order first encountered. If an + element's count is less than one, :meth:`elements` will ignore it. + + >>> c = Counter(a=4, b=2, c=0, d=-2) + >>> sorted(c.elements()) + ['a', 'a', 'a', 'a', 'b', 'b'] + + .. method:: most_common([n]) + + Return a list of the *n* most common elements and their counts from the + most common to the least. If *n* is omitted or ``None``, + :meth:`most_common` returns *all* elements in the counter. + Elements with equal counts are ordered in the order first encountered: + + >>> Counter('abracadabra').most_common(3) + [('a', 5), ('b', 2), ('r', 2)] + + .. method:: subtract([iterable-or-mapping]) + + Elements are subtracted from an *iterable* or from another *mapping* + (or counter). Like :meth:`dict.update` but subtracts counts instead + of replacing them. Both inputs and outputs may be zero or negative. + + >>> c = Counter(a=4, b=2, c=0, d=-2) + >>> d = Counter(a=1, b=2, c=3, d=4) + >>> c.subtract(d) + >>> c + Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6}) + + .. versionadded:: 3.2 + + The usual dictionary methods are available for :class:`Counter` objects + except for two which work differently for counters. + + .. method:: fromkeys(iterable) + + This class method is not implemented for :class:`Counter` objects. + + .. method:: update([iterable-or-mapping]) + + Elements are counted from an *iterable* or added-in from another + *mapping* (or counter). Like :meth:`dict.update` but adds counts + instead of replacing them. Also, the *iterable* is expected to be a + sequence of elements, not a sequence of ``(key, value)`` pairs. + +Counters support rich comparison operators for equality, subset, and +superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. +All of those tests treat missing elements as having zero counts so that +``Counter(a=1) == Counter(a=1, b=0)`` returns true. + +.. versionadded:: 3.10 + Rich comparison operations we were added + +.. versionchanged:: 3.10 + In equality tests, missing elements are treated as having zero counts. + Formerly, ``Counter(a=3)`` and ``Counter(a=3, b=0)`` were considered + distinct. + +Common patterns for working with :class:`Counter` objects:: + + sum(c.values()) # total of all counts + c.clear() # reset all counts + list(c) # list unique elements + set(c) # convert to a set + dict(c) # convert to a regular dictionary + c.items() # convert to a list of (elem, cnt) pairs + Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs + c.most_common()[:-n-1:-1] # n least common elements + +c # remove zero and negative counts + +Several mathematical operations are provided for combining :class:`Counter` +objects to produce multisets (counters that have counts greater than zero). +Addition and subtraction combine counters by adding or subtracting the counts +of corresponding elements. Intersection and union return the minimum and +maximum of corresponding counts. Each operation can accept inputs with signed +counts, but the output will exclude results with counts of zero or less. + + >>> c = Counter(a=3, b=1) + >>> d = Counter(a=1, b=2) + >>> c + d # add two counters together: c[x] + d[x] + Counter({'a': 4, 'b': 3}) + >>> c - d # subtract (keeping only positive counts) + Counter({'a': 2}) + >>> c & d # intersection: min(c[x], d[x]) # doctest: +SKIP + Counter({'a': 1, 'b': 1}) + >>> c | d # union: max(c[x], d[x]) + Counter({'a': 3, 'b': 2}) + +Unary addition and subtraction are shortcuts for adding an empty counter +or subtracting from an empty counter. + + >>> c = Counter(a=2, b=-4) + >>> +c + Counter({'a': 2}) + >>> -c + Counter({'b': 4}) + +.. versionadded:: 3.3 + Added support for unary plus, unary minus, and in-place multiset operations. + +.. note:: + + Counters were primarily designed to work with positive integers to represent + running counts; however, care was taken to not unnecessarily preclude use + cases needing other types or negative values. To help with those use cases, + this section documents the minimum range and type restrictions. + + * The :class:`Counter` class itself is a dictionary subclass with no + restrictions on its keys and values. The values are intended to be numbers + representing counts, but you *could* store anything in the value field. + + * The :meth:`~Counter.most_common` method requires only that the values be orderable. + + * For in-place operations such as ``c[key] += 1``, the value type need only + support addition and subtraction. So fractions, floats, and decimals would + work and negative values are supported. The same is also true for + :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values + for both inputs and outputs. + + * The multiset methods are designed only for use cases with positive values. + The inputs may be negative or zero, but only outputs with positive values + are created. There are no type restrictions, but the value type needs to + support addition, subtraction, and comparison. + + * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and + negative counts. + +.. seealso:: + + * `Bag class `_ + in Smalltalk. + + * Wikipedia entry for `Multisets `_. + + * `C++ multisets `_ + tutorial with examples. + + * For mathematical operations on multisets and their use cases, see + *Knuth, Donald. The Art of Computer Programming Volume II, + Section 4.6.3, Exercise 19*. + + * To enumerate all distinct multisets of a given size over a given set of + elements, see :func:`itertools.combinations_with_replacement`:: + + map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC + + +:class:`deque` objects +---------------------- + +.. class:: deque([iterable, [maxlen]]) + + Returns a new deque object initialized left-to-right (using :meth:`append`) with + data from *iterable*. If *iterable* is not specified, the new deque is empty. + + Deques are a generalization of stacks and queues (the name is pronounced "deck" + and is short for "double-ended queue"). Deques support thread-safe, memory + efficient appends and pops from either side of the deque with approximately the + same O(1) performance in either direction. + + Though :class:`list` objects support similar operations, they are optimized for + fast fixed-length operations and incur O(n) memory movement costs for + ``pop(0)`` and ``insert(0, v)`` operations which change both the size and + position of the underlying data representation. + + + If *maxlen* is not specified or is ``None``, deques may grow to an + arbitrary length. Otherwise, the deque is bounded to the specified maximum + length. Once a bounded length deque is full, when new items are added, a + corresponding number of items are discarded from the opposite end. Bounded + length deques provide functionality similar to the ``tail`` filter in + Unix. They are also useful for tracking transactions and other pools of data + where only the most recent activity is of interest. + + + Deque objects support the following methods: + + .. method:: append(x) + + Add *x* to the right side of the deque. + + + .. method:: appendleft(x) + + Add *x* to the left side of the deque. + + + .. method:: clear() + + Remove all elements from the deque leaving it with length 0. + + + .. method:: copy() + + Create a shallow copy of the deque. + + .. versionadded:: 3.5 + + + .. method:: count(x) + + Count the number of deque elements equal to *x*. + + .. versionadded:: 3.2 + + + .. method:: extend(iterable) + + Extend the right side of the deque by appending elements from the iterable + argument. + + + .. method:: extendleft(iterable) + + Extend the left side of the deque by appending elements from *iterable*. + Note, the series of left appends results in reversing the order of + elements in the iterable argument. + + + .. method:: index(x[, start[, stop]]) + + Return the position of *x* in the deque (at or after index *start* + and before index *stop*). Returns the first match or raises + :exc:`ValueError` if not found. + + .. versionadded:: 3.5 + + + .. method:: insert(i, x) + + Insert *x* into the deque at position *i*. + + If the insertion would cause a bounded deque to grow beyond *maxlen*, + an :exc:`IndexError` is raised. + + .. versionadded:: 3.5 + + + .. method:: pop() + + Remove and return an element from the right side of the deque. If no + elements are present, raises an :exc:`IndexError`. + + + .. method:: popleft() + + Remove and return an element from the left side of the deque. If no + elements are present, raises an :exc:`IndexError`. + + + .. method:: remove(value) + + Remove the first occurrence of *value*. If not found, raises a + :exc:`ValueError`. + + + .. method:: reverse() + + Reverse the elements of the deque in-place and then return ``None``. + + .. versionadded:: 3.2 + + + .. method:: rotate(n=1) + + Rotate the deque *n* steps to the right. If *n* is negative, rotate + to the left. + + When the deque is not empty, rotating one step to the right is equivalent + to ``d.appendleft(d.pop())``, and rotating one step to the left is + equivalent to ``d.append(d.popleft())``. + + + Deque objects also provide one read-only attribute: + + .. attribute:: maxlen + + Maximum size of a deque or ``None`` if unbounded. + + .. versionadded:: 3.1 + + +In addition to the above, deques support iteration, pickling, ``len(d)``, +``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with +the :keyword:`in` operator, and subscript references such as ``d[0]`` to access +the first element. Indexed access is O(1) at both ends but slows to O(n) in +the middle. For fast random access, use lists instead. + +Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, +and ``__imul__()``. + +Example: + +.. doctest:: + + >>> from collections import deque + >>> d = deque('ghi') # make a new deque with three items + >>> for elem in d: # iterate over the deque's elements + ... print(elem.upper()) + G + H + I + + >>> d.append('j') # add a new entry to the right side + >>> d.appendleft('f') # add a new entry to the left side + >>> d # show the representation of the deque + deque(['f', 'g', 'h', 'i', 'j']) + + >>> d.pop() # return and remove the rightmost item + 'j' + >>> d.popleft() # return and remove the leftmost item + 'f' + >>> list(d) # list the contents of the deque + ['g', 'h', 'i'] + >>> d[0] # peek at leftmost item + 'g' + >>> d[-1] # peek at rightmost item + 'i' + + >>> list(reversed(d)) # list the contents of a deque in reverse + ['i', 'h', 'g'] + >>> 'h' in d # search the deque + True + >>> d.extend('jkl') # add multiple elements at once + >>> d + deque(['g', 'h', 'i', 'j', 'k', 'l']) + >>> d.rotate(1) # right rotation + >>> d + deque(['l', 'g', 'h', 'i', 'j', 'k']) + >>> d.rotate(-1) # left rotation + >>> d + deque(['g', 'h', 'i', 'j', 'k', 'l']) + + >>> deque(reversed(d)) # make a new deque in reverse order + deque(['l', 'k', 'j', 'i', 'h', 'g']) + >>> d.clear() # empty the deque + >>> d.pop() # cannot pop from an empty deque + Traceback (most recent call last): + File "", line 1, in -toplevel- + d.pop() + IndexError: pop from an empty deque + + >>> d.extendleft('abc') # extendleft() reverses the input order + >>> d + deque(['c', 'b', 'a']) + + +:class:`deque` Recipes +^^^^^^^^^^^^^^^^^^^^^^ + +This section shows various approaches to working with deques. + +Bounded length deques provide functionality similar to the ``tail`` filter +in Unix:: + + def tail(filename, n=10): + 'Return the last n lines of a file' + with open(filename) as f: + return deque(f, n) + +Another approach to using deques is to maintain a sequence of recently +added elements by appending to the right and popping to the left:: + + def moving_average(iterable, n=3): + # moving_average([40, 30, 50, 46, 39, 44]) --> 40.0 42.0 45.0 43.0 + # http://en.wikipedia.org/wiki/Moving_average + it = iter(iterable) + d = deque(itertools.islice(it, n-1)) + d.appendleft(0) + s = sum(d) + for elem in it: + s += elem - d.popleft() + d.append(elem) + yield s / n + +A `round-robin scheduler +`_ can be implemented with +input iterators stored in a :class:`deque`. Values are yielded from the active +iterator in position zero. If that iterator is exhausted, it can be removed +with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with +the :meth:`~deque.rotate` method:: + + def roundrobin(*iterables): + "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + iterators = deque(map(iter, iterables)) + while iterators: + try: + while True: + yield next(iterators[0]) + iterators.rotate(-1) + except StopIteration: + # Remove an exhausted iterator. + iterators.popleft() + +The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and +deletion. For example, a pure Python implementation of ``del d[n]`` relies on +the ``rotate()`` method to position elements to be popped:: + + def delete_nth(d, n): + d.rotate(-n) + d.popleft() + d.rotate(n) + +To implement :class:`deque` slicing, use a similar approach applying +:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove +old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then +reverse the rotation. +With minor variations on that approach, it is easy to implement Forth style +stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, +``rot``, and ``roll``. + + +:class:`defaultdict` objects +---------------------------- + +.. class:: defaultdict([default_factory[, ...]]) + + Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the + built-in :class:`dict` class. It overrides one method and adds one writable + instance variable. The remaining functionality is the same as for the + :class:`dict` class and is not documented here. + + The first argument provides the initial value for the :attr:`default_factory` + attribute; it defaults to ``None``. All remaining arguments are treated the same + as if they were passed to the :class:`dict` constructor, including keyword + arguments. + + + :class:`defaultdict` objects support the following method in addition to the + standard :class:`dict` operations: + + .. method:: __missing__(key) + + If the :attr:`default_factory` attribute is ``None``, this raises a + :exc:`KeyError` exception with the *key* as argument. + + If :attr:`default_factory` is not ``None``, it is called without arguments + to provide a default value for the given *key*, this value is inserted in + the dictionary for the *key*, and returned. + + If calling :attr:`default_factory` raises an exception this exception is + propagated unchanged. + + This method is called by the :meth:`__getitem__` method of the + :class:`dict` class when the requested key is not found; whatever it + returns or raises is then returned or raised by :meth:`__getitem__`. + + Note that :meth:`__missing__` is *not* called for any operations besides + :meth:`__getitem__`. This means that :meth:`get` will, like normal + dictionaries, return ``None`` as a default rather than using + :attr:`default_factory`. + + + :class:`defaultdict` objects support the following instance variable: + + + .. attribute:: default_factory + + This attribute is used by the :meth:`__missing__` method; it is + initialized from the first argument to the constructor, if present, or to + ``None``, if absent. + + .. versionchanged:: 3.9 + Added merge (``|``) and update (``|=``) operators, specified in + :pep:`584`. + + +:class:`defaultdict` Examples +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a +sequence of key-value pairs into a dictionary of lists: + + >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] + >>> d = defaultdict(list) + >>> for k, v in s: + ... d[k].append(v) + ... + >>> sorted(d.items()) + [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] + +When each key is encountered for the first time, it is not already in the +mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` +function which returns an empty :class:`list`. The :meth:`list.append` +operation then attaches the value to the new list. When keys are encountered +again, the look-up proceeds normally (returning the list for that key) and the +:meth:`list.append` operation adds another value to the list. This technique is +simpler and faster than an equivalent technique using :meth:`dict.setdefault`: + + >>> d = {} + >>> for k, v in s: + ... d.setdefault(k, []).append(v) + ... + >>> sorted(d.items()) + [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] + +Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the +:class:`defaultdict` useful for counting (like a bag or multiset in other +languages): + + >>> s = 'mississippi' + >>> d = defaultdict(int) + >>> for k in s: + ... d[k] += 1 + ... + >>> sorted(d.items()) + [('i', 4), ('m', 1), ('p', 2), ('s', 4)] + +When a letter is first encountered, it is missing from the mapping, so the +:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of +zero. The increment operation then builds up the count for each letter. + +The function :func:`int` which always returns zero is just a special case of +constant functions. A faster and more flexible way to create constant functions +is to use a lambda function which can supply any constant value (not just +zero): + + >>> def constant_factory(value): + ... return lambda: value + >>> d = defaultdict(constant_factory('')) + >>> d.update(name='John', action='ran') + >>> '%(name)s %(action)s to %(object)s' % d + 'John ran to ' + +Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the +:class:`defaultdict` useful for building a dictionary of sets: + + >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] + >>> d = defaultdict(set) + >>> for k, v in s: + ... d[k].add(v) + ... + >>> sorted(d.items()) + [('blue', {2, 4}), ('red', {1, 3})] + + +:func:`namedtuple` Factory Function for Tuples with Named Fields +---------------------------------------------------------------- + +Named tuples assign meaning to each position in a tuple and allow for more readable, +self-documenting code. They can be used wherever regular tuples are used, and +they add the ability to access fields by name instead of position index. + +.. function:: namedtuple(typename, field_names, *, rename=False, defaults=None, module=None) + + Returns a new tuple subclass named *typename*. The new subclass is used to + create tuple-like objects that have fields accessible by attribute lookup as + well as being indexable and iterable. Instances of the subclass also have a + helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` + method which lists the tuple contents in a ``name=value`` format. + + The *field_names* are a sequence of strings such as ``['x', 'y']``. + Alternatively, *field_names* can be a single string with each fieldname + separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. + + Any valid Python identifier may be used for a fieldname except for names + starting with an underscore. Valid identifiers consist of letters, digits, + and underscores but do not start with a digit or underscore and cannot be + a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, + or *raise*. + + If *rename* is true, invalid fieldnames are automatically replaced + with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is + converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword + ``def`` and the duplicate fieldname ``abc``. + + *defaults* can be ``None`` or an :term:`iterable` of default values. + Since fields with a default value must come after any fields without a + default, the *defaults* are applied to the rightmost parameters. For + example, if the fieldnames are ``['x', 'y', 'z']`` and the defaults are + ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to + ``1``, and ``z`` will default to ``2``. + + If *module* is defined, the ``__module__`` attribute of the named tuple is + set to that value. + + Named tuple instances do not have per-instance dictionaries, so they are + lightweight and require no more memory than regular tuples. + + To support pickling, the named tuple class should be assigned to a variable + that matches *typename*. + + .. versionchanged:: 3.1 + Added support for *rename*. + + .. versionchanged:: 3.6 + The *verbose* and *rename* parameters became + :ref:`keyword-only arguments `. + + .. versionchanged:: 3.6 + Added the *module* parameter. + + .. versionchanged:: 3.7 + Removed the *verbose* parameter and the :attr:`_source` attribute. + + .. versionchanged:: 3.7 + Added the *defaults* parameter and the :attr:`_field_defaults` + attribute. + +.. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> # Basic example + >>> Point = namedtuple('Point', ['x', 'y']) + >>> p = Point(11, y=22) # instantiate with positional or keyword arguments + >>> p[0] + p[1] # indexable like the plain tuple (11, 22) + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessible by name + 33 + >>> p # readable __repr__ with a name=value style + Point(x=11, y=22) + +Named tuples are especially useful for assigning field names to result tuples returned +by the :mod:`csv` or :mod:`sqlite3` modules:: + + EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') + + import csv + for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): + print(emp.name, emp.title) + + import sqlite3 + conn = sqlite3.connect('/companydata') + cursor = conn.cursor() + cursor.execute('SELECT name, age, title, department, paygrade FROM employees') + for emp in map(EmployeeRecord._make, cursor.fetchall()): + print(emp.name, emp.title) + +In addition to the methods inherited from tuples, named tuples support +three additional methods and two attributes. To prevent conflicts with +field names, the method and attribute names start with an underscore. + +.. classmethod:: somenamedtuple._make(iterable) + + Class method that makes a new instance from an existing sequence or iterable. + + .. doctest:: + + >>> t = [11, 22] + >>> Point._make(t) + Point(x=11, y=22) + +.. method:: somenamedtuple._asdict() + + Return a new :class:`dict` which maps field names to their corresponding + values: + + .. doctest:: + + >>> p = Point(x=11, y=22) + >>> p._asdict() + {'x': 11, 'y': 22} + + .. versionchanged:: 3.1 + Returns an :class:`OrderedDict` instead of a regular :class:`dict`. + + .. versionchanged:: 3.8 + Returns a regular :class:`dict` instead of an :class:`OrderedDict`. + As of Python 3.7, regular dicts are guaranteed to be ordered. If the + extra features of :class:`OrderedDict` are required, the suggested + remediation is to cast the result to the desired type: + ``OrderedDict(nt._asdict())``. + +.. method:: somenamedtuple._replace(**kwargs) + + Return a new instance of the named tuple replacing specified fields with new + values:: + + >>> p = Point(x=11, y=22) + >>> p._replace(x=33) + Point(x=33, y=22) + + >>> for partnum, record in inventory.items(): + ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) + +.. attribute:: somenamedtuple._fields + + Tuple of strings listing the field names. Useful for introspection + and for creating new named tuple types from existing named tuples. + + .. doctest:: + + >>> p._fields # view the field names + ('x', 'y') + + >>> Color = namedtuple('Color', 'red green blue') + >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) + >>> Pixel(11, 22, 128, 255, 0) + Pixel(x=11, y=22, red=128, green=255, blue=0) + +.. attribute:: somenamedtuple._field_defaults + + Dictionary mapping field names to default values. + + .. doctest:: + + >>> Account = namedtuple('Account', ['type', 'balance'], defaults=[0]) + >>> Account._field_defaults + {'balance': 0} + >>> Account('premium') + Account(type='premium', balance=0) + +To retrieve a field whose name is stored in a string, use the :func:`getattr` +function: + + >>> getattr(p, 'x') + 11 + +To convert a dictionary to a named tuple, use the double-star-operator +(as described in :ref:`tut-unpacking-arguments`): + + >>> d = {'x': 11, 'y': 22} + >>> Point(**d) + Point(x=11, y=22) + +Since a named tuple is a regular Python class, it is easy to add or change +functionality with a subclass. Here is how to add a calculated field and +a fixed-width print format: + +.. doctest:: + + >>> class Point(namedtuple('Point', ['x', 'y'])): + ... __slots__ = () + ... @property + ... def hypot(self): + ... return (self.x ** 2 + self.y ** 2) ** 0.5 + ... def __str__(self): + ... return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) + + >>> for p in Point(3, 4), Point(14, 5/7): + ... print(p) + Point: x= 3.000 y= 4.000 hypot= 5.000 + Point: x=14.000 y= 0.714 hypot=14.018 + +The subclass shown above sets ``__slots__`` to an empty tuple. This helps +keep memory requirements low by preventing the creation of instance dictionaries. + +Subclassing is not useful for adding new, stored fields. Instead, simply +create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: + + >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) + +Docstrings can be customized by making direct assignments to the ``__doc__`` +fields: + + >>> Book = namedtuple('Book', ['id', 'title', 'authors']) + >>> Book.__doc__ += ': Hardcover book in active collection' + >>> Book.id.__doc__ = '13-digit ISBN' + >>> Book.title.__doc__ = 'Title of first printing' + >>> Book.authors.__doc__ = 'List of authors sorted by last name' + +.. versionchanged:: 3.5 + Property docstrings became writeable. + +.. seealso:: + + * See :class:`typing.NamedTuple` for a way to add type hints for named + tuples. It also provides an elegant notation using the :keyword:`class` + keyword:: + + class Component(NamedTuple): + part_number: int + weight: float + description: Optional[str] = None + + * See :meth:`types.SimpleNamespace` for a mutable namespace based on an + underlying dictionary instead of a tuple. + + * The :mod:`dataclasses` module provides a decorator and functions for + automatically adding generated special methods to user-defined classes. + + +:class:`OrderedDict` objects +---------------------------- + +Ordered dictionaries are just like regular dictionaries but have some extra +capabilities relating to ordering operations. They have become less +important now that the built-in :class:`dict` class gained the ability +to remember insertion order (this new behavior became guaranteed in +Python 3.7). + +Some differences from :class:`dict` still remain: + +* The regular :class:`dict` was designed to be very good at mapping + operations. Tracking insertion order was secondary. + +* The :class:`OrderedDict` was designed to be good at reordering operations. + Space efficiency, iteration speed, and the performance of update + operations were secondary. + +* Algorithmically, :class:`OrderedDict` can handle frequent reordering + operations better than :class:`dict`. This makes it suitable for tracking + recent accesses (for example in an `LRU cache + `_). + +* The equality operation for :class:`OrderedDict` checks for matching order. + +* The :meth:`popitem` method of :class:`OrderedDict` has a different + signature. It accepts an optional argument to specify which item is popped. + +* :class:`OrderedDict` has a :meth:`move_to_end` method to + efficiently reposition an element to an endpoint. + +* Until Python 3.8, :class:`dict` lacked a :meth:`__reversed__` method. + + +.. class:: OrderedDict([items]) + + Return an instance of a :class:`dict` subclass that has methods + specialized for rearranging dictionary order. + + .. versionadded:: 3.1 + + .. method:: popitem(last=True) + + The :meth:`popitem` method for ordered dictionaries returns and removes a + (key, value) pair. The pairs are returned in + :abbr:`LIFO (last-in, first-out)` order if *last* is true + or :abbr:`FIFO (first-in, first-out)` order if false. + + .. method:: move_to_end(key, last=True) + + Move an existing *key* to either end of an ordered dictionary. The item + is moved to the right end if *last* is true (the default) or to the + beginning if *last* is false. Raises :exc:`KeyError` if the *key* does + not exist:: + + >>> d = OrderedDict.fromkeys('abcde') + >>> d.move_to_end('b') + >>> ''.join(d.keys()) + 'acdeb' + >>> d.move_to_end('b', last=False) + >>> ''.join(d.keys()) + 'bacde' + + .. versionadded:: 3.2 + +In addition to the usual mapping methods, ordered dictionaries also support +reverse iteration using :func:`reversed`. + +Equality tests between :class:`OrderedDict` objects are order-sensitive +and are implemented as ``list(od1.items())==list(od2.items())``. +Equality tests between :class:`OrderedDict` objects and other +:class:`~collections.abc.Mapping` objects are order-insensitive like regular +dictionaries. This allows :class:`OrderedDict` objects to be substituted +anywhere a regular dictionary is used. + +.. versionchanged:: 3.5 + The items, keys, and values :term:`views ` + of :class:`OrderedDict` now support reverse iteration using :func:`reversed`. + +.. versionchanged:: 3.6 + With the acceptance of :pep:`468`, order is retained for keyword arguments + passed to the :class:`OrderedDict` constructor and its :meth:`update` + method. + +.. versionchanged:: 3.9 + Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. + + +:class:`OrderedDict` Examples and Recipes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is straightforward to create an ordered dictionary variant +that remembers the order the keys were *last* inserted. +If a new entry overwrites an existing entry, the +original insertion position is changed and moved to the end:: + + class LastUpdatedOrderedDict(OrderedDict): + 'Store items in the order the keys were last added' + + def __setitem__(self, key, value): + super().__setitem__(key, value) + self.move_to_end(key) + +An :class:`OrderedDict` would also be useful for implementing +variants of :func:`functools.lru_cache`:: + + class LRU(OrderedDict): + 'Limit size, evicting the least recently looked-up key when full' + + def __init__(self, maxsize=128, /, *args, **kwds): + self.maxsize = maxsize + super().__init__(*args, **kwds) + + def __getitem__(self, key): + value = super().__getitem__(key) + self.move_to_end(key) + return value + + def __setitem__(self, key, value): + if key in self: + self.move_to_end(key) + super().__setitem__(key, value) + if len(self) > self.maxsize: + oldest = next(iter(self)) + del self[oldest] + + +:class:`UserDict` objects +------------------------- + +The class, :class:`UserDict` acts as a wrapper around dictionary objects. +The need for this class has been partially supplanted by the ability to +subclass directly from :class:`dict`; however, this class can be easier +to work with because the underlying dictionary is accessible as an +attribute. + +.. class:: UserDict([initialdata]) + + Class that simulates a dictionary. The instance's contents are kept in a + regular dictionary, which is accessible via the :attr:`data` attribute of + :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is + initialized with its contents; note that a reference to *initialdata* will not + be kept, allowing it be used for other purposes. + + In addition to supporting the methods and operations of mappings, + :class:`UserDict` instances provide the following attribute: + + .. attribute:: data + + A real dictionary used to store the contents of the :class:`UserDict` + class. + + + +:class:`UserList` objects +------------------------- + +This class acts as a wrapper around list objects. It is a useful base class +for your own list-like classes which can inherit from them and override +existing methods or add new ones. In this way, one can add new behaviors to +lists. + +The need for this class has been partially supplanted by the ability to +subclass directly from :class:`list`; however, this class can be easier +to work with because the underlying list is accessible as an attribute. + +.. class:: UserList([list]) + + Class that simulates a list. The instance's contents are kept in a regular + list, which is accessible via the :attr:`data` attribute of :class:`UserList` + instances. The instance's contents are initially set to a copy of *list*, + defaulting to the empty list ``[]``. *list* can be any iterable, for + example a real Python list or a :class:`UserList` object. + + In addition to supporting the methods and operations of mutable sequences, + :class:`UserList` instances provide the following attribute: + + .. attribute:: data + + A real :class:`list` object used to store the contents of the + :class:`UserList` class. + +**Subclassing requirements:** Subclasses of :class:`UserList` are expected to +offer a constructor which can be called with either no arguments or one +argument. List operations which return a new sequence attempt to create an +instance of the actual implementation class. To do so, it assumes that the +constructor can be called with a single parameter, which is a sequence object +used as a data source. + +If a derived class does not wish to comply with this requirement, all of the +special methods supported by this class will need to be overridden; please +consult the sources for information about the methods which need to be provided +in that case. + +:class:`UserString` objects +--------------------------- + +The class, :class:`UserString` acts as a wrapper around string objects. +The need for this class has been partially supplanted by the ability to +subclass directly from :class:`str`; however, this class can be easier +to work with because the underlying string is accessible as an +attribute. + +.. class:: UserString(seq) + + Class that simulates a string object. The instance's + content is kept in a regular string object, which is accessible via the + :attr:`data` attribute of :class:`UserString` instances. The instance's + contents are initially set to a copy of *seq*. The *seq* argument can + be any object which can be converted into a string using the built-in + :func:`str` function. + + In addition to supporting the methods and operations of strings, + :class:`UserString` instances provide the following attribute: + + .. attribute:: data + + A real :class:`str` object used to store the contents of the + :class:`UserString` class. + + .. versionchanged:: 3.5 + New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, + ``format_map``, ``isprintable``, and ``maketrans``. diff --git a/Doc/library/dataclasses.rst.bak b/Doc/library/dataclasses.rst.bak new file mode 100644 index 00000000000000..e706f7fcc566d8 --- /dev/null +++ b/Doc/library/dataclasses.rst.bak @@ -0,0 +1,595 @@ +:mod:`dataclasses` --- Data Classes +=================================== + +.. module:: dataclasses + :synopsis: Generate special methods on user-defined classes. + +.. moduleauthor:: Eric V. Smith +.. sectionauthor:: Eric V. Smith + +**Source code:** :source:`Lib/dataclasses.py` + +-------------- + +This module provides a decorator and functions for automatically +adding generated :term:`special method`\s such as :meth:`__init__` and +:meth:`__repr__` to user-defined classes. It was originally described +in :pep:`557`. + +The member variables to use in these generated methods are defined +using :pep:`526` type annotations. For example this code:: + + from dataclasses import dataclass + + @dataclass + class InventoryItem: + """Class for keeping track of an item in inventory.""" + name: str + unit_price: float + quantity_on_hand: int = 0 + + def total_cost(self) -> float: + return self.unit_price * self.quantity_on_hand + +Will add, among other things, a :meth:`__init__` that looks like:: + + def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0): + self.name = name + self.unit_price = unit_price + self.quantity_on_hand = quantity_on_hand + +Note that this method is automatically added to the class: it is not +directly specified in the ``InventoryItem`` definition shown above. + +.. versionadded:: 3.7 + +Module-level decorators, classes, and functions +----------------------------------------------- + +.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + + This function is a :term:`decorator` that is used to add generated + :term:`special method`\s to classes, as described below. + + The :func:`dataclass` decorator examines the class to find + ``field``\s. A ``field`` is defined as class variable that has a + :term:`type annotation `. With two + exceptions described below, nothing in :func:`dataclass` + examines the type specified in the variable annotation. + + The order of the fields in all of the generated methods is the + order in which they appear in the class definition. + + The :func:`dataclass` decorator will add various "dunder" methods to + the class, described below. If any of the added methods already + exist on the class, the behavior depends on the parameter, as documented + below. The decorator returns the same class that is called on; no new + class is created. + + If :func:`dataclass` is used just as a simple decorator with no parameters, + it acts as if it has the default values documented in this + signature. That is, these three uses of :func:`dataclass` are + equivalent:: + + @dataclass + class C: + ... + + @dataclass() + class C: + ... + + @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + class C: + ... + + The parameters to :func:`dataclass` are: + + - ``init``: If true (the default), a :meth:`__init__` method will be + generated. + + If the class already defines :meth:`__init__`, this parameter is + ignored. + + - ``repr``: If true (the default), a :meth:`__repr__` method will be + generated. The generated repr string will have the class name and + the name and repr of each field, in the order they are defined in + the class. Fields that are marked as being excluded from the repr + are not included. For example: + ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. + + If the class already defines :meth:`__repr__`, this parameter is + ignored. + + - ``eq``: If true (the default), an :meth:`__eq__` method will be + generated. This method compares the class as if it were a tuple + of its fields, in order. Both instances in the comparison must + be of the identical type. + + If the class already defines :meth:`__eq__`, this parameter is + ignored. + + - ``order``: If true (the default is ``False``), :meth:`__lt__`, + :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` methods will be + generated. These compare the class as if it were a tuple of its + fields, in order. Both instances in the comparison must be of the + identical type. If ``order`` is true and ``eq`` is false, a + :exc:`ValueError` is raised. + + If the class already defines any of :meth:`__lt__`, + :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then + :exc:`TypeError` is raised. + + - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method + is generated according to how ``eq`` and ``frozen`` are set. + + :meth:`__hash__` is used by built-in :meth:`hash()`, and when objects are + added to hashed collections such as dictionaries and sets. Having a + :meth:`__hash__` implies that instances of the class are immutable. + Mutability is a complicated property that depends on the programmer's + intent, the existence and behavior of :meth:`__eq__`, and the values of + the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. + + By default, :func:`dataclass` will not implicitly add a :meth:`__hash__` + method unless it is safe to do so. Neither will it add or change an + existing explicitly defined :meth:`__hash__` method. Setting the class + attribute ``__hash__ = None`` has a specific meaning to Python, as + described in the :meth:`__hash__` documentation. + + If :meth:`__hash__` is not explicit defined, or if it is set to ``None``, + then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. + Although not recommended, you can force :func:`dataclass` to create a + :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case + if your class is logically immutable but can nonetheless be mutated. + This is a specialized use case and should be considered carefully. + + Here are the rules governing implicit creation of a :meth:`__hash__` + method. Note that you cannot both have an explicit :meth:`__hash__` + method in your dataclass and set ``unsafe_hash=True``; this will result + in a :exc:`TypeError`. + + If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will + generate a :meth:`__hash__` method for you. If ``eq`` is true and + ``frozen`` is false, :meth:`__hash__` will be set to ``None``, marking it + unhashable (which it is, since it is mutable). If ``eq`` is false, + :meth:`__hash__` will be left untouched meaning the :meth:`__hash__` + method of the superclass will be used (if the superclass is + :class:`object`, this means it will fall back to id-based hashing). + + - ``frozen``: If true (the default is ``False``), assigning to fields will + generate an exception. This emulates read-only frozen instances. If + :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then + :exc:`TypeError` is raised. See the discussion below. + + ``field``\s may optionally specify a default value, using normal + Python syntax:: + + @dataclass + class C: + a: int # 'a' has no default value + b: int = 0 # assign a default value for 'b' + + In this example, both ``a`` and ``b`` will be included in the added + :meth:`__init__` method, which will be defined as:: + + def __init__(self, a: int, b: int = 0): + + :exc:`TypeError` will be raised if a field without a default value + follows a field with a default value. This is true either when this + occurs in a single class, or as a result of class inheritance. + +.. function:: field(*, default=MISSING, default_factory=MISSING, repr=True, hash=None, init=True, compare=True, metadata=None) + + For common and simple use cases, no other functionality is + required. There are, however, some dataclass features that + require additional per-field information. To satisfy this need for + additional information, you can replace the default field value + with a call to the provided :func:`field` function. For example:: + + @dataclass + class C: + mylist: list[int] = field(default_factory=list) + + c = C() + c.mylist += [1, 2, 3] + + As shown above, the ``MISSING`` value is a sentinel object used to + detect if the ``default`` and ``default_factory`` parameters are + provided. This sentinel is used because ``None`` is a valid value + for ``default``. No code should directly use the ``MISSING`` + value. + + The parameters to :func:`field` are: + + - ``default``: If provided, this will be the default value for this + field. This is needed because the :meth:`field` call itself + replaces the normal position of the default value. + + - ``default_factory``: If provided, it must be a zero-argument + callable that will be called when a default value is needed for + this field. Among other purposes, this can be used to specify + fields with mutable default values, as discussed below. It is an + error to specify both ``default`` and ``default_factory``. + + - ``init``: If true (the default), this field is included as a + parameter to the generated :meth:`__init__` method. + + - ``repr``: If true (the default), this field is included in the + string returned by the generated :meth:`__repr__` method. + + - ``compare``: If true (the default), this field is included in the + generated equality and comparison methods (:meth:`__eq__`, + :meth:`__gt__`, et al.). + + - ``hash``: This can be a bool or ``None``. If true, this field is + included in the generated :meth:`__hash__` method. If ``None`` (the + default), use the value of ``compare``: this would normally be + the expected behavior. A field should be considered in the hash + if it's used for comparisons. Setting this value to anything + other than ``None`` is discouraged. + + One possible reason to set ``hash=False`` but ``compare=True`` + would be if a field is expensive to compute a hash value for, + that field is needed for equality testing, and there are other + fields that contribute to the type's hash value. Even if a field + is excluded from the hash, it will still be used for comparisons. + + - ``metadata``: This can be a mapping or None. None is treated as + an empty dict. This value is wrapped in + :func:`~types.MappingProxyType` to make it read-only, and exposed + on the :class:`Field` object. It is not used at all by Data + Classes, and is provided as a third-party extension mechanism. + Multiple third-parties can each have their own key, to use as a + namespace in the metadata. + + If the default value of a field is specified by a call to + :func:`field()`, then the class attribute for this field will be + replaced by the specified ``default`` value. If no ``default`` is + provided, then the class attribute will be deleted. The intent is + that after the :func:`dataclass` decorator runs, the class + attributes will all contain the default values for the fields, just + as if the default value itself were specified. For example, + after:: + + @dataclass + class C: + x: int + y: int = field(repr=False) + z: int = field(repr=False, default=10) + t: int = 20 + + The class attribute ``C.z`` will be ``10``, the class attribute + ``C.t`` will be ``20``, and the class attributes ``C.x`` and + ``C.y`` will not be set. + +.. class:: Field + + :class:`Field` objects describe each defined field. These objects + are created internally, and are returned by the :func:`fields` + module-level method (see below). Users should never instantiate a + :class:`Field` object directly. Its documented attributes are: + + - ``name``: The name of the field. + + - ``type``: The type of the field. + + - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, + ``compare``, and ``metadata`` have the identical meaning and + values as they do in the :func:`field` declaration. + + Other attributes may exist, but they are private and must not be + inspected or relied on. + +.. function:: fields(class_or_instance) + + Returns a tuple of :class:`Field` objects that define the fields for this + dataclass. Accepts either a dataclass, or an instance of a dataclass. + Raises :exc:`TypeError` if not passed a dataclass or instance of one. + Does not return pseudo-fields which are ``ClassVar`` or ``InitVar``. + +.. function:: asdict(instance, *, dict_factory=dict) + + Converts the dataclass ``instance`` to a dict (by using the + factory function ``dict_factory``). Each dataclass is converted + to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, + lists, and tuples are recursed into. For example:: + + @dataclass + class Point: + x: int + y: int + + @dataclass + class C: + mylist: list[Point] + + p = Point(10, 20) + assert asdict(p) == {'x': 10, 'y': 20} + + c = C([Point(0, 0), Point(10, 4)]) + assert asdict(c) == {'mylist': [{'x': 0, 'y': 0}, {'x': 10, 'y': 4}]} + + Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. + +.. function:: astuple(instance, *, tuple_factory=tuple) + + Converts the dataclass ``instance`` to a tuple (by using the + factory function ``tuple_factory``). Each dataclass is converted + to a tuple of its field values. dataclasses, dicts, lists, and + tuples are recursed into. + + Continuing from the previous example:: + + assert astuple(p) == (10, 20) + assert astuple(c) == ([(0, 0), (10, 4)],) + + Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. + +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + + Creates a new dataclass with name ``cls_name``, fields as defined + in ``fields``, base classes as given in ``bases``, and initialized + with a namespace as given in ``namespace``. ``fields`` is an + iterable whose elements are each either ``name``, ``(name, type)``, + or ``(name, type, Field)``. If just ``name`` is supplied, + ``typing.Any`` is used for ``type``. The values of ``init``, + ``repr``, ``eq``, ``order``, ``unsafe_hash``, and ``frozen`` have + the same meaning as they do in :func:`dataclass`. + + This function is not strictly required, because any Python + mechanism for creating a new class with ``__annotations__`` can + then apply the :func:`dataclass` function to convert that class to + a dataclass. This function is provided as a convenience. For + example:: + + C = make_dataclass('C', + [('x', int), + 'y', + ('z', int, field(default=5))], + namespace={'add_one': lambda self: self.x + 1}) + + Is equivalent to:: + + @dataclass + class C: + x: int + y: 'typing.Any' + z: int = 5 + + def add_one(self): + return self.x + 1 + +.. function:: replace(instance, /, **changes) + + Creates a new object of the same type of ``instance``, replacing + fields with values from ``changes``. If ``instance`` is not a Data + Class, raises :exc:`TypeError`. If values in ``changes`` do not + specify fields, raises :exc:`TypeError`. + + The newly returned object is created by calling the :meth:`__init__` + method of the dataclass. This ensures that + :meth:`__post_init__`, if present, is also called. + + Init-only variables without default values, if any exist, must be + specified on the call to :func:`replace` so that they can be passed to + :meth:`__init__` and :meth:`__post_init__`. + + It is an error for ``changes`` to contain any fields that are + defined as having ``init=False``. A :exc:`ValueError` will be raised + in this case. + + Be forewarned about how ``init=False`` fields work during a call to + :func:`replace`. They are not copied from the source object, but + rather are initialized in :meth:`__post_init__`, if they're + initialized at all. It is expected that ``init=False`` fields will + be rarely and judiciously used. If they are used, it might be wise + to have alternate class constructors, or perhaps a custom + ``replace()`` (or similarly named) method which handles instance + copying. + +.. function:: is_dataclass(class_or_instance) + + Return ``True`` if its parameter is a dataclass or an instance of one, + otherwise return ``False``. + + If you need to know if a class is an instance of a dataclass (and + not a dataclass itself), then add a further check for ``not + isinstance(obj, type)``:: + + def is_dataclass_instance(obj): + return is_dataclass(obj) and not isinstance(obj, type) + +Post-init processing +-------------------- + +The generated :meth:`__init__` code will call a method named +:meth:`__post_init__`, if :meth:`__post_init__` is defined on the +class. It will normally be called as ``self.__post_init__()``. +However, if any ``InitVar`` fields are defined, they will also be +passed to :meth:`__post_init__` in the order they were defined in the +class. If no :meth:`__init__` method is generated, then +:meth:`__post_init__` will not automatically be called. + +Among other uses, this allows for initializing field values that +depend on one or more other fields. For example:: + + @dataclass + class C: + a: float + b: float + c: float = field(init=False) + + def __post_init__(self): + self.c = self.a + self.b + +See the section below on init-only variables for ways to pass +parameters to :meth:`__post_init__`. Also see the warning about how +:func:`replace` handles ``init=False`` fields. + +Class variables +--------------- + +One of two places where :func:`dataclass` actually inspects the type +of a field is to determine if a field is a class variable as defined +in :pep:`526`. It does this by checking if the type of the field is +``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded +from consideration as a field and is ignored by the dataclass +mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the +module-level :func:`fields` function. + +Init-only variables +------------------- + +The other place where :func:`dataclass` inspects a type annotation is to +determine if a field is an init-only variable. It does this by seeing +if the type of a field is of type ``dataclasses.InitVar``. If a field +is an ``InitVar``, it is considered a pseudo-field called an init-only +field. As it is not a true field, it is not returned by the +module-level :func:`fields` function. Init-only fields are added as +parameters to the generated :meth:`__init__` method, and are passed to +the optional :meth:`__post_init__` method. They are not otherwise used +by dataclasses. + +For example, suppose a field will be initialized from a database, if a +value is not provided when creating the class:: + + @dataclass + class C: + i: int + j: int = None + database: InitVar[DatabaseType] = None + + def __post_init__(self, database): + if self.j is None and database is not None: + self.j = database.lookup('j') + + c = C(10, database=my_database) + +In this case, :func:`fields` will return :class:`Field` objects for ``i`` and +``j``, but not for ``database``. + +Frozen instances +---------------- + +It is not possible to create truly immutable Python objects. However, +by passing ``frozen=True`` to the :meth:`dataclass` decorator you can +emulate immutability. In that case, dataclasses will add +:meth:`__setattr__` and :meth:`__delattr__` methods to the class. These +methods will raise a :exc:`FrozenInstanceError` when invoked. + +There is a tiny performance penalty when using ``frozen=True``: +:meth:`__init__` cannot use simple assignment to initialize fields, and +must use :meth:`object.__setattr__`. + +Inheritance +----------- + +When the dataclass is being created by the :meth:`dataclass` decorator, +it looks through all of the class's base classes in reverse MRO (that +is, starting at :class:`object`) and, for each dataclass that it finds, +adds the fields from that base class to an ordered mapping of fields. +After all of the base class fields are added, it adds its own fields +to the ordered mapping. All of the generated methods will use this +combined, calculated ordered mapping of fields. Because the fields +are in insertion order, derived classes override base classes. An +example:: + + @dataclass + class Base: + x: Any = 15.0 + y: int = 0 + + @dataclass + class C(Base): + z: int = 10 + x: int = 15 + +The final list of fields is, in order, ``x``, ``y``, ``z``. The final +type of ``x`` is ``int``, as specified in class ``C``. + +The generated :meth:`__init__` method for ``C`` will look like:: + + def __init__(self, x: int = 15, y: int = 0, z: int = 10): + +Default factory functions +------------------------- + + If a :func:`field` specifies a ``default_factory``, it is called with + zero arguments when a default value for the field is needed. For + example, to create a new instance of a list, use:: + + mylist: list = field(default_factory=list) + + If a field is excluded from :meth:`__init__` (using ``init=False``) + and the field also specifies ``default_factory``, then the default + factory function will always be called from the generated + :meth:`__init__` function. This happens because there is no other + way to give the field an initial value. + +Mutable default values +---------------------- + + Python stores default member variable values in class attributes. + Consider this example, not using dataclasses:: + + class C: + x = [] + def add(self, element): + self.x.append(element) + + o1 = C() + o2 = C() + o1.add(1) + o2.add(2) + assert o1.x == [1, 2] + assert o1.x is o2.x + + Note that the two instances of class ``C`` share the same class + variable ``x``, as expected. + + Using dataclasses, *if* this code was valid:: + + @dataclass + class D: + x: List = [] + def add(self, element): + self.x += element + + it would generate code similar to:: + + class D: + x = [] + def __init__(self, x=x): + self.x = x + def add(self, element): + self.x += element + + assert D().x is D().x + + This has the same issue as the original example using class ``C``. + That is, two instances of class ``D`` that do not specify a value for + ``x`` when creating a class instance will share the same copy of + ``x``. Because dataclasses just use normal Python class creation + they also share this behavior. There is no general way for Data + Classes to detect this condition. Instead, dataclasses will raise a + :exc:`TypeError` if it detects a default parameter of type ``list``, + ``dict``, or ``set``. This is a partial solution, but it does protect + against many common errors. + + Using default factory functions is a way to create new instances of + mutable types as default values for fields:: + + @dataclass + class D: + x: list = field(default_factory=list) + + assert D().x is not D().x + +Exceptions +---------- + +.. exception:: FrozenInstanceError + + Raised when an implicitly defined :meth:`__setattr__` or + :meth:`__delattr__` is called on a dataclass which was defined with + ``frozen=True``. diff --git a/Doc/library/gc.rst.bak b/Doc/library/gc.rst.bak new file mode 100644 index 00000000000000..a3d201d5055c8e --- /dev/null +++ b/Doc/library/gc.rst.bak @@ -0,0 +1,322 @@ +:mod:`gc` --- Garbage Collector interface +========================================= + +.. module:: gc + :synopsis: Interface to the cycle-detecting garbage collector. + +.. moduleauthor:: Neil Schemenauer +.. sectionauthor:: Neil Schemenauer + +-------------- + +This module provides an interface to the optional garbage collector. It +provides the ability to disable the collector, tune the collection frequency, +and set debugging options. It also provides access to unreachable objects that +the collector found but cannot free. Since the collector supplements the +reference counting already used in Python, you can disable the collector if you +are sure your program does not create reference cycles. Automatic collection +can be disabled by calling ``gc.disable()``. To debug a leaking program call +``gc.set_debug(gc.DEBUG_LEAK)``. Notice that this includes +``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in +gc.garbage for inspection. + +The :mod:`gc` module provides the following functions: + + +.. function:: enable() + + Enable automatic garbage collection. + + +.. function:: disable() + + Disable automatic garbage collection. + + +.. function:: isenabled() + + Return ``True`` if automatic collection is enabled. + + +.. function:: collect(generation=2) + + With no arguments, run a full collection. The optional argument *generation* + may be an integer specifying which generation to collect (from 0 to 2). A + :exc:`ValueError` is raised if the generation number is invalid. The number of + unreachable objects found is returned. + + The free lists maintained for a number of built-in types are cleared + whenever a full collection or collection of the highest generation (2) + is run. Not all items in some free lists may be freed due to the + particular implementation, in particular :class:`float`. + + +.. function:: set_debug(flags) + + Set the garbage collection debugging flags. Debugging information will be + written to ``sys.stderr``. See below for a list of debugging flags which can be + combined using bit operations to control debugging. + + +.. function:: get_debug() + + Return the debugging flags currently set. + + +.. function:: get_objects(generation=None) + + Returns a list of all objects tracked by the collector, excluding the list + returned. If *generation* is not None, return only the objects tracked by + the collector that are in that generation. + + .. versionchanged:: 3.8 + New *generation* parameter. + +.. function:: get_stats() + + Return a list of three per-generation dictionaries containing collection + statistics since interpreter start. The number of keys may change + in the future, but currently each dictionary will contain the following + items: + + * ``collections`` is the number of times this generation was collected; + + * ``collected`` is the total number of objects collected inside this + generation; + + * ``uncollectable`` is the total number of objects which were found + to be uncollectable (and were therefore moved to the :data:`garbage` + list) inside this generation. + + .. versionadded:: 3.4 + + +.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) + + Set the garbage collection thresholds (the collection frequency). Setting + *threshold0* to zero disables collection. + + The GC classifies objects into three generations depending on how many + collection sweeps they have survived. New objects are placed in the youngest + generation (generation ``0``). If an object survives a collection it is moved + into the next older generation. Since generation ``2`` is the oldest + generation, objects in that generation remain there after a collection. In + order to decide when to run, the collector keeps track of the number object + allocations and deallocations since the last collection. When the number of + allocations minus the number of deallocations exceeds *threshold0*, collection + starts. Initially only generation ``0`` is examined. If generation ``0`` has + been examined more than *threshold1* times since generation ``1`` has been + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. + + +.. function:: get_count() + + Return the current collection counts as a tuple of ``(count0, count1, + count2)``. + + +.. function:: get_threshold() + + Return the current collection thresholds as a tuple of ``(threshold0, + threshold1, threshold2)``. + + +.. function:: get_referrers(*objs) + + Return the list of objects that directly refer to any of objs. This function + will only locate those containers which support garbage collection; extension + types which do refer to other objects but do not support garbage collection will + not be found. + + Note that objects which have already been dereferenced, but which live in cycles + and have not yet been collected by the garbage collector can be listed among the + resulting referrers. To get only currently live objects, call :func:`collect` + before calling :func:`get_referrers`. + + .. warning:: + Care must be taken when using objects returned by :func:`get_referrers` because + some of them could still be under construction and hence in a temporarily + invalid state. Avoid using :func:`get_referrers` for any purpose other than + debugging. + + +.. function:: get_referents(*objs) + + Return a list of objects directly referred to by any of the arguments. The + referents returned are those objects visited by the arguments' C-level + :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually + directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects + that support garbage collection, and are only required to visit objects that may + be involved in a cycle. So, for example, if an integer is directly reachable + from an argument, that integer object may or may not appear in the result list. + + +.. function:: is_tracked(obj) + + Returns ``True`` if the object is currently tracked by the garbage collector, + ``False`` otherwise. As a general rule, instances of atomic types aren't + tracked and instances of non-atomic types (containers, user-defined + objects...) are. However, some type-specific optimizations can be present + in order to suppress the garbage collector footprint of simple instances + (e.g. dicts containing only atomic keys and values):: + + >>> gc.is_tracked(0) + False + >>> gc.is_tracked("a") + False + >>> gc.is_tracked([]) + True + >>> gc.is_tracked({}) + False + >>> gc.is_tracked({"a": 1}) + False + >>> gc.is_tracked({"a": []}) + True + + .. versionadded:: 3.1 + + +.. function:: is_finalized(obj) + + Returns ``True`` if the given object has been finalized by the + garbage collector, ``False`` otherwise. :: + + >>> x = None + >>> class Lazarus: + ... def __del__(self): + ... global x + ... x = self + ... + >>> lazarus = Lazarus() + >>> gc.is_finalized(lazarus) + False + >>> del lazarus + >>> gc.is_finalized(x) + True + + .. versionadded:: 3.9 + + +.. function:: freeze() + + Freeze all the objects tracked by gc - move them to a permanent generation + and ignore all the future collections. This can be used before a POSIX + fork() call to make the gc copy-on-write friendly or to speed up collection. + Also collection before a POSIX fork() call may free pages for future + allocation which can cause copy-on-write too so it's advised to disable gc + in parent process and freeze before fork and enable gc in child process. + + .. versionadded:: 3.7 + + +.. function:: unfreeze() + + Unfreeze the objects in the permanent generation, put them back into the + oldest generation. + + .. versionadded:: 3.7 + + +.. function:: get_freeze_count() + + Return the number of objects in the permanent generation. + + .. versionadded:: 3.7 + + +The following variables are provided for read-only access (you can mutate the +values but should not rebind them): + +.. data:: garbage + + A list of objects which the collector found to be unreachable but could + not be freed (uncollectable objects). Starting with Python 3.4, this + list should be empty most of the time, except when using instances of + C extension types with a non-``NULL`` ``tp_del`` slot. + + If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be + added to this list rather than freed. + + .. versionchanged:: 3.2 + If this list is non-empty at :term:`interpreter shutdown`, a + :exc:`ResourceWarning` is emitted, which is silent by default. If + :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects + are printed. + + .. versionchanged:: 3.4 + Following :pep:`442`, objects with a :meth:`__del__` method don't end + up in :attr:`gc.garbage` anymore. + +.. data:: callbacks + + A list of callbacks that will be invoked by the garbage collector before and + after collection. The callbacks will be called with two arguments, + *phase* and *info*. + + *phase* can be one of two values: + + "start": The garbage collection is about to start. + + "stop": The garbage collection has finished. + + *info* is a dict providing more information for the callback. The following + keys are currently defined: + + "generation": The oldest generation being collected. + + "collected": When *phase* is "stop", the number of objects + successfully collected. + + "uncollectable": When *phase* is "stop", the number of objects + that could not be collected and were put in :data:`garbage`. + + Applications can add their own callbacks to this list. The primary + use cases are: + + Gathering statistics about garbage collection, such as how often + various generations are collected, and how long the collection + takes. + + Allowing applications to identify and clear their own uncollectable + types when they appear in :data:`garbage`. + + .. versionadded:: 3.3 + + +The following constants are provided for use with :func:`set_debug`: + + +.. data:: DEBUG_STATS + + Print statistics during collection. This information can be useful when tuning + the collection frequency. + + +.. data:: DEBUG_COLLECTABLE + + Print information on collectable objects found. + + +.. data:: DEBUG_UNCOLLECTABLE + + Print information of uncollectable objects found (objects which are not + reachable but cannot be freed by the collector). These objects will be added + to the ``garbage`` list. + + .. versionchanged:: 3.2 + Also print the contents of the :data:`garbage` list at + :term:`interpreter shutdown`, if it isn't empty. + +.. data:: DEBUG_SAVEALL + + When set, all unreachable objects found will be appended to *garbage* rather + than being freed. This can be useful for debugging a leaking program. + + +.. data:: DEBUG_LEAK + + The debugging flags necessary for the collector to print information about a + leaking program (equal to ``DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | + DEBUG_SAVEALL``). diff --git a/Doc/library/importlib.metadata.rst.bak b/Doc/library/importlib.metadata.rst.bak new file mode 100644 index 00000000000000..7f154ea02cc4f2 --- /dev/null +++ b/Doc/library/importlib.metadata.rst.bak @@ -0,0 +1,262 @@ +.. _using: + +================================= + Using :mod:`!importlib.metadata` +================================= + +.. note:: + This functionality is provisional and may deviate from the usual + version semantics of the standard library. + +``importlib.metadata`` is a library that provides for access to installed +package metadata. Built in part on Python's import system, this library +intends to replace similar functionality in the `entry point +API`_ and `metadata API`_ of ``pkg_resources``. Along with +:mod:`importlib.resources` in Python 3.7 +and newer (backported as `importlib_resources`_ for older versions of +Python), this can eliminate the need to use the older and less efficient +``pkg_resources`` package. + +By "installed package" we generally mean a third-party package installed into +Python's ``site-packages`` directory via tools such as `pip +`_. Specifically, +it means a package with either a discoverable ``dist-info`` or ``egg-info`` +directory, and metadata defined by :pep:`566` or its older specifications. +By default, package metadata can live on the file system or in zip archives on +:data:`sys.path`. Through an extension mechanism, the metadata can live almost +anywhere. + + +Overview +======== + +Let's say you wanted to get the version string for a package you've installed +using ``pip``. We start by creating a virtual environment and installing +something into it: + +.. code-block:: shell-session + + $ python3 -m venv example + $ source example/bin/activate + (example) $ pip install wheel + +You can get the version string for ``wheel`` by running the following: + +.. code-block:: pycon + + (example) $ python + >>> from importlib.metadata import version # doctest: +SKIP + >>> version('wheel') # doctest: +SKIP + '0.32.3' + +You can also get the set of entry points keyed by group, such as +``console_scripts``, ``distutils.commands`` and others. Each group contains a +sequence of :ref:`EntryPoint ` objects. + +You can get the :ref:`metadata for a distribution `:: + + >>> list(metadata('wheel')) # doctest: +SKIP + ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] + +You can also get a :ref:`distribution's version number `, list its +:ref:`constituent files `, and get a list of the distribution's +:ref:`requirements`. + + +Functional API +============== + +This package provides the following functionality via its public API. + + +.. _entry-points: + +Entry points +------------ + +The ``entry_points()`` function returns a dictionary of all entry points, +keyed by group. Entry points are represented by ``EntryPoint`` instances; +each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and +a ``.load()`` method to resolve the value. There are also ``.module``, +``.attr``, and ``.extras`` attributes for getting the components of the +``.value`` attribute:: + + >>> eps = entry_points() # doctest: +SKIP + >>> list(eps) # doctest: +SKIP + ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] + >>> scripts = eps['console_scripts'] # doctest: +SKIP + >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP + >>> wheel # doctest: +SKIP + EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> wheel.module # doctest: +SKIP + 'wheel.cli' + >>> wheel.attr # doctest: +SKIP + 'main' + >>> wheel.extras # doctest: +SKIP + [] + >>> main = wheel.load() # doctest: +SKIP + >>> main # doctest: +SKIP + + +The ``group`` and ``name`` are arbitrary values defined by the package author +and usually a client will wish to resolve all entry points for a particular +group. Read `the setuptools docs +`_ +for more information on entry points, their definition, and usage. + + +.. _metadata: + +Distribution metadata +--------------------- + +Every distribution includes some metadata, which you can extract using the +``metadata()`` function:: + + >>> wheel_metadata = metadata('wheel') # doctest: +SKIP + +The keys of the returned data structure, a ``PackageMetadata``, +name the metadata keywords, and +the values are returned unparsed from the distribution metadata:: + + >>> wheel_metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + + +.. _version: + +Distribution versions +--------------------- + +The ``version()`` function is the quickest way to get a distribution's version +number, as a string:: + + >>> version('wheel') # doctest: +SKIP + '0.32.3' + + +.. _files: + +Distribution files +------------------ + +You can also get the full set of files contained within a distribution. The +``files()`` function takes a distribution package name and returns all of the +files installed by this distribution. Each file object returned is a +``PackagePath``, a :class:`pathlib.Path` derived object with additional ``dist``, +``size``, and ``hash`` properties as indicated by the metadata. For example:: + + >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP + >>> util # doctest: +SKIP + PackagePath('wheel/util.py') + >>> util.size # doctest: +SKIP + 859 + >>> util.dist # doctest: +SKIP + + >>> util.hash # doctest: +SKIP + + +Once you have the file, you can also read its contents:: + + >>> print(util.read_text()) # doctest: +SKIP + import base64 + import sys + ... + def as_bytes(s): + if isinstance(s, text_type): + return s.encode('utf-8') + return s + +In the case where the metadata file listing files +(RECORD or SOURCES.txt) is missing, ``files()`` will +return ``None``. The caller may wish to wrap calls to +``files()`` in `always_iterable +`_ +or otherwise guard against this condition if the target +distribution is not known to have the metadata present. + +.. _requirements: + +Distribution requirements +------------------------- + +To get the full set of requirements for a distribution, use the ``requires()`` +function:: + + >>> requires('wheel') # doctest: +SKIP + ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] + + +Distributions +============= + +While the above API is the most common and convenient usage, you can get all +of that information from the ``Distribution`` class. A ``Distribution`` is an +abstract object that represents the metadata for a Python package. You can +get the ``Distribution`` instance:: + + >>> from importlib.metadata import distribution # doctest: +SKIP + >>> dist = distribution('wheel') # doctest: +SKIP + +Thus, an alternative way to get the version number is through the +``Distribution`` instance:: + + >>> dist.version # doctest: +SKIP + '0.32.3' + +There are all kinds of additional metadata available on the ``Distribution`` +instance:: + + >>> dist.metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> dist.metadata['License'] # doctest: +SKIP + 'MIT' + +The full set of available metadata is not described here. See :pep:`566` +for additional details. + + +Extending the search algorithm +============================== + +Because package metadata is not available through :data:`sys.path` searches, or +package loaders directly, the metadata for a package is found through import +system :ref:`finders `. To find a distribution package's metadata, +``importlib.metadata`` queries the list of :term:`meta path finders ` on +:data:`sys.meta_path`. + +The default ``PathFinder`` for Python includes a hook that calls into +``importlib.metadata.MetadataPathFinder`` for finding distributions +loaded from typical file-system-based paths. + +The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the +interface expected of finders by Python's import system. +``importlib.metadata`` extends this protocol by looking for an optional +``find_distributions`` callable on the finders from +:data:`sys.meta_path` and presents this extended interface as the +``DistributionFinder`` abstract base class, which defines this abstract +method:: + + @abc.abstractmethod + def find_distributions(context=DistributionFinder.Context()): + """Return an iterable of all Distribution instances capable of + loading the metadata for packages for the indicated ``context``. + """ + +The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` +properties indicating the path to search and name to match and may +supply other relevant context. + +What this means in practice is that to support finding distribution package +metadata in locations other than the file system, subclass +``Distribution`` and implement the abstract methods. Then from +a custom finder, return instances of this derived ``Distribution`` in the +``find_distributions()`` method. + + +.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points +.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html + + +.. rubric:: Footnotes diff --git a/Doc/library/logging.rst.bak b/Doc/library/logging.rst.bak new file mode 100644 index 00000000000000..431a5849fa9bff --- /dev/null +++ b/Doc/library/logging.rst.bak @@ -0,0 +1,1350 @@ +:mod:`logging` --- Logging facility for Python +============================================== + +.. module:: logging + :synopsis: Flexible event logging system for applications. + +.. moduleauthor:: Vinay Sajip +.. sectionauthor:: Vinay Sajip + +**Source code:** :source:`Lib/logging/__init__.py` + +.. index:: pair: Errors; logging + +.. sidebar:: Important + + This page contains the API reference information. For tutorial + information and discussion of more advanced topics, see + + * :ref:`Basic Tutorial ` + * :ref:`Advanced Tutorial ` + * :ref:`Logging Cookbook ` + +-------------- + +This module defines functions and classes which implement a flexible event +logging system for applications and libraries. + +The key benefit of having the logging API provided by a standard library module +is that all Python modules can participate in logging, so your application log +can include your own messages integrated with messages from third-party +modules. + +The module provides a lot of functionality and flexibility. If you are +unfamiliar with logging, the best way to get to grips with it is to see the +tutorials (see the links on the right). + +The basic classes defined by the module, together with their functions, are +listed below. + +* Loggers expose the interface that application code directly uses. +* Handlers send the log records (created by loggers) to the appropriate + destination. +* Filters provide a finer grained facility for determining which log records + to output. +* Formatters specify the layout of log records in the final output. + + +.. _logger: + +Logger Objects +-------------- + +Loggers have the following attributes and methods. Note that Loggers should +*NEVER* be instantiated directly, but always through the module-level function +``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same +name will always return a reference to the same Logger object. + +The ``name`` is potentially a period-separated hierarchical value, like +``foo.bar.baz`` (though it could also be just plain ``foo``, for example). +Loggers that are further down in the hierarchical list are children of loggers +higher up in the list. For example, given a logger with a name of ``foo``, +loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all +descendants of ``foo``. The logger name hierarchy is analogous to the Python +package hierarchy, and identical to it if you organise your loggers on a +per-module basis using the recommended construction +``logging.getLogger(__name__)``. That's because in a module, ``__name__`` +is the module's name in the Python package namespace. + + +.. class:: Logger + + .. attribute:: Logger.propagate + + If this attribute evaluates to true, events logged to this logger will be + passed to the handlers of higher level (ancestor) loggers, in addition to + any handlers attached to this logger. Messages are passed directly to the + ancestor loggers' handlers - neither the level nor filters of the ancestor + loggers in question are considered. + + If this evaluates to false, logging messages are not passed to the handlers + of ancestor loggers. + + The constructor sets this attribute to ``True``. + + .. note:: If you attach a handler to a logger *and* one or more of its + ancestors, it may emit the same record multiple times. In general, you + should not need to attach a handler to more than one logger - if you just + attach it to the appropriate logger which is highest in the logger + hierarchy, then it will see all events logged by all descendant loggers, + provided that their propagate setting is left set to ``True``. A common + scenario is to attach handlers only to the root logger, and to let + propagation take care of the rest. + + .. method:: Logger.setLevel(level) + + Sets the threshold for this logger to *level*. Logging messages which are less + severe than *level* will be ignored; logging messages which have severity *level* + or higher will be emitted by whichever handler or handlers service this logger, + unless a handler's level has been set to a higher severity level than *level*. + + When a logger is created, the level is set to :const:`NOTSET` (which causes + all messages to be processed when the logger is the root logger, or delegation + to the parent when the logger is a non-root logger). Note that the root logger + is created with level :const:`WARNING`. + + The term 'delegation to the parent' means that if a logger has a level of + NOTSET, its chain of ancestor loggers is traversed until either an ancestor with + a level other than NOTSET is found, or the root is reached. + + If an ancestor is found with a level other than NOTSET, then that ancestor's + level is treated as the effective level of the logger where the ancestor search + began, and is used to determine how a logging event is handled. + + If the root is reached, and it has a level of NOTSET, then all messages will be + processed. Otherwise, the root's level will be used as the effective level. + + See :ref:`levels` for a list of levels. + + .. versionchanged:: 3.2 + The *level* parameter now accepts a string representation of the + level such as 'INFO' as an alternative to the integer constants + such as :const:`INFO`. Note, however, that levels are internally stored + as integers, and methods such as e.g. :meth:`getEffectiveLevel` and + :meth:`isEnabledFor` will return/expect to be passed integers. + + + .. method:: Logger.isEnabledFor(level) + + Indicates if a message of severity *level* would be processed by this logger. + This method checks first the module-level level set by + ``logging.disable(level)`` and then the logger's effective level as determined + by :meth:`getEffectiveLevel`. + + + .. method:: Logger.getEffectiveLevel() + + Indicates the effective level for this logger. If a value other than + :const:`NOTSET` has been set using :meth:`setLevel`, it is returned. Otherwise, + the hierarchy is traversed towards the root until a value other than + :const:`NOTSET` is found, and that value is returned. The value returned is + an integer, typically one of :const:`logging.DEBUG`, :const:`logging.INFO` + etc. + + + .. method:: Logger.getChild(suffix) + + Returns a logger which is a descendant to this logger, as determined by the suffix. + Thus, ``logging.getLogger('abc').getChild('def.ghi')`` would return the same + logger as would be returned by ``logging.getLogger('abc.def.ghi')``. This is a + convenience method, useful when the parent logger is named using e.g. ``__name__`` + rather than a literal string. + + .. versionadded:: 3.2 + + + .. method:: Logger.debug(msg, *args, **kwargs) + + Logs a message with level :const:`DEBUG` on this logger. The *msg* is the + message format string, and the *args* are the arguments which are merged into + *msg* using the string formatting operator. (Note that this means that you can + use keywords in the format string, together with a single dictionary argument.) + No % formatting operation is performed on *msg* when no *args* are supplied. + + There are four keyword arguments in *kwargs* which are inspected: + *exc_info*, *stack_info*, *stacklevel* and *extra*. + + If *exc_info* does not evaluate as false, it causes exception information to be + added to the logging message. If an exception tuple (in the format returned by + :func:`sys.exc_info`) or an exception instance is provided, it is used; + otherwise, :func:`sys.exc_info` is called to get the exception information. + + The second optional keyword argument is *stack_info*, which defaults to + ``False``. If true, stack information is added to the logging + message, including the actual logging call. Note that this is not the same + stack information as that displayed through specifying *exc_info*: The + former is stack frames from the bottom of the stack up to the logging call + in the current thread, whereas the latter is information about stack frames + which have been unwound, following an exception, while searching for + exception handlers. + + You can specify *stack_info* independently of *exc_info*, e.g. to just show + how you got to a certain point in your code, even when no exceptions were + raised. The stack frames are printed following a header line which says: + + .. code-block:: none + + Stack (most recent call last): + + This mimics the ``Traceback (most recent call last):`` which is used when + displaying exception frames. + + The third optional keyword argument is *stacklevel*, which defaults to ``1``. + If greater than 1, the corresponding number of stack frames are skipped + when computing the line number and function name set in the :class:`LogRecord` + created for the logging event. This can be used in logging helpers so that + the function name, filename and line number recorded are not the information + for the helper function/method, but rather its caller. The name of this + parameter mirrors the equivalent one in the :mod:`warnings` module. + + The fourth keyword argument is *extra* which can be used to pass a + dictionary which is used to populate the __dict__ of the :class:`LogRecord` + created for the logging event with user-defined attributes. These custom + attributes can then be used as you like. For example, they could be + incorporated into logged messages. For example:: + + FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' + logging.basicConfig(format=FORMAT) + d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} + logger = logging.getLogger('tcpserver') + logger.warning('Protocol problem: %s', 'connection reset', extra=d) + + would print something like + + .. code-block:: none + + 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset + + The keys in the dictionary passed in *extra* should not clash with the keys used + by the logging system. (See the :class:`Formatter` documentation for more + information on which keys are used by the logging system.) + + If you choose to use these attributes in logged messages, you need to exercise + some care. In the above example, for instance, the :class:`Formatter` has been + set up with a format string which expects 'clientip' and 'user' in the attribute + dictionary of the :class:`LogRecord`. If these are missing, the message will + not be logged because a string formatting exception will occur. So in this case, + you always need to pass the *extra* dictionary with these keys. + + While this might be annoying, this feature is intended for use in specialized + circumstances, such as multi-threaded servers where the same code executes in + many contexts, and interesting conditions which arise are dependent on this + context (such as remote client IP address and authenticated user name, in the + above example). In such circumstances, it is likely that specialized + :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. + + .. versionchanged:: 3.2 + The *stack_info* parameter was added. + + .. versionchanged:: 3.5 + The *exc_info* parameter can now accept exception instances. + + .. versionchanged:: 3.8 + The *stacklevel* parameter was added. + + + .. method:: Logger.info(msg, *args, **kwargs) + + Logs a message with level :const:`INFO` on this logger. The arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.warning(msg, *args, **kwargs) + + Logs a message with level :const:`WARNING` on this logger. The arguments are + interpreted as for :meth:`debug`. + + .. note:: There is an obsolete method ``warn`` which is functionally + identical to ``warning``. As ``warn`` is deprecated, please do not use + it - use ``warning`` instead. + + .. method:: Logger.error(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on this logger. The arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.critical(msg, *args, **kwargs) + + Logs a message with level :const:`CRITICAL` on this logger. The arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.log(level, msg, *args, **kwargs) + + Logs a message with integer level *level* on this logger. The other arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.exception(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on this logger. The arguments are + interpreted as for :meth:`debug`. Exception info is added to the logging + message. This method should only be called from an exception handler. + + + .. method:: Logger.addFilter(filter) + + Adds the specified filter *filter* to this logger. + + + .. method:: Logger.removeFilter(filter) + + Removes the specified filter *filter* from this logger. + + + .. method:: Logger.filter(record) + + Apply this logger's filters to the record and return ``True`` if the + record is to be processed. The filters are consulted in turn, until one of + them returns a false value. If none of them return a false value, the record + will be processed (passed to handlers). If one returns a false value, no + further processing of the record occurs. + + + .. method:: Logger.addHandler(hdlr) + + Adds the specified handler *hdlr* to this logger. + + + .. method:: Logger.removeHandler(hdlr) + + Removes the specified handler *hdlr* from this logger. + + + .. method:: Logger.findCaller(stack_info=False, stacklevel=1) + + Finds the caller's source filename and line number. Returns the filename, line + number, function name and stack information as a 4-element tuple. The stack + information is returned as ``None`` unless *stack_info* is ``True``. + + The *stacklevel* parameter is passed from code calling the :meth:`debug` + and other APIs. If greater than 1, the excess is used to skip stack frames + before determining the values to be returned. This will generally be useful + when calling logging APIs from helper/wrapper code, so that the information + in the event log refers not to the helper/wrapper code, but to the code that + calls it. + + + .. method:: Logger.handle(record) + + Handles a record by passing it to all handlers associated with this logger and + its ancestors (until a false value of *propagate* is found). This method is used + for unpickled records received from a socket, as well as those created locally. + Logger-level filtering is applied using :meth:`~Logger.filter`. + + + .. method:: Logger.makeRecord(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) + + This is a factory method which can be overridden in subclasses to create + specialized :class:`LogRecord` instances. + + .. method:: Logger.hasHandlers() + + Checks to see if this logger has any handlers configured. This is done by + looking for handlers in this logger and its parents in the logger hierarchy. + Returns ``True`` if a handler was found, else ``False``. The method stops searching + up the hierarchy whenever a logger with the 'propagate' attribute set to + false is found - that will be the last logger which is checked for the + existence of handlers. + + .. versionadded:: 3.2 + + .. versionchanged:: 3.7 + Loggers can now be pickled and unpickled. + +.. _levels: + +Logging Levels +-------------- + +The numeric values of logging levels are given in the following table. These are +primarily of interest if you want to define your own levels, and need them to +have specific values relative to the predefined levels. If you define a level +with the same numeric value, it overwrites the predefined value; the predefined +name is lost. + ++--------------+---------------+ +| Level | Numeric value | ++==============+===============+ +| ``CRITICAL`` | 50 | ++--------------+---------------+ +| ``ERROR`` | 40 | ++--------------+---------------+ +| ``WARNING`` | 30 | ++--------------+---------------+ +| ``INFO`` | 20 | ++--------------+---------------+ +| ``DEBUG`` | 10 | ++--------------+---------------+ +| ``NOTSET`` | 0 | ++--------------+---------------+ + + +.. _handler: + +Handler Objects +--------------- + +Handlers have the following attributes and methods. Note that :class:`Handler` +is never instantiated directly; this class acts as a base for more useful +subclasses. However, the :meth:`__init__` method in subclasses needs to call +:meth:`Handler.__init__`. + +.. class:: Handler + + .. method:: Handler.__init__(level=NOTSET) + + Initializes the :class:`Handler` instance by setting its level, setting the list + of filters to the empty list and creating a lock (using :meth:`createLock`) for + serializing access to an I/O mechanism. + + + .. method:: Handler.createLock() + + Initializes a thread lock which can be used to serialize access to underlying + I/O functionality which may not be threadsafe. + + + .. method:: Handler.acquire() + + Acquires the thread lock created with :meth:`createLock`. + + + .. method:: Handler.release() + + Releases the thread lock acquired with :meth:`acquire`. + + + .. method:: Handler.setLevel(level) + + Sets the threshold for this handler to *level*. Logging messages which are + less severe than *level* will be ignored. When a handler is created, the + level is set to :const:`NOTSET` (which causes all messages to be + processed). + + See :ref:`levels` for a list of levels. + + .. versionchanged:: 3.2 + The *level* parameter now accepts a string representation of the + level such as 'INFO' as an alternative to the integer constants + such as :const:`INFO`. + + + .. method:: Handler.setFormatter(fmt) + + Sets the :class:`Formatter` for this handler to *fmt*. + + + .. method:: Handler.addFilter(filter) + + Adds the specified filter *filter* to this handler. + + + .. method:: Handler.removeFilter(filter) + + Removes the specified filter *filter* from this handler. + + + .. method:: Handler.filter(record) + + Apply this handler's filters to the record and return ``True`` if the + record is to be processed. The filters are consulted in turn, until one of + them returns a false value. If none of them return a false value, the record + will be emitted. If one returns a false value, the handler will not emit the + record. + + + .. method:: Handler.flush() + + Ensure all logging output has been flushed. This version does nothing and is + intended to be implemented by subclasses. + + + .. method:: Handler.close() + + Tidy up any resources used by the handler. This version does no output but + removes the handler from an internal list of handlers which is closed when + :func:`shutdown` is called. Subclasses should ensure that this gets called + from overridden :meth:`close` methods. + + + .. method:: Handler.handle(record) + + Conditionally emits the specified logging record, depending on filters which may + have been added to the handler. Wraps the actual emission of the record with + acquisition/release of the I/O thread lock. + + + .. method:: Handler.handleError(record) + + This method should be called from handlers when an exception is encountered + during an :meth:`emit` call. If the module-level attribute + ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is + what is mostly wanted for a logging system - most users will not care about + errors in the logging system, they are more interested in application + errors. You could, however, replace this with a custom handler if you wish. + The specified record is the one which was being processed when the exception + occurred. (The default value of ``raiseExceptions`` is ``True``, as that is + more useful during development). + + + .. method:: Handler.format(record) + + Do formatting for a record - if a formatter is set, use it. Otherwise, use the + default formatter for the module. + + + .. method:: Handler.emit(record) + + Do whatever it takes to actually log the specified logging record. This version + is intended to be implemented by subclasses and so raises a + :exc:`NotImplementedError`. + +For a list of handlers included as standard, see :mod:`logging.handlers`. + +.. _formatter-objects: + +Formatter Objects +----------------- + +.. currentmodule:: logging + +:class:`Formatter` objects have the following attributes and methods. They are +responsible for converting a :class:`LogRecord` to (usually) a string which can +be interpreted by either a human or an external system. The base +:class:`Formatter` allows a formatting string to be specified. If none is +supplied, the default value of ``'%(message)s'`` is used, which just includes +the message in the logging call. To have additional items of information in the +formatted output (such as a timestamp), keep reading. + +A Formatter can be initialized with a format string which makes use of knowledge +of the :class:`LogRecord` attributes - such as the default value mentioned above +making use of the fact that the user's message and arguments are pre-formatted +into a :class:`LogRecord`'s *message* attribute. This format string contains +standard Python %-style mapping keys. See section :ref:`old-string-formatting` +for more information on string formatting. + +The useful mapping keys in a :class:`LogRecord` are given in the section on +:ref:`logrecord-attributes`. + + +.. class:: Formatter(fmt=None, datefmt=None, style='%', validate=True, *, defaults=None) + + Returns a new instance of the :class:`Formatter` class. The instance is + initialized with a format string for the message as a whole, as well as a + format string for the date/time portion of a message. If no *fmt* is + specified, ``'%(message)s'`` is used. If no *datefmt* is specified, a format + is used which is described in the :meth:`formatTime` documentation. + + The *style* parameter can be one of '%', '{' or '$' and determines how + the format string will be merged with its data: using one of %-formatting, + :meth:`str.format` or :class:`string.Template`. This only applies to the + format string *fmt* (e.g. ``'%(message)s'`` or ``{message}``), not to the + actual log messages passed to ``Logger.debug`` etc; see + :ref:`formatting-styles` for more information on using {- and $-formatting + for log messages. + + The *defaults* parameter can be a dictionary with default values to use in + custom fields. For example: + ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` + + .. versionchanged:: 3.2 + The *style* parameter was added. + + .. versionchanged:: 3.8 + The *validate* parameter was added. Incorrect or mismatched style and fmt + will raise a ``ValueError``. + For example: ``logging.Formatter('%(asctime)s - %(message)s', style='{')``. + + .. versionchanged:: 3.10 + The *defaults* parameter was added. + + .. method:: format(record) + + The record's attribute dictionary is used as the operand to a string + formatting operation. Returns the resulting string. Before formatting the + dictionary, a couple of preparatory steps are carried out. The *message* + attribute of the record is computed using *msg* % *args*. If the + formatting string contains ``'(asctime)'``, :meth:`formatTime` is called + to format the event time. If there is exception information, it is + formatted using :meth:`formatException` and appended to the message. Note + that the formatted exception information is cached in attribute + *exc_text*. This is useful because the exception information can be + pickled and sent across the wire, but you should be careful if you have + more than one :class:`Formatter` subclass which customizes the formatting + of exception information. In this case, you will have to clear the cached + value (by setting the *exc_text* attribute to ``None``) after a formatter + has done its formatting, so that the next formatter to handle the event + doesn't use the cached value, but recalculates it afresh. + + If stack information is available, it's appended after the exception + information, using :meth:`formatStack` to transform it if necessary. + + + .. method:: formatTime(record, datefmt=None) + + This method should be called from :meth:`format` by a formatter which + wants to make use of a formatted time. This method can be overridden in + formatters to provide for any specific requirement, but the basic behavior + is as follows: if *datefmt* (a string) is specified, it is used with + :func:`time.strftime` to format the creation time of the + record. Otherwise, the format '%Y-%m-%d %H:%M:%S,uuu' is used, where the + uuu part is a millisecond value and the other letters are as per the + :func:`time.strftime` documentation. An example time in this format is + ``2003-01-23 00:29:50,411``. The resulting string is returned. + + This function uses a user-configurable function to convert the creation + time to a tuple. By default, :func:`time.localtime` is used; to change + this for a particular formatter instance, set the ``converter`` attribute + to a function with the same signature as :func:`time.localtime` or + :func:`time.gmtime`. To change it for all formatters, for example if you + want all logging times to be shown in GMT, set the ``converter`` + attribute in the ``Formatter`` class. + + .. versionchanged:: 3.3 + Previously, the default format was hard-coded as in this example: + ``2010-09-06 22:38:15,292`` where the part before the comma is + handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the + part after the comma is a millisecond value. Because strptime does not + have a format placeholder for milliseconds, the millisecond value is + appended using another format string, ``'%s,%03d'`` --- and both of these + format strings have been hardcoded into this method. With the change, + these strings are defined as class-level attributes which can be + overridden at the instance level when desired. The names of the + attributes are ``default_time_format`` (for the strptime format string) + and ``default_msec_format`` (for appending the millisecond value). + + .. versionchanged:: 3.9 + The ``default_msec_format`` can be ``None``. + + .. method:: formatException(exc_info) + + Formats the specified exception information (a standard exception tuple as + returned by :func:`sys.exc_info`) as a string. This default implementation + just uses :func:`traceback.print_exception`. The resulting string is + returned. + + .. method:: formatStack(stack_info) + + Formats the specified stack information (a string as returned by + :func:`traceback.print_stack`, but with the last newline removed) as a + string. This default implementation just returns the input value. + +.. _filter: + +Filter Objects +-------------- + +``Filters`` can be used by ``Handlers`` and ``Loggers`` for more sophisticated +filtering than is provided by levels. The base filter class only allows events +which are below a certain point in the logger hierarchy. For example, a filter +initialized with 'A.B' will allow events logged by loggers 'A.B', 'A.B.C', +'A.B.C.D', 'A.B.D' etc. but not 'A.BB', 'B.A.B' etc. If initialized with the +empty string, all events are passed. + + +.. class:: Filter(name='') + + Returns an instance of the :class:`Filter` class. If *name* is specified, it + names a logger which, together with its children, will have its events allowed + through the filter. If *name* is the empty string, allows every event. + + + .. method:: filter(record) + + Is the specified record to be logged? Returns zero for no, nonzero for + yes. If deemed appropriate, the record may be modified in-place by this + method. + +Note that filters attached to handlers are consulted before an event is +emitted by the handler, whereas filters attached to loggers are consulted +whenever an event is logged (using :meth:`debug`, :meth:`info`, +etc.), before sending an event to handlers. This means that events which have +been generated by descendant loggers will not be filtered by a logger's filter +setting, unless the filter has also been applied to those descendant loggers. + +You don't actually need to subclass ``Filter``: you can pass any instance +which has a ``filter`` method with the same semantics. + +.. versionchanged:: 3.2 + You don't need to create specialized ``Filter`` classes, or use other + classes with a ``filter`` method: you can use a function (or other + callable) as a filter. The filtering logic will check to see if the filter + object has a ``filter`` attribute: if it does, it's assumed to be a + ``Filter`` and its :meth:`~Filter.filter` method is called. Otherwise, it's + assumed to be a callable and called with the record as the single + parameter. The returned value should conform to that returned by + :meth:`~Filter.filter`. + +Although filters are used primarily to filter records based on more +sophisticated criteria than levels, they get to see every record which is +processed by the handler or logger they're attached to: this can be useful if +you want to do things like counting how many records were processed by a +particular logger or handler, or adding, changing or removing attributes in +the :class:`LogRecord` being processed. Obviously changing the LogRecord needs +to be done with some care, but it does allow the injection of contextual +information into logs (see :ref:`filters-contextual`). + +.. _log-record: + +LogRecord Objects +----------------- + +:class:`LogRecord` instances are created automatically by the :class:`Logger` +every time something is logged, and can be created manually via +:func:`makeLogRecord` (for example, from a pickled event received over the +wire). + + +.. class:: LogRecord(name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None) + + Contains all the information pertinent to the event being logged. + + The primary information is passed in :attr:`msg` and :attr:`args`, which + are combined using ``msg % args`` to create the :attr:`message` field of the + record. + + :param name: The name of the logger used to log the event represented by + this LogRecord. Note that this name will always have this + value, even though it may be emitted by a handler attached to + a different (ancestor) logger. + :param level: The numeric level of the logging event (one of DEBUG, INFO etc.) + Note that this is converted to *two* attributes of the LogRecord: + ``levelno`` for the numeric value and ``levelname`` for the + corresponding level name. + :param pathname: The full pathname of the source file where the logging call + was made. + :param lineno: The line number in the source file where the logging call was + made. + :param msg: The event description message, possibly a format string with + placeholders for variable data. + :param args: Variable data to merge into the *msg* argument to obtain the + event description. + :param exc_info: An exception tuple with the current exception information, + or ``None`` if no exception information is available. + :param func: The name of the function or method from which the logging call + was invoked. + :param sinfo: A text string representing stack information from the base of + the stack in the current thread, up to the logging call. + + .. method:: getMessage() + + Returns the message for this :class:`LogRecord` instance after merging any + user-supplied arguments with the message. If the user-supplied message + argument to the logging call is not a string, :func:`str` is called on it to + convert it to a string. This allows use of user-defined classes as + messages, whose ``__str__`` method can return the actual format string to + be used. + + .. versionchanged:: 3.2 + The creation of a :class:`LogRecord` has been made more configurable by + providing a factory which is used to create the record. The factory can be + set using :func:`getLogRecordFactory` and :func:`setLogRecordFactory` + (see this for the factory's signature). + + This functionality can be used to inject your own values into a + :class:`LogRecord` at creation time. You can use the following pattern:: + + old_factory = logging.getLogRecordFactory() + + def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.custom_attribute = 0xdecafbad + return record + + logging.setLogRecordFactory(record_factory) + + With this pattern, multiple factories could be chained, and as long + as they don't overwrite each other's attributes or unintentionally + overwrite the standard attributes listed above, there should be no + surprises. + + +.. _logrecord-attributes: + +LogRecord attributes +-------------------- + +The LogRecord has a number of attributes, most of which are derived from the +parameters to the constructor. (Note that the names do not always correspond +exactly between the LogRecord constructor parameters and the LogRecord +attributes.) These attributes can be used to merge data from the record into +the format string. The following table lists (in alphabetical order) the +attribute names, their meanings and the corresponding placeholder in a %-style +format string. + +If you are using {}-formatting (:func:`str.format`), you can use +``{attrname}`` as the placeholder in the format string. If you are using +$-formatting (:class:`string.Template`), use the form ``${attrname}``. In +both cases, of course, replace ``attrname`` with the actual attribute name +you want to use. + +In the case of {}-formatting, you can specify formatting flags by placing them +after the attribute name, separated from it with a colon. For example: a +placeholder of ``{msecs:03d}`` would format a millisecond value of ``4`` as +``004``. Refer to the :meth:`str.format` documentation for full details on +the options available to you. + ++----------------+-------------------------+-----------------------------------------------+ +| Attribute name | Format | Description | ++================+=========================+===============================================+ +| args | You shouldn't need to | The tuple of arguments merged into ``msg`` to | +| | format this yourself. | produce ``message``, or a dict whose values | +| | | are used for the merge (when there is only one| +| | | argument, and it is a dictionary). | ++----------------+-------------------------+-----------------------------------------------+ +| asctime | ``%(asctime)s`` | Human-readable time when the | +| | | :class:`LogRecord` was created. By default | +| | | this is of the form '2003-07-08 16:49:45,896' | +| | | (the numbers after the comma are millisecond | +| | | portion of the time). | ++----------------+-------------------------+-----------------------------------------------+ +| created | ``%(created)f`` | Time when the :class:`LogRecord` was created | +| | | (as returned by :func:`time.time`). | ++----------------+-------------------------+-----------------------------------------------+ +| exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | +| | format this yourself. | if no exception has occurred, ``None``. | ++----------------+-------------------------+-----------------------------------------------+ +| filename | ``%(filename)s`` | Filename portion of ``pathname``. | ++----------------+-------------------------+-----------------------------------------------+ +| funcName | ``%(funcName)s`` | Name of function containing the logging call. | ++----------------+-------------------------+-----------------------------------------------+ +| levelname | ``%(levelname)s`` | Text logging level for the message | +| | | (``'DEBUG'``, ``'INFO'``, ``'WARNING'``, | +| | | ``'ERROR'``, ``'CRITICAL'``). | ++----------------+-------------------------+-----------------------------------------------+ +| levelno | ``%(levelno)s`` | Numeric logging level for the message | +| | | (:const:`DEBUG`, :const:`INFO`, | +| | | :const:`WARNING`, :const:`ERROR`, | +| | | :const:`CRITICAL`). | ++----------------+-------------------------+-----------------------------------------------+ +| lineno | ``%(lineno)d`` | Source line number where the logging call was | +| | | issued (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| message | ``%(message)s`` | The logged message, computed as ``msg % | +| | | args``. This is set when | +| | | :meth:`Formatter.format` is invoked. | ++----------------+-------------------------+-----------------------------------------------+ +| module | ``%(module)s`` | Module (name portion of ``filename``). | ++----------------+-------------------------+-----------------------------------------------+ +| msecs | ``%(msecs)d`` | Millisecond portion of the time when the | +| | | :class:`LogRecord` was created. | ++----------------+-------------------------+-----------------------------------------------+ +| msg | You shouldn't need to | The format string passed in the original | +| | format this yourself. | logging call. Merged with ``args`` to | +| | | produce ``message``, or an arbitrary object | +| | | (see :ref:`arbitrary-object-messages`). | ++----------------+-------------------------+-----------------------------------------------+ +| name | ``%(name)s`` | Name of the logger used to log the call. | ++----------------+-------------------------+-----------------------------------------------+ +| pathname | ``%(pathname)s`` | Full pathname of the source file where the | +| | | logging call was issued (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| process | ``%(process)d`` | Process ID (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| processName | ``%(processName)s`` | Process name (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| relativeCreated| ``%(relativeCreated)d`` | Time in milliseconds when the LogRecord was | +| | | created, relative to the time the logging | +| | | module was loaded. | ++----------------+-------------------------+-----------------------------------------------+ +| stack_info | You shouldn't need to | Stack frame information (where available) | +| | format this yourself. | from the bottom of the stack in the current | +| | | thread, up to and including the stack frame | +| | | of the logging call which resulted in the | +| | | creation of this record. | ++----------------+-------------------------+-----------------------------------------------+ +| thread | ``%(thread)d`` | Thread ID (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| threadName | ``%(threadName)s`` | Thread name (if available). | ++----------------+-------------------------+-----------------------------------------------+ + +.. versionchanged:: 3.1 + *processName* was added. + + +.. _logger-adapter: + +LoggerAdapter Objects +--------------------- + +:class:`LoggerAdapter` instances are used to conveniently pass contextual +information into logging calls. For a usage example, see the section on +:ref:`adding contextual information to your logging output `. + +.. class:: LoggerAdapter(logger, extra) + + Returns an instance of :class:`LoggerAdapter` initialized with an + underlying :class:`Logger` instance and a dict-like object. + + .. method:: process(msg, kwargs) + + Modifies the message and/or keyword arguments passed to a logging call in + order to insert contextual information. This implementation takes the object + passed as *extra* to the constructor and adds it to *kwargs* using key + 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the + (possibly modified) versions of the arguments passed in. + +In addition to the above, :class:`LoggerAdapter` supports the following +methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, +:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, +:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, +:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and +:meth:`~Logger.hasHandlers`. These methods have the same signatures as their +counterparts in :class:`Logger`, so you can use the two types of instances +interchangeably. + +.. versionchanged:: 3.2 + The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, + :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added + to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + + +Thread Safety +------------- + +The logging module is intended to be thread-safe without any special work +needing to be done by its clients. It achieves this though using threading +locks; there is one lock to serialize access to the module's shared data, and +each handler also creates a lock to serialize access to its underlying I/O. + +If you are implementing asynchronous signal handlers using the :mod:`signal` +module, you may not be able to use logging from within such handlers. This is +because lock implementations in the :mod:`threading` module are not always +re-entrant, and so cannot be invoked from such signal handlers. + + +Module-Level Functions +---------------------- + +In addition to the classes described above, there are a number of module-level +functions. + + +.. function:: getLogger(name=None) + + Return a logger with the specified name or, if name is ``None``, return a + logger which is the root logger of the hierarchy. If specified, the name is + typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. + Choice of these names is entirely up to the developer who is using logging. + + All calls to this function with a given name return the same logger instance. + This means that logger instances never need to be passed between different parts + of an application. + + +.. function:: getLoggerClass() + + Return either the standard :class:`Logger` class, or the last class passed to + :func:`setLoggerClass`. This function may be called from within a new class + definition, to ensure that installing a customized :class:`Logger` class will + not undo customizations already applied by other code. For example:: + + class MyLogger(logging.getLoggerClass()): + # ... override behaviour here + + +.. function:: getLogRecordFactory() + + Return a callable which is used to create a :class:`LogRecord`. + + .. versionadded:: 3.2 + This function has been provided, along with :func:`setLogRecordFactory`, + to allow developers more control over how the :class:`LogRecord` + representing a logging event is constructed. + + See :func:`setLogRecordFactory` for more information about the how the + factory is called. + +.. function:: debug(msg, *args, **kwargs) + + Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the + message format string, and the *args* are the arguments which are merged into + *msg* using the string formatting operator. (Note that this means that you can + use keywords in the format string, together with a single dictionary argument.) + + There are three keyword arguments in *kwargs* which are inspected: *exc_info* + which, if it does not evaluate as false, causes exception information to be + added to the logging message. If an exception tuple (in the format returned by + :func:`sys.exc_info`) or an exception instance is provided, it is used; + otherwise, :func:`sys.exc_info` is called to get the exception information. + + The second optional keyword argument is *stack_info*, which defaults to + ``False``. If true, stack information is added to the logging + message, including the actual logging call. Note that this is not the same + stack information as that displayed through specifying *exc_info*: The + former is stack frames from the bottom of the stack up to the logging call + in the current thread, whereas the latter is information about stack frames + which have been unwound, following an exception, while searching for + exception handlers. + + You can specify *stack_info* independently of *exc_info*, e.g. to just show + how you got to a certain point in your code, even when no exceptions were + raised. The stack frames are printed following a header line which says: + + .. code-block:: none + + Stack (most recent call last): + + This mimics the ``Traceback (most recent call last):`` which is used when + displaying exception frames. + + The third optional keyword argument is *extra* which can be used to pass a + dictionary which is used to populate the __dict__ of the LogRecord created for + the logging event with user-defined attributes. These custom attributes can then + be used as you like. For example, they could be incorporated into logged + messages. For example:: + + FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' + logging.basicConfig(format=FORMAT) + d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} + logging.warning('Protocol problem: %s', 'connection reset', extra=d) + + would print something like: + + .. code-block:: none + + 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset + + The keys in the dictionary passed in *extra* should not clash with the keys used + by the logging system. (See the :class:`Formatter` documentation for more + information on which keys are used by the logging system.) + + If you choose to use these attributes in logged messages, you need to exercise + some care. In the above example, for instance, the :class:`Formatter` has been + set up with a format string which expects 'clientip' and 'user' in the attribute + dictionary of the LogRecord. If these are missing, the message will not be + logged because a string formatting exception will occur. So in this case, you + always need to pass the *extra* dictionary with these keys. + + While this might be annoying, this feature is intended for use in specialized + circumstances, such as multi-threaded servers where the same code executes in + many contexts, and interesting conditions which arise are dependent on this + context (such as remote client IP address and authenticated user name, in the + above example). In such circumstances, it is likely that specialized + :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. + + .. versionchanged:: 3.2 + The *stack_info* parameter was added. + +.. function:: info(msg, *args, **kwargs) + + Logs a message with level :const:`INFO` on the root logger. The arguments are + interpreted as for :func:`debug`. + + +.. function:: warning(msg, *args, **kwargs) + + Logs a message with level :const:`WARNING` on the root logger. The arguments + are interpreted as for :func:`debug`. + + .. note:: There is an obsolete function ``warn`` which is functionally + identical to ``warning``. As ``warn`` is deprecated, please do not use + it - use ``warning`` instead. + + +.. function:: error(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on the root logger. The arguments are + interpreted as for :func:`debug`. + + +.. function:: critical(msg, *args, **kwargs) + + Logs a message with level :const:`CRITICAL` on the root logger. The arguments + are interpreted as for :func:`debug`. + + +.. function:: exception(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on the root logger. The arguments are + interpreted as for :func:`debug`. Exception info is added to the logging + message. This function should only be called from an exception handler. + +.. function:: log(level, msg, *args, **kwargs) + + Logs a message with level *level* on the root logger. The other arguments are + interpreted as for :func:`debug`. + + .. note:: The above module-level convenience functions, which delegate to the + root logger, call :func:`basicConfig` to ensure that at least one handler + is available. Because of this, they should *not* be used in threads, + in versions of Python earlier than 2.7.1 and 3.2, unless at least one + handler has been added to the root logger *before* the threads are + started. In earlier versions of Python, due to a thread safety shortcoming + in :func:`basicConfig`, this can (under rare circumstances) lead to + handlers being added multiple times to the root logger, which can in turn + lead to multiple messages for the same event. + +.. function:: disable(level=CRITICAL) + + Provides an overriding level *level* for all loggers which takes precedence over + the logger's own level. When the need arises to temporarily throttle logging + output down across the whole application, this function can be useful. Its + effect is to disable all logging calls of severity *level* and below, so that + if you call it with a value of INFO, then all INFO and DEBUG events would be + discarded, whereas those of severity WARNING and above would be processed + according to the logger's effective level. If + ``logging.disable(logging.NOTSET)`` is called, it effectively removes this + overriding level, so that logging output again depends on the effective + levels of individual loggers. + + Note that if you have defined any custom logging level higher than + ``CRITICAL`` (this is not recommended), you won't be able to rely on the + default value for the *level* parameter, but will have to explicitly supply a + suitable value. + + .. versionchanged:: 3.7 + The *level* parameter was defaulted to level ``CRITICAL``. See + :issue:`28524` for more information about this change. + +.. function:: addLevelName(level, levelName) + + Associates level *level* with text *levelName* in an internal dictionary, which is + used to map numeric levels to a textual representation, for example when a + :class:`Formatter` formats a message. This function can also be used to define + your own levels. The only constraints are that all levels used must be + registered using this function, levels should be positive integers and they + should increase in increasing order of severity. + + .. note:: If you are thinking of defining your own levels, please see the + section on :ref:`custom-levels`. + +.. function:: getLevelName(level) + + Returns the textual representation of logging level *level*. If the level is one + of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, + :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you + have associated levels with names using :func:`addLevelName` then the name you + have associated with *level* is returned. If a numeric value corresponding to one + of the defined levels is passed in, the corresponding string representation is + returned. Otherwise, the string 'Level %s' % level is returned. + + .. note:: Levels are internally integers (as they need to be compared in the + logging logic). This function is used to convert between an integer level + and the level name displayed in the formatted log output by means of the + ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). + + .. versionchanged:: 3.4 + In Python versions earlier than 3.4, this function could also be passed a + text level, and would return the corresponding numeric value of the level. + This undocumented behaviour was considered a mistake, and was removed in + Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. + +.. function:: makeLogRecord(attrdict) + + Creates and returns a new :class:`LogRecord` instance whose attributes are + defined by *attrdict*. This function is useful for taking a pickled + :class:`LogRecord` attribute dictionary, sent over a socket, and reconstituting + it as a :class:`LogRecord` instance at the receiving end. + + +.. function:: basicConfig(**kwargs) + + Does basic configuration for the logging system by creating a + :class:`StreamHandler` with a default :class:`Formatter` and adding it to the + root logger. The functions :func:`debug`, :func:`info`, :func:`warning`, + :func:`error` and :func:`critical` will call :func:`basicConfig` automatically + if no handlers are defined for the root logger. + + This function does nothing if the root logger already has handlers + configured, unless the keyword argument *force* is set to ``True``. + + .. note:: This function should be called from the main thread + before other threads are started. In versions of Python prior to + 2.7.1 and 3.2, if this function is called from multiple threads, + it is possible (in rare circumstances) that a handler will be added + to the root logger more than once, leading to unexpected results + such as messages being duplicated in the log. + + The following keyword arguments are supported. + + .. tabularcolumns:: |l|L| + + +--------------+---------------------------------------------+ + | Format | Description | + +==============+=============================================+ + | *filename* | Specifies that a :class:`FileHandler` be | + | | created, using the specified filename, | + | | rather than a :class:`StreamHandler`. | + +--------------+---------------------------------------------+ + | *filemode* | If *filename* is specified, open the file | + | | in this :ref:`mode `. Defaults | + | | to ``'a'``. | + +--------------+---------------------------------------------+ + | *format* | Use the specified format string for the | + | | handler. Defaults to attributes | + | | ``levelname``, ``name`` and ``message`` | + | | separated by colons. | + +--------------+---------------------------------------------+ + | *datefmt* | Use the specified date/time format, as | + | | accepted by :func:`time.strftime`. | + +--------------+---------------------------------------------+ + | *style* | If *format* is specified, use this style | + | | for the format string. One of ``'%'``, | + | | ``'{'`` or ``'$'`` for :ref:`printf-style | + | | `, | + | | :meth:`str.format` or | + | | :class:`string.Template` respectively. | + | | Defaults to ``'%'``. | + +--------------+---------------------------------------------+ + | *level* | Set the root logger level to the specified | + | | :ref:`level `. | + +--------------+---------------------------------------------+ + | *stream* | Use the specified stream to initialize the | + | | :class:`StreamHandler`. Note that this | + | | argument is incompatible with *filename* - | + | | if both are present, a ``ValueError`` is | + | | raised. | + +--------------+---------------------------------------------+ + | *handlers* | If specified, this should be an iterable of | + | | already created handlers to add to the root | + | | logger. Any handlers which don't already | + | | have a formatter set will be assigned the | + | | default formatter created in this function. | + | | Note that this argument is incompatible | + | | with *filename* or *stream* - if both | + | | are present, a ``ValueError`` is raised. | + +--------------+---------------------------------------------+ + | *force* | If this keyword argument is specified as | + | | true, any existing handlers attached to the | + | | root logger are removed and closed, before | + | | carrying out the configuration as specified | + | | by the other arguments. | + +--------------+---------------------------------------------+ + | *encoding* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | :class:`FileHandler` is created, and thus | + | | used when opening the output file. | + +--------------+---------------------------------------------+ + | *errors* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | :class:`FileHandler` is created, and thus | + | | used when opening the output file. If not | + | | specified, the value 'backslashreplace' is | + | | used. Note that if ``None`` is specified, | + | | it will be passed as such to :func:`open`, | + | | which means that it will be treated the | + | | same as passing 'errors'. | + +--------------+---------------------------------------------+ + + .. versionchanged:: 3.2 + The *style* argument was added. + + .. versionchanged:: 3.3 + The *handlers* argument was added. Additional checks were added to + catch situations where incompatible arguments are specified (e.g. + *handlers* together with *stream* or *filename*, or *stream* + together with *filename*). + + .. versionchanged:: 3.8 + The *force* argument was added. + + .. versionchanged:: 3.9 + The *encoding* and *errors* arguments were added. + +.. function:: shutdown() + + Informs the logging system to perform an orderly shutdown by flushing and + closing all handlers. This should be called at application exit and no + further use of the logging system should be made after this call. + + When the logging module is imported, it registers this function as an exit + handler (see :mod:`atexit`), so normally there's no need to do that + manually. + + +.. function:: setLoggerClass(klass) + + Tells the logging system to use the class *klass* when instantiating a logger. + The class should define :meth:`__init__` such that only a name argument is + required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This + function is typically called before any loggers are instantiated by applications + which need to use custom logger behavior. After this call, as at any other + time, do not instantiate loggers directly using the subclass: continue to use + the :func:`logging.getLogger` API to get your loggers. + + +.. function:: setLogRecordFactory(factory) + + Set a callable which is used to create a :class:`LogRecord`. + + :param factory: The factory callable to be used to instantiate a log record. + + .. versionadded:: 3.2 + This function has been provided, along with :func:`getLogRecordFactory`, to + allow developers more control over how the :class:`LogRecord` representing + a logging event is constructed. + + The factory has the following signature: + + ``factory(name, level, fn, lno, msg, args, exc_info, func=None, sinfo=None, **kwargs)`` + + :name: The logger name. + :level: The logging level (numeric). + :fn: The full pathname of the file where the logging call was made. + :lno: The line number in the file where the logging call was made. + :msg: The logging message. + :args: The arguments for the logging message. + :exc_info: An exception tuple, or ``None``. + :func: The name of the function or method which invoked the logging + call. + :sinfo: A stack traceback such as is provided by + :func:`traceback.print_stack`, showing the call hierarchy. + :kwargs: Additional keyword arguments. + + +Module-Level Attributes +----------------------- + +.. attribute:: lastResort + + A "handler of last resort" is available through this attribute. This + is a :class:`StreamHandler` writing to ``sys.stderr`` with a level of + ``WARNING``, and is used to handle logging events in the absence of any + logging configuration. The end result is to just print the message to + ``sys.stderr``. This replaces the earlier error message saying that + "no handlers could be found for logger XYZ". If you need the earlier + behaviour for some reason, ``lastResort`` can be set to ``None``. + + .. versionadded:: 3.2 + +Integration with the warnings module +------------------------------------ + +The :func:`captureWarnings` function can be used to integrate :mod:`logging` +with the :mod:`warnings` module. + +.. function:: captureWarnings(capture) + + This function is used to turn the capture of warnings by logging on and + off. + + If *capture* is ``True``, warnings issued by the :mod:`warnings` module will + be redirected to the logging system. Specifically, a warning will be + formatted using :func:`warnings.formatwarning` and the resulting string + logged to a logger named ``'py.warnings'`` with a severity of :const:`WARNING`. + + If *capture* is ``False``, the redirection of warnings to the logging system + will stop, and warnings will be redirected to their original destinations + (i.e. those in effect before ``captureWarnings(True)`` was called). + + +.. seealso:: + + Module :mod:`logging.config` + Configuration API for the logging module. + + Module :mod:`logging.handlers` + Useful handlers included with the logging module. + + :pep:`282` - A Logging System + The proposal which described this feature for inclusion in the Python standard + library. + + `Original Python logging package `_ + This is the original source for the :mod:`logging` package. The version of the + package available from this site is suitable for use with Python 1.5.2, 2.1.x + and 2.2.x, which do not include the :mod:`logging` package in the standard + library. diff --git a/Doc/library/readline.rst.bak b/Doc/library/readline.rst.bak new file mode 100644 index 00000000000000..3ff64885f7fced --- /dev/null +++ b/Doc/library/readline.rst.bak @@ -0,0 +1,361 @@ +:mod:`readline` --- GNU readline interface +========================================== + +.. module:: readline + :platform: Unix + :synopsis: GNU readline support for Python. + +.. sectionauthor:: Skip Montanaro + +-------------- + +The :mod:`readline` module defines a number of functions to facilitate +completion and reading/writing of history files from the Python interpreter. +This module can be used directly, or via the :mod:`rlcompleter` module, which +supports completion of Python identifiers at the interactive prompt. Settings +made using this module affect the behaviour of both the interpreter's +interactive prompt and the prompts offered by the built-in :func:`input` +function. + +Readline keybindings may be configured via an initialization file, typically +``.inputrc`` in your home directory. See `Readline Init File +`_ +in the GNU Readline manual for information about the format and +allowable constructs of that file, and the capabilities of the +Readline library in general. + +.. note:: + + The underlying Readline library API may be implemented by + the ``libedit`` library instead of GNU readline. + On macOS the :mod:`readline` module detects which library is being used + at run time. + + The configuration file for ``libedit`` is different from that + of GNU readline. If you programmatically load configuration strings + you can check for the text "libedit" in :const:`readline.__doc__` + to differentiate between GNU readline and libedit. + + If you use *editline*/``libedit`` readline emulation on macOS, the + initialization file located in your home directory is named + ``.editrc``. For example, the following content in ``~/.editrc`` will + turn ON *vi* keybindings and TAB completion:: + + python:bind -v + python:bind ^I rl_complete + + +Init file +--------- + +The following functions relate to the init file and user configuration: + + +.. function:: parse_and_bind(string) + + Execute the init line provided in the *string* argument. This calls + :c:func:`rl_parse_and_bind` in the underlying library. + + +.. function:: read_init_file([filename]) + + Execute a readline initialization file. The default filename is the last filename + used. This calls :c:func:`rl_read_init_file` in the underlying library. + + +Line buffer +----------- + +The following functions operate on the line buffer: + + +.. function:: get_line_buffer() + + Return the current contents of the line buffer (:c:data:`rl_line_buffer` + in the underlying library). + + +.. function:: insert_text(string) + + Insert text into the line buffer at the cursor position. This calls + :c:func:`rl_insert_text` in the underlying library, but ignores + the return value. + + +.. function:: redisplay() + + Change what's displayed on the screen to reflect the current contents of the + line buffer. This calls :c:func:`rl_redisplay` in the underlying library. + + +History file +------------ + +The following functions operate on a history file: + + +.. function:: read_history_file([filename]) + + Load a readline history file, and append it to the history list. + The default filename is :file:`~/.history`. This calls + :c:func:`read_history` in the underlying library. + + +.. function:: write_history_file([filename]) + + Save the history list to a readline history file, overwriting any + existing file. The default filename is :file:`~/.history`. This calls + :c:func:`write_history` in the underlying library. + + +.. function:: append_history_file(nelements[, filename]) + + Append the last *nelements* items of history to a file. The default filename is + :file:`~/.history`. The file must already exist. This calls + :c:func:`append_history` in the underlying library. This function + only exists if Python was compiled for a version of the library + that supports it. + + .. versionadded:: 3.5 + + +.. function:: get_history_length() + set_history_length(length) + + Set or return the desired number of lines to save in the history file. + The :func:`write_history_file` function uses this value to truncate + the history file, by calling :c:func:`history_truncate_file` in + the underlying library. Negative values imply + unlimited history file size. + + +History list +------------ + +The following functions operate on a global history list: + + +.. function:: clear_history() + + Clear the current history. This calls :c:func:`clear_history` in the + underlying library. The Python function only exists if Python was + compiled for a version of the library that supports it. + + +.. function:: get_current_history_length() + + Return the number of items currently in the history. (This is different from + :func:`get_history_length`, which returns the maximum number of lines that will + be written to a history file.) + + +.. function:: get_history_item(index) + + Return the current contents of history item at *index*. The item index + is one-based. This calls :c:func:`history_get` in the underlying library. + + +.. function:: remove_history_item(pos) + + Remove history item specified by its position from the history. + The position is zero-based. This calls :c:func:`remove_history` in + the underlying library. + + +.. function:: replace_history_item(pos, line) + + Replace history item specified by its position with *line*. + The position is zero-based. This calls :c:func:`replace_history_entry` + in the underlying library. + + +.. function:: add_history(line) + + Append *line* to the history buffer, as if it was the last line typed. + This calls :c:func:`add_history` in the underlying library. + + +.. function:: set_auto_history(enabled) + + Enable or disable automatic calls to :c:func:`add_history` when reading + input via readline. The *enabled* argument should be a Boolean value + that when true, enables auto history, and that when false, disables + auto history. + + .. versionadded:: 3.6 + + .. impl-detail:: + Auto history is enabled by default, and changes to this do not persist + across multiple sessions. + + +Startup hooks +------------- + + +.. function:: set_startup_hook([function]) + + Set or remove the function invoked by the :c:data:`rl_startup_hook` + callback of the underlying library. If *function* is specified, it will + be used as the new hook function; if omitted or ``None``, any function + already installed is removed. The hook is called with no + arguments just before readline prints the first prompt. + + +.. function:: set_pre_input_hook([function]) + + Set or remove the function invoked by the :c:data:`rl_pre_input_hook` + callback of the underlying library. If *function* is specified, it will + be used as the new hook function; if omitted or ``None``, any + function already installed is removed. The hook is called + with no arguments after the first prompt has been printed and just before + readline starts reading input characters. This function only exists + if Python was compiled for a version of the library that supports it. + + +Completion +---------- + +The following functions relate to implementing a custom word completion +function. This is typically operated by the Tab key, and can suggest and +automatically complete a word being typed. By default, Readline is set up +to be used by :mod:`rlcompleter` to complete Python identifiers for +the interactive interpreter. If the :mod:`readline` module is to be used +with a custom completer, a different set of word delimiters should be set. + + +.. function:: set_completer([function]) + + Set or remove the completer function. If *function* is specified, it will be + used as the new completer function; if omitted or ``None``, any completer + function already installed is removed. The completer function is called as + ``function(text, state)``, for *state* in ``0``, ``1``, ``2``, ..., until it + returns a non-string value. It should return the next possible completion + starting with *text*. + + The installed completer function is invoked by the *entry_func* callback + passed to :c:func:`rl_completion_matches` in the underlying library. + The *text* string comes from the first parameter to the + :c:data:`rl_attempted_completion_function` callback of the + underlying library. + + +.. function:: get_completer() + + Get the completer function, or ``None`` if no completer function has been set. + + +.. function:: get_completion_type() + + Get the type of completion being attempted. This returns the + :c:data:`rl_completion_type` variable in the underlying library as + an integer. + + +.. function:: get_begidx() + get_endidx() + + Get the beginning or ending index of the completion scope. + These indexes are the *start* and *end* arguments passed to the + :c:data:`rl_attempted_completion_function` callback of the + underlying library. The values may be different in the same + input editing scenario based on the underlying C readline implemtation. + Ex: libedit is known to behave differently than libreadline. + + +.. function:: set_completer_delims(string) + get_completer_delims() + + Set or get the word delimiters for completion. These determine the + start of the word to be considered for completion (the completion scope). + These functions access the :c:data:`rl_completer_word_break_characters` + variable in the underlying library. + + +.. function:: set_completion_display_matches_hook([function]) + + Set or remove the completion display function. If *function* is + specified, it will be used as the new completion display function; + if omitted or ``None``, any completion display function already + installed is removed. This sets or clears the + :c:data:`rl_completion_display_matches_hook` callback in the + underlying library. The completion display function is called as + ``function(substitution, [matches], longest_match_length)`` once + each time matches need to be displayed. + + +.. _readline-example: + +Example +------- + +The following example demonstrates how to use the :mod:`readline` module's +history reading and writing functions to automatically load and save a history +file named :file:`.python_history` from the user's home directory. The code +below would normally be executed automatically during interactive sessions +from the user's :envvar:`PYTHONSTARTUP` file. :: + + import atexit + import os + import readline + + histfile = os.path.join(os.path.expanduser("~"), ".python_history") + try: + readline.read_history_file(histfile) + # default history len is -1 (infinite), which may grow unruly + readline.set_history_length(1000) + except FileNotFoundError: + pass + + atexit.register(readline.write_history_file, histfile) + +This code is actually automatically run when Python is run in +:ref:`interactive mode ` (see :ref:`rlcompleter-config`). + +The following example achieves the same goal but supports concurrent interactive +sessions, by only appending the new history. :: + + import atexit + import os + import readline + histfile = os.path.join(os.path.expanduser("~"), ".python_history") + + try: + readline.read_history_file(histfile) + h_len = readline.get_current_history_length() + except FileNotFoundError: + open(histfile, 'wb').close() + h_len = 0 + + def save(prev_h_len, histfile): + new_h_len = readline.get_current_history_length() + readline.set_history_length(1000) + readline.append_history_file(new_h_len - prev_h_len, histfile) + atexit.register(save, h_len, histfile) + +The following example extends the :class:`code.InteractiveConsole` class to +support history save/restore. :: + + import atexit + import code + import os + import readline + + class HistoryConsole(code.InteractiveConsole): + def __init__(self, locals=None, filename="", + histfile=os.path.expanduser("~/.console-history")): + code.InteractiveConsole.__init__(self, locals, filename) + self.init_history(histfile) + + def init_history(self, histfile): + readline.parse_and_bind("tab: complete") + if hasattr(readline, "read_history_file"): + try: + readline.read_history_file(histfile) + except FileNotFoundError: + pass + atexit.register(self.save_history, histfile) + + def save_history(self, histfile): + readline.set_history_length(1000) + readline.write_history_file(histfile) diff --git a/Doc/library/sqlite3.rst.bak b/Doc/library/sqlite3.rst.bak new file mode 100644 index 00000000000000..172ce6c6bb03ba --- /dev/null +++ b/Doc/library/sqlite3.rst.bak @@ -0,0 +1,1094 @@ +:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases +============================================================ + +.. module:: sqlite3 + :synopsis: A DB-API 2.0 implementation using SQLite 3.x. + +.. sectionauthor:: Gerhard Häring + +**Source code:** :source:`Lib/sqlite3/` + +-------------- + +SQLite is a C library that provides a lightweight disk-based database that +doesn't require a separate server process and allows accessing the database +using a nonstandard variant of the SQL query language. Some applications can use +SQLite for internal data storage. It's also possible to prototype an +application using SQLite and then port the code to a larger database such as +PostgreSQL or Oracle. + +The sqlite3 module was written by Gerhard Häring. It provides a SQL interface +compliant with the DB-API 2.0 specification described by :pep:`249`, and +requires SQLite 3.7.15 or newer. + +To use the module, you must first create a :class:`Connection` object that +represents the database. Here the data will be stored in the +:file:`example.db` file:: + + import sqlite3 + con = sqlite3.connect('example.db') + +You can also supply the special name ``:memory:`` to create a database in RAM. + +Once you have a :class:`Connection`, you can create a :class:`Cursor` object +and call its :meth:`~Cursor.execute` method to perform SQL commands:: + + cur = con.cursor() + + # Create table + cur.execute('''CREATE TABLE stocks + (date text, trans text, symbol text, qty real, price real)''') + + # Insert a row of data + cur.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") + + # Save (commit) the changes + con.commit() + + # We can also close the connection if we are done with it. + # Just be sure any changes have been committed or they will be lost. + con.close() + +The data you've saved is persistent and is available in subsequent sessions:: + + import sqlite3 + con = sqlite3.connect('example.db') + cur = con.cursor() + +Usually your SQL operations will need to use values from Python variables. You +shouldn't assemble your query using Python's string operations because doing so +is insecure; it makes your program vulnerable to an SQL injection attack +(see https://xkcd.com/327/ for humorous example of what can go wrong). + +Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder +wherever you want to use a value, and then provide a tuple of values as the +second argument to the cursor's :meth:`~Cursor.execute` method. (Other database +modules may use a different placeholder, such as ``%s`` or ``:1``.) For +example:: + + # Never do this -- insecure! + symbol = 'RHAT' + cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) + + # Do this instead + t = ('RHAT',) + cur.execute('SELECT * FROM stocks WHERE symbol=?', t) + print(cur.fetchone()) + + # Larger example that inserts many records at a time + purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), + ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), + ('2006-04-06', 'SELL', 'IBM', 500, 53.00), + ] + cur.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) + +To retrieve data after executing a SELECT statement, you can either treat the +cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to +retrieve a single matching row, or call :meth:`~Cursor.fetchall` to get a list of the +matching rows. + +This example uses the iterator form:: + + >>> for row in cur.execute('SELECT * FROM stocks ORDER BY price'): + print(row) + + ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) + ('2006-03-28', 'BUY', 'IBM', 1000, 45.0) + ('2006-04-06', 'SELL', 'IBM', 500, 53.0) + ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) + + +.. seealso:: + + https://www.sqlite.org + The SQLite web page; the documentation describes the syntax and the + available data types for the supported SQL dialect. + + https://www.w3schools.com/sql/ + Tutorial, reference and examples for learning SQL syntax. + + :pep:`249` - Database API Specification 2.0 + PEP written by Marc-André Lemburg. + + +.. _sqlite3-module-contents: + +Module functions and constants +------------------------------ + + +.. data:: version + + The version number of this module, as a string. This is not the version of + the SQLite library. + + +.. data:: version_info + + The version number of this module, as a tuple of integers. This is not the + version of the SQLite library. + + +.. data:: sqlite_version + + The version number of the run-time SQLite library, as a string. + + +.. data:: sqlite_version_info + + The version number of the run-time SQLite library, as a tuple of integers. + + +.. data:: PARSE_DECLTYPES + + This constant is meant to be used with the *detect_types* parameter of the + :func:`connect` function. + + Setting it makes the :mod:`sqlite3` module parse the declared type for each + column it returns. It will parse out the first word of the declared type, + i. e. for "integer primary key", it will parse out "integer", or for + "number(10)" it will parse out "number". Then for that column, it will look + into the converters dictionary and use the converter function registered for + that type there. + + +.. data:: PARSE_COLNAMES + + This constant is meant to be used with the *detect_types* parameter of the + :func:`connect` function. + + Setting this makes the SQLite interface parse the column name for each column it + returns. It will look for a string formed [mytype] in there, and then decide + that 'mytype' is the type of the column. It will try to find an entry of + 'mytype' in the converters dictionary and then use the converter function found + there to return the value. The column name found in :attr:`Cursor.description` + does not include the type, i. e. if you use something like + ``'as "Expiration date [datetime]"'`` in your SQL, then we will parse out + everything until the first ``'['`` for the column name and strip + the preceeding space: the column name would simply be "Expiration date". + + +.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) + + Opens a connection to the SQLite database file *database*. By default returns a + :class:`Connection` object, unless a custom *factory* is given. + + *database* is a :term:`path-like object` giving the pathname (absolute or + relative to the current working directory) of the database file to be opened. + You can use ``":memory:"`` to open a database connection to a database that + resides in RAM instead of on disk. + + When a database is accessed by multiple connections, and one of the processes + modifies the database, the SQLite database is locked until that transaction is + committed. The *timeout* parameter specifies how long the connection should wait + for the lock to go away until raising an exception. The default for the timeout + parameter is 5.0 (five seconds). + + For the *isolation_level* parameter, please see the + :attr:`~Connection.isolation_level` property of :class:`Connection` objects. + + SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If + you want to use other types you must add support for them yourself. The + *detect_types* parameter and the using custom **converters** registered with the + module-level :func:`register_converter` function allow you to easily do that. + + *detect_types* defaults to 0 (i. e. off, no type detection), you can set it to + any combination of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to turn + type detection on. Due to SQLite behaviour, types can't be detected for generated + fields (for example ``max(data)``), even when *detect_types* parameter is set. In + such case, the returned type is :class:`str`. + + By default, *check_same_thread* is :const:`True` and only the creating thread may + use the connection. If set :const:`False`, the returned connection may be shared + across multiple threads. When using multiple threads with the same connection + writing operations should be serialized by the user to avoid data corruption. + + By default, the :mod:`sqlite3` module uses its :class:`Connection` class for the + connect call. You can, however, subclass the :class:`Connection` class and make + :func:`connect` use your class instead by providing your class for the *factory* + parameter. + + Consult the section :ref:`sqlite3-types` of this manual for details. + + The :mod:`sqlite3` module internally uses a statement cache to avoid SQL parsing + overhead. If you want to explicitly set the number of statements that are cached + for the connection, you can set the *cached_statements* parameter. The currently + implemented default is to cache 100 statements. + + If *uri* is true, *database* is interpreted as a URI. This allows you + to specify options. For example, to open a database in read-only mode + you can use:: + + db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) + + More information about this feature, including a list of recognized options, can + be found in the `SQLite URI documentation `_. + + .. audit-event:: sqlite3.connect database sqlite3.connect + + .. versionchanged:: 3.4 + Added the *uri* parameter. + + .. versionchanged:: 3.7 + *database* can now also be a :term:`path-like object`, not only a string. + + +.. function:: register_converter(typename, callable) + + Registers a callable to convert a bytestring from the database into a custom + Python type. The callable will be invoked for all database values that are of + the type *typename*. Confer the parameter *detect_types* of the :func:`connect` + function for how the type detection works. Note that *typename* and the name of + the type in your query are matched in case-insensitive manner. + + +.. function:: register_adapter(type, callable) + + Registers a callable to convert the custom Python type *type* into one of + SQLite's supported types. The callable *callable* accepts as single parameter + the Python value, and must return a value of the following types: int, + float, str or bytes. + + +.. function:: complete_statement(sql) + + Returns :const:`True` if the string *sql* contains one or more complete SQL + statements terminated by semicolons. It does not verify that the SQL is + syntactically correct, only that there are no unclosed string literals and the + statement is terminated by a semicolon. + + This can be used to build a shell for SQLite, as in the following example: + + + .. literalinclude:: ../includes/sqlite3/complete_statement.py + + +.. function:: enable_callback_tracebacks(flag) + + By default you will not get any tracebacks in user-defined functions, + aggregates, converters, authorizer callbacks etc. If you want to debug them, + you can call this function with *flag* set to ``True``. Afterwards, you will + get tracebacks from callbacks on ``sys.stderr``. Use :const:`False` to + disable the feature again. + + +.. _sqlite3-connection-objects: + +Connection Objects +------------------ + +.. class:: Connection + + A SQLite database connection has the following attributes and methods: + + .. attribute:: isolation_level + + Get or set the current default isolation level. :const:`None` for autocommit mode or + one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section + :ref:`sqlite3-controlling-transactions` for a more detailed explanation. + + .. attribute:: in_transaction + + :const:`True` if a transaction is active (there are uncommitted changes), + :const:`False` otherwise. Read-only attribute. + + .. versionadded:: 3.2 + + .. method:: cursor(factory=Cursor) + + The cursor method accepts a single optional parameter *factory*. If + supplied, this must be a callable returning an instance of :class:`Cursor` + or its subclasses. + + .. method:: commit() + + This method commits the current transaction. If you don't call this method, + anything you did since the last call to ``commit()`` is not visible from + other database connections. If you wonder why you don't see the data you've + written to the database, please check you didn't forget to call this method. + + .. method:: rollback() + + This method rolls back any changes to the database since the last call to + :meth:`commit`. + + .. method:: close() + + This closes the database connection. Note that this does not automatically + call :meth:`commit`. If you just close your database connection without + calling :meth:`commit` first, your changes will be lost! + + .. method:: execute(sql[, parameters]) + + This is a nonstandard shortcut that creates a cursor object by calling + the :meth:`~Connection.cursor` method, calls the cursor's + :meth:`~Cursor.execute` method with the *parameters* given, and returns + the cursor. + + .. method:: executemany(sql[, parameters]) + + This is a nonstandard shortcut that creates a cursor object by + calling the :meth:`~Connection.cursor` method, calls the cursor's + :meth:`~Cursor.executemany` method with the *parameters* given, and + returns the cursor. + + .. method:: executescript(sql_script) + + This is a nonstandard shortcut that creates a cursor object by + calling the :meth:`~Connection.cursor` method, calls the cursor's + :meth:`~Cursor.executescript` method with the given *sql_script*, and + returns the cursor. + + .. method:: create_function(name, num_params, func, *, deterministic=False) + + Creates a user-defined function that you can later use from within SQL + statements under the function name *name*. *num_params* is the number of + parameters the function accepts (if *num_params* is -1, the function may + take any number of arguments), and *func* is a Python callable that is + called as the SQL function. If *deterministic* is true, the created function + is marked as `deterministic `_, which + allows SQLite to perform additional optimizations. This flag is supported by + SQLite 3.8.3 or higher, :exc:`NotSupportedError` will be raised if used + with older versions. + + The function can return any of the types supported by SQLite: bytes, str, int, + float and ``None``. + + .. versionchanged:: 3.8 + The *deterministic* parameter was added. + + Example: + + .. literalinclude:: ../includes/sqlite3/md5func.py + + + .. method:: create_aggregate(name, num_params, aggregate_class) + + Creates a user-defined aggregate function. + + The aggregate class must implement a ``step`` method, which accepts the number + of parameters *num_params* (if *num_params* is -1, the function may take + any number of arguments), and a ``finalize`` method which will return the + final result of the aggregate. + + The ``finalize`` method can return any of the types supported by SQLite: + bytes, str, int, float and ``None``. + + Example: + + .. literalinclude:: ../includes/sqlite3/mysumaggr.py + + + .. method:: create_collation(name, callable) + + Creates a collation with the specified *name* and *callable*. The callable will + be passed two string arguments. It should return -1 if the first is ordered + lower than the second, 0 if they are ordered equal and 1 if the first is ordered + higher than the second. Note that this controls sorting (ORDER BY in SQL) so + your comparisons don't affect other SQL operations. + + Note that the callable will get its parameters as Python bytestrings, which will + normally be encoded in UTF-8. + + The following example shows a custom collation that sorts "the wrong way": + + .. literalinclude:: ../includes/sqlite3/collation_reverse.py + + To remove a collation, call ``create_collation`` with ``None`` as callable:: + + con.create_collation("reverse", None) + + + .. method:: interrupt() + + You can call this method from a different thread to abort any queries that might + be executing on the connection. The query will then abort and the caller will + get an exception. + + + .. method:: set_authorizer(authorizer_callback) + + This routine registers a callback. The callback is invoked for each attempt to + access a column of a table in the database. The callback should return + :const:`SQLITE_OK` if access is allowed, :const:`SQLITE_DENY` if the entire SQL + statement should be aborted with an error and :const:`SQLITE_IGNORE` if the + column should be treated as a NULL value. These constants are available in the + :mod:`sqlite3` module. + + The first argument to the callback signifies what kind of operation is to be + authorized. The second and third argument will be arguments or :const:`None` + depending on the first argument. The 4th argument is the name of the database + ("main", "temp", etc.) if applicable. The 5th argument is the name of the + inner-most trigger or view that is responsible for the access attempt or + :const:`None` if this access attempt is directly from input SQL code. + + Please consult the SQLite documentation about the possible values for the first + argument and the meaning of the second and third argument depending on the first + one. All necessary constants are available in the :mod:`sqlite3` module. + + + .. method:: set_progress_handler(handler, n) + + This routine registers a callback. The callback is invoked for every *n* + instructions of the SQLite virtual machine. This is useful if you want to + get called from SQLite during long-running operations, for example to update + a GUI. + + If you want to clear any previously installed progress handler, call the + method with :const:`None` for *handler*. + + Returning a non-zero value from the handler function will terminate the + currently executing query and cause it to raise an :exc:`OperationalError` + exception. + + + .. method:: set_trace_callback(trace_callback) + + Registers *trace_callback* to be called for each SQL statement that is + actually executed by the SQLite backend. + + The only argument passed to the callback is the statement (as string) that + is being executed. The return value of the callback is ignored. Note that + the backend does not only run statements passed to the :meth:`Cursor.execute` + methods. Other sources include the transaction management of the Python + module and the execution of triggers defined in the current database. + + Passing :const:`None` as *trace_callback* will disable the trace callback. + + .. versionadded:: 3.3 + + + .. method:: enable_load_extension(enabled) + + This routine allows/disallows the SQLite engine to load SQLite extensions + from shared libraries. SQLite extensions can define new functions, + aggregates or whole new virtual table implementations. One well-known + extension is the fulltext-search extension distributed with SQLite. + + Loadable extensions are disabled by default. See [#f1]_. + + .. versionadded:: 3.2 + + .. literalinclude:: ../includes/sqlite3/load_extension.py + + .. method:: load_extension(path) + + This routine loads a SQLite extension from a shared library. You have to + enable extension loading with :meth:`enable_load_extension` before you can + use this routine. + + Loadable extensions are disabled by default. See [#f1]_. + + .. versionadded:: 3.2 + + .. attribute:: row_factory + + You can change this attribute to a callable that accepts the cursor and the + original row as a tuple and will return the real result row. This way, you can + implement more advanced ways of returning results, such as returning an object + that can also access columns by name. + + Example: + + .. literalinclude:: ../includes/sqlite3/row_factory.py + + If returning a tuple doesn't suffice and you want name-based access to + columns, you should consider setting :attr:`row_factory` to the + highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both + index-based and case-insensitive name-based access to columns with almost no + memory overhead. It will probably be better than your own custom + dictionary-based approach or even a db_row based solution. + + .. XXX what's a db_row-based solution? + + + .. attribute:: text_factory + + Using this attribute you can control what objects are returned for the ``TEXT`` + data type. By default, this attribute is set to :class:`str` and the + :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to + return bytestrings instead, you can set it to :class:`bytes`. + + You can also set it to any other callable that accepts a single bytestring + parameter and returns the resulting object. + + See the following example code for illustration: + + .. literalinclude:: ../includes/sqlite3/text_factory.py + + + .. attribute:: total_changes + + Returns the total number of database rows that have been modified, inserted, or + deleted since the database connection was opened. + + + .. method:: iterdump + + Returns an iterator to dump the database in an SQL text format. Useful when + saving an in-memory database for later restoration. This function provides + the same capabilities as the :kbd:`.dump` command in the :program:`sqlite3` + shell. + + Example:: + + # Convert file existing_db.db to SQL dump file dump.sql + import sqlite3 + + con = sqlite3.connect('existing_db.db') + with open('dump.sql', 'w') as f: + for line in con.iterdump(): + f.write('%s\n' % line) + con.close() + + + .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) + + This method makes a backup of a SQLite database even while it's being accessed + by other clients, or concurrently by the same connection. The copy will be + written into the mandatory argument *target*, that must be another + :class:`Connection` instance. + + By default, or when *pages* is either ``0`` or a negative integer, the entire + database is copied in a single step; otherwise the method performs a loop + copying up to *pages* pages at a time. + + If *progress* is specified, it must either be ``None`` or a callable object that + will be executed at each iteration with three integer arguments, respectively + the *status* of the last iteration, the *remaining* number of pages still to be + copied and the *total* number of pages. + + The *name* argument specifies the database name that will be copied: it must be + a string containing either ``"main"``, the default, to indicate the main + database, ``"temp"`` to indicate the temporary database or the name specified + after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached + database. + + The *sleep* argument specifies the number of seconds to sleep by between + successive attempts to backup remaining pages, can be specified either as an + integer or a floating point value. + + Example 1, copy an existing database into another:: + + import sqlite3 + + def progress(status, remaining, total): + print(f'Copied {total-remaining} of {total} pages...') + + con = sqlite3.connect('existing_db.db') + bck = sqlite3.connect('backup.db') + with bck: + con.backup(bck, pages=1, progress=progress) + bck.close() + con.close() + + Example 2, copy an existing database into a transient copy:: + + import sqlite3 + + source = sqlite3.connect('existing_db.db') + dest = sqlite3.connect(':memory:') + source.backup(dest) + + .. versionadded:: 3.7 + + +.. _sqlite3-cursor-objects: + +Cursor Objects +-------------- + +.. class:: Cursor + + A :class:`Cursor` instance has the following attributes and methods. + + .. index:: single: ? (question mark); in SQL statements + .. index:: single: : (colon); in SQL statements + + .. method:: execute(sql[, parameters]) + + Executes an SQL statement. The SQL statement may be parameterized (i. e. + placeholders instead of SQL literals). The :mod:`sqlite3` module supports two + kinds of placeholders: question marks (qmark style) and named placeholders + (named style). + + Here's an example of both styles: + + .. literalinclude:: ../includes/sqlite3/execute_1.py + + :meth:`execute` will only execute a single SQL statement. If you try to execute + more than one statement with it, it will raise a :exc:`.Warning`. Use + :meth:`executescript` if you want to execute multiple SQL statements with one + call. + + + .. method:: executemany(sql, seq_of_parameters) + + Executes an SQL command against all parameter sequences or mappings found in + the sequence *seq_of_parameters*. The :mod:`sqlite3` module also allows + using an :term:`iterator` yielding parameters instead of a sequence. + + .. literalinclude:: ../includes/sqlite3/executemany_1.py + + Here's a shorter example using a :term:`generator`: + + .. literalinclude:: ../includes/sqlite3/executemany_2.py + + + .. method:: executescript(sql_script) + + This is a nonstandard convenience method for executing multiple SQL statements + at once. It issues a ``COMMIT`` statement first, then executes the SQL script it + gets as a parameter. + + *sql_script* can be an instance of :class:`str`. + + Example: + + .. literalinclude:: ../includes/sqlite3/executescript.py + + + .. method:: fetchone() + + Fetches the next row of a query result set, returning a single sequence, + or :const:`None` when no more data is available. + + + .. method:: fetchmany(size=cursor.arraysize) + + Fetches the next set of rows of a query result, returning a list. An empty + list is returned when no more rows are available. + + The number of rows to fetch per call is specified by the *size* parameter. + If it is not given, the cursor's arraysize determines the number of rows + to be fetched. The method should try to fetch as many rows as indicated by + the size parameter. If this is not possible due to the specified number of + rows not being available, fewer rows may be returned. + + Note there are performance considerations involved with the *size* parameter. + For optimal performance, it is usually best to use the arraysize attribute. + If the *size* parameter is used, then it is best for it to retain the same + value from one :meth:`fetchmany` call to the next. + + .. method:: fetchall() + + Fetches all (remaining) rows of a query result, returning a list. Note that + the cursor's arraysize attribute can affect the performance of this operation. + An empty list is returned when no rows are available. + + .. method:: close() + + Close the cursor now (rather than whenever ``__del__`` is called). + + The cursor will be unusable from this point forward; a :exc:`ProgrammingError` + exception will be raised if any operation is attempted with the cursor. + + .. attribute:: rowcount + + Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this + attribute, the database engine's own support for the determination of "rows + affected"/"rows selected" is quirky. + + For :meth:`executemany` statements, the number of modifications are summed up + into :attr:`rowcount`. + + As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in + case no ``executeXX()`` has been performed on the cursor or the rowcount of the + last operation is not determinable by the interface". This includes ``SELECT`` + statements because we cannot determine the number of rows a query produced + until all rows were fetched. + + .. attribute:: lastrowid + + This read-only attribute provides the rowid of the last modified row. It is + only set if you issued an ``INSERT`` or a ``REPLACE`` statement using the + :meth:`execute` method. For operations other than ``INSERT`` or + ``REPLACE`` or when :meth:`executemany` is called, :attr:`lastrowid` is + set to :const:`None`. + + If the ``INSERT`` or ``REPLACE`` statement failed to insert the previous + successful rowid is returned. + + .. versionchanged:: 3.6 + Added support for the ``REPLACE`` statement. + + .. attribute:: arraysize + + Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. + The default value is 1 which means a single row would be fetched per call. + + .. attribute:: description + + This read-only attribute provides the column names of the last query. To + remain compatible with the Python DB API, it returns a 7-tuple for each + column where the last six items of each tuple are :const:`None`. + + It is set for ``SELECT`` statements without any matching rows as well. + + .. attribute:: connection + + This read-only attribute provides the SQLite database :class:`Connection` + used by the :class:`Cursor` object. A :class:`Cursor` object created by + calling :meth:`con.cursor() ` will have a + :attr:`connection` attribute that refers to *con*:: + + >>> con = sqlite3.connect(":memory:") + >>> cur = con.cursor() + >>> cur.connection == con + True + +.. _sqlite3-row-objects: + +Row Objects +----------- + +.. class:: Row + + A :class:`Row` instance serves as a highly optimized + :attr:`~Connection.row_factory` for :class:`Connection` objects. + It tries to mimic a tuple in most of its features. + + It supports mapping access by column name and index, iteration, + representation, equality testing and :func:`len`. + + If two :class:`Row` objects have exactly the same columns and their + members are equal, they compare equal. + + .. method:: keys + + This method returns a list of column names. Immediately after a query, + it is the first member of each tuple in :attr:`Cursor.description`. + + .. versionchanged:: 3.5 + Added support of slicing. + +Let's assume we initialize a table as in the example given above:: + + con = sqlite3.connect(":memory:") + cur = con.cursor() + cur.execute('''create table stocks + (date text, trans text, symbol text, + qty real, price real)''') + cur.execute("""insert into stocks + values ('2006-01-05','BUY','RHAT',100,35.14)""") + con.commit() + cur.close() + +Now we plug :class:`Row` in:: + + >>> con.row_factory = sqlite3.Row + >>> cur = con.cursor() + >>> cur.execute('select * from stocks') + + >>> r = cur.fetchone() + >>> type(r) + + >>> tuple(r) + ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14) + >>> len(r) + 5 + >>> r[2] + 'RHAT' + >>> r.keys() + ['date', 'trans', 'symbol', 'qty', 'price'] + >>> r['qty'] + 100.0 + >>> for member in r: + ... print(member) + ... + 2006-01-05 + BUY + RHAT + 100.0 + 35.14 + + +.. _sqlite3-exceptions: + +Exceptions +---------- + +.. exception:: Warning + + A subclass of :exc:`Exception`. + +.. exception:: Error + + The base class of the other exceptions in this module. It is a subclass + of :exc:`Exception`. + +.. exception:: DatabaseError + + Exception raised for errors that are related to the database. + +.. exception:: IntegrityError + + Exception raised when the relational integrity of the database is affected, + e.g. a foreign key check fails. It is a subclass of :exc:`DatabaseError`. + +.. exception:: ProgrammingError + + Exception raised for programming errors, e.g. table not found or already + exists, syntax error in the SQL statement, wrong number of parameters + specified, etc. It is a subclass of :exc:`DatabaseError`. + +.. exception:: OperationalError + + Exception raised for errors that are related to the database's operation + and not necessarily under the control of the programmer, e.g. an unexpected + disconnect occurs, the data source name is not found, a transaction could + not be processed, etc. It is a subclass of :exc:`DatabaseError`. + +.. exception:: NotSupportedError + + Exception raised in case a method or database API was used which is not + supported by the database, e.g. calling the :meth:`~Connection.rollback` + method on a connection that does not support transaction or has + transactions turned off. It is a subclass of :exc:`DatabaseError`. + + +.. _sqlite3-types: + +SQLite and Python types +----------------------- + + +Introduction +^^^^^^^^^^^^ + +SQLite natively supports the following types: ``NULL``, ``INTEGER``, +``REAL``, ``TEXT``, ``BLOB``. + +The following Python types can thus be sent to SQLite without any problem: + ++-------------------------------+-------------+ +| Python type | SQLite type | ++===============================+=============+ +| :const:`None` | ``NULL`` | ++-------------------------------+-------------+ +| :class:`int` | ``INTEGER`` | ++-------------------------------+-------------+ +| :class:`float` | ``REAL`` | ++-------------------------------+-------------+ +| :class:`str` | ``TEXT`` | ++-------------------------------+-------------+ +| :class:`bytes` | ``BLOB`` | ++-------------------------------+-------------+ + + +This is how SQLite types are converted to Python types by default: + ++-------------+----------------------------------------------+ +| SQLite type | Python type | ++=============+==============================================+ +| ``NULL`` | :const:`None` | ++-------------+----------------------------------------------+ +| ``INTEGER`` | :class:`int` | ++-------------+----------------------------------------------+ +| ``REAL`` | :class:`float` | ++-------------+----------------------------------------------+ +| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | +| | :class:`str` by default | ++-------------+----------------------------------------------+ +| ``BLOB`` | :class:`bytes` | ++-------------+----------------------------------------------+ + +The type system of the :mod:`sqlite3` module is extensible in two ways: you can +store additional Python types in a SQLite database via object adaptation, and +you can let the :mod:`sqlite3` module convert SQLite types to different Python +types via converters. + + +Using adapters to store additional Python types in SQLite databases +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As described before, SQLite supports only a limited set of types natively. To +use other Python types with SQLite, you must **adapt** them to one of the +sqlite3 module's supported types for SQLite: one of NoneType, int, float, +str, bytes. + +There are two ways to enable the :mod:`sqlite3` module to adapt a custom Python +type to one of the supported ones. + + +Letting your object adapt itself +"""""""""""""""""""""""""""""""" + +This is a good approach if you write the class yourself. Let's suppose you have +a class like this:: + + class Point: + def __init__(self, x, y): + self.x, self.y = x, y + +Now you want to store the point in a single SQLite column. First you'll have to +choose one of the supported types to be used for representing the point. +Let's just use str and separate the coordinates using a semicolon. Then you need +to give your class a method ``__conform__(self, protocol)`` which must return +the converted value. The parameter *protocol* will be :class:`PrepareProtocol`. + +.. literalinclude:: ../includes/sqlite3/adapter_point_1.py + + +Registering an adapter callable +""""""""""""""""""""""""""""""" + +The other possibility is to create a function that converts the type to the +string representation and register the function with :meth:`register_adapter`. + +.. literalinclude:: ../includes/sqlite3/adapter_point_2.py + +The :mod:`sqlite3` module has two default adapters for Python's built-in +:class:`datetime.date` and :class:`datetime.datetime` types. Now let's suppose +we want to store :class:`datetime.datetime` objects not in ISO representation, +but as a Unix timestamp. + +.. literalinclude:: ../includes/sqlite3/adapter_datetime.py + + +Converting SQLite values to custom Python types +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Writing an adapter lets you send custom Python types to SQLite. But to make it +really useful we need to make the Python to SQLite to Python roundtrip work. + +Enter converters. + +Let's go back to the :class:`Point` class. We stored the x and y coordinates +separated via semicolons as strings in SQLite. + +First, we'll define a converter function that accepts the string as a parameter +and constructs a :class:`Point` object from it. + +.. note:: + + Converter functions **always** get called with a :class:`bytes` object, no + matter under which data type you sent the value to SQLite. + +:: + + def convert_point(s): + x, y = map(float, s.split(b";")) + return Point(x, y) + +Now you need to make the :mod:`sqlite3` module know that what you select from +the database is actually a point. There are two ways of doing this: + +* Implicitly via the declared type + +* Explicitly via the column name + +Both ways are described in section :ref:`sqlite3-module-contents`, in the entries +for the constants :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES`. + +The following example illustrates both approaches. + +.. literalinclude:: ../includes/sqlite3/converter_point.py + + +Default adapters and converters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are default adapters for the date and datetime types in the datetime +module. They will be sent as ISO dates/ISO timestamps to SQLite. + +The default converters are registered under the name "date" for +:class:`datetime.date` and under the name "timestamp" for +:class:`datetime.datetime`. + +This way, you can use date/timestamps from Python without any additional +fiddling in most cases. The format of the adapters is also compatible with the +experimental SQLite date/time functions. + +The following example demonstrates this. + +.. literalinclude:: ../includes/sqlite3/pysqlite_datetime.py + +If a timestamp stored in SQLite has a fractional part longer than 6 +numbers, its value will be truncated to microsecond precision by the +timestamp converter. + + +.. _sqlite3-controlling-transactions: + +Controlling Transactions +------------------------ + +The underlying ``sqlite3`` library operates in ``autocommit`` mode by default, +but the Python :mod:`sqlite3` module by default does not. + +``autocommit`` mode means that statements that modify the database take effect +immediately. A ``BEGIN`` or ``SAVEPOINT`` statement disables ``autocommit`` +mode, and a ``COMMIT``, a ``ROLLBACK``, or a ``RELEASE`` that ends the +outermost transaction, turns ``autocommit`` mode back on. + +The Python :mod:`sqlite3` module by default issues a ``BEGIN`` statement +implicitly before a Data Modification Language (DML) statement (i.e. +``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). + +You can control which kind of ``BEGIN`` statements :mod:`sqlite3` implicitly +executes via the *isolation_level* parameter to the :func:`connect` +call, or via the :attr:`isolation_level` property of connections. +If you specify no *isolation_level*, a plain ``BEGIN`` is used, which is +equivalent to specifying ``DEFERRED``. Other possible values are ``IMMEDIATE`` +and ``EXCLUSIVE``. + +You can disable the :mod:`sqlite3` module's implicit transaction management by +setting :attr:`isolation_level` to ``None``. This will leave the underlying +``sqlite3`` library operating in ``autocommit`` mode. You can then completely +control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, +``SAVEPOINT``, and ``RELEASE`` statements in your code. + +.. versionchanged:: 3.6 + :mod:`sqlite3` used to implicitly commit an open transaction before DDL + statements. This is no longer the case. + + +Using :mod:`sqlite3` efficiently +-------------------------------- + + +Using shortcut methods +^^^^^^^^^^^^^^^^^^^^^^ + +Using the nonstandard :meth:`execute`, :meth:`executemany` and +:meth:`executescript` methods of the :class:`Connection` object, your code can +be written more concisely because you don't have to create the (often +superfluous) :class:`Cursor` objects explicitly. Instead, the :class:`Cursor` +objects are created implicitly and these shortcut methods return the cursor +objects. This way, you can execute a ``SELECT`` statement and iterate over it +directly using only a single call on the :class:`Connection` object. + +.. literalinclude:: ../includes/sqlite3/shortcut_methods.py + + +Accessing columns by name instead of by index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +One useful feature of the :mod:`sqlite3` module is the built-in +:class:`sqlite3.Row` class designed to be used as a row factory. + +Rows wrapped with this class can be accessed both by index (like tuples) and +case-insensitively by name: + +.. literalinclude:: ../includes/sqlite3/rowclass.py + + +Using the connection as a context manager +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Connection objects can be used as context managers +that automatically commit or rollback transactions. In the event of an +exception, the transaction is rolled back; otherwise, the transaction is +committed: + +.. literalinclude:: ../includes/sqlite3/ctx_manager.py + + +.. rubric:: Footnotes + +.. [#f1] The sqlite3 module is not built with loadable extension support by + default, because some platforms (notably Mac OS X) have SQLite + libraries which are compiled without this feature. To get loadable + extension support, you must pass --enable-loadable-sqlite-extensions to + configure. diff --git a/Doc/library/statistics.rst.bak b/Doc/library/statistics.rst.bak new file mode 100644 index 00000000000000..6b6d3154a28810 --- /dev/null +++ b/Doc/library/statistics.rst.bak @@ -0,0 +1,879 @@ +:mod:`statistics` --- Mathematical statistics functions +======================================================= + +.. module:: statistics + :synopsis: Mathematical statistics functions + +.. moduleauthor:: Steven D'Aprano +.. sectionauthor:: Steven D'Aprano + +.. versionadded:: 3.4 + +**Source code:** :source:`Lib/statistics.py` + +.. testsetup:: * + + from statistics import * + __name__ = '' + +-------------- + +This module provides functions for calculating mathematical statistics of +numeric (:class:`~numbers.Real`-valued) data. + +The module is not intended to be a competitor to third-party libraries such +as `NumPy `_, `SciPy `_, or +proprietary full-featured statistics packages aimed at professional +statisticians such as Minitab, SAS and Matlab. It is aimed at the level of +graphing and scientific calculators. + +Unless explicitly noted, these functions support :class:`int`, +:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. +Behaviour with other types (whether in the numeric tower or not) is +currently unsupported. Collections with a mix of types are also undefined +and implementation-dependent. If your input data consists of mixed types, +you may be able to use :func:`map` to ensure a consistent result, for +example: ``map(float, input_data)``. + +Averages and measures of central location +----------------------------------------- + +These functions calculate an average or typical value from a population +or sample. + +======================= =============================================================== +:func:`mean` Arithmetic mean ("average") of data. +:func:`fmean` Fast, floating point arithmetic mean. +:func:`geometric_mean` Geometric mean of data. +:func:`harmonic_mean` Harmonic mean of data. +:func:`median` Median (middle value) of data. +:func:`median_low` Low median of data. +:func:`median_high` High median of data. +:func:`median_grouped` Median, or 50th percentile, of grouped data. +:func:`mode` Single mode (most common value) of discrete or nominal data. +:func:`multimode` List of modes (most common values) of discrete or nomimal data. +:func:`quantiles` Divide data into intervals with equal probability. +======================= =============================================================== + +Measures of spread +------------------ + +These functions calculate a measure of how much the population or sample +tends to deviate from the typical or average values. + +======================= ============================================= +:func:`pstdev` Population standard deviation of data. +:func:`pvariance` Population variance of data. +:func:`stdev` Sample standard deviation of data. +:func:`variance` Sample variance of data. +======================= ============================================= + + +Function details +---------------- + +Note: The functions do not require the data given to them to be sorted. +However, for reading convenience, most of the examples show sorted sequences. + +.. function:: mean(data) + + Return the sample arithmetic mean of *data* which can be a sequence or iterable. + + The arithmetic mean is the sum of the data divided by the number of data + points. It is commonly called "the average", although it is only one of many + different mathematical averages. It is a measure of the central location of + the data. + + If *data* is empty, :exc:`StatisticsError` will be raised. + + Some examples of use: + + .. doctest:: + + >>> mean([1, 2, 3, 4, 4]) + 2.8 + >>> mean([-1.0, 2.5, 3.25, 5.75]) + 2.625 + + >>> from fractions import Fraction as F + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) + + >>> from decimal import Decimal as D + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + .. note:: + + The mean is strongly affected by outliers and is not a robust estimator + for central location: the mean is not necessarily a typical example of + the data points. For more robust measures of central location, see + :func:`median` and :func:`mode`. + + The sample mean gives an unbiased estimate of the true population mean, + so that when taken on average over all the possible samples, + ``mean(sample)`` converges on the true mean of the entire population. If + *data* represents the entire population rather than a sample, then + ``mean(data)`` is equivalent to calculating the true population mean μ. + + +.. function:: fmean(data) + + Convert *data* to floats and compute the arithmetic mean. + + This runs faster than the :func:`mean` function and it always returns a + :class:`float`. The *data* may be a sequence or iterable. If the input + dataset is empty, raises a :exc:`StatisticsError`. + + .. doctest:: + + >>> fmean([3.5, 4.0, 5.25]) + 4.25 + + .. versionadded:: 3.8 + + +.. function:: geometric_mean(data) + + Convert *data* to floats and compute the geometric mean. + + The geometric mean indicates the central tendency or typical value of the + *data* using the product of the values (as opposed to the arithmetic mean + which uses their sum). + + Raises a :exc:`StatisticsError` if the input dataset is empty, + if it contains a zero, or if it contains a negative value. + The *data* may be a sequence or iterable. + + No special efforts are made to achieve exact results. + (However, this may change in the future.) + + .. doctest:: + + >>> round(geometric_mean([54, 24, 36]), 1) + 36.0 + + .. versionadded:: 3.8 + + +.. function:: harmonic_mean(data, weights=None) + + Return the harmonic mean of *data*, a sequence or iterable of + real-valued numbers. If *weights* is omitted or *None*, then + equal weighting is assumed. + + The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the + reciprocals of the data. For example, the harmonic mean of three values *a*, + *b* and *c* will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the + values is zero, the result will be zero. + + The harmonic mean is a type of average, a measure of the central + location of the data. It is often appropriate when averaging + ratios or rates, for example speeds. + + Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. + What is the average speed? + + .. doctest:: + + >>> harmonic_mean([40, 60]) + 48.0 + + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? + + .. doctest:: + + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 + + :exc:`StatisticsError` is raised if *data* is empty, any element + is less than zero, or if the weighted sum isn't positive. + + The current algorithm has an early-out when it encounters a zero + in the input. This means that the subsequent inputs are not tested + for validity. (This behavior may change in the future.) + + .. versionadded:: 3.6 + + .. versionchanged:: 3.10 + Added support for *weights*. + +.. function:: median(data) + + Return the median (middle value) of numeric data, using the common "mean of + middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. + *data* can be a sequence or iterable. + + The median is a robust measure of central location and is less affected by + the presence of outliers. When the number of data points is odd, the + middle data point is returned: + + .. doctest:: + + >>> median([1, 3, 5]) + 3 + + When the number of data points is even, the median is interpolated by taking + the average of the two middle values: + + .. doctest:: + + >>> median([1, 3, 5, 7]) + 4.0 + + This is suited for when your data is discrete, and you don't mind that the + median may not be an actual data point. + + If the data is ordinal (supports order operations) but not numeric (doesn't + support addition), consider using :func:`median_low` or :func:`median_high` + instead. + +.. function:: median_low(data) + + Return the low median of numeric data. If *data* is empty, + :exc:`StatisticsError` is raised. *data* can be a sequence or iterable. + + The low median is always a member of the data set. When the number of data + points is odd, the middle value is returned. When it is even, the smaller of + the two middle values is returned. + + .. doctest:: + + >>> median_low([1, 3, 5]) + 3 + >>> median_low([1, 3, 5, 7]) + 3 + + Use the low median when your data are discrete and you prefer the median to + be an actual data point rather than interpolated. + + +.. function:: median_high(data) + + Return the high median of data. If *data* is empty, :exc:`StatisticsError` + is raised. *data* can be a sequence or iterable. + + The high median is always a member of the data set. When the number of data + points is odd, the middle value is returned. When it is even, the larger of + the two middle values is returned. + + .. doctest:: + + >>> median_high([1, 3, 5]) + 3 + >>> median_high([1, 3, 5, 7]) + 5 + + Use the high median when your data are discrete and you prefer the median to + be an actual data point rather than interpolated. + + +.. function:: median_grouped(data, interval=1) + + Return the median of grouped continuous data, calculated as the 50th + percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` + is raised. *data* can be a sequence or iterable. + + .. doctest:: + + >>> median_grouped([52, 52, 53, 54]) + 52.5 + + In the following example, the data are rounded, so that each value represents + the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 + is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data + given, the middle value falls somewhere in the class 3.5--4.5, and + interpolation is used to estimate it: + + .. doctest:: + + >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) + 3.7 + + Optional argument *interval* represents the class interval, and defaults + to 1. Changing the class interval naturally will change the interpolation: + + .. doctest:: + + >>> median_grouped([1, 3, 3, 5, 7], interval=1) + 3.25 + >>> median_grouped([1, 3, 3, 5, 7], interval=2) + 3.5 + + This function does not check whether the data points are at least + *interval* apart. + + .. impl-detail:: + + Under some circumstances, :func:`median_grouped` may coerce data points to + floats. This behaviour is likely to change in the future. + + .. seealso:: + + * "Statistics for the Behavioral Sciences", Frederick J Gravetter and + Larry B Wallnau (8th Edition). + + * The `SSMEDIAN + `_ + function in the Gnome Gnumeric spreadsheet, including `this discussion + `_. + + +.. function:: mode(data) + + Return the single most common data point from discrete or nominal *data*. + The mode (when it exists) is the most typical value and serves as a + measure of central location. + + If there are multiple modes with the same frequency, returns the first one + encountered in the *data*. If the smallest or largest of those is + desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``. + If the input *data* is empty, :exc:`StatisticsError` is raised. + + ``mode`` assumes discrete data and returns a single value. This is the + standard treatment of the mode as commonly taught in schools: + + .. doctest:: + + >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) + 3 + + The mode is unique in that it is the only statistic in this package that + also applies to nominal (non-numeric) data: + + .. doctest:: + + >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) + 'red' + + .. versionchanged:: 3.8 + Now handles multimodal datasets by returning the first mode encountered. + Formerly, it raised :exc:`StatisticsError` when more than one mode was + found. + + +.. function:: multimode(data) + + Return a list of the most frequently occurring values in the order they + were first encountered in the *data*. Will return more than one result if + there are multiple modes or an empty list if the *data* is empty: + + .. doctest:: + + >>> multimode('aabbbbccddddeeffffgg') + ['b', 'd', 'f'] + >>> multimode('') + [] + + .. versionadded:: 3.8 + + +.. function:: pstdev(data, mu=None) + + Return the population standard deviation (the square root of the population + variance). See :func:`pvariance` for arguments and other details. + + .. doctest:: + + >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) + 0.986893273527251 + + +.. function:: pvariance(data, mu=None) + + Return the population variance of *data*, a non-empty sequence or iterable + of real-valued numbers. Variance, or second moment about the mean, is a + measure of the variability (spread or dispersion) of data. A large + variance indicates that the data is spread out; a small variance indicates + it is clustered closely around the mean. + + If the optional second argument *mu* is given, it is typically the mean of + the *data*. It can also be used to compute the second moment around a + point that is not the mean. If it is missing or ``None`` (the default), + the arithmetic mean is automatically calculated. + + Use this function to calculate the variance from the entire population. To + estimate the variance from a sample, the :func:`variance` function is usually + a better choice. + + Raises :exc:`StatisticsError` if *data* is empty. + + Examples: + + .. doctest:: + + >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] + >>> pvariance(data) + 1.25 + + If you have already calculated the mean of your data, you can pass it as the + optional second argument *mu* to avoid recalculation: + + .. doctest:: + + >>> mu = mean(data) + >>> pvariance(data, mu) + 1.25 + + Decimals and Fractions are supported: + + .. doctest:: + + >>> from decimal import Decimal as D + >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('24.815') + + >>> from fractions import Fraction as F + >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) + Fraction(13, 72) + + .. note:: + + When called with the entire population, this gives the population variance + σ². When called on a sample instead, this is the biased sample variance + s², also known as variance with N degrees of freedom. + + If you somehow know the true population mean μ, you may use this + function to calculate the variance of a sample, giving the known + population mean as the second argument. Provided the data points are a + random sample of the population, the result will be an unbiased estimate + of the population variance. + + +.. function:: stdev(data, xbar=None) + + Return the sample standard deviation (the square root of the sample + variance). See :func:`variance` for arguments and other details. + + .. doctest:: + + >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) + 1.0810874155219827 + + +.. function:: variance(data, xbar=None) + + Return the sample variance of *data*, an iterable of at least two real-valued + numbers. Variance, or second moment about the mean, is a measure of the + variability (spread or dispersion) of data. A large variance indicates that + the data is spread out; a small variance indicates it is clustered closely + around the mean. + + If the optional second argument *xbar* is given, it should be the mean of + *data*. If it is missing or ``None`` (the default), the mean is + automatically calculated. + + Use this function when your data is a sample from a population. To calculate + the variance from the entire population, see :func:`pvariance`. + + Raises :exc:`StatisticsError` if *data* has fewer than two values. + + Examples: + + .. doctest:: + + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 + + If you have already calculated the mean of your data, you can pass it as the + optional second argument *xbar* to avoid recalculation: + + .. doctest:: + + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 + + This function does not attempt to verify that you have passed the actual mean + as *xbar*. Using arbitrary values for *xbar* can lead to invalid or + impossible results. + + Decimal and Fraction values are supported: + + .. doctest:: + + >>> from decimal import Decimal as D + >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('31.01875') + + >>> from fractions import Fraction as F + >>> variance([F(1, 6), F(1, 2), F(5, 3)]) + Fraction(67, 108) + + .. note:: + + This is the sample variance s² with Bessel's correction, also known as + variance with N-1 degrees of freedom. Provided that the data points are + representative (e.g. independent and identically distributed), the result + should be an unbiased estimate of the true population variance. + + If you somehow know the actual population mean μ you should pass it to the + :func:`pvariance` function as the *mu* parameter to get the variance of a + sample. + +.. function:: quantiles(data, *, n=4, method='exclusive') + + Divide *data* into *n* continuous intervals with equal probability. + Returns a list of ``n - 1`` cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. Set + *n* to 100 for percentiles which gives the 99 cuts points that separate + *data* into 100 equal sized groups. Raises :exc:`StatisticsError` if *n* + is not least 1. + + The *data* can be any iterable containing sample data. For meaningful + results, the number of data points in *data* should be larger than *n*. + Raises :exc:`StatisticsError` if there are not at least two data points. + + The cut points are linearly interpolated from the + two nearest data points. For example, if a cut point falls one-third + of the distance between two sample values, ``100`` and ``112``, the + cut-point will evaluate to ``104``. + + The *method* for computing quantiles can be varied depending on + whether the *data* includes or excludes the lowest and + highest possible values from the population. + + The default *method* is "exclusive" and is used for data sampled from + a population that can have more extreme values than found in the + samples. The portion of the population falling below the *i-th* of + *m* sorted data points is computed as ``i / (m + 1)``. Given nine + sample values, the method sorts them and assigns the following + percentiles: 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%. + + Setting the *method* to "inclusive" is used for describing population + data or for samples that are known to include the most extreme values + from the population. The minimum value in *data* is treated as the 0th + percentile and the maximum value is treated as the 100th percentile. + The portion of the population falling below the *i-th* of *m* sorted + data points is computed as ``(i - 1) / (m - 1)``. Given 11 sample + values, the method sorts them and assigns the following percentiles: + 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%. + + .. doctest:: + + # Decile cut points for empirically sampled data + >>> data = [105, 129, 87, 86, 111, 111, 89, 81, 108, 92, 110, + ... 100, 75, 105, 103, 109, 76, 119, 99, 91, 103, 129, + ... 106, 101, 84, 111, 74, 87, 86, 103, 103, 106, 86, + ... 111, 75, 87, 102, 121, 111, 88, 89, 101, 106, 95, + ... 103, 107, 101, 81, 109, 104] + >>> [round(q, 1) for q in quantiles(data, n=10)] + [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] + + .. versionadded:: 3.8 + + +Exceptions +---------- + +A single exception is defined: + +.. exception:: StatisticsError + + Subclass of :exc:`ValueError` for statistics-related exceptions. + + +:class:`NormalDist` objects +--------------------------- + +:class:`NormalDist` is a tool for creating and manipulating normal +distributions of a `random variable +`_. It is a +class that treats the mean and standard deviation of data +measurements as a single entity. + +Normal distributions arise from the `Central Limit Theorem +`_ and have a wide range +of applications in statistics. + +.. class:: NormalDist(mu=0.0, sigma=1.0) + + Returns a new *NormalDist* object where *mu* represents the `arithmetic + mean `_ and *sigma* + represents the `standard deviation + `_. + + If *sigma* is negative, raises :exc:`StatisticsError`. + + .. attribute:: mean + + A read-only property for the `arithmetic mean + `_ of a normal + distribution. + + .. attribute:: median + + A read-only property for the `median + `_ of a normal + distribution. + + .. attribute:: mode + + A read-only property for the `mode + `_ of a normal + distribution. + + .. attribute:: stdev + + A read-only property for the `standard deviation + `_ of a normal + distribution. + + .. attribute:: variance + + A read-only property for the `variance + `_ of a normal + distribution. Equal to the square of the standard deviation. + + .. classmethod:: NormalDist.from_samples(data) + + Makes a normal distribution instance with *mu* and *sigma* parameters + estimated from the *data* using :func:`fmean` and :func:`stdev`. + + The *data* can be any :term:`iterable` and should consist of values + that can be converted to type :class:`float`. If *data* does not + contain at least two elements, raises :exc:`StatisticsError` because it + takes at least one point to estimate a central value and at least two + points to estimate dispersion. + + .. method:: NormalDist.samples(n, *, seed=None) + + Generates *n* random samples for a given mean and standard deviation. + Returns a :class:`list` of :class:`float` values. + + If *seed* is given, creates a new instance of the underlying random + number generator. This is useful for creating reproducible results, + even in a multi-threading context. + + .. method:: NormalDist.pdf(x) + + Using a `probability density function (pdf) + `_, compute + the relative likelihood that a random variable *X* will be near the + given value *x*. Mathematically, it is the limit of the ratio ``P(x <= + X < x+dx) / dx`` as *dx* approaches zero. + + The relative likelihood is computed as the probability of a sample + occurring in a narrow range divided by the width of the range (hence + the word "density"). Since the likelihood is relative to other points, + its value can be greater than `1.0`. + + .. method:: NormalDist.cdf(x) + + Using a `cumulative distribution function (cdf) + `_, + compute the probability that a random variable *X* will be less than or + equal to *x*. Mathematically, it is written ``P(X <= x)``. + + .. method:: NormalDist.inv_cdf(p) + + Compute the inverse cumulative distribution function, also known as the + `quantile function `_ + or the `percent-point + `_ + function. Mathematically, it is written ``x : P(X <= x) = p``. + + Finds the value *x* of the random variable *X* such that the + probability of the variable being less than or equal to that value + equals the given probability *p*. + + .. method:: NormalDist.overlap(other) + + Measures the agreement between two normal probability distributions. + Returns a value between 0.0 and 1.0 giving `the overlapping area for + the two probability density functions + `_. + + .. method:: NormalDist.quantiles(n=4) + + Divide the normal distribution into *n* continuous intervals with + equal probability. Returns a list of (n - 1) cut points separating + the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution into 100 equal sized groups. + + .. method:: NormalDist.zscore(x) + + Compute the + `Standard Score `_ + describing *x* in terms of the number of standard deviations + above or below the mean of the normal distribution: + ``(x - mean) / stdev``. + + .. versionadded:: 3.9 + + Instances of :class:`NormalDist` support addition, subtraction, + multiplication and division by a constant. These operations + are used for translation and scaling. For example: + + .. doctest:: + + >>> temperature_february = NormalDist(5, 2.5) # Celsius + >>> temperature_february * (9/5) + 32 # Fahrenheit + NormalDist(mu=41.0, sigma=4.5) + + Dividing a constant by an instance of :class:`NormalDist` is not supported + because the result wouldn't be normally distributed. + + Since normal distributions arise from additive effects of independent + variables, it is possible to `add and subtract two independent normally + distributed random variables + `_ + represented as instances of :class:`NormalDist`. For example: + + .. doctest:: + + >>> birth_weights = NormalDist.from_samples([2.5, 3.1, 2.1, 2.4, 2.7, 3.5]) + >>> drug_effects = NormalDist(0.4, 0.15) + >>> combined = birth_weights + drug_effects + >>> round(combined.mean, 1) + 3.1 + >>> round(combined.stdev, 1) + 0.5 + + .. versionadded:: 3.8 + + +:class:`NormalDist` Examples and Recipes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:class:`NormalDist` readily solves classic probability problems. + +For example, given `historical data for SAT exams +`_ showing +that scores are normally distributed with a mean of 1060 and a standard +deviation of 195, determine the percentage of students with test scores +between 1100 and 1200, after rounding to the nearest whole number: + +.. doctest:: + + >>> sat = NormalDist(1060, 195) + >>> fraction = sat.cdf(1200 + 0.5) - sat.cdf(1100 - 0.5) + >>> round(fraction * 100.0, 1) + 18.4 + +Find the `quartiles `_ and `deciles +`_ for the SAT scores: + +.. doctest:: + + >>> list(map(round, sat.quantiles())) + [928, 1060, 1192] + >>> list(map(round, sat.quantiles(n=10))) + [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] + +To estimate the distribution for a model than isn't easy to solve +analytically, :class:`NormalDist` can generate input samples for a `Monte +Carlo simulation `_: + +.. doctest:: + + >>> def model(x, y, z): + ... return (3*x + 7*x*y - 5*y) / (11 * z) + ... + >>> n = 100_000 + >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728) + >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471) + >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453) + >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP + [1.4591308524824727, 1.8035946855390597, 2.175091447274739] + +Normal distributions can be used to approximate `Binomial +distributions `_ +when the sample size is large and when the probability of a successful +trial is near 50%. + +For example, an open source conference has 750 attendees and two rooms with a +500 person capacity. There is a talk about Python and another about Ruby. +In previous conferences, 65% of the attendees preferred to listen to Python +talks. Assuming the population preferences haven't changed, what is the +probability that the Python room will stay within its capacity limits? + +.. doctest:: + + >>> n = 750 # Sample size + >>> p = 0.65 # Preference for Python + >>> q = 1.0 - p # Preference for Ruby + >>> k = 500 # Room capacity + + >>> # Approximation using the cumulative normal distribution + >>> from math import sqrt + >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) + 0.8402 + + >>> # Solution using the cumulative binomial distribution + >>> from math import comb, fsum + >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) + 0.8402 + + >>> # Approximation using a simulation + >>> from random import seed, choices + >>> seed(8675309) + >>> def trial(): + ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') + >>> mean(trial() <= k for i in range(10_000)) + 0.8398 + +Normal distributions commonly arise in machine learning problems. + +Wikipedia has a `nice example of a Naive Bayesian Classifier +`_. +The challenge is to predict a person's gender from measurements of normally +distributed features including height, weight, and foot size. + +We're given a training dataset with measurements for eight people. The +measurements are assumed to be normally distributed, so we summarize the data +with :class:`NormalDist`: + +.. doctest:: + + >>> height_male = NormalDist.from_samples([6, 5.92, 5.58, 5.92]) + >>> height_female = NormalDist.from_samples([5, 5.5, 5.42, 5.75]) + >>> weight_male = NormalDist.from_samples([180, 190, 170, 165]) + >>> weight_female = NormalDist.from_samples([100, 150, 130, 150]) + >>> foot_size_male = NormalDist.from_samples([12, 11, 12, 10]) + >>> foot_size_female = NormalDist.from_samples([6, 8, 7, 9]) + +Next, we encounter a new person whose feature measurements are known but whose +gender is unknown: + +.. doctest:: + + >>> ht = 6.0 # height + >>> wt = 130 # weight + >>> fs = 8 # foot size + +Starting with a 50% `prior probability +`_ of being male or female, +we compute the posterior as the prior times the product of likelihoods for the +feature measurements given the gender: + +.. doctest:: + + >>> prior_male = 0.5 + >>> prior_female = 0.5 + >>> posterior_male = (prior_male * height_male.pdf(ht) * + ... weight_male.pdf(wt) * foot_size_male.pdf(fs)) + + >>> posterior_female = (prior_female * height_female.pdf(ht) * + ... weight_female.pdf(wt) * foot_size_female.pdf(fs)) + +The final prediction goes to the largest posterior. This is known as the +`maximum a posteriori +`_ or MAP: + +.. doctest:: + + >>> 'male' if posterior_male > posterior_female else 'female' + 'female' + + +.. + # This modelines must appear within the last ten lines of the file. + kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/tempfile.rst.bak b/Doc/library/tempfile.rst.bak new file mode 100644 index 00000000000000..2b8a35e2651e73 --- /dev/null +++ b/Doc/library/tempfile.rst.bak @@ -0,0 +1,374 @@ +:mod:`tempfile` --- Generate temporary files and directories +============================================================ + +.. module:: tempfile + :synopsis: Generate temporary files and directories. + +.. sectionauthor:: Zack Weinberg + +**Source code:** :source:`Lib/tempfile.py` + +.. index:: + pair: temporary; file name + pair: temporary; file + +-------------- + +This module creates temporary files and directories. It works on all +supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, +:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level +interfaces which provide automatic cleanup and can be used as +context managers. :func:`mkstemp` and +:func:`mkdtemp` are lower-level functions which require manual cleanup. + +All the user-callable functions and constructors take additional arguments which +allow direct control over the location and name of temporary files and +directories. Files names used by this module include a string of +random characters which allows those files to be securely created in +shared temporary directories. +To maintain backward compatibility, the argument order is somewhat odd; it +is recommended to use keyword arguments for clarity. + +The module defines the following user-callable items: + +.. function:: TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) + + Return a :term:`file-like object` that can be used as a temporary storage area. + The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon + as it is closed (including an implicit close when the object is garbage + collected). Under Unix, the directory entry for the file is either not created at all or is removed + immediately after the file is created. Other platforms do not support + this; your code should not rely on a temporary file created using this + function having or not having a visible name in the file system. + + The resulting object can be used as a context manager (see + :ref:`tempfile-examples`). On completion of the context or + destruction of the file object the temporary file will be removed + from the filesystem. + + The *mode* parameter defaults to ``'w+b'`` so that the file created can + be read and written without being closed. Binary mode is used so that it + behaves consistently on all platforms without regard for the data that is + stored. *buffering*, *encoding*, *errors* and *newline* are interpreted as for + :func:`open`. + + The *dir*, *prefix* and *suffix* parameters have the same meaning and + defaults as with :func:`mkstemp`. + + The returned object is a true file object on POSIX platforms. On other + platforms, it is a file-like object whose :attr:`!file` attribute is the + underlying true file object. + + The :py:data:`os.O_TMPFILE` flag is used if it is available and works + (Linux-specific, requires Linux kernel 3.11 or later). + + .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile + + .. versionchanged:: 3.5 + + The :py:data:`os.O_TMPFILE` flag is now used if available. + + .. versionchanged:: 3.8 + Added *errors* parameter. + + +.. function:: NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) + + This function operates exactly as :func:`TemporaryFile` does, except that + the file is guaranteed to have a visible name in the file system (on + Unix, the directory entry is not unlinked). That name can be retrieved + from the :attr:`name` attribute of the returned + file-like object. Whether the name can be + used to open the file a second time, while the named temporary file is + still open, varies across platforms (it can be so used on Unix; it cannot + on Windows NT or later). If *delete* is true (the default), the file is + deleted as soon as it is closed. + The returned object is always a file-like object whose :attr:`!file` + attribute is the underlying true file object. This file-like object can + be used in a :keyword:`with` statement, just like a normal file. + + .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile + + .. versionchanged:: 3.8 + Added *errors* parameter. + + +.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) + + This function operates exactly as :func:`TemporaryFile` does, except that + data is spooled in memory until the file size exceeds *max_size*, or + until the file's :func:`fileno` method is called, at which point the + contents are written to disk and operation proceeds as with + :func:`TemporaryFile`. + + The resulting file has one additional method, :func:`rollover`, which + causes the file to roll over to an on-disk file regardless of its size. + + The returned object is a file-like object whose :attr:`_file` attribute + is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object + (depending on whether binary or text *mode* was specified) or a true file + object, depending on whether :func:`rollover` has been called. This + file-like object can be used in a :keyword:`with` statement, just like + a normal file. + + .. versionchanged:: 3.3 + the truncate method now accepts a ``size`` argument. + + .. versionchanged:: 3.8 + Added *errors* parameter. + + +.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) + + This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. + The resulting object can be used as a context manager (see + :ref:`tempfile-examples`). On completion of the context or destruction + of the temporary directory object the newly created temporary directory + and all its contents are removed from the filesystem. + + The directory name can be retrieved from the :attr:`name` attribute of the + returned object. When the returned object is used as a context manager, the + :attr:`name` will be assigned to the target of the :keyword:`!as` clause in + the :keyword:`with` statement, if there is one. + + The directory can be explicitly cleaned up by calling the + :func:`cleanup` method. + + .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory + + .. versionadded:: 3.2 + + +.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) + + Creates a temporary file in the most secure manner possible. There are + no race conditions in the file's creation, assuming that the platform + properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The + file is readable and writable only by the creating user ID. If the + platform uses permission bits to indicate whether a file is executable, + the file is executable by no one. The file descriptor is not inherited + by child processes. + + Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible + for deleting the temporary file when done with it. + + If *suffix* is not ``None``, the file name will end with that suffix, + otherwise there will be no suffix. :func:`mkstemp` does not put a dot + between the file name and the suffix; if you need one, put it at the + beginning of *suffix*. + + If *prefix* is not ``None``, the file name will begin with that prefix; + otherwise, a default prefix is used. The default is the return value of + :func:`gettempprefix` or :func:`gettempprefixb`, as appropriate. + + If *dir* is not ``None``, the file will be created in that directory; + otherwise, a default directory is used. The default directory is chosen + from a platform-dependent list, but the user of the application can + control the directory location by setting the *TMPDIR*, *TEMP* or *TMP* + environment variables. There is thus no guarantee that the generated + filename will have any nice properties, such as not requiring quoting + when passed to external commands via ``os.popen()``. + + If any of *suffix*, *prefix*, and *dir* are not + ``None``, they must be the same type. + If they are bytes, the returned name will be bytes instead of str. + If you want to force a bytes return value with otherwise default behavior, + pass ``suffix=b''``. + + If *text* is specified and true, the file is opened in text mode. + Otherwise, (the default) the file is opened in binary mode. + + :func:`mkstemp` returns a tuple containing an OS-level handle to an open + file (as would be returned by :func:`os.open`) and the absolute pathname + of that file, in that order. + + .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp + + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + *suffix* and *prefix* now accept and default to ``None`` to cause + an appropriate default value to be used. + + .. versionchanged:: 3.6 + The *dir* parameter now accepts a :term:`path-like object`. + + +.. function:: mkdtemp(suffix=None, prefix=None, dir=None) + + Creates a temporary directory in the most secure manner possible. There + are no race conditions in the directory's creation. The directory is + readable, writable, and searchable only by the creating user ID. + + The user of :func:`mkdtemp` is responsible for deleting the temporary + directory and its contents when done with it. + + The *prefix*, *suffix*, and *dir* arguments are the same as for + :func:`mkstemp`. + + :func:`mkdtemp` returns the absolute pathname of the new directory. + + .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp + + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + *suffix* and *prefix* now accept and default to ``None`` to cause + an appropriate default value to be used. + + .. versionchanged:: 3.6 + The *dir* parameter now accepts a :term:`path-like object`. + + +.. function:: gettempdir() + + Return the name of the directory used for temporary files. This + defines the default value for the *dir* argument to all functions + in this module. + + Python searches a standard list of directories to find one which + the calling user can create files in. The list is: + + #. The directory named by the :envvar:`TMPDIR` environment variable. + + #. The directory named by the :envvar:`TEMP` environment variable. + + #. The directory named by the :envvar:`TMP` environment variable. + + #. A platform-specific location: + + * On Windows, the directories :file:`C:\\TEMP`, :file:`C:\\TMP`, + :file:`\\TEMP`, and :file:`\\TMP`, in that order. + + * On all other platforms, the directories :file:`/tmp`, :file:`/var/tmp`, and + :file:`/usr/tmp`, in that order. + + #. As a last resort, the current working directory. + + The result of this search is cached, see the description of + :data:`tempdir` below. + + .. versionchanged:: 3.10 + + Always returns a str. Previously it would return any :data:`tempdir` + value regardless of type so long as it was not ``None``. + +.. function:: gettempdirb() + + Same as :func:`gettempdir` but the return value is in bytes. + + .. versionadded:: 3.5 + +.. function:: gettempprefix() + + Return the filename prefix used to create temporary files. This does not + contain the directory component. + +.. function:: gettempprefixb() + + Same as :func:`gettempprefix` but the return value is in bytes. + + .. versionadded:: 3.5 + +The module uses a global variable to store the name of the directory +used for temporary files returned by :func:`gettempdir`. It can be +set directly to override the selection process, but this is discouraged. +All functions in this module take a *dir* argument which can be used +to specify the directory. This is the recommended approach that does +not surprise other unsuspecting code by changing global API behavior. + +.. data:: tempdir + + When set to a value other than ``None``, this variable defines the + default value for the *dir* argument to the functions defined in this + module, including its type, bytes or str. It cannot be a + :term:`path-like object`. + + If ``tempdir`` is ``None`` (the default) at any call to any of the above + functions except :func:`gettempprefix` it is initialized following the + algorithm described in :func:`gettempdir`. + + .. note:: + + Beware that if you set ``tempdir`` to a bytes value, there is a + nasty side effect: The global default return type of + :func:`mkstemp` and :func:`mkdtemp` changes to bytes when no + explicit ``prefix``, ``suffix``, or ``dir`` arguments of type + str are supplied. Please do not write code expecting or + depending on this. This awkward behavior is maintained for + compatibility with the historcal implementation. + +.. _tempfile-examples: + +Examples +-------- + +Here are some examples of typical usage of the :mod:`tempfile` module:: + + >>> import tempfile + + # create a temporary file and write some data to it + >>> fp = tempfile.TemporaryFile() + >>> fp.write(b'Hello world!') + # read data from file + >>> fp.seek(0) + >>> fp.read() + b'Hello world!' + # close the file, it will be removed + >>> fp.close() + + # create a temporary file using a context manager + >>> with tempfile.TemporaryFile() as fp: + ... fp.write(b'Hello world!') + ... fp.seek(0) + ... fp.read() + b'Hello world!' + >>> + # file is now closed and removed + + # create a temporary directory using the context manager + >>> with tempfile.TemporaryDirectory() as tmpdirname: + ... print('created temporary directory', tmpdirname) + >>> + # directory and contents have been removed + + +Deprecated functions and variables +---------------------------------- + +A historical way to create temporary files was to first generate a +file name with the :func:`mktemp` function and then create a file +using this name. Unfortunately this is not secure, because a different +process may create a file with this name in the time between the call +to :func:`mktemp` and the subsequent attempt to create the file by the +first process. The solution is to combine the two steps and create the +file immediately. This approach is used by :func:`mkstemp` and the +other functions described above. + +.. function:: mktemp(suffix='', prefix='tmp', dir=None) + + .. deprecated:: 2.3 + Use :func:`mkstemp` instead. + + Return an absolute pathname of a file that did not exist at the time the + call is made. The *prefix*, *suffix*, and *dir* arguments are similar + to those of :func:`mkstemp`, except that bytes file names, ``suffix=None`` + and ``prefix=None`` are not supported. + + .. warning:: + + Use of this function may introduce a security hole in your program. By + the time you get around to doing anything with the file name it returns, + someone else may have beaten you to the punch. :func:`mktemp` usage can + be replaced easily with :func:`NamedTemporaryFile`, passing it the + ``delete=False`` parameter:: + + >>> f = NamedTemporaryFile(delete=False) + >>> f.name + '/tmp/tmptjujjt' + >>> f.write(b"Hello World!\n") + 13 + >>> f.close() + >>> os.unlink(f.name) + >>> os.path.exists(f.name) + False diff --git a/Doc/library/venv.rst.bak b/Doc/library/venv.rst.bak new file mode 100644 index 00000000000000..5d4a36481f1dcc --- /dev/null +++ b/Doc/library/venv.rst.bak @@ -0,0 +1,496 @@ +:mod:`venv` --- Creation of virtual environments +================================================ + +.. module:: venv + :synopsis: Creation of virtual environments. + +.. moduleauthor:: Vinay Sajip +.. sectionauthor:: Vinay Sajip + +.. versionadded:: 3.3 + +**Source code:** :source:`Lib/venv/` + +.. index:: pair: Environments; virtual + +-------------- + +The :mod:`venv` module provides support for creating lightweight "virtual +environments" with their own site directories, optionally isolated from system +site directories. Each virtual environment has its own Python binary (which +matches the version of the binary that was used to create this environment) and +can have its own independent set of installed Python packages in its site +directories. + +See :pep:`405` for more information about Python virtual environments. + +.. seealso:: + + `Python Packaging User Guide: Creating and using virtual environments + `__ + + +Creating virtual environments +----------------------------- + +.. include:: /using/venv-create.inc + + +.. _venv-def: + +.. note:: A virtual environment is a Python environment such that the Python + interpreter, libraries and scripts installed into it are isolated from those + installed in other virtual environments, and (by default) any libraries + installed in a "system" Python, i.e., one which is installed as part of your + operating system. + + A virtual environment is a directory tree which contains Python executable + files and other files which indicate that it is a virtual environment. + + Common installation tools such as setuptools_ and pip_ work as + expected with virtual environments. In other words, when a virtual + environment is active, they install Python packages into the virtual + environment without needing to be told to do so explicitly. + + When a virtual environment is active (i.e., the virtual environment's Python + interpreter is running), the attributes :attr:`sys.prefix` and + :attr:`sys.exec_prefix` point to the base directory of the virtual + environment, whereas :attr:`sys.base_prefix` and + :attr:`sys.base_exec_prefix` point to the non-virtual environment Python + installation which was used to create the virtual environment. If a virtual + environment is not active, then :attr:`sys.prefix` is the same as + :attr:`sys.base_prefix` and :attr:`sys.exec_prefix` is the same as + :attr:`sys.base_exec_prefix` (they all point to a non-virtual environment + Python installation). + + When a virtual environment is active, any options that change the + installation path will be ignored from all :mod:`distutils` configuration + files to prevent projects being inadvertently installed outside of the + virtual environment. + + When working in a command shell, users can make a virtual environment active + by running an ``activate`` script in the virtual environment's executables + directory (the precise filename and command to use the file is + shell-dependent), which prepends the virtual environment's directory for + executables to the ``PATH`` environment variable for the running shell. There + should be no need in other circumstances to activate a virtual + environment; scripts installed into virtual environments have a "shebang" + line which points to the virtual environment's Python interpreter. This means + that the script will run with that interpreter regardless of the value of + ``PATH``. On Windows, "shebang" line processing is supported if you have the + Python Launcher for Windows installed (this was added to Python in 3.3 - see + :pep:`397` for more details). Thus, double-clicking an installed script in a + Windows Explorer window should run the script with the correct interpreter + without there needing to be any reference to its virtual environment in + ``PATH``. + + +.. _venv-api: + +API +--- + +.. highlight:: python + +The high-level method described above makes use of a simple API which provides +mechanisms for third-party virtual environment creators to customize environment +creation according to their needs, the :class:`EnvBuilder` class. + +.. class:: EnvBuilder(system_site_packages=False, clear=False, \ + symlinks=False, upgrade=False, with_pip=False, \ + prompt=None, upgrade_deps=False) + + The :class:`EnvBuilder` class accepts the following keyword arguments on + instantiation: + + * ``system_site_packages`` -- a Boolean value indicating that the system Python + site-packages should be available to the environment (defaults to ``False``). + + * ``clear`` -- a Boolean value which, if true, will delete the contents of + any existing target directory, before creating the environment. + + * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the + Python binary rather than copying. + + * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing + environment with the running Python - for use when that Python has been + upgraded in-place (defaults to ``False``). + + * ``with_pip`` -- a Boolean value which, if true, ensures pip is + installed in the virtual environment. This uses :mod:`ensurepip` with + the ``--default-pip`` option. + + * ``prompt`` -- a String to be used after virtual environment is activated + (defaults to ``None`` which means directory name of the environment would + be used). If the special string ``"."`` is provided, the basename of the + current directory is used as the prompt. + + * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + + .. versionchanged:: 3.4 + Added the ``with_pip`` parameter + + .. versionadded:: 3.6 + Added the ``prompt`` parameter + + .. versionadded:: 3.9 + Added the ``upgrade_deps`` parameter + + Creators of third-party virtual environment tools will be free to use the + provided :class:`EnvBuilder` class as a base class. + + The returned env-builder is an object which has a method, ``create``: + + .. method:: create(env_dir) + + Create a virtual environment by specifying the target directory + (absolute or relative to the current directory) which is to contain the + virtual environment. The ``create`` method will either create the + environment in the specified directory, or raise an appropriate + exception. + + The ``create`` method of the :class:`EnvBuilder` class illustrates the + hooks available for subclass customization:: + + def create(self, env_dir): + """ + Create a virtualized Python environment in a directory. + env_dir is the target directory to create an environment in. + """ + env_dir = os.path.abspath(env_dir) + context = self.ensure_directories(env_dir) + self.create_configuration(context) + self.setup_python(context) + self.setup_scripts(context) + self.post_setup(context) + + Each of the methods :meth:`ensure_directories`, + :meth:`create_configuration`, :meth:`setup_python`, + :meth:`setup_scripts` and :meth:`post_setup` can be overridden. + + .. method:: ensure_directories(env_dir) + + Creates the environment directory and all necessary directories, and + returns a context object. This is just a holder for attributes (such as + paths), for use by the other methods. The directories are allowed to + exist already, as long as either ``clear`` or ``upgrade`` were + specified to allow operating on an existing environment directory. + + .. method:: create_configuration(context) + + Creates the ``pyvenv.cfg`` configuration file in the environment. + + .. method:: setup_python(context) + + Creates a copy or symlink to the Python executable in the environment. + On POSIX systems, if a specific executable ``python3.x`` was used, + symlinks to ``python`` and ``python3`` will be created pointing to that + executable, unless files with those names already exist. + + .. method:: setup_scripts(context) + + Installs activation scripts appropriate to the platform into the virtual + environment. + + .. method:: upgrade_dependencies(context) + + Upgrades the core venv dependency packages (currently ``pip`` and + ``setuptools``) in the environment. This is done by shelling out to the + ``pip`` executable in the environment. + + .. versionadded:: 3.9 + + .. method:: post_setup(context) + + A placeholder method which can be overridden in third party + implementations to pre-install packages in the virtual environment or + perform other post-creation steps. + + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. + + In addition, :class:`EnvBuilder` provides this utility method that can be + called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to + assist in installing custom scripts into the virtual environment. + + .. method:: install_scripts(context, path) + + *path* is the path to a directory that should contain subdirectories + "common", "posix", "nt", each containing scripts destined for the bin + directory in the environment. The contents of "common" and the + directory corresponding to :data:`os.name` are copied after some text + replacement of placeholders: + + * ``__VENV_DIR__`` is replaced with the absolute path of the environment + directory. + + * ``__VENV_NAME__`` is replaced with the environment name (final path + segment of environment directory). + + * ``__VENV_PROMPT__`` is replaced with the prompt (the environment + name surrounded by parentheses and with a following space) + + * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory + (either ``bin`` or ``Scripts``). + + * ``__VENV_PYTHON__`` is replaced with the absolute path of the + environment's executable. + + The directories are allowed to exist (for when an existing environment + is being upgraded). + +There is also a module-level convenience function: + +.. function:: create(env_dir, system_site_packages=False, clear=False, \ + symlinks=False, with_pip=False, prompt=None, \ + upgrade_deps=False) + + Create an :class:`EnvBuilder` with the given keyword arguments, and call its + :meth:`~EnvBuilder.create` method with the *env_dir* argument. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.4 + Added the ``with_pip`` parameter + + .. versionchanged:: 3.6 + Added the ``prompt`` parameter + + .. versionchanged:: 3.9 + Added the ``upgrade_deps`` parameter + +An example of extending ``EnvBuilder`` +-------------------------------------- + +The following script shows how to extend :class:`EnvBuilder` by implementing a +subclass which installs setuptools and pip into a created virtual environment:: + + import os + import os.path + from subprocess import Popen, PIPE + import sys + from threading import Thread + from urllib.parse import urlparse + from urllib.request import urlretrieve + import venv + + class ExtendedEnvBuilder(venv.EnvBuilder): + """ + This builder installs setuptools and pip so that you can pip or + easy_install other packages into the created virtual environment. + + :param nodist: If true, setuptools and pip are not installed into the + created virtual environment. + :param nopip: If true, pip is not installed into the created + virtual environment. + :param progress: If setuptools or pip are installed, the progress of the + installation can be monitored by passing a progress + callable. If specified, it is called with two + arguments: a string indicating some progress, and a + context indicating where the string is coming from. + The context argument can have one of three values: + 'main', indicating that it is called from virtualize() + itself, and 'stdout' and 'stderr', which are obtained + by reading lines from the output streams of a subprocess + which is used to install the app. + + If a callable is not specified, default progress + information is output to sys.stderr. + """ + + def __init__(self, *args, **kwargs): + self.nodist = kwargs.pop('nodist', False) + self.nopip = kwargs.pop('nopip', False) + self.progress = kwargs.pop('progress', None) + self.verbose = kwargs.pop('verbose', False) + super().__init__(*args, **kwargs) + + def post_setup(self, context): + """ + Set up any packages which need to be pre-installed into the + virtual environment being created. + + :param context: The information for the virtual environment + creation request being processed. + """ + os.environ['VIRTUAL_ENV'] = context.env_dir + if not self.nodist: + self.install_setuptools(context) + # Can't install pip without setuptools + if not self.nopip and not self.nodist: + self.install_pip(context) + + def reader(self, stream, context): + """ + Read lines from a subprocess' output stream and either pass to a progress + callable (if specified) or write progress information to sys.stderr. + """ + progress = self.progress + while True: + s = stream.readline() + if not s: + break + if progress is not None: + progress(s, context) + else: + if not self.verbose: + sys.stderr.write('.') + else: + sys.stderr.write(s.decode('utf-8')) + sys.stderr.flush() + stream.close() + + def install_script(self, context, name, url): + _, _, path, _, _, _ = urlparse(url) + fn = os.path.split(path)[-1] + binpath = context.bin_path + distpath = os.path.join(binpath, fn) + # Download script into the virtual environment's binaries folder + urlretrieve(url, distpath) + progress = self.progress + if self.verbose: + term = '\n' + else: + term = '' + if progress is not None: + progress('Installing %s ...%s' % (name, term), 'main') + else: + sys.stderr.write('Installing %s ...%s' % (name, term)) + sys.stderr.flush() + # Install in the virtual environment + args = [context.env_exe, fn] + p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath) + t1 = Thread(target=self.reader, args=(p.stdout, 'stdout')) + t1.start() + t2 = Thread(target=self.reader, args=(p.stderr, 'stderr')) + t2.start() + p.wait() + t1.join() + t2.join() + if progress is not None: + progress('done.', 'main') + else: + sys.stderr.write('done.\n') + # Clean up - no longer needed + os.unlink(distpath) + + def install_setuptools(self, context): + """ + Install setuptools in the virtual environment. + + :param context: The information for the virtual environment + creation request being processed. + """ + url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' + self.install_script(context, 'setuptools', url) + # clear up the setuptools archive which gets downloaded + pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') + files = filter(pred, os.listdir(context.bin_path)) + for f in files: + f = os.path.join(context.bin_path, f) + os.unlink(f) + + def install_pip(self, context): + """ + Install pip in the virtual environment. + + :param context: The information for the virtual environment + creation request being processed. + """ + url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' + self.install_script(context, 'pip', url) + + def main(args=None): + compatible = True + if sys.version_info < (3, 3): + compatible = False + elif not hasattr(sys, 'base_prefix'): + compatible = False + if not compatible: + raise ValueError('This script is only for use with ' + 'Python 3.3 or later') + else: + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory in which to create the + 'virtual environment.') + parser.add_argument('--no-setuptools', default=False, + action='store_true', dest='nodist', + help="Don't install setuptools or pip in the " + "virtual environment.") + parser.add_argument('--no-pip', default=False, + action='store_true', dest='nopip', + help="Don't install pip in the virtual " + "environment.") + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False + else: + use_symlinks = True + parser.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'virtual environment ' + 'directory if it already ' + 'exists, before virtual ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the virtual ' + 'environment directory to ' + 'use this version of ' + 'Python, assuming Python ' + 'has been upgraded ' + 'in-place.') + parser.add_argument('--verbose', default=False, action='store_true', + dest='verbose', help='Display the output ' + 'from the scripts which ' + 'install setuptools and pip.') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = ExtendedEnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + nodist=options.nodist, + nopip=options.nopip, + verbose=options.verbose) + for d in options.dirs: + builder.create(d) + + if __name__ == '__main__': + rc = 1 + try: + main() + rc = 0 + except Exception as e: + print('Error: %s' % e, file=sys.stderr) + sys.exit(rc) + + +This script is also available for download `online +`_. + + +.. _setuptools: https://pypi.org/project/setuptools/ +.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/xml.sax.handler.rst.bak b/Doc/library/xml.sax.handler.rst.bak new file mode 100644 index 00000000000000..3746a58c9b9558 --- /dev/null +++ b/Doc/library/xml.sax.handler.rst.bak @@ -0,0 +1,463 @@ +:mod:`xml.sax.handler` --- Base classes for SAX handlers +======================================================== + +.. module:: xml.sax.handler + :synopsis: Base classes for SAX event handlers. + +.. moduleauthor:: Lars Marius Garshol +.. sectionauthor:: Martin v. Löwis + +**Source code:** :source:`Lib/xml/sax/handler.py` + +-------------- + +The SAX API defines five kinds of handlers: content handlers, DTD handlers, +error handlers, entity resolvers and lexical handlers. Applications normally +only need to implement those interfaces whose events they are interested in; +they can implement the interfaces in a single object or in multiple objects. +Handler implementations should inherit from the base classes provided in the +module :mod:`xml.sax.handler`, so that all methods get default implementations. + + +.. class:: ContentHandler + + This is the main callback interface in SAX, and the one most important to + applications. The order of events in this interface mirrors the order of the + information in the document. + + +.. class:: DTDHandler + + Handle DTD events. + + This interface specifies only those DTD events required for basic parsing + (unparsed entities and attributes). + + +.. class:: EntityResolver + + Basic interface for resolving entities. If you create an object implementing + this interface, then register the object with your Parser, the parser will call + the method in your object to resolve all external entities. + + +.. class:: ErrorHandler + + Interface used by the parser to present error and warning messages to the + application. The methods of this object control whether errors are immediately + converted to exceptions or are handled in some other way. + + +.. class:: LexicalHandler + + Interface used by the parser to represent low freqency events which may not + be of interest to many applications. + +In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants +for the feature and property names. + + +.. data:: feature_namespaces + + | value: ``"http://xml.org/sax/features/namespaces"`` + | true: Perform Namespace processing. + | false: Optionally do not perform Namespace processing (implies + namespace-prefixes; default). + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_namespace_prefixes + + | value: ``"http://xml.org/sax/features/namespace-prefixes"`` + | true: Report the original prefixed names and attributes used for Namespace + declarations. + | false: Do not report attributes used for Namespace declarations, and + optionally do not report original prefixed names (default). + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_string_interning + + | value: ``"http://xml.org/sax/features/string-interning"`` + | true: All element names, prefixes, attribute names, Namespace URIs, and + local names are interned using the built-in intern function. + | false: Names are not necessarily interned, although they may be (default). + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_validation + + | value: ``"http://xml.org/sax/features/validation"`` + | true: Report all validation errors (implies external-general-entities and + external-parameter-entities). + | false: Do not report validation errors. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_external_ges + + | value: ``"http://xml.org/sax/features/external-general-entities"`` + | true: Include all external general (text) entities. + | false: Do not include external general entities. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_external_pes + + | value: ``"http://xml.org/sax/features/external-parameter-entities"`` + | true: Include all external parameter entities, including the external DTD + subset. + | false: Do not include any external parameter entities, even the external + DTD subset. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: all_features + + List of all features. + + +.. data:: property_lexical_handler + + | value: ``"http://xml.org/sax/properties/lexical-handler"`` + | data type: xml.sax.handler.LexicalHandler (not supported in Python 2) + | description: An optional extension handler for lexical events like + comments. + | access: read/write + + +.. data:: property_declaration_handler + + | value: ``"http://xml.org/sax/properties/declaration-handler"`` + | data type: xml.sax.sax2lib.DeclHandler (not supported in Python 2) + | description: An optional extension handler for DTD-related events other + than notations and unparsed entities. + | access: read/write + + +.. data:: property_dom_node + + | value: ``"http://xml.org/sax/properties/dom-node"`` + | data type: org.w3c.dom.Node (not supported in Python 2) + | description: When parsing, the current DOM node being visited if this is + a DOM iterator; when not parsing, the root DOM node for iteration. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: property_xml_string + + | value: ``"http://xml.org/sax/properties/xml-string"`` + | data type: String + | description: The literal string of characters that was the source for the + current event. + | access: read-only + + +.. data:: all_properties + + List of all known property names. + + +.. _content-handler-objects: + +ContentHandler Objects +---------------------- + +Users are expected to subclass :class:`ContentHandler` to support their +application. The following methods are called by the parser on the appropriate +events in the input document: + + +.. method:: ContentHandler.setDocumentLocator(locator) + + Called by the parser to give the application a locator for locating the origin + of document events. + + SAX parsers are strongly encouraged (though not absolutely required) to supply a + locator: if it does so, it must supply the locator to the application by + invoking this method before invoking any of the other methods in the + DocumentHandler interface. + + The locator allows the application to determine the end position of any + document-related event, even if the parser is not reporting an error. Typically, + the application will use this information for reporting its own errors (such as + character content that does not match an application's business rules). The + information returned by the locator is probably not sufficient for use with a + search engine. + + Note that the locator will return correct information only during the invocation + of the events in this interface. The application should not attempt to use it at + any other time. + + +.. method:: ContentHandler.startDocument() + + Receive notification of the beginning of a document. + + The SAX parser will invoke this method only once, before any other methods in + this interface or in DTDHandler (except for :meth:`setDocumentLocator`). + + +.. method:: ContentHandler.endDocument() + + Receive notification of the end of a document. + + The SAX parser will invoke this method only once, and it will be the last method + invoked during the parse. The parser shall not invoke this method until it has + either abandoned parsing (because of an unrecoverable error) or reached the end + of input. + + +.. method:: ContentHandler.startPrefixMapping(prefix, uri) + + Begin the scope of a prefix-URI Namespace mapping. + + The information from this event is not necessary for normal Namespace + processing: the SAX XML reader will automatically replace prefixes for element + and attribute names when the ``feature_namespaces`` feature is enabled (the + default). + + There are cases, however, when applications need to use prefixes in character + data or in attribute values, where they cannot safely be expanded automatically; + the :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events supply the + information to the application to expand prefixes in those contexts itself, if + necessary. + + .. XXX This is not really the default, is it? MvL + + Note that :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events are not + guaranteed to be properly nested relative to each-other: all + :meth:`startPrefixMapping` events will occur before the corresponding + :meth:`startElement` event, and all :meth:`endPrefixMapping` events will occur + after the corresponding :meth:`endElement` event, but their order is not + guaranteed. + + +.. method:: ContentHandler.endPrefixMapping(prefix) + + End the scope of a prefix-URI mapping. + + See :meth:`startPrefixMapping` for details. This event will always occur after + the corresponding :meth:`endElement` event, but the order of + :meth:`endPrefixMapping` events is not otherwise guaranteed. + + +.. method:: ContentHandler.startElement(name, attrs) + + Signals the start of an element in non-namespace mode. + + The *name* parameter contains the raw XML 1.0 name of the element type as a + string and the *attrs* parameter holds an object of the + :class:`~xml.sax.xmlreader.Attributes` + interface (see :ref:`attributes-objects`) containing the attributes of + the element. The object passed as *attrs* may be re-used by the parser; holding + on to a reference to it is not a reliable way to keep a copy of the attributes. + To keep a copy of the attributes, use the :meth:`copy` method of the *attrs* + object. + + +.. method:: ContentHandler.endElement(name) + + Signals the end of an element in non-namespace mode. + + The *name* parameter contains the name of the element type, just as with the + :meth:`startElement` event. + + +.. method:: ContentHandler.startElementNS(name, qname, attrs) + + Signals the start of an element in namespace mode. + + The *name* parameter contains the name of the element type as a ``(uri, + localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in + the source document, and the *attrs* parameter holds an instance of the + :class:`~xml.sax.xmlreader.AttributesNS` interface (see + :ref:`attributes-ns-objects`) + containing the attributes of the element. If no namespace is associated with + the element, the *uri* component of *name* will be ``None``. The object passed + as *attrs* may be re-used by the parser; holding on to a reference to it is not + a reliable way to keep a copy of the attributes. To keep a copy of the + attributes, use the :meth:`copy` method of the *attrs* object. + + Parsers may set the *qname* parameter to ``None``, unless the + ``feature_namespace_prefixes`` feature is activated. + + +.. method:: ContentHandler.endElementNS(name, qname) + + Signals the end of an element in namespace mode. + + The *name* parameter contains the name of the element type, just as with the + :meth:`startElementNS` method, likewise the *qname* parameter. + + +.. method:: ContentHandler.characters(content) + + Receive notification of character data. + + The Parser will call this method to report each chunk of character data. SAX + parsers may return all contiguous character data in a single chunk, or they may + split it into several chunks; however, all of the characters in any single event + must come from the same external entity so that the Locator provides useful + information. + + *content* may be a string or bytes instance; the ``expat`` reader module + always produces strings. + + .. note:: + + The earlier SAX 1 interface provided by the Python XML Special Interest Group + used a more Java-like interface for this method. Since most parsers used from + Python did not take advantage of the older interface, the simpler signature was + chosen to replace it. To convert old code to the new interface, use *content* + instead of slicing content with the old *offset* and *length* parameters. + + +.. method:: ContentHandler.ignorableWhitespace(whitespace) + + Receive notification of ignorable whitespace in element content. + + Validating Parsers must use this method to report each chunk of ignorable + whitespace (see the W3C XML 1.0 recommendation, section 2.10): non-validating + parsers may also use this method if they are capable of parsing and using + content models. + + SAX parsers may return all contiguous whitespace in a single chunk, or they may + split it into several chunks; however, all of the characters in any single event + must come from the same external entity, so that the Locator provides useful + information. + + +.. method:: ContentHandler.processingInstruction(target, data) + + Receive notification of a processing instruction. + + The Parser will invoke this method once for each processing instruction found: + note that processing instructions may occur before or after the main document + element. + + A SAX parser should never report an XML declaration (XML 1.0, section 2.8) or a + text declaration (XML 1.0, section 4.3.1) using this method. + + +.. method:: ContentHandler.skippedEntity(name) + + Receive notification of a skipped entity. + + The Parser will invoke this method once for each entity skipped. Non-validating + processors may skip entities if they have not seen the declarations (because, + for example, the entity was declared in an external DTD subset). All processors + may skip external entities, depending on the values of the + ``feature_external_ges`` and the ``feature_external_pes`` properties. + + +.. _dtd-handler-objects: + +DTDHandler Objects +------------------ + +:class:`DTDHandler` instances provide the following methods: + + +.. method:: DTDHandler.notationDecl(name, publicId, systemId) + + Handle a notation declaration event. + + +.. method:: DTDHandler.unparsedEntityDecl(name, publicId, systemId, ndata) + + Handle an unparsed entity declaration event. + + +.. _entity-resolver-objects: + +EntityResolver Objects +---------------------- + + +.. method:: EntityResolver.resolveEntity(publicId, systemId) + + Resolve the system identifier of an entity and return either the system + identifier to read from as a string, or an InputSource to read from. The default + implementation returns *systemId*. + + +.. _sax-error-handler: + +ErrorHandler Objects +-------------------- + +Objects with this interface are used to receive error and warning information +from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that +implements this interface, then register the object with your +:class:`~xml.sax.xmlreader.XMLReader`, the parser +will call the methods in your object to report all warnings and errors. There +are three levels of errors available: warnings, (possibly) recoverable errors, +and unrecoverable errors. All methods take a :exc:`SAXParseException` as the +only parameter. Errors and warnings may be converted to an exception by raising +the passed-in exception object. + + +.. method:: ErrorHandler.error(exception) + + Called when the parser encounters a recoverable error. If this method does not + raise an exception, parsing may continue, but further document information + should not be expected by the application. Allowing the parser to continue may + allow additional errors to be discovered in the input document. + + +.. method:: ErrorHandler.fatalError(exception) + + Called when the parser encounters an error it cannot recover from; parsing is + expected to terminate when this method returns. + + +.. method:: ErrorHandler.warning(exception) + + Called when the parser presents minor warning information to the application. + Parsing is expected to continue when this method returns, and document + information will continue to be passed to the application. Raising an exception + in this method will cause parsing to end. + + +.. _lexical-handler-objects: + +LexicalHandler Objects +---------------------- +Optional SAX2 handler for lexical events. + +This handler is used to obtain lexical information about an XML +document. Lexical information includes information describing the +document encoding used and XML comments embedded in the document, as +well as section boundaries for the DTD and for any CDATA sections. +The lexical handlers are used in the same manner as content handlers. + +Set the LexicalHandler of an XMLReader by using the setProperty method +with the property identifier +``'http://xml.org/sax/properties/lexical-handler'``. + + +.. method:: LexicalHandler.comment(content) + + Reports a comment anywhere in the document (including the DTD and + outside the document element). + +.. method:: LexicalHandler.startDTD(name, public_id, system_id) + + Reports the start of the DTD declarations if the document has an + associated DTD. + +.. method:: LexicalHandler.endDTD() + + Reports the end of DTD declaration. + +.. method:: LexicalHandler.startCDATA() + + Reports the start of a CDATA marked section. + + The contents of the CDATA marked section will be reported through + the characters handler. + +.. method:: LexicalHandler.endCDATA() + + Reports the end of a CDATA marked section. diff --git a/Doc/library/zipimport.rst.bak b/Doc/library/zipimport.rst.bak new file mode 100644 index 00000000000000..62e1e47e980e17 --- /dev/null +++ b/Doc/library/zipimport.rst.bak @@ -0,0 +1,209 @@ +:mod:`zipimport` --- Import modules from Zip archives +===================================================== + +.. module:: zipimport + :synopsis: Support for importing Python modules from ZIP archives. + +.. moduleauthor:: Just van Rossum + +**Source code:** :source:`Lib/zipimport.py` + +-------------- + +This module adds the ability to import Python modules (:file:`\*.py`, +:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not +needed to use the :mod:`zipimport` module explicitly; it is automatically used +by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths +to ZIP archives. + +Typically, :data:`sys.path` is a list of directory names as strings. This module +also allows an item of :data:`sys.path` to be a string naming a ZIP file archive. +The ZIP archive can contain a subdirectory structure to support package imports, +and a path within the archive can be specified to only import from a +subdirectory. For example, the path :file:`example.zip/lib/` would only +import from the :file:`lib/` subdirectory within the archive. + +Any files may be present in the ZIP archive, but only files :file:`.py` and +:file:`.pyc` are available for import. ZIP import of dynamic modules +(:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains +:file:`.py` files, Python will not attempt to modify the archive by adding the +corresponding :file:`.pyc` file, meaning that if a ZIP archive +doesn't contain :file:`.pyc` files, importing may be rather slow. + +.. versionchanged:: 3.8 + Previously, ZIP archives with an archive comment were not supported. + +.. seealso:: + + `PKZIP Application Note `_ + Documentation on the ZIP file format by Phil Katz, the creator of the format and + algorithms used. + + :pep:`273` - Import Modules from Zip Archives + Written by James C. Ahlstrom, who also provided an implementation. Python 2.3 + follows the specification in :pep:`273`, but uses an implementation written by Just + van Rossum that uses the import hooks described in :pep:`302`. + + :mod:`importlib` - The implementation of the import machinery + Package providing the relevant protocols for all importers to + implement. + + +This module defines an exception: + +.. exception:: ZipImportError + + Exception raised by zipimporter objects. It's a subclass of :exc:`ImportError`, + so it can be caught as :exc:`ImportError`, too. + + +.. _zipimporter-objects: + +zipimporter Objects +------------------- + +:class:`zipimporter` is the class for importing ZIP files. + +.. class:: zipimporter(archivepath) + + Create a new zipimporter instance. *archivepath* must be a path to a ZIP + file, or to a specific path within a ZIP file. For example, an *archivepath* + of :file:`foo/bar.zip/lib` will look for modules in the :file:`lib` directory + inside the ZIP file :file:`foo/bar.zip` (provided that it exists). + + :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP + archive. + + .. method:: create_module(spec) + + Implementation of :meth:`importlib.abc.Loader.create_module` that returns + :const:`None` to explicitly request the default semantics. + + .. versionadded:: 3.10 + + + .. method:: exec_module(module) + + Implementation of :meth:`importlib.abc.Loader.exec_module`. + + .. versionadded:: 3.10 + + + .. method:: find_loader(fullname, path=None) + + An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. + + .. deprecated:: 3.10 + + Use :meth:`find_spec` instead. + + + .. method:: find_module(fullname, path=None) + + Search for a module specified by *fullname*. *fullname* must be the fully + qualified (dotted) module name. It returns the zipimporter instance itself + if the module was found, or :const:`None` if it wasn't. The optional + *path* argument is ignored---it's there for compatibility with the + importer protocol. + + .. deprecated:: 3.10 + + Use :meth:`find_spec` instead. + + + .. method:: find_spec(fullname, target=None) + + An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. + + .. versionadded:: 3.10 + + + .. method:: get_code(fullname) + + Return the code object for the specified module. Raise + :exc:`ZipImportError` if the module couldn't be imported. + + + .. method:: get_data(pathname) + + Return the data associated with *pathname*. Raise :exc:`OSError` if the + file wasn't found. + + .. versionchanged:: 3.3 + :exc:`IOError` used to be raised instead of :exc:`OSError`. + + + .. method:: get_filename(fullname) + + Return the value ``__file__`` would be set to if the specified module + was imported. Raise :exc:`ZipImportError` if the module couldn't be + imported. + + .. versionadded:: 3.1 + + + .. method:: get_source(fullname) + + Return the source code for the specified module. Raise + :exc:`ZipImportError` if the module couldn't be found, return + :const:`None` if the archive does contain the module, but has no source + for it. + + + .. method:: is_package(fullname) + + Return ``True`` if the module specified by *fullname* is a package. Raise + :exc:`ZipImportError` if the module couldn't be found. + + + .. method:: load_module(fullname) + + Load the module specified by *fullname*. *fullname* must be the fully + qualified (dotted) module name. Returns the imported module on success, + raises :exc:`ZipImportError` on failure. + + .. deprecated:: 3.10 + + Use :meth:`exec_module` instead. + + .. attribute:: archive + + The file name of the importer's associated ZIP file, without a possible + subpath. + + + .. attribute:: prefix + + The subpath within the ZIP file where modules are searched. This is the + empty string for zipimporter objects which point to the root of the ZIP + file. + + The :attr:`archive` and :attr:`prefix` attributes, when combined with a + slash, equal the original *archivepath* argument given to the + :class:`zipimporter` constructor. + + +.. _zipimport-examples: + +Examples +-------- + +Here is an example that imports a module from a ZIP archive - note that the +:mod:`zipimport` module is not explicitly used. + +.. code-block:: shell-session + + $ unzip -l example.zip + Archive: example.zip + Length Date Time Name + -------- ---- ---- ---- + 8467 11-26-02 22:30 jwzthreading.py + -------- ------- + 8467 1 file + $ ./python + Python 2.3 (#1, Aug 1 2003, 19:54:32) + >>> import sys + >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path + >>> import jwzthreading + >>> jwzthreading.__file__ + 'example.zip/jwzthreading.py' diff --git a/Doc/whatsnew/3.10.rst.bak b/Doc/whatsnew/3.10.rst.bak new file mode 100644 index 00000000000000..db71f061f14df4 --- /dev/null +++ b/Doc/whatsnew/3.10.rst.bak @@ -0,0 +1,1315 @@ +**************************** + What's New In Python 3.10 +**************************** + +:Release: |release| +:Date: |today| + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the bug/patch number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :issue:`12345`.) + + This saves the maintainer the effort of going through the Mercurial log + when researching a change. + +This article explains the new features in Python 3.10, compared to 3.9. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.10 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.10. + Brevity is key. + + +.. PEP-sized items next. + + + +New Features +============ + +.. _whatsnew310-pep563: + +Parenthesized context managers +------------------------------ + +Using enclosing parentheses for continuation across multiple lines +in context managers is now supported. This allows formatting a long +collection of context managers in multiple lines in a similar way +as it was previously possible with import statements. For instance, +all these examples are now valid: + +.. code-block:: python + + with (CtxManager() as example): + ... + + with ( + CtxManager1(), + CtxManager2() + ): + ... + + with (CtxManager1() as example, + CtxManager2()): + ... + + with (CtxManager1(), + CtxManager2() as example): + ... + + with ( + CtxManager1() as example1, + CtxManager2() as example2 + ): + ... + +it is also possible to use a trailing comma at the end of the +enclosed group: + +.. code-block:: python + + with ( + CtxManager1() as example1, + CtxManager2() as example2, + CtxManager3() as example3, + ): + ... + +This new syntax uses the non LL(1) capacities of the new parser. +Check :pep:`617` for more details. + +(Contributed by Guido van Rossum, Pablo Galindo and Lysandros Nikolaou +in :issue:`12782` and :issue:`40334`.) + + +Better error messages in the parser +----------------------------------- + +When parsing code that contains unclosed parentheses or brackets the interpreter +now includes the location of the unclosed bracket of parentheses instead of displaying +*SyntaxError: unexpected EOF while parsing* or pointing to some incorrect location. +For instance, consider the following code (notice the unclosed '{'): + +.. code-block:: python + + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + 38: 4, 39: 4, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 54: 6, + some_other_code = foo() + +previous versions of the interpreter reported confusing places as the location of +the syntax error: + +.. code-block:: text + + File "example.py", line 3 + some_other_code = foo() + ^ + SyntaxError: invalid syntax + +but in Python3.10 a more informative error is emitted: + +.. code-block:: text + + File "example.py", line 1 + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + ^ + SyntaxError: '{' was never closed + + +In a similar way, errors involving unclosed string literals (single and triple +quoted) now point to the start of the string instead of reporting EOF/EOL. + +These improvements are inspired by previous work in the PyPy interpreter. + +(Contributed by Pablo Galindo in :issue:`42864` and Batuhan Taskaya in +:issue:`40176`.) + + +PEP 634: Structural Pattern Matching +------------------------------------ + +Structural pattern matching has been added in the form of a *match statement* +and *case statements* of patterns with associated actions. Patterns +consist of sequences, mappings, primitive data types as well as class instances. +Pattern matching enables programs to extract information from complex data types, +branch on the structure of data, and apply specific actions based on different +forms of data. + +Syntax and operations +~~~~~~~~~~~~~~~~~~~~~ + +The generic syntax of pattern matching is:: + + match subject: + case : + + case : + + case : + + case _: + + +A match statement takes an expression and compares its value to successive +patterns given as one or more case blocks. Specifically, pattern matching +operates by: + + 1. using data with type and shape (the ``subject``) + 2. evaluating the ``subject`` in the ``match`` statement + 3. comparing the subject with each pattern in a ``case`` statement + from top to bottom until a match is confirmed. + 4. executing the action associated with the pattern of the confirmed + match + 5. If an exact match is not confirmed, the last case, a wildcard ``_``, + if provided, will be used as the matching case. If an exact match is + not confirmed and a wildcard case does not exists, the entire match + block is a no-op. + +Declarative approach +~~~~~~~~~~~~~~~~~~~~ + +Readers may be aware of pattern matching through the simple example of matching +a subject (data object) to a literal (pattern) with the switch statement found +in C, Java or JavaScript (and many other languages). Often the switch statement +is used for comparison of an object/expression with case statements containing +literals. + +More powerful examples of pattern matching can be found in languages, such as +Scala and Elixir. With structural pattern matching, the approach is "declarative" and +explicitly states the conditions (the patterns) for data to match. + +While an "imperative" series of instructions using nested "if" statements +could be used to accomplish something similar to structural pattern matching, +it is less clear than the "declarative" approach. Instead the "declarative" +approach states the conditions to meet for a match and is more readable through +its explicit patterns. While structural pattern matching can be used in its +simplest form comparing a variable to a literal in a case statement, its +true value for Python lies in its handling of the subject's type and shape. + +Simple pattern: match to a literal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's look at this example as pattern matching in its simplest form: a value, +the subject, being matched to several literals, the patterns. In the example +below, ``status`` is the subject of the match statement. The patterns are +each of the case statements, where literals represent request status codes. +The associated action to the case is executed after a match:: + + def http_error(status): + match status: + case 400: + return "Bad request" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + case _: + return "Something's wrong with the Internet" + +If the above function is passed a ``status`` of 418, "I'm a teapot" is returned. +If the above function is passed a ``status`` of 500, the case statement with +``_`` will match as a wildcard, and "Something's wrong with the Internet" is +returned. +Note the last block: the variable name, ``_``, acts as a *wildcard* and insures +the subject will always match. The use of ``_`` is optional. + +You can combine several literals in a single pattern using ``|`` ("or"):: + + case 401 | 403 | 404: + return "Not allowed" + +Behavior without the wildcard +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If we modify the above example by removing the last case block, the example +becomes:: + + def http_error(status): + match status: + case 400: + return "Bad request" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + +Without the use of ``_`` in a case statement, a match may not exist. If no +match exists, the behavior is a no-op. For example, if ``status`` of 500 is +passed, a no-op occurs. + +Patterns with a literal and variable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Patterns can look like unpacking assignments, and a pattern may be used to bind +variables. In this example, a data point can be unpacked to its x-coordinate +and y-coordinate:: + + # point is an (x, y) tuple + match point: + case (0, 0): + print("Origin") + case (0, y): + print(f"Y={y}") + case (x, 0): + print(f"X={x}") + case (x, y): + print(f"X={x}, Y={y}") + case _: + raise ValueError("Not a point") + +The first pattern has two literals, ``(0, 0)``, and may be thought of as an +extension of the literal pattern shown above. The next two patterns combine a +literal and a variable, and the variable *binds* a value from the subject +(``point``). The fourth pattern captures two values, which makes it +conceptually similar to the unpacking assignment ``(x, y) = point``. + +Patterns and classes +~~~~~~~~~~~~~~~~~~~~ + +If you are using classes to structure your data, you can use as a pattern +the class name followed by an argument list resembling a constructor. This +pattern has the ability to capture class attributes into variables:: + + class Point: + x: int + y: int + + def location(point): + match point: + case Point(x=0, y=0): + print("Origin is the point's location.") + case Point(x=0, y=y): + print(f"Y={y} and the point is on the y-axis.") + case Point(x=x, y=0): + print(f"X={x} and the point is on the x-axis.") + case Point(): + print("The point is located somewhere else on the plane.") + case _: + print("Not a point") + +Patterns with positional parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can use positional parameters with some builtin classes that provide an +ordering for their attributes (e.g. dataclasses). You can also define a specific +position for attributes in patterns by setting the ``__match_args__`` special +attribute in your classes. If it's set to ("x", "y"), the following patterns +are all equivalent (and all bind the ``y`` attribute to the ``var`` variable):: + + Point(1, var) + Point(1, y=var) + Point(x=1, y=var) + Point(y=var, x=1) + +Nested patterns +~~~~~~~~~~~~~~~ + +Patterns can be arbitrarily nested. For example, if our data is a short +list of points, it could be matched like this:: + + match points: + case []: + print("No points in the list.") + case [Point(0, 0)]: + print("The origin is the only point in the list.") + case [Point(x, y)]: + print(f"A single point {x}, {y} is in the list.") + case [Point(0, y1), Point(0, y2)]: + print(f"Two points on the Y axis at {y1}, {y2} are in the list.") + case _: + print("Something else is found in the list.") + +Complex patterns and the wildcard +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To this point, the examples have used ``_`` alone in the last case statement. +A wildcard can be used in more complex patterns, such as ``('error', code, _)``. +For example:: + + match test_variable: + case ('warning', code, 40): + print("A warning has been received.") + case ('error', code, _): + print(f"An error {code} occured.") + +In the above case, ``test_variable`` will match for ('error', code, 100) and +('error', code, 800). + +Guard +~~~~~ + +We can add an ``if`` clause to a pattern, known as a "guard". If the +guard is false, ``match`` goes on to try the next case block. Note +that value capture happens before the guard is evaluated:: + + match point: + case Point(x, y) if x == y: + print(f"The point is located on the diagonal Y=X at {x}.") + case Point(x, y): + print(f"Point is not on the diagonal.") + +Other Key Features +~~~~~~~~~~~~~~~~~~ + +Several other key features: + +- Like unpacking assignments, tuple and list patterns have exactly the + same meaning and actually match arbitrary sequences. Technically, + the subject must be an instance of ``collections.abc.Sequence``. + Therefore, an important exception is that patterns don't match iterators. + Also, to prevent a common mistake, sequence patterns don't match strings. + +- Sequence patterns support wildcards: ``[x, y, *rest]`` and ``(x, y, + *rest)`` work similar to wildcards in unpacking assignments. The + name after ``*`` may also be ``_``, so ``(x, y, *_)`` matches a sequence + of at least two items without binding the remaining items. + +- Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the + ``"bandwidth"`` and ``"latency"`` values from a dict. Unlike sequence + patterns, extra keys are ignored. A wildcard ``**rest`` is also + supported. (But ``**_`` would be redundant, so it not allowed.) + +- Subpatterns may be captured using the ``as`` keyword:: + + case (Point(x1, y1), Point(x2, y2) as p2): ... + + This binds x1, y1, x2, y2 like you would expect without the ``as`` clause, + and p2 to the entire second item of the subject. + +- Most literals are compared by equality. However, the singletons ``True``, + ``False`` and ``None`` are compared by identity. + +- Named constants may be used in patterns. These named constants must be + dotted names to prevent the constant from being interpreted as a capture + variable:: + + from enum import Enum + class Color(Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + + match color: + case Color.RED: + print("I see red!") + case Color.GREEN: + print("Grass is green") + case Color.BLUE: + print("I'm feeling the blues :(") + +For the full specification see :pep:`634`. Motivation and rationale +are in :pep:`635`, and a longer tutorial is in :pep:`636`. + + +New Features Related to Type Annotations +======================================== + +This section covers major changes affecting :pep:`484` type annotations and +the :mod:`typing` module. + + +PEP 563: Postponed Evaluation of Annotations Becomes Default +------------------------------------------------------------ + +In Python 3.7, postponed evaluation of annotations was added, +to be enabled with a ``from __future__ import annotations`` +directive. In 3.10 this became the default behavior, even +without that future directive. With this being default, all +annotations stored in :attr:`__annotations__` will be strings. +If needed, annotations can be resolved at runtime using +:func:`typing.get_type_hints`. See :pep:`563` for a full +description. Also, the :func:`inspect.signature` will try to +resolve types from now on, and when it fails it will fall back to +showing the string annotations. (Contributed by Batuhan Taskaya +in :issue:`38605`.) + + +PEP 604: New Type Union Operator +-------------------------------- + +A new type union operator was introduced which enables the syntax ``X | Y``. +This provides a cleaner way of expressing 'either type X or type Y' instead of +using :data:`typing.Union`, especially in type hints (annotations). + +In previous versions of Python, to apply a type hint for functions accepting +arguments of multiple types, :data:`typing.Union` was used:: + + def square(number: Union[int, float]) -> Union[int, float]: + return number ** 2 + + +Type hints can now be written in a more succinct manner:: + + def square(number: int | float) -> int | float: + return number ** 2 + + +This new syntax is also accepted as the second argument to :func:`isinstance` +and :func:`issubclass`:: + + >>> isinstance(1, int | str) + True + +See :ref:`types-union` and :pep:`604` for more details. + +(Contributed by Maggie Moss and Philippe Prados in :issue:`41428`.) + + +PEP 612: Parameter Specification Variables +------------------------------------------ + +Two new options to improve the information provided to static type checkers for +:pep:`484`\ 's ``Callable`` have been added to the :mod:`typing` module. + +The first is the parameter specification variable. They are used to forward the +parameter types of one callable to another callable -- a pattern commonly +found in higher order functions and decorators. Examples of usage can be found +in :class:`typing.ParamSpec`. Previously, there was no easy way to type annotate +dependency of parameter types in such a precise manner. + +The second option is the new ``Concatenate`` operator. It's used in conjunction +with parameter specification variables to type annotate a higher order callable +which adds or removes parameters of another callable. Examples of usage can +be found in :class:`typing.Concatenate`. + +See :class:`typing.Callable`, :class:`typing.ParamSpec`, +:class:`typing.Concatenate` and :pep:`612` for more details. + +(Contributed by Ken Jin in :issue:`41559`.) + + +PEP 613: TypeAlias Annotation +----------------------------- + +:pep:`484` introduced the concept of type aliases, only requiring them to be +top-level unannotated assignments. This simplicity sometimes made it difficult +for type checkers to distinguish between type aliases and ordinary assignments, +especially when forward references or invalid types were involved. Compare:: + + StrCache = 'Cache[str]' # a type alias + LOG_PREFIX = 'LOG[DEBUG]' # a module constant + +Now the :mod:`typing` module has a special annotation :data:`TypeAlias` to +declare type aliases more explicitly:: + + StrCache: TypeAlias = 'Cache[str]' # a type alias + LOG_PREFIX = 'LOG[DEBUG]' # a module constant + +See :pep:`613` for more details. + +(Contributed by Mikhail Golubev in :issue:`41923`.) + + +Other Language Changes +====================== + +* The :class:`int` type has a new method :meth:`int.bit_count`, returning the + number of ones in the binary expansion of a given integer, also known + as the population count. (Contributed by Niklas Fiekas in :issue:`29882`.) + +* The views returned by :meth:`dict.keys`, :meth:`dict.values` and + :meth:`dict.items` now all have a ``mapping`` attribute that gives a + :class:`types.MappingProxyType` object wrapping the original + dictionary. (Contributed by Dennis Sweeney in :issue:`40890`.) + +* :pep:`618`: The :func:`zip` function now has an optional ``strict`` flag, used + to require that all the iterables have an equal length. + +* Builtin and extension functions that take integer arguments no longer accept + :class:`~decimal.Decimal`\ s, :class:`~fractions.Fraction`\ s and other + objects that can be converted to integers only with a loss (e.g. that have + the :meth:`~object.__int__` method but do not have the + :meth:`~object.__index__` method). + (Contributed by Serhiy Storchaka in :issue:`37999`.) + +* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will + correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. + (Contributed by Alex Shkop in :issue:`38302`.) + +* Assignment expressions can now be used unparenthesized within set literals + and set comprehensions, as well as in sequence indexes (but not slices). + +* Functions have a new ``__builtins__`` attribute which is used to look for + builtin symbols when a function is executed, instead of looking into + ``__globals__['__builtins__']``. The attribute is initialized from + ``__globals__["__builtins__"]`` if it exists, else from the current builtins. + (Contributed by Mark Shannon in :issue:`42990`.) + + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + +argparse +-------- + +Misleading phrase "optional arguments" was replaced with "options" in argparse help. Some tests might require adaptation if they rely on exact output match. +(Contributed by Raymond Hettinger in :issue:`9694`.) + +base64 +------ + +Add :func:`base64.b32hexencode` and :func:`base64.b32hexdecode` to support the +Base32 Encoding with Extended Hex Alphabet. + +codecs +------ + +Add a :func:`codecs.unregister` function to unregister a codec search function. +(Contributed by Hai Shi in :issue:`41842`.) + +collections.abc +--------------- + +The ``__args__`` of the :ref:`parameterized generic ` for +:class:`collections.abc.Callable` are now consistent with :data:`typing.Callable`. +:class:`collections.abc.Callable` generic now flattens type parameters, similar +to what :data:`typing.Callable` currently does. This means that +``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of +``(int, str, str)``; previously this was ``([int, str], str)``. To allow this +change, :class:`types.GenericAlias` can now be subclassed, and a subclass will +be returned when subscripting the :class:`collections.abc.Callable` type. Note +that a :exc:`TypeError` may be raised for invalid forms of parameterizing +:class:`collections.abc.Callable` which may have passed silently in Python 3.9. +(Contributed by Ken Jin in :issue:`42195`.) + +contextlib +---------- + +Add a :func:`contextlib.aclosing` context manager to safely close async generators +and objects representing asynchronously released resources. +(Contributed by Joongi Kim and John Belmonte in :issue:`41229`.) + +Add asynchronous context manager support to :func:`contextlib.nullcontext`. +(Contributed by Tom Gringauz in :issue:`41543`.) + +curses +------ + +The extended color functions added in ncurses 6.1 will be used transparently +by :func:`curses.color_content`, :func:`curses.init_color`, +:func:`curses.init_pair`, and :func:`curses.pair_content`. A new function, +:func:`curses.has_extended_color_support`, indicates whether extended color +support is provided by the underlying ncurses library. +(Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) + +The ``BUTTON5_*`` constants are now exposed in the :mod:`curses` module if +they are provided by the underlying curses library. +(Contributed by Zackery Spytz in :issue:`39273`.) + +.. _distutils-deprecated: + +distutils +--------- + +The entire ``distutils`` package is deprecated, to be removed in Python +3.12. Its functionality for specifying package builds has already been +completely replaced by third-party packages ``setuptools`` and +``packaging``, and most other commonly used APIs are available elsewhere +in the standard library (such as :mod:`platform`, :mod:`shutil`, +:mod:`subprocess` or :mod:`sysconfig`). There are no plans to migrate +any other functionality from ``distutils``, and applications that are +using other functions should plan to make private copies of the code. +Refer to :pep:`632` for discussion. + +The ``bdist_wininst`` command deprecated in Python 3.8 has been removed. +The ``bdist_wheel`` command is now recommended to distribute binary packages +on Windows. +(Contributed by Victor Stinner in :issue:`42802`.) + +doctest +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +encodings +--------- +:func:`encodings.normalize_encoding` now ignores non-ASCII characters. +(Contributed by Hai Shi in :issue:`39337`.) + +glob +---- + +Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and +:func:`~glob.iglob` which allow to specify the root directory for searching. +(Contributed by Serhiy Storchaka in :issue:`38144`.) + +inspect +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +Added *globalns* and *localns* parameters in :func:`~inspect.signature` and +:meth:`inspect.Signature.from_callable` to retrieve the annotations in given +local and global namespaces. +(Contributed by Batuhan Taskaya in :issue:`41960`.) + +linecache +--------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +os +-- + +Added :func:`os.cpu_count()` support for VxWorks RTOS. +(Contributed by Peixing Xin in :issue:`41440`.) + +Added a new function :func:`os.eventfd` and related helpers to wrap the +``eventfd2`` syscall on Linux. +(Contributed by Christian Heimes in :issue:`41001`.) + +Added :func:`os.splice()` that allows to move data between two file +descriptors without copying between kernel address space and user +address space, where one of the file descriptors must refer to a +pipe. (Contributed by Pablo Galindo in :issue:`41625`.) + +Added :data:`~os.O_EVTONLY`, :data:`~os.O_FSYNC`, :data:`~os.O_SYMLINK` +and :data:`~os.O_NOFOLLOW_ANY` for macOS. +(Contributed by Dong-hee Na in :issue:`43106`.) + +pathlib +------- + +Added slice support to :attr:`PurePath.parents `. +(Contributed by Joshua Cannon in :issue:`35498`) + +Added negative indexing support to :attr:`PurePath.parents +`. +(Contributed by Yaroslav Pankovych in :issue:`21041`) + +platform +-------- + +Added :func:`platform.freedesktop_os_release()` to retrieve operation system +identification from `freedesktop.org os-release +`_ standard file. +(Contributed by Christian Heimes in :issue:`28468`) + +py_compile +---------- + +Added ``--quiet`` option to command-line interface of :mod:`py_compile`. +(Contributed by Gregory Schevchenko in :issue:`38731`.) + +pyclbr +------ + +Added an ``end_lineno`` attribute to the ``Function`` and ``Class`` +objects in the tree returned by :func:`pyclbr.readline` and +:func:`pyclbr.readline_ex`. It matches the existing (start) ``lineno``. +(Contributed by Aviral Srivastava in :issue:`38307`.) + +shelve +------ + +The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default +instead of :mod:`pickle` protocol ``3`` when creating shelves. +(Contributed by Zackery Spytz in :issue:`34204`.) + +site +---- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +socket +------ + +The exception :exc:`socket.timeout` is now an alias of :exc:`TimeoutError`. +(Contributed by Christian Heimes in :issue:`42413`.) + +sys +--- + +Add :data:`sys.orig_argv` attribute: the list of the original command line +arguments passed to the Python executable. +(Contributed by Victor Stinner in :issue:`23427`.) + +Add :data:`sys.stdlib_module_names`, containing the list of the standard library +module names. +(Contributed by Victor Stinner in :issue:`42955`.) + +threading +--------- + +Added :func:`threading.gettrace` and :func:`threading.getprofile` to +retrieve the functions set by :func:`threading.settrace` and +:func:`threading.setprofile` respectively. +(Contributed by Mario Corchero in :issue:`42251`.) + +Add :data:`threading.__excepthook__` to allow retrieving the original value +of :func:`threading.excepthook` in case it is set to a broken or a different +value. +(Contributed by Mario Corchero in :issue:`42308`.) + +traceback +--------- + +The :func:`~traceback.format_exception`, +:func:`~traceback.format_exception_only`, and +:func:`~traceback.print_exception` functions can now take an exception object +as a positional-only argument. +(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + +types +----- + +Reintroduced the :data:`types.EllipsisType`, :data:`types.NoneType` +and :data:`types.NotImplementedType` classes, providing a new set +of types readily interpretable by type checkers. +(Contributed by Bas van Beek in :issue:`41810`.) + +typing +------ + +For major changes, see `New Features Related to Type Annotations`_. + +The behavior of :class:`typing.Literal` was changed to conform with :pep:`586` +and to match the behavior of static type checkers specified in the PEP. + +1. ``Literal`` now de-duplicates parameters. +2. Equality comparisons between ``Literal`` objects are now order independent. +3. ``Literal`` comparisons now respects types. For example, + ``Literal[0] == Literal[False]`` previously evaluated to ``True``. It is + now ``False``. To support this change, the internally used type cache now + supports differentiating types. +4. ``Literal`` objects will now raise a :exc:`TypeError` exception during + equality comparisons if one of their parameters are not :term:`immutable`. + Note that declaring ``Literal`` with mutable parameters will not throw + an error:: + + >>> from typing import Literal + >>> Literal[{0}] + >>> Literal[{0}] == Literal[{False}] + Traceback (most recent call last): + File "", line 1, in + TypeError: unhashable type: 'set' + +(Contributed by Yurii Karabas in :issue:`42345`.) + +unittest +-------- + +Add new method :meth:`~unittest.TestCase.assertNoLogs` to complement the +existing :meth:`~unittest.TestCase.assertLogs`. (Contributed by Kit Yan Choi +in :issue:`39385`.) + +urllib.parse +------------ + +Python versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +xml +--- + +Add a :class:`~xml.sax.handler.LexicalHandler` class to the +:mod:`xml.sax.handler` module. +(Contributed by Jonathan Gossage and Zackery Spytz in :issue:`35018`.) + +zipimport +--------- +Add methods related to :pep:`451`: :meth:`~zipimport.zipimporter.find_spec`, +:meth:`zipimport.zipimporter.create_module`, and +:meth:`zipimport.zipimporter.exec_module`. +(Contributed by Brett Cannon in :issue:`42131`. + + +Optimizations +============= + +* Constructors :func:`str`, :func:`bytes` and :func:`bytearray` are now faster + (around 30--40% for small objects). + (Contributed by Serhiy Storchaka in :issue:`41334`.) + +* The :mod:`runpy` module now imports fewer modules. + The ``python3 -m module-name`` command startup time is 1.3x faster in + average. + (Contributed by Victor Stinner in :issue:`41006`.) + +* The ``LOAD_ATTR`` instruction now uses new "per opcode cache" mechanism. It + is about 36% faster now for regular attributes and 44% faster for slots. + (Contributed by Pablo Galindo and Yury Selivanov in :issue:`42093` and Guido + van Rossum in :issue:`42927`, based on ideas implemented originally in PyPy + and MicroPython.) + +* When building Python with ``--enable-optimizations`` now + ``-fno-semantic-interposition`` is added to both the compile and link line. + This speeds builds of the Python interpreter created with ``--enable-shared`` + with ``gcc`` by up to 30%. See `this article + `_ + for more details. (Contributed by Victor Stinner and Pablo Galindo in + :issue:`38980`.) + + +* Function parameters and their annotations are no longer computed at runtime, + but rather at compilation time. They are stored as a tuple of strings at the + bytecode level. It is now around 2 times faster to create a function with + parameter annotations. (Contributed by Yurii Karabas and Inada Naoki + in :issue:`42202`) + +* Substring search functions such as ``str1 in str2`` and ``str2.find(str1)`` + now sometimes use Crochemore & Perrin's "Two-Way" string searching + algorithm to avoid quadratic behavior on long strings. (Contributed + by Dennis Sweeney in :issue:`41972`) + +Deprecated +========== + +* Starting in this release, there will be a concerted effort to begin + cleaning up old import semantics that were kept for Python 2.7 + compatibility. Specifically, + :meth:`~importlib.abc.PathEntryFinder.find_loader`/:meth:`~importlib.abc.Finder.find_module` + (superseded by :meth:`~importlib.abc.Finder.find_spec`), + :meth:`~importlib.abc.Loader.load_module` + (superseded by :meth:`~importlib.abc.Loader.exec_module`), + :meth:`~importlib.abc.Loader.module_repr` (which the import system + takes care of for you), the ``__package__`` attribute + (superseded by ``__spec__.parent``), the ``__loader__`` attribute + (superseded by ``__spec__.loader``), and the ``__cached__`` attribute + (superseded by ``__spec__.cached``) will slowly be removed (as well + as other classes and methods in :mod:`importlib`). + :exc:`ImportWarning` and/or :exc:`DeprecationWarning` will be raised + as appropriate to help identify code which needs updating during + this transition. + +* The entire ``distutils`` namespace is deprecated, to be removed in + Python 3.12. Refer to the :ref:`module changes ` + section for more information. + +* Non-integer arguments to :func:`random.randrange` are deprecated. + The :exc:`ValueError` is deprecated in favor of a :exc:`TypeError`. + (Contributed by Serhiy Storchaka and Raymond Hettinger in :issue:`37319`.) + +* The various ``load_module()`` methods of :mod:`importlib` have been + documented as deprecated since Python 3.6, but will now also trigger + a :exc:`DeprecationWarning`. Use + :meth:`~importlib.abc.Loader.exec_module` instead. + (Contributed by Brett Cannon in :issue:`26131`.) + +* :meth:`zimport.zipimporter.load_module` has been deprecated in + preference for :meth:`~zipimport.zipimporter.exec_module`. + (Contributed by Brett Cannon in :issue:`26131`.) + +* The use of :meth:`~importlib.abc.Loader.load_module` by the import + system now triggers an :exc:`ImportWarning` as + :meth:`~importlib.abc.Loader.exec_module` is preferred. + (Contributed by Brett Cannon in :issue:`26131`.) + +* ``sqlite3.OptimizedUnicode`` has been undocumented and obsolete since Python + 3.3, when it was made an alias to :class:`str`. It is now deprecated, + scheduled for removal in Python 3.12. + (Contributed by Erlend E. Aasland in :issue:`42264`.) + +* The undocumented built-in function ``sqlite3.enable_shared_cache`` is now + deprecated, scheduled for removal in Python 3.12. Its use is strongly + discouraged by the SQLite3 documentation. See `the SQLite3 docs + `_ for more details. + If shared cache must be used, open the database in URI mode using the + ``cache=shared`` query parameter. + (Contributed by Erlend E. Aasland in :issue:`24464`.) + + +Removed +======= + +* Removed special methods ``__int__``, ``__float__``, ``__floordiv__``, + ``__mod__``, ``__divmod__``, ``__rfloordiv__``, ``__rmod__`` and + ``__rdivmod__`` of the :class:`complex` class. They always raised + a :exc:`TypeError`. + (Contributed by Serhiy Storchaka in :issue:`41974`.) + +* The ``ParserBase.error()`` method from the private and undocumented ``_markupbase`` + module has been removed. :class:`html.parser.HTMLParser` is the only subclass of + ``ParserBase`` and its ``error()`` implementation has already been removed in + Python 3.5. + (Contributed by Berker Peksag in :issue:`31844`.) + +* Removed the ``unicodedata.ucnhash_CAPI`` attribute which was an internal + PyCapsule object. The related private ``_PyUnicode_Name_CAPI`` structure was + moved to the internal C API. + (Contributed by Victor Stinner in :issue:`42157`.) + +* Removed the ``parser`` module, which was deprecated in 3.9 due to the + switch to the new PEG parser, as well as all the C source and header files + that were only being used by the old parser, including ``node.h``, ``parser.h``, + ``graminit.h`` and ``grammar.h``. + +* Removed the Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, + :c:func:`PyParser_SimpleParseStringFlagsFilename`, + :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` + that were deprecated in 3.9 due to the switch to the new PEG parser. + +* Removed the ``formatter`` module, which was deprecated in Python 3.4. + It is somewhat obsolete, little used, and not tested. It was originally + scheduled to be removed in Python 3.6, but such removals were delayed until + after Python 2.7 EOL. Existing users should copy whatever classes they use + into their code. + (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) + +* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless + now due to the _warnings module was converted to a builtin module in 2.6. + (Contributed by Hai Shi in :issue:`42599`.) + +* Remove deprecated aliases to :ref:`collections-abstract-base-classes` from + the :mod:`collections` module. + (Contributed by Victor Stinner in :issue:`37324`.) + +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. The motivation behind this change is multifold: + + 1. This simplifies the high-level API. + 2. The functions in the high-level API have been implicitly getting the + current thread's running event loop since Python 3.7. There isn't a need to + pass the event loop to the API in most normal use cases. + 3. Event loop passing is error-prone especially when dealing with loops + running in different threads. + + Note that the low-level API will still accept ``loop``. + See `Changes in the Python API`_ for examples of how to replace existing code. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) + + +Porting to Python 3.10 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Changes in the Python API +------------------------- + +* The *etype* parameters of the :func:`~traceback.format_exception`, + :func:`~traceback.format_exception_only`, and + :func:`~traceback.print_exception` functions in the :mod:`traceback` module + have been renamed to *exc*. + (Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + +* :mod:`atexit`: At Python exit, if a callback registered with + :func:`atexit.register` fails, its exception is now logged. Previously, only + some exceptions were logged, and the last exception was always silently + ignored. + (Contributed by Victor Stinner in :issue:`42639`.) + +* :class:`collections.abc.Callable` generic now flattens type parameters, similar + to what :data:`typing.Callable` currently does. This means that + ``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of + ``(int, str, str)``; previously this was ``([int, str], str)``. Code which + accesses the arguments via :func:`typing.get_args` or ``__args__`` need to account + for this change. Furthermore, :exc:`TypeError` may be raised for invalid forms + of parameterizing :class:`collections.abc.Callable` which may have passed + silently in Python 3.9. + (Contributed by Ken Jin in :issue:`42195`.) + +* :meth:`socket.htons` and :meth:`socket.ntohs` now raise :exc:`OverflowError` + instead of :exc:`DeprecationWarning` if the given parameter will not fit in + a 16-bit unsigned integer. + (Contributed by Erlend E. Aasland in :issue:`42393`.) + +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. + + A coroutine that currently look like this:: + + async def foo(loop): + await asyncio.sleep(1, loop=loop) + + Should be replaced with this:: + + async def foo(): + await asyncio.sleep(1) + + If ``foo()`` was specifically designed *not* to run in the current thread's + running event loop (e.g. running in another thread's event loop), consider + using :func:`asyncio.run_coroutine_threadsafe` instead. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) + +* The :data:`types.FunctionType` constructor now inherits the current builtins + if the *globals* dictionary has no ``"__builtins__"`` key, rather than using + ``{"None": None}`` as builtins: same behavior as :func:`eval` and + :func:`exec` functions. Defining a function with ``def function(...): ...`` + in Python is not affected, globals cannot be overriden with this syntax: it + also inherits the current builtins. + (Contributed by Victor Stinner in :issue:`42990`.) + +CPython bytecode changes +======================== + +* The ``MAKE_FUNCTION`` instruction accepts tuple of strings as annotations + instead of dictionary. + (Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`) + +Build Changes +============= + +* The C99 functions :c:func:`snprintf` and :c:func:`vsnprintf` are now required + to build Python. + (Contributed by Victor Stinner in :issue:`36020`.) + +* :mod:`sqlite3` requires SQLite 3.7.15 or higher. (Contributed by Sergey Fedoseev + and Erlend E. Aasland :issue:`40744` and :issue:`40810`.) + +* The :mod:`atexit` module must now always be built as a built-in module. + (Contributed by Victor Stinner in :issue:`42639`.) + +* Added ``--disable-test-modules`` option to the ``configure`` script: + don't build nor install test modules. + (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) + +* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If + specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` + wheel packages in this directory: if both are present, these wheel packages + are used instead of ensurepip bundled wheel packages. + + Some Linux distribution packaging policies recommend against bundling + dependencies. For example, Fedora installs wheel packages in the + ``/usr/share/python-wheels/`` directory and don't install the + ``ensurepip._bundled`` package. + + (Contributed by Victor Stinner in :issue:`42856`.) + +* Add a new configure ``--without-static-libpython`` option to not build the + ``libpythonMAJOR.MINOR.a`` static library and not install the ``python.o`` + object file. + + (Contributed by Victor Stinner in :issue:`43103`.) + +* The ``configure`` script now uses the ``pkg-config`` utility, if available, + to detect the location of Tcl/Tk headers and libraries. As before, those + locations can be explicitly specified with the ``--with-tcltk-includes`` + and ``--with-tcltk-libs`` configuration options. + (Contributed by Manolis Stamatogiannakis in :issue:`42603`.) + + +C API Changes +============= + +New Features +------------ + +* The result of :c:func:`PyNumber_Index` now always has exact type :class:`int`. + Previously, the result could have been an instance of a subclass of ``int``. + (Contributed by Serhiy Storchaka in :issue:`40792`.) + +* Add a new :c:member:`~PyConfig.orig_argv` member to the :c:type:`PyConfig` + structure: the list of the original command line arguments passed to the + Python executable. + (Contributed by Victor Stinner in :issue:`23427`.) + +* The :c:func:`PyDateTime_DATE_GET_TZINFO` and + :c:func:`PyDateTime_TIME_GET_TZINFO` macros have been added for accessing + the ``tzinfo`` attributes of :class:`datetime.datetime` and + :class:`datetime.time` objects. + (Contributed by Zackery Spytz in :issue:`30155`.) + +* Add a :c:func:`PyCodec_Unregister` function to unregister a codec + search function. + (Contributed by Hai Shi in :issue:`41842`.) + +* The :c:func:`PyIter_Send` function was added to allow + sending value into iterator without raising ``StopIteration`` exception. + (Contributed by Vladimir Matveev in :issue:`41756`.) + +* Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API. + (Contributed by Alex Gaynor in :issue:`41784`.) + +* Added :c:func:`PyModule_AddObjectRef` function: similar to + :c:func:`PyModule_AddObject` but don't steal a reference to the value on + success. + (Contributed by Victor Stinner in :issue:`1635741`.) + +* Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the + reference count of an object and return the object. + (Contributed by Victor Stinner in :issue:`42262`.) + +* The :c:func:`PyType_FromSpecWithBases` and :c:func:`PyType_FromModuleAndSpec` + functions now accept a single class as the *bases* argument. + (Contributed by Serhiy Storchaka in :issue:`42423`.) + +* The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc`` + slot. + (Contributed by Hai Shi in :issue:`41832`.) + +* The :c:func:`PyType_GetSlot` function can accept static types. + (Contributed by Hai Shi and Petr Viktorin in :issue:`41073`.) + +* Add a new :c:func:`PySet_CheckExact` function to the C-API to check if an + object is an instance of :class:`set` but not an instance of a subtype. + (Contributed by Pablo Galindo in :issue:`43277`.) + +Porting to Python 3.10 +---------------------- + +* The ``PY_SSIZE_T_CLEAN`` macro must now be defined to use + :c:func:`PyArg_ParseTuple` and :c:func:`Py_BuildValue` formats which use + ``#``: ``es#``, ``et#``, ``s#``, ``u#``, ``y#``, ``z#``, ``U#`` and ``Z#``. + See :ref:`Parsing arguments and building values + ` and the :pep:`353`. + (Contributed by Victor Stinner in :issue:`40943`.) + +* Since :c:func:`Py_REFCNT()` is changed to the inline static function, + ``Py_REFCNT(obj) = new_refcnt`` must be replaced with ``Py_SET_REFCNT(obj, new_refcnt)``: + see :c:func:`Py_SET_REFCNT()` (available since Python 3.9). For backward + compatibility, this macro can be used:: + + #if PY_VERSION_HEX < 0x030900A4 + # define Py_SET_REFCNT(obj, refcnt) ((Py_REFCNT(obj) = (refcnt)), (void)0) + #endif + + (Contributed by Victor Stinner in :issue:`39573`.) + +* Calling :c:func:`PyDict_GetItem` without :term:`GIL` held had been allowed + for historical reason. It is no longer allowed. + (Contributed by Victor Stinner in :issue:`40839`.) + +* ``PyUnicode_FromUnicode(NULL, size)`` and ``PyUnicode_FromStringAndSize(NULL, size)`` + raise ``DeprecationWarning`` now. Use :c:func:`PyUnicode_New` to allocate + Unicode object without initial data. + (Contributed by Inada Naoki in :issue:`36346`.) + +* The private ``_PyUnicode_Name_CAPI`` structure of the PyCapsule API + ``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. + (Contributed by Victor Stinner in :issue:`42157`.) + +* :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, + :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and + :c:func:`Py_GetProgramName` functions now return ``NULL`` if called before + :c:func:`Py_Initialize` (before Python is initialized). Use the new + :ref:`Python Initialization Configuration API ` to get the + :ref:`Python Path Configuration. `. + (Contributed by Victor Stinner in :issue:`42260`.) + +* :c:func:`PyList_SET_ITEM`, :c:func:`PyTuple_SET_ITEM` and + :c:func:`PyCell_SET` macros can no longer be used as l-value or r-value. + For example, ``x = PyList_SET_ITEM(a, b, c)`` and + ``PyList_SET_ITEM(a, b, c) = x`` now fail with a compiler error. It prevents + bugs like ``if (PyList_SET_ITEM (a, b, c) < 0) ...`` test. + (Contributed by Zackery Spytz and Victor Stinner in :issue:`30459`.) + +* The non-limited API files ``odictobject.h``, ``parser_interface.h``, + ``picklebufobject.h``, ``pyarena.h``, ``pyctype.h``, ``pydebug.h``, + ``pyfpe.h``, and ``pytime.h`` have been moved to the ``Include/cpython`` + directory. These files must not be included directly, as they are already + included in ``Python.h``: :ref:`Include Files `. If they have + been included directly, consider including ``Python.h`` instead. + (Contributed by Nicholas Sim in :issue:`35134`) + +Deprecated +---------- + +* The ``PyUnicode_InternImmortal()`` function is now deprecated + and will be removed in Python 3.12: use :c:func:`PyUnicode_InternInPlace` + instead. + (Contributed by Victor Stinner in :issue:`41692`.) + +Removed +------- + +* ``PyObject_AsCharBuffer()``, ``PyObject_AsReadBuffer()``, ``PyObject_CheckReadBuffer()``, + and ``PyObject_AsWriteBuffer()`` are removed. Please migrate to new buffer protocol; + :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release`. + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``Py_UNICODE_str*`` functions manipulating ``Py_UNICODE*`` strings. + (Contributed by Inada Naoki in :issue:`41123`.) + + * ``Py_UNICODE_strlen``: use :c:func:`PyUnicode_GetLength` or + :c:macro:`PyUnicode_GET_LENGTH` + * ``Py_UNICODE_strcat``: use :c:func:`PyUnicode_CopyCharacters` or + :c:func:`PyUnicode_FromFormat` + * ``Py_UNICODE_strcpy``, ``Py_UNICODE_strncpy``: use + :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` + * ``Py_UNICODE_strcmp``: use :c:func:`PyUnicode_Compare` + * ``Py_UNICODE_strncmp``: use :c:func:`PyUnicode_Tailmatch` + * ``Py_UNICODE_strchr``, ``Py_UNICODE_strrchr``: use + :c:func:`PyUnicode_FindChar` + +* Removed ``PyUnicode_GetMax()``. Please migrate to new (:pep:`393`) APIs. + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``PyLong_FromUnicode()``. Please migrate to :c:func:`PyLong_FromUnicodeObject`. + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``PyUnicode_AsUnicodeCopy()``. Please use :c:func:`PyUnicode_AsUCS4Copy` or + :c:func:`PyUnicode_AsWideCharString` + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``_Py_CheckRecursionLimit`` variable: it has been replaced by + ``ceval.recursion_limit`` of the :c:type:`PyInterpreterState` structure. + (Contributed by Victor Stinner in :issue:`41834`.) + +* Removed undocumented macros ``Py_ALLOW_RECURSION`` and + ``Py_END_ALLOW_RECURSION`` and the ``recursion_critical`` field of the + :c:type:`PyInterpreterState` structure. + (Contributed by Serhiy Storchaka in :issue:`41936`.) + +* Removed the undocumented ``PyOS_InitInterrupts()`` function. Initializing + Python already implicitly installs signal handlers: see + :c:member:`PyConfig.install_signal_handlers`. + (Contributed by Victor Stinner in :issue:`41713`.) From f4bb33a418cd5d881d34540ea93959bfe281514d Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 10:37:20 +0100 Subject: [PATCH 04/93] Update test_locks.py --- Lib/test/test_asyncio/test_locks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 42ef44db2d3758..26c16b3850d296 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -11,8 +11,8 @@ STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' - r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?(, count:\d+/\d+)?(, state:^-?\d)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier object + r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -71,7 +71,7 @@ def test_lock_doesnt_accept_loop_parameter(self): cls(loop=self.loop) # Barrier object has a positional paramater - # so check alone + # so test alone cls = asyncio.Barrier with self.assertRaisesRegex( TypeError, From 5f14a7df63fd4e7fb7bb321bd1ded9cac9049ef3 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 16:09:54 +0100 Subject: [PATCH 05/93] Update locks.py Add comment when testing self._state Change if to while instruction in _block method --- Lib/asyncio/locks.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 4bc69100ed225d..e7f2561468e8cf 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -452,7 +452,7 @@ def __init__(self, parties, action=None, *, loop=mixins._marker): raise ValueError('parties must be > 0') self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block tasks while wainting tasks are draining or broken + self._blocking = Event() # used block new tasks while waiting tasks are draining or broken self._action = action self._parties = parties self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken @@ -491,7 +491,7 @@ async def wait(self): # Block until the barrier is ready for us, or raise an exception # if it is broken. async def _block (self): - if self._state in (-1, 1): + while self._state in (-1, 1): # It is draining or resetting, wait until done await self._blocking.wait() @@ -522,7 +522,7 @@ def _release(self): async def _wait(self): await self._waiting.wait() # no timeout so - if self._state < 0: + if self._state < 0: # resetting or broken raise BrokenBarrierError assert self._state == 1, repr(self) @@ -530,9 +530,9 @@ async def _wait(self): # waiting for the barrier to drain. def _exit(self): if self._count == 0: - if self._state == 1: + if self._state == 1: # draining self._state = 0 - elif self._state == -1: + elif self._state == -1: # resetting self._state = 0 self._waiting.clear() self._blocking.set() @@ -544,12 +544,12 @@ def reset(self): raised. """ if self._count > 0: - if self._state in (0, 1): - #reset the barrier, waking up tasks + if self._state in (0, 1): # filling or draining + # reset the barrier, waking up tasks self._state = -1 elif self._state == -2: - #was broken, set it to reset state - #which clears when the last tasks exits + # was broken, set it to reset state + # which clears when the last tasks exits self._state = -1 self._waiting.set() self._blocking.clear() From 214fd353cfd18e24909de3d3dc984a9083e02515 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 16:13:24 +0100 Subject: [PATCH 06/93] Update test_locks.py Refactoring all test names of BarrierTests to be more readable Add a method cancel_coros Refactoring and simplifyiing some tests Correction of RGX_REPR --- Lib/test/test_asyncio/test_locks.py | 253 +++++++++++++++++----------- 1 file changed, 155 insertions(+), 98 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 26c16b3850d296..ec90698b6155aa 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -12,7 +12,7 @@ r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # new line dedicated to repr barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -71,7 +71,7 @@ def test_lock_doesnt_accept_loop_parameter(self): cls(loop=self.loop) # Barrier object has a positional paramater - # so test alone + # so tests alone cls = asyncio.Barrier with self.assertRaisesRegex( TypeError, @@ -954,6 +954,11 @@ async def run_coros(self, n, coro): res = await asyncio.gather(*tasks) return res, tasks + def cancel_coros(self): + for task in asyncio.all_tasks(self.loop): + task.cancel() + test_utils.run_briefly(self.loop) + def test_repr(self): async def coro(): try: @@ -963,7 +968,7 @@ async def coro(): else: return await asyncio.sleep(1, True) - self.N = 999 + self.N = 1011001 barrier = asyncio.Barrier(self.N) self.assertTrue("[unset," in repr(barrier)) self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) @@ -998,23 +1003,24 @@ async def coro(): self.assertTrue(f"0/{self.N}" in repr(barrier)) self.assertTrue(repr(barrier).endswith(':-2]>')) + self.cancel_coros() + def test_init(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - def test_wait_more(self): - async def coro(result): - r = await barrier.wait() - result.append(r) - + def test_wait_task_once(self): + self.N = 1 barrier = asyncio.Barrier(self.N) - results = [] - _ = [self.loop.create_task(coro(results)) for _ in range(self.N)] - test_utils.run_briefly(self.loop) - self.assertEqual(sum(results), sum(range(self.N))) + self.assertEqual(barrier.n_waiting, 0) + r1 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + r2 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + self.assertEqual(r1, r2) self.assertFalse(barrier.broken) - def test_wait_task_multi(self): + def test_wait_task_by_task(self): self.N = 3 barrier = asyncio.Barrier(self.N) self.assertEqual(barrier.n_waiting, 0) @@ -1029,144 +1035,181 @@ def test_wait_task_multi(self): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_wait_task_one(self): - self.N = 1 - barrier = asyncio.Barrier(self.N) - r1 = self.loop.run_until_complete(barrier.wait()) - self.assertEqual(barrier.n_waiting, 0) - r2 = self.loop.run_until_complete(barrier.wait()) - self.assertEqual(barrier.n_waiting, 0) - self.assertEqual(r1, r2) - self.assertFalse(barrier.broken) - - def test_wait_step_by_step(self): - async def coro(result, value): + def test_wait_tasks_twice(self): + results = [] + async def coro(): await barrier.wait() - result.append(value) + results.append(True) + await barrier.wait() + results.append(False) - results = [] barrier = asyncio.Barrier(self.N) - value = 1 - tasks = [] - for n in range(self.N-1): - tasks.append(self.loop.create_task(coro(results, value))) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, n+1) - - tasks.append(self.loop.create_task(coro(results, value))) - test_utils.run_briefly(self.loop) - self.assertEqual(results, [value]* self.N) - self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(all(t.done() for t in tasks)) - self.assertTrue(all(t.result() is None for t in tasks)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertEqual(len(results), self.N*2) + self.assertEqual(results.count(True), self.N) + self.assertEqual(results.count(False), self.N) self.assertFalse(barrier.broken) + self.assertEqual(barrier.n_waiting, 0) - def test_wait_multi_return(self): + def test_wait_return_from_wait(self): results1 = [] results2 = [] - results3 = [] async def coro(): await barrier.wait() results1.append(True) i = await barrier.wait() results2.append(True) - results3.append(i) return i barrier = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + res = self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertEqual(len(results1), self.N) self.assertTrue(all(results1)) self.assertEqual(len(results2), self.N) self.assertTrue(all(results2)) - self.assertEqual(sum(res), sum(results3)) + self.assertEqual(sum(res[0]), sum(range(self.N))) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) - def test_barrier(self, multipass=1, nn=5): - results = [[] for _ in range(multipass)] + + def test_wait_repeat_more(self, more=5): async def coro(result, value): ret = await barrier.wait() result.append(value) - return ret - barrier = asyncio.Barrier(nn) - for i in range(multipass): - _ = [self.loop.create_task(coro(results[i], value)) for value in range(nn)] + results = [[] for _ in range(more)] + barrier = asyncio.Barrier(self.N) + for i in range(more): + _ = [self.loop.create_task(coro(results[i], value)) for value in range(self.N)] test_utils.run_briefly(self.loop) - self.assertEqual(len(results[i]), nn) - self.assertEqual(sum(results[i]), sum(range(nn))) + self.assertEqual(len(results[i]), self.N) + self.assertEqual(sum(results[i]), sum(range(self.N))) if i > 0: self.assertEqual(sum(results[i]), sum(results[i-1])) self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_wait_state_draining(self): + result = [] + state = [] + async def coro(): + i = await barrier.wait() + if i == 0: + result.append(True) + # Do we need to add a Barrie.draining() property ? + state.append(repr(barrier).endswith("state:1]>")) + rest = self.N-1 + nb_tasks = self.N + rest + barrier = asyncio.Barrier(self.N) + for _ in range(nb_tasks): + self.loop.create_task(coro()) + for _ in range((nb_tasks//self.N)+1): + test_utils.run_briefly(self.loop) + self.assertEqual(len(result), nb_tasks//self.N) + self.assertTrue(all(result)) + self.assertTrue(all(state)) + self.assertEqual(barrier.n_waiting, rest) self.assertFalse(barrier.broken) - def test_barrier_multipass(self): - self.test_barrier(10) + self.cancel_coros() - def test_action_callback(self): - async def coro(result, value): - ret = await barrier.wait() - result.append(value) - return ret + def test_wait_tasks_cancel_one_task(self): + results = [] + async def coro(): + await barrier.wait() + results.append(True) - result = [] - result1 = [] - value = 1 - barrier = asyncio.Barrier(1, action=lambda: result1.append(True)) - _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + self.N=3 + barrier = asyncio.Barrier(self.N) + t1 = self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 2) - self.assertEqual(len(result1), self.N) - self.assertTrue(all(result1)) - self.assertEqual(len(result), self.N) - self.assertEqual(sum(result), self.N*value) + t1.cancel() + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 2) + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(len(results), self.N) + self.assertTrue(all(results)) + self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_action_callback_n(self): - async def coro(result, value): + def test_wait_action_callback(self): + result = [] + async def coro(): ret = await barrier.wait() - result.append(value) - return ret + result.append(True) - result = [] result1 = [] - value = 1 + self.N = 2 barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) - _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) self.assertEqual(len(result1), 1) self.assertTrue(result1[0]) - self.assertEqual(len(result), self.N) - self.assertEqual(sum(result), value*self.N) + self.assertEqual(len(result), 2) + self.assertTrue(all(result)) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_wait_action_callback_more(self): + result = [] + async def coro(): + ret = await barrier.wait() + result.append(True) + + result1 = [] + + self.N = 2 + more = 3 + barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) + for _ in range(more): + _ = [self.loop.create_task(coro()) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(result1), more) + self.assertTrue(all(result1)) + self.assertEqual(len(result), self.N*more) + self.assertTrue(all(result)) + self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_action_callback_error(self): + def test_wait_action_callback_error_broken(self): + ERROR = ZeroDivisionError results1 = [] results2 = [] - results3 = [] - def raise_except(e): - raise e - - async def coro(e): + async def coro(): try: ret = await barrier.wait() - except e: + except ERROR: results1.append(False) except: results2.append(True) - else: - results3.append(None) - barrier = asyncio.Barrier(self.N, lambda: raise_except(ZeroDivisionError)) - _ = [self.loop.create_task(coro(ZeroDivisionError)) for _ in range(self.N)] + def raise_except(): + raise ERROR + + barrier = asyncio.Barrier(self.N, lambda: raise_except()) + _ = [self.loop.create_task(coro()) for _ in range(self.N)] test_utils.run_briefly(self.loop) self.assertEqual(len(results1), 1) self.assertFalse(results1[0]) self.assertEqual(len(results2), self.N-1) self.assertTrue(all(results2)) - self.assertEqual(len(results3), 0) + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(barrier.broken) def test_reset(self): @@ -1196,16 +1239,21 @@ async def coro2(): self.assertFalse(barrier.broken) self.assertEqual(len(results1), self.N-1) self.assertEqual(len(results2), self.N) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) - def test_reset_multi_wait(self): + def test_reset_and_wait(self): results1 = [] results2 = [] results3 = [] - alls = [] + count = 0 async def coro(): + nonlocal count + i = await barrier1.wait() - alls.append(i) - if len(alls) == self.N: # i == self.N//2: + count += 1 + if count == self.N: # i == self.N//2: barrier.reset() else: try: @@ -1220,13 +1268,16 @@ async def coro(): barrier = asyncio.Barrier(self.N) barrier1 = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertFalse(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertEqual(len(results3), self.N) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) - def test_abort(self): + def test_abort_broken(self): results1 = [] results2 = [] async def coro(): @@ -1242,10 +1293,13 @@ async def coro(): barrier.abort() barrier = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertTrue(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) + self.assertEqual(barrier.n_waiting, 0) + + self.assertTrue(barrier.broken) def test_abort_and_reset(self): results1 = [] @@ -1275,11 +1329,14 @@ async def coro(): barrier = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertFalse(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertEqual(len(results3), self.N) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) if __name__ == '__main__': From 619f60afa4174b88fcadea74c41c11b9b4226232 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 16:14:05 +0100 Subject: [PATCH 07/93] Update from patch after run python.bat Tools\scripts\patchcheck.py --- Doc/whatsnew/3.10.rst.bak | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Doc/whatsnew/3.10.rst.bak b/Doc/whatsnew/3.10.rst.bak index db71f061f14df4..c4c282e5a04eae 100644 --- a/Doc/whatsnew/3.10.rst.bak +++ b/Doc/whatsnew/3.10.rst.bak @@ -673,9 +673,16 @@ When a module does not define ``__loader__``, fall back to ``__spec__.loader``. encodings --------- + :func:`encodings.normalize_encoding` now ignores non-ASCII characters. (Contributed by Hai Shi in :issue:`39337`.) +gc +-- + +Added audit hooks for :func:`gc.get_objects`, :func:`gc.get_referrers` and +:func:`gc.get_referents`. (Contributed by Pablo Galindo in :issue:`43439`.) + glob ---- @@ -683,6 +690,19 @@ Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and :func:`~glob.iglob` which allow to specify the root directory for searching. (Contributed by Serhiy Storchaka in :issue:`38144`.) +importlib.metadata +------------------ + +Feature parity with ``importlib_metadata`` 3.7. + +:func:`importlib.metadata.entry_points` now provides a nicer experience +for selecting entry points by group and name through a new +:class:`importlib.metadata.EntryPoints` class. + +Added :func:`importlib.metadata.packages_distributions` for resolving +top-level Python modules and packages to their +:class:`importlib.metadata.Distribution`. + inspect ------- @@ -781,6 +801,13 @@ Add :data:`sys.stdlib_module_names`, containing the list of the standard library module names. (Contributed by Victor Stinner in :issue:`42955`.) +_thread +------- + +:func:`_thread.interrupt_main` now takes an optional signal number to +simulate (the default is still :data:`signal.SIGINT`). +(Contributed by Antoine Pitrou in :issue:`43356`.) + threading --------- @@ -1203,6 +1230,11 @@ New Features object is an instance of :class:`set` but not an instance of a subtype. (Contributed by Pablo Galindo in :issue:`43277`.) +* Added :c:func:`PyErr_SetInterruptEx` which allows passing a signal number + to simulate. + (Contributed by Antoine Pitrou in :issue:`43356`.) + + Porting to Python 3.10 ---------------------- From 04d4f691d502018059124f793e1ee0d1fed32d1d Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 31 Mar 2021 16:24:52 +0200 Subject: [PATCH 08/93] remove cpython/doc/*.rst.bak --- Doc/c-api/exceptions.rst.bak | 1054 ------------- Doc/c-api/memory.rst.bak | 642 -------- Doc/faq/design.rst.bak | 759 --------- Doc/howto/descriptor.rst.bak | 1575 ------------------- Doc/library/_thread.rst.bak | 215 --- Doc/library/ast.rst.bak | 1993 ------------------------ Doc/library/asyncio-api-index.rst.bak | 221 --- Doc/library/collections.rst.bak | 1278 --------------- Doc/library/dataclasses.rst.bak | 595 ------- Doc/library/gc.rst.bak | 322 ---- Doc/library/importlib.metadata.rst.bak | 262 ---- Doc/library/logging.rst.bak | 1350 ---------------- Doc/library/readline.rst.bak | 361 ----- Doc/library/sqlite3.rst.bak | 1094 ------------- Doc/library/statistics.rst.bak | 879 ----------- Doc/library/tempfile.rst.bak | 374 ----- Doc/library/venv.rst.bak | 496 ------ Doc/library/xml.sax.handler.rst.bak | 463 ------ Doc/library/zipimport.rst.bak | 209 --- Doc/whatsnew/3.10.rst.bak | 1347 ---------------- 20 files changed, 15489 deletions(-) delete mode 100644 Doc/c-api/exceptions.rst.bak delete mode 100644 Doc/c-api/memory.rst.bak delete mode 100644 Doc/faq/design.rst.bak delete mode 100644 Doc/howto/descriptor.rst.bak delete mode 100644 Doc/library/_thread.rst.bak delete mode 100644 Doc/library/ast.rst.bak delete mode 100644 Doc/library/asyncio-api-index.rst.bak delete mode 100644 Doc/library/collections.rst.bak delete mode 100644 Doc/library/dataclasses.rst.bak delete mode 100644 Doc/library/gc.rst.bak delete mode 100644 Doc/library/importlib.metadata.rst.bak delete mode 100644 Doc/library/logging.rst.bak delete mode 100644 Doc/library/readline.rst.bak delete mode 100644 Doc/library/sqlite3.rst.bak delete mode 100644 Doc/library/statistics.rst.bak delete mode 100644 Doc/library/tempfile.rst.bak delete mode 100644 Doc/library/venv.rst.bak delete mode 100644 Doc/library/xml.sax.handler.rst.bak delete mode 100644 Doc/library/zipimport.rst.bak delete mode 100644 Doc/whatsnew/3.10.rst.bak diff --git a/Doc/c-api/exceptions.rst.bak b/Doc/c-api/exceptions.rst.bak deleted file mode 100644 index 4e99a0167a632d..00000000000000 --- a/Doc/c-api/exceptions.rst.bak +++ /dev/null @@ -1,1054 +0,0 @@ -.. highlight:: c - - -.. _exceptionhandling: - -****************** -Exception Handling -****************** - -The functions described in this chapter will let you handle and raise Python -exceptions. It is important to understand some of the basics of Python -exception handling. It works somewhat like the POSIX :c:data:`errno` variable: -there is a global indicator (per thread) of the last error that occurred. Most -C API functions don't clear this on success, but will set it to indicate the -cause of the error on failure. Most C API functions also return an error -indicator, usually ``NULL`` if they are supposed to return a pointer, or ``-1`` -if they return an integer (exception: the :c:func:`PyArg_\*` functions -return ``1`` for success and ``0`` for failure). - -Concretely, the error indicator consists of three object pointers: the -exception's type, the exception's value, and the traceback object. Any -of those pointers can be ``NULL`` if non-set (although some combinations are -forbidden, for example you can't have a non-``NULL`` traceback if the exception -type is ``NULL``). - -When a function must fail because some function it called failed, it generally -doesn't set the error indicator; the function it called already set it. It is -responsible for either handling the error and clearing the exception or -returning after cleaning up any resources it holds (such as object references or -memory allocations); it should *not* continue normally if it is not prepared to -handle the error. If returning due to an error, it is important to indicate to -the caller that an error has been set. If the error is not handled or carefully -propagated, additional calls into the Python/C API may not behave as intended -and may fail in mysterious ways. - -.. note:: - The error indicator is **not** the result of :func:`sys.exc_info()`. - The former corresponds to an exception that is not yet caught (and is - therefore still propagating), while the latter returns an exception after - it is caught (and has therefore stopped propagating). - - -Printing and clearing -===================== - - -.. c:function:: void PyErr_Clear() - - Clear the error indicator. If the error indicator is not set, there is no - effect. - - -.. c:function:: void PyErr_PrintEx(int set_sys_last_vars) - - Print a standard traceback to ``sys.stderr`` and clear the error indicator. - **Unless** the error is a ``SystemExit``, in that case no traceback is - printed and the Python process will exit with the error code specified by - the ``SystemExit`` instance. - - Call this function **only** when the error indicator is set. Otherwise it - will cause a fatal error! - - If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, - :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the - type, value and traceback of the printed exception, respectively. - - -.. c:function:: void PyErr_Print() - - Alias for ``PyErr_PrintEx(1)``. - - -.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) - - Call :func:`sys.unraisablehook` using the current exception and *obj* - argument. - - This utility function prints a warning message to ``sys.stderr`` when an - exception has been set but it is impossible for the interpreter to actually - raise the exception. It is used, for example, when an exception occurs in an - :meth:`__del__` method. - - The function is called with a single argument *obj* that identifies the context - in which the unraisable exception occurred. If possible, - the repr of *obj* will be printed in the warning message. - - An exception must be set when calling this function. - - -Raising exceptions -================== - -These functions help you set the current thread's error indicator. -For convenience, some of these functions will always return a -``NULL`` pointer for use in a ``return`` statement. - - -.. c:function:: void PyErr_SetString(PyObject *type, const char *message) - - This is the most common way to set the error indicator. The first argument - specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. - The second argument is an error message; it is decoded from ``'utf-8``'. - - -.. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) - - This function is similar to :c:func:`PyErr_SetString` but lets you specify an - arbitrary Python object for the "value" of the exception. - - -.. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) - - This function sets the error indicator and returns ``NULL``. *exception* - should be a Python exception class. The *format* and subsequent - parameters help format the error message; they have the same meaning and - values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded - string. - - -.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs) - - Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather - than a variable number of arguments. - - .. versionadded:: 3.5 - - -.. c:function:: void PyErr_SetNone(PyObject *type) - - This is a shorthand for ``PyErr_SetObject(type, Py_None)``. - - -.. c:function:: int PyErr_BadArgument() - - This is a shorthand for ``PyErr_SetString(PyExc_TypeError, message)``, where - *message* indicates that a built-in operation was invoked with an illegal - argument. It is mostly for internal use. - - -.. c:function:: PyObject* PyErr_NoMemory() - - This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns ``NULL`` - so an object allocation function can write ``return PyErr_NoMemory();`` when it - runs out of memory. - - -.. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) - - .. index:: single: strerror() - - This is a convenience function to raise an exception when a C library function - has returned an error and set the C variable :c:data:`errno`. It constructs a - tuple object whose first item is the integer :c:data:`errno` value and whose - second item is the corresponding error message (gotten from :c:func:`strerror`), - and then calls ``PyErr_SetObject(type, object)``. On Unix, when the - :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, - this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, - leaves it set to that. The function always returns ``NULL``, so a wrapper - function around a system call can write ``return PyErr_SetFromErrno(type);`` - when the system call returns an error. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) - - Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as - a third parameter. In the case of :exc:`OSError` exception, - this is used to define the :attr:`filename` attribute of the - exception instance. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second - filename object, for raising errors when a function that takes two filenames - fails. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename - is given as a C string. *filename* is decoded from the :term:`filesystem - encoding and error handler`. - - -.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) - - This is a convenience function to raise :exc:`WindowsError`. If called with - *ierr* of :c:data:`0`, the error code returned by a call to :c:func:`GetLastError` - is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve - the Windows description of error code given by *ierr* or :c:func:`GetLastError`, - then it constructs a tuple object whose first item is the *ierr* value and whose - second item is the corresponding error message (gotten from - :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, - object)``. This function always returns ``NULL``. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr) - - Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter - specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) - - Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, - but accepts a second filename object. - - .. availability:: Windows. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional - parameter specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path) - - This is a convenience function to raise :exc:`ImportError`. *msg* will be - set as the exception's message string. *name* and *path*, both of which can - be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name`` - and ``path`` attributes. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) - - Set file, line, and offset information for the current exception. If the - current exception is not a :exc:`SyntaxError`, then it sets additional - attributes, which make the exception printing subsystem think the exception - is a :exc:`SyntaxError`. - - .. versionadded:: 3.4 - - -.. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) - - Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string - decoded from the :term:`filesystem encoding and error handler`. - - .. versionadded:: 3.2 - - -.. c:function:: void PyErr_SyntaxLocation(const char *filename, int lineno) - - Like :c:func:`PyErr_SyntaxLocationEx`, but the col_offset parameter is - omitted. - - -.. c:function:: void PyErr_BadInternalCall() - - This is a shorthand for ``PyErr_SetString(PyExc_SystemError, message)``, - where *message* indicates that an internal operation (e.g. a Python/C API - function) was invoked with an illegal argument. It is mostly for internal - use. - - -Issuing warnings -================ - -Use these functions to issue warnings from C code. They mirror similar -functions exported by the Python :mod:`warnings` module. They normally -print a warning message to *sys.stderr*; however, it is -also possible that the user has specified that warnings are to be turned into -errors, and in that case they will raise an exception. It is also possible that -the functions raise an exception because of a problem with the warning machinery. -The return value is ``0`` if no exception is raised, or ``-1`` if an exception -is raised. (It is not possible to determine whether a warning message is -actually printed, nor what the reason is for the exception; this is -intentional.) If an exception is raised, the caller should do its normal -exception handling (for example, :c:func:`Py_DECREF` owned references and return -an error value). - -.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) - - Issue a warning message. The *category* argument is a warning category (see - below) or ``NULL``; the *message* argument is a UTF-8 encoded string. *stack_level* is a - positive number giving a number of stack frames; the warning will be issued from - the currently executing line of code in that stack frame. A *stack_level* of 1 - is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, - and so forth. - - Warning categories must be subclasses of :c:data:`PyExc_Warning`; - :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; - the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard - Python warning categories are available as global variables whose names are - enumerated at :ref:`standardwarningcategories`. - - For information about warning control, see the documentation for the - :mod:`warnings` module and the :option:`-W` option in the command line - documentation. There is no C API for warning control. - -.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) - - Much like :c:func:`PyErr_SetImportError` but this function allows for - specifying a subclass of :exc:`ImportError` to raise. - - .. versionadded:: 3.6 - - -.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) - - Issue a warning message with explicit control over all warning attributes. This - is a straightforward wrapper around the Python function - :func:`warnings.warn_explicit`, see there for more information. The *module* - and *registry* arguments may be set to ``NULL`` to get the default effect - described there. - - .. versionadded:: 3.4 - - -.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) - - Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and - *module* are UTF-8 encoded strings, and *filename* is decoded from the - :term:`filesystem encoding and error handler`. - - -.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnEx`, but use - :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is - an ASCII-encoded string. - - .. versionadded:: 3.2 - - -.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. - - .. versionadded:: 3.6 - - -Querying the error indicator -============================ - -.. c:function:: PyObject* PyErr_Occurred() - - Test whether the error indicator is set. If set, return the exception *type* - (the first argument to the last call to one of the :c:func:`PyErr_Set\*` - functions or to :c:func:`PyErr_Restore`). If not set, return ``NULL``. You do not - own a reference to the return value, so you do not need to :c:func:`Py_DECREF` - it. - - The caller must hold the GIL. - - .. note:: - - Do not compare the return value to a specific exception; use - :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could - easily fail since the exception may be an instance instead of a class, in the - case of a class exception, or it may be a subclass of the expected exception.) - - -.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) - - Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This - should only be called when an exception is actually set; a memory access - violation will occur if no exception has been raised. - - -.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) - - Return true if the *given* exception matches the exception type in *exc*. If - *exc* is a class object, this also returns true when *given* is an instance - of a subclass. If *exc* is a tuple, all exception types in the tuple (and - recursively in subtuples) are searched for a match. - - -.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the error indicator into three variables whose addresses are passed. - If the error indicator is not set, set all three variables to ``NULL``. If it is - set, it will be cleared and you own a reference to each object retrieved. The - value and traceback object may be ``NULL`` even when the type object is not. - - .. note:: - - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: - - { - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - - /* ... code that might produce other errors ... */ - - PyErr_Restore(type, value, traceback); - } - - -.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) - - Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are ``NULL``, the error - indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or - traceback. The exception type should be a class. Do not pass an invalid - exception type or value. (Violating these rules will cause subtle problems - later.) This call takes away a reference to each object: you must own a - reference to each object before the call and after the call you no longer own - these references. (If you don't understand this, don't use this function. I - warned you.) - - .. note:: - - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current - error indicator. - - -.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) - - Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below - can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance. - - .. note:: - - This function *does not* implicitly set the ``__traceback__`` - attribute on the exception value. If setting the traceback - appropriately is desired, the following additional snippet is needed:: - - if (tb != NULL) { - PyException_SetTraceback(val, tb); - } - - -.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. Returns new references for the three objects, any of which - may be ``NULL``. Does not modify the exception info state. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the - exception state. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback) - - Set the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. This function steals the references of the arguments. - To clear the exception state, pass ``NULL`` for all three arguments. - For general rules about the three arguments, see :c:func:`PyErr_Restore`. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception - state. - - .. versionadded:: 3.3 - - -Signal Handling -=============== - - -.. c:function:: int PyErr_CheckSignals() - - .. index:: - module: signal - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - This function interacts with Python's signal handling. It checks whether a - signal has been sent to the processes and if so, invokes the corresponding - signal handler. If the :mod:`signal` module is supported, this can invoke a - signal handler written in Python. In all cases, the default effect for - :const:`SIGINT` is to raise the :exc:`KeyboardInterrupt` exception. If an - exception is raised the error indicator is set and the function returns ``-1``; - otherwise the function returns ``0``. The error indicator may or may not be - cleared if it was previously set. - - -.. c:function:: void PyErr_SetInterrupt() - - .. index:: - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - Simulate the effect of a :const:`SIGINT` signal arriving. The next time - :c:func:`PyErr_CheckSignals` is called, the Python signal handler for - :const:`SIGINT` will be called. - - If :const:`SIGINT` isn't handled by Python (it was set to - :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does - nothing. - -.. c:function:: int PySignal_SetWakeupFd(int fd) - - This utility function specifies a file descriptor to which the signal number - is written as a single byte whenever a signal is received. *fd* must be - non-blocking. It returns the previous such file descriptor. - - The value ``-1`` disables the feature; this is the initial state. - This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any - error checking. *fd* should be a valid file descriptor. The function should - only be called from the main thread. - - .. versionchanged:: 3.5 - On Windows, the function now also supports socket handles. - - -Exception Classes -================= - -.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict) - - This utility function creates and returns a new exception class. The *name* - argument must be the name of the new exception, a C string of the form - ``module.classname``. The *base* and *dict* arguments are normally ``NULL``. - This creates a class object derived from :exc:`Exception` (accessible in C as - :c:data:`PyExc_Exception`). - - The :attr:`__module__` attribute of the new class is set to the first part (up - to the last dot) of the *name* argument, and the class name is set to the last - part (after the last dot). The *base* argument can be used to specify alternate - base classes; it can either be only one class or a tuple of classes. The *dict* - argument can be used to specify a dictionary of class variables and methods. - - -.. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) - - Same as :c:func:`PyErr_NewException`, except that the new exception class can - easily be given a docstring: If *doc* is non-``NULL``, it will be used as the - docstring for the exception class. - - .. versionadded:: 3.2 - - -Exception Objects -================= - -.. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) - - Return the traceback associated with the exception as a new reference, as - accessible from Python through :attr:`__traceback__`. If there is no - traceback associated, this returns ``NULL``. - - -.. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) - - Set the traceback associated with the exception to *tb*. Use ``Py_None`` to - clear it. - - -.. c:function:: PyObject* PyException_GetContext(PyObject *ex) - - Return the context (another exception instance during whose handling *ex* was - raised) associated with the exception as a new reference, as accessible from - Python through :attr:`__context__`. If there is no context associated, this - returns ``NULL``. - - -.. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) - - Set the context associated with the exception to *ctx*. Use ``NULL`` to clear - it. There is no type check to make sure that *ctx* is an exception instance. - This steals a reference to *ctx*. - - -.. c:function:: PyObject* PyException_GetCause(PyObject *ex) - - Return the cause (either an exception instance, or :const:`None`, - set by ``raise ... from ...``) associated with the exception as a new - reference, as accessible from Python through :attr:`__cause__`. - - -.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) - - Set the cause associated with the exception to *cause*. Use ``NULL`` to clear - it. There is no type check to make sure that *cause* is either an exception - instance or :const:`None`. This steals a reference to *cause*. - - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. - - -.. _unicodeexceptions: - -Unicode Exception Objects -========================= - -The following functions are used to create and modify Unicode exceptions from C. - -.. c:function:: PyObject* PyUnicodeDecodeError_Create(const char *encoding, const char *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeDecodeError` object with the attributes *encoding*, - *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are - UTF-8 encoded strings. - -.. c:function:: PyObject* PyUnicodeEncodeError_Create(const char *encoding, const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeEncodeError` object with the attributes *encoding*, - *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are - UTF-8 encoded strings. - - .. deprecated:: 3.3 3.11 - - ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to - ``PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", ...)``. - -.. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeTranslateError` object with the attributes *object*, - *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. - - .. deprecated:: 3.3 3.11 - - ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to - ``PyObject_CallFunction(PyExc_UnicodeTranslateError, "Onns", ...)``. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) - - Return the *encoding* attribute of the given exception object. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetObject(PyObject *exc) - - Return the *object* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) - - Get the *start* attribute of the given exception object and place it into - *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. - -.. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) - - Get the *end* attribute of the given exception object and place it into - *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) - - Set the *end* attribute of the given exception object to *end*. Return ``0`` - on success, ``-1`` on failure. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) - - Return the *reason* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) - - Set the *reason* attribute of the given exception object to *reason*. Return - ``0`` on success, ``-1`` on failure. - - -.. _recursion: - -Recursion Control -================= - -These two functions provide a way to perform safe recursive calls at the C -level, both in the core and in extension modules. They are needed if the -recursive code does not necessarily invoke Python code (which tracks its -recursion depth automatically). -They are also not needed for *tp_call* implementations -because the :ref:`call protocol ` takes care of recursion handling. - -.. c:function:: int Py_EnterRecursiveCall(const char *where) - - Marks a point where a recursive C-level call is about to be performed. - - If :const:`USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it - sets a :exc:`MemoryError` and returns a nonzero value. - - The function then checks if the recursion limit is reached. If this is the - case, a :exc:`RecursionError` is set and a nonzero value is returned. - Otherwise, zero is returned. - - *where* should be a UTF-8 encoded string such as ``" in instance check"`` to - be concatenated to the :exc:`RecursionError` message caused by the recursion - depth limit. - - .. versionchanged:: 3.9 - This function is now also available in the limited API. - -.. c:function:: void Py_LeaveRecursiveCall(void) - - Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each - *successful* invocation of :c:func:`Py_EnterRecursiveCall`. - - .. versionchanged:: 3.9 - This function is now also available in the limited API. - -Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires -special recursion handling. In addition to protecting the stack, -:c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The -following two functions facilitate this functionality. Effectively, -these are the C equivalent to :func:`reprlib.recursive_repr`. - -.. c:function:: int Py_ReprEnter(PyObject *object) - - Called at the beginning of the :c:member:`~PyTypeObject.tp_repr` implementation to - detect cycles. - - If the object has already been processed, the function returns a - positive integer. In that case the :c:member:`~PyTypeObject.tp_repr` implementation - should return a string object indicating a cycle. As examples, - :class:`dict` objects return ``{...}`` and :class:`list` objects - return ``[...]``. - - The function will return a negative integer if the recursion limit - is reached. In that case the :c:member:`~PyTypeObject.tp_repr` implementation should - typically return ``NULL``. - - Otherwise, the function returns zero and the :c:member:`~PyTypeObject.tp_repr` - implementation can continue normally. - -.. c:function:: void Py_ReprLeave(PyObject *object) - - Ends a :c:func:`Py_ReprEnter`. Must be called once for each - invocation of :c:func:`Py_ReprEnter` that returns zero. - - -.. _standardexceptions: - -Standard Exceptions -=================== - -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_BufferError - single: PyExc_ChildProcessError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_GeneratorExit - single: PyExc_ImportError - single: PyExc_IndentationError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_LookupError - single: PyExc_MemoryError - single: PyExc_ModuleNotFoundError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_StopAsyncIteration - single: PyExc_StopIteration - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TabError - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_UnboundLocalError - single: PyExc_UnicodeDecodeError - single: PyExc_UnicodeEncodeError - single: PyExc_UnicodeError - single: PyExc_UnicodeTranslateError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.3 - :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, - :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`, - :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`, - :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`, - :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`, - :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`, - :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` - and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. - -.. versionadded:: 3.5 - :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. - -.. versionadded:: 3.6 - :c:data:`PyExc_ModuleNotFoundError`. - -These are compatibility aliases to :c:data:`PyExc_OSError`: - -.. index:: - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | \(3) | -+-------------------------------------+----------+ - -.. versionchanged:: 3.3 - These aliases used to be separate exception types. - -Notes: - -(1) - This is a base class for other standard exceptions. - -(2) - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. - -.. _standardwarningcategories: - -Standard Warning Categories -=========================== - -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_Warning - single: PyExc_BytesWarning - single: PyExc_DeprecationWarning - single: PyExc_FutureWarning - single: PyExc_ImportWarning - single: PyExc_PendingDeprecationWarning - single: PyExc_ResourceWarning - single: PyExc_RuntimeWarning - single: PyExc_SyntaxWarning - single: PyExc_UnicodeWarning - single: PyExc_UserWarning - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.2 - :c:data:`PyExc_ResourceWarning`. - -Notes: - -(1) - This is a base class for other standard warning categories. diff --git a/Doc/c-api/memory.rst.bak b/Doc/c-api/memory.rst.bak deleted file mode 100644 index 588668ee853c65..00000000000000 --- a/Doc/c-api/memory.rst.bak +++ /dev/null @@ -1,642 +0,0 @@ -.. highlight:: c - - -.. _memory: - -***************** -Memory Management -***************** - -.. sectionauthor:: Vladimir Marangozov - - - -.. _memoryoverview: - -Overview -======== - -Memory management in Python involves a private heap containing all Python -objects and data structures. The management of this private heap is ensured -internally by the *Python memory manager*. The Python memory manager has -different components which deal with various dynamic storage management aspects, -like sharing, segmentation, preallocation or caching. - -At the lowest level, a raw memory allocator ensures that there is enough room in -the private heap for storing all Python-related data by interacting with the -memory manager of the operating system. On top of the raw memory allocator, -several object-specific allocators operate on the same heap and implement -distinct memory management policies adapted to the peculiarities of every object -type. For example, integer objects are managed differently within the heap than -strings, tuples or dictionaries because integers imply different storage -requirements and speed/space tradeoffs. The Python memory manager thus delegates -some of the work to the object-specific allocators, but ensures that the latter -operate within the bounds of the private heap. - -It is important to understand that the management of the Python heap is -performed by the interpreter itself and that the user has no control over it, -even if they regularly manipulate object pointers to memory blocks inside that -heap. The allocation of heap space for Python objects and other internal -buffers is performed on demand by the Python memory manager through the Python/C -API functions listed in this document. - -.. index:: - single: malloc() - single: calloc() - single: realloc() - single: free() - -To avoid memory corruption, extension writers should never try to operate on -Python objects with the functions exported by the C library: :c:func:`malloc`, -:c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. This will result in mixed -calls between the C allocator and the Python memory manager with fatal -consequences, because they implement different algorithms and operate on -different heaps. However, one may safely allocate and release memory blocks -with the C library allocator for individual purposes, as shown in the following -example:: - - PyObject *res; - char *buf = (char *) malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - ...Do some I/O operation involving buf... - res = PyBytes_FromString(buf); - free(buf); /* malloc'ed */ - return res; - -In this example, the memory request for the I/O buffer is handled by the C -library allocator. The Python memory manager is involved only in the allocation -of the bytes object returned as a result. - -In most situations, however, it is recommended to allocate memory from the -Python heap specifically because the latter is under control of the Python -memory manager. For example, this is required when the interpreter is extended -with new object types written in C. Another reason for using the Python heap is -the desire to *inform* the Python memory manager about the memory needs of the -extension module. Even when the requested memory is used exclusively for -internal, highly-specific purposes, delegating all memory requests to the Python -memory manager causes the interpreter to have a more accurate image of its -memory footprint as a whole. Consequently, under certain circumstances, the -Python memory manager may or may not trigger appropriate actions, like garbage -collection, memory compaction or other preventive procedures. Note that by using -the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. - -.. seealso:: - - The :envvar:`PYTHONMALLOC` environment variable can be used to configure - the memory allocators used by Python. - - The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print - statistics of the :ref:`pymalloc memory allocator ` every time a - new pymalloc object arena is created, and on shutdown. - -Allocator Domains -================= - -All allocating functions belong to one of three different "domains" (see also -:c:type:`PyMemAllocatorDomain`). These domains represent different allocation -strategies and are optimized for different purposes. The specific details on -how every domain allocates memory or what internal functions each domain calls -is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. There is no hard -requirement to use the memory returned by the allocation functions belonging to -a given domain for only the purposes hinted by that domain (although this is the -recommended practice). For example, one could use the memory returned by -:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned -by :c:func:`PyObject_Malloc` for allocating memory for buffers. - -The three allocation domains are: - -* Raw domain: intended for allocating memory for general-purpose memory - buffers where the allocation *must* go to the system allocator or where the - allocator can operate without the :term:`GIL`. The memory is requested directly - to the system. - -* "Mem" domain: intended for allocating memory for Python buffers and - general-purpose memory buffers where the allocation must be performed with - the :term:`GIL` held. The memory is taken from the Python private heap. - -* Object domain: intended for allocating memory belonging to Python objects. The - memory is taken from the Python private heap. - -When freeing memory previously allocated by the allocating functions belonging to a -given domain,the matching specific deallocating functions must be used. For example, -:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. - -Raw Memory Interface -==================== - -The following function sets are wrappers to the system allocator. These -functions are thread-safe, the :term:`GIL ` does not -need to be held. - -The :ref:`default raw memory allocator ` uses -the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` -and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting -zero bytes. - -.. versionadded:: 3.4 - -.. c:function:: void* PyMem_RawMalloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been - called instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will - be unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyMem_RawMalloc(n)``; else if - *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. - - If the request fails, :c:func:`PyMem_RawRealloc` returns ``NULL`` and *p* - remains a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_RawFree(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been - called before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - - -.. _memoryinterface: - -Memory Interface -================ - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -The :ref:`default memory allocator ` uses the -:ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. versionchanged:: 3.6 - - The default allocator is now pymalloc instead of system :c:func:`malloc`. - -.. c:function:: void* PyMem_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyMem_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. - - If the request fails, :c:func:`PyMem_Realloc` returns ``NULL`` and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or - :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - -The following type-oriented macros are provided for convenience. Note that -*TYPE* refers to any C type. - - -.. c:function:: TYPE* PyMem_New(TYPE, size_t n) - - Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:type:`TYPE*`. The memory will not have - been initialized in any way. - - -.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) - - Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE*`. On return, - *p* will be a pointer to the new memory area, or ``NULL`` in the event of - failure. - - This is a C preprocessor macro; *p* is always reassigned. Save the original - value of *p* to avoid losing memory when handling errors. - - -.. c:function:: void PyMem_Del(void *p) - - Same as :c:func:`PyMem_Free`. - -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. - -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` - - -Object allocators -================= - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -.. note:: - There is no guarantee that the memory returned by these allocators can be - succesfully casted to a Python object when intercepting the allocating - functions in this domain by the methods described in - the :ref:`Customize Memory Allocators ` section. - -The :ref:`default object allocator ` uses the -:ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. c:function:: void* PyObject_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyObject_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyObject_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. - - If the request fails, :c:func:`PyObject_Realloc` returns ``NULL`` and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyObject_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or - :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - - -.. _default-memory-allocators: - -Default Memory Allocators -========================= - -Default memory allocators: - -=============================== ==================== ================== ===================== ==================== -Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc -=============================== ==================== ================== ===================== ==================== -Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` -Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug -Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` -Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug -=============================== ==================== ================== ===================== ==================== - -Legend: - -* Name: value for :envvar:`PYTHONMALLOC` environment variable -* ``malloc``: system allocators from the standard C library, C functions: - :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free` -* ``pymalloc``: :ref:`pymalloc memory allocator ` -* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` - -.. _customize-memory-allocators: - -Customize Memory Allocators -=========================== - -.. versionadded:: 3.4 - -.. c:type:: PyMemAllocatorEx - - Structure used to describe a memory block allocator. The structure has - four fields: - - +----------------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==========================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +----------------------------------------------------------+---------------------------------------+ - | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized | - | | with zeros | - +----------------------------------------------------------+---------------------------------------+ - | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, void *ptr)`` | free a memory block | - +----------------------------------------------------------+---------------------------------------+ - - .. versionchanged:: 3.5 - The :c:type:`PyMemAllocator` structure was renamed to - :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. - - -.. c:type:: PyMemAllocatorDomain - - Enum used to identify an allocator domain. Domains: - - .. c:macro:: PYMEM_DOMAIN_RAW - - Functions: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - - .. c:macro:: PYMEM_DOMAIN_MEM - - Functions: - - * :c:func:`PyMem_Malloc`, - * :c:func:`PyMem_Realloc` - * :c:func:`PyMem_Calloc` - * :c:func:`PyMem_Free` - - .. c:macro:: PYMEM_DOMAIN_OBJ - - Functions: - - * :c:func:`PyObject_Malloc` - * :c:func:`PyObject_Realloc` - * :c:func:`PyObject_Calloc` - * :c:func:`PyObject_Free` - -.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Get the memory block allocator of the specified domain. - - -.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Set the memory block allocator of the specified domain. - - The new allocator must return a distinct non-``NULL`` pointer when requesting - zero bytes. - - For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be - thread-safe: the :term:`GIL ` is not held when the - allocator is called. - - If the new allocator is not a hook (does not call the previous allocator), - the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the - debug hooks on top on the new allocator. - - -.. c:function:: void PyMem_SetupDebugHooks(void) - - Setup hooks to detect bugs in the Python memory allocator functions. - - Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``), - freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks - are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``). - - Runtime checks: - - - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer - allocated by :c:func:`PyMem_Malloc` - - Detect write before the start of the buffer (buffer underflow) - - Detect write after the end of the buffer (buffer overflow) - - Check that the :term:`GIL ` is held when - allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: - :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: - :c:func:`PyMem_Malloc`) domains are called - - On error, the debug hooks use the :mod:`tracemalloc` module to get the - traceback where a memory block was allocated. The traceback is only - displayed if :mod:`tracemalloc` is tracing Python memory allocations and the - memory block was traced. - - These hooks are :ref:`installed by default ` if - Python is compiled in debug - mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install - debug hooks on a Python compiled in release mode. - - .. versionchanged:: 3.6 - This function now also works on Python compiled in release mode. - On error, the debug hooks now use :mod:`tracemalloc` to get the traceback - where a memory block was allocated. The debug hooks now also check - if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and - :c:data:`PYMEM_DOMAIN_MEM` domains are called. - - .. versionchanged:: 3.8 - Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and - ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` - and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` - and ``free()``. - - -.. _pymalloc: - -The pymalloc allocator -====================== - -Python has a *pymalloc* allocator optimized for small objects (smaller or equal -to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" -with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and -:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. - -*pymalloc* is the :ref:`default allocator ` of the -:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and -:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. - -The arena allocator uses the following functions: - -* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, -* :c:func:`mmap` and :c:func:`munmap` if available, -* :c:func:`malloc` and :c:func:`free` otherwise. - -Customize pymalloc Arena Allocator ----------------------------------- - -.. versionadded:: 3.4 - -.. c:type:: PyObjectArenaAllocator - - Structure used to describe an arena allocator. The structure has - three fields: - - +--------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +--------------------------------------------------+---------------------------------------+ - | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | - +--------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | - +--------------------------------------------------+---------------------------------------+ - -.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) - - Get the arena allocator. - -.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) - - Set the arena allocator. - - -tracemalloc C API -================= - -.. versionadded:: 3.7 - -.. c:function:: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) - - Track an allocated memory block in the :mod:`tracemalloc` module. - - Return ``0`` on success, return ``-1`` on error (failed to allocate memory to - store the trace). Return ``-2`` if tracemalloc is disabled. - - If memory block is already tracked, update the existing trace. - -.. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) - - Untrack an allocated memory block in the :mod:`tracemalloc` module. - Do nothing if the block was not tracked. - - Return ``-2`` if tracemalloc is disabled, otherwise return ``0``. - - -.. _memoryexamples: - -Examples -======== - -Here is the example from section :ref:`memoryoverview`, rewritten so that the -I/O buffer is allocated from the Python heap by using the first function set:: - - PyObject *res; - char *buf = (char *) PyMem_Malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Free(buf); /* allocated with PyMem_Malloc */ - return res; - -The same code using the type-oriented function set:: - - PyObject *res; - char *buf = PyMem_New(char, BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Del(buf); /* allocated with PyMem_New */ - return res; - -Note that in the two examples above, the buffer is always manipulated via -functions belonging to the same set. Indeed, it is required to use the same -memory API family for a given memory block, so that the risk of mixing different -allocators is reduced to a minimum. The following code sequence contains two -errors, one of which is labeled as *fatal* because it mixes two different -allocators operating on different heaps. :: - - char *buf1 = PyMem_New(char, BUFSIZ); - char *buf2 = (char *) malloc(BUFSIZ); - char *buf3 = (char *) PyMem_Malloc(BUFSIZ); - ... - PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ - free(buf2); /* Right -- allocated via malloc() */ - free(buf1); /* Fatal -- should be PyMem_Del() */ - -In addition to the functions aimed at handling raw memory blocks from the Python -heap, objects in Python are allocated and released with :c:func:`PyObject_New`, -:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`. - -These will be explained in the next chapter on defining and implementing new -object types in C. diff --git a/Doc/faq/design.rst.bak b/Doc/faq/design.rst.bak deleted file mode 100644 index 7fe1c6d58f58a1..00000000000000 --- a/Doc/faq/design.rst.bak +++ /dev/null @@ -1,759 +0,0 @@ -====================== -Design and History FAQ -====================== - -.. only:: html - - .. contents:: - - -Why does Python use indentation for grouping of statements? ------------------------------------------------------------ - -Guido van Rossum believes that using indentation for grouping is extremely -elegant and contributes a lot to the clarity of the average Python program. -Most people learn to love this feature after a while. - -Since there are no begin/end brackets there cannot be a disagreement between -grouping perceived by the parser and the human reader. Occasionally C -programmers will encounter a fragment of code like this:: - - if (x <= y) - x++; - y--; - z++; - -Only the ``x++`` statement is executed if the condition is true, but the -indentation leads many to believe otherwise. Even experienced C programmers will -sometimes stare at it a long time wondering as to why ``y`` is being decremented even -for ``x > y``. - -Because there are no begin/end brackets, Python is much less prone to -coding-style conflicts. In C there are many different ways to place the braces. -After becoming used to reading and writing code using a particular style, -it is normal to feel somewhat uneasy when reading (or being required to write) -in a different one. - - -Many coding styles place begin/end brackets on a line by themselves. This makes -programs considerably longer and wastes valuable screen space, making it harder -to get a good overview of a program. Ideally, a function should fit on one -screen (say, 20--30 lines). 20 lines of Python can do a lot more work than 20 -lines of C. This is not solely due to the lack of begin/end brackets -- the -lack of declarations and the high-level data types are also responsible -- but -the indentation-based syntax certainly helps. - - -Why am I getting strange results with simple arithmetic operations? -------------------------------------------------------------------- - -See the next question. - - -Why are floating-point calculations so inaccurate? --------------------------------------------------- - -Users are often surprised by results like this:: - - >>> 1.2 - 1.0 - 0.19999999999999996 - -and think it is a bug in Python. It's not. This has little to do with Python, -and much more to do with how the underlying platform handles floating-point -numbers. - -The :class:`float` type in CPython uses a C ``double`` for storage. A -:class:`float` object's value is stored in binary floating-point with a fixed -precision (typically 53 bits) and Python uses C operations, which in turn rely -on the hardware implementation in the processor, to perform floating-point -operations. This means that as far as floating-point operations are concerned, -Python behaves like many popular languages including C and Java. - -Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: - - >>> x = 1.2 - -the value stored for ``x`` is a (very good) approximation to the decimal value -``1.2``, but is not exactly equal to it. On a typical machine, the actual -stored value is:: - - 1.0011001100110011001100110011001100110011001100110011 (binary) - -which is exactly:: - - 1.1999999999999999555910790149937383830547332763671875 (decimal) - -The typical precision of 53 bits provides Python floats with 15--16 -decimal digits of accuracy. - -For a fuller explanation, please see the :ref:`floating point arithmetic -` chapter in the Python tutorial. - - -Why are Python strings immutable? ---------------------------------- - -There are several advantages. - -One is performance: knowing that a string is immutable means we can allocate -space for it at creation time, and the storage requirements are fixed and -unchanging. This is also one of the reasons for the distinction between tuples -and lists. - -Another advantage is that strings in Python are considered as "elemental" as -numbers. No amount of activity will change the value 8 to anything else, and in -Python, no amount of activity will change the string "eight" to anything else. - - -.. _why-self: - -Why must 'self' be used explicitly in method definitions and calls? -------------------------------------------------------------------- - -The idea was borrowed from Modula-3. It turns out to be very useful, for a -variety of reasons. - -First, it's more obvious that you are using a method or instance attribute -instead of a local variable. Reading ``self.x`` or ``self.meth()`` makes it -absolutely clear that an instance variable or method is used even if you don't -know the class definition by heart. In C++, you can sort of tell by the lack of -a local variable declaration (assuming globals are rare or easily recognizable) --- but in Python, there are no local variable declarations, so you'd have to -look up the class definition to be sure. Some C++ and Java coding standards -call for instance attributes to have an ``m_`` prefix, so this explicitness is -still useful in those languages, too. - -Second, it means that no special syntax is necessary if you want to explicitly -reference or call the method from a particular class. In C++, if you want to -use a method from a base class which is overridden in a derived class, you have -to use the ``::`` operator -- in Python you can write -``baseclass.methodname(self, )``. This is particularly useful -for :meth:`__init__` methods, and in general in cases where a derived class -method wants to extend the base class method of the same name and thus has to -call the base class method somehow. - -Finally, for instance variables it solves a syntactic problem with assignment: -since local variables in Python are (by definition!) those variables to which a -value is assigned in a function body (and that aren't explicitly declared -global), there has to be some way to tell the interpreter that an assignment was -meant to assign to an instance variable instead of to a local variable, and it -should preferably be syntactic (for efficiency reasons). C++ does this through -declarations, but Python doesn't have declarations and it would be a pity having -to introduce them just for this purpose. Using the explicit ``self.var`` solves -this nicely. Similarly, for using instance variables, having to write -``self.var`` means that references to unqualified names inside a method don't -have to search the instance's directories. To put it another way, local -variables and instance variables live in two different namespaces, and you need -to tell Python which namespace to use. - - -.. _why-can-t-i-use-an-assignment-in-an-expression: - -Why can't I use an assignment in an expression? ------------------------------------------------ - -Starting in Python 3.8, you can! - -Assignment expressions using the walrus operator `:=` assign a variable in an -expression:: - - while chunk := fp.read(200): - print(chunk) - -See :pep:`572` for more information. - - - -Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))? ----------------------------------------------------------------------------------------------------------------- - -As Guido said: - - (a) For some operations, prefix notation just reads better than - postfix -- prefix (and infix!) operations have a long tradition in - mathematics which likes notations where the visuals help the - mathematician thinking about a problem. Compare the easy with which we - rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of - doing the same thing using a raw OO notation. - - (b) When I read code that says len(x) I *know* that it is asking for - the length of something. This tells me two things: the result is an - integer, and the argument is some kind of container. To the contrary, - when I read x.len(), I have to already know that x is some kind of - container implementing an interface or inheriting from a class that - has a standard len(). Witness the confusion we occasionally have when - a class that is not implementing a mapping has a get() or keys() - method, or something that isn't a file has a write() method. - - -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html - - -Why is join() a string method instead of a list or tuple method? ----------------------------------------------------------------- - -Strings became much more like other standard types starting in Python 1.6, when -methods were added which give the same functionality that has always been -available using the functions of the string module. Most of these new methods -have been widely accepted, but the one which appears to make some programmers -feel uncomfortable is:: - - ", ".join(['1', '2', '4', '8', '16']) - -which gives the result:: - - "1, 2, 4, 8, 16" - -There are two common arguments against this usage. - -The first runs along the lines of: "It looks really ugly using a method of a -string literal (string constant)", to which the answer is that it might, but a -string literal is just a fixed value. If the methods are to be allowed on names -bound to strings there is no logical reason to make them unavailable on -literals. - -The second objection is typically cast as: "I am really telling a sequence to -join its members together with a string constant". Sadly, you aren't. For some -reason there seems to be much less difficulty with having :meth:`~str.split` as -a string method, since in that case it is easy to see that :: - - "1, 2, 4, 8, 16".split(", ") - -is an instruction to a string literal to return the substrings delimited by the -given separator (or, by default, arbitrary runs of white space). - -:meth:`~str.join` is a string method because in using it you are telling the -separator string to iterate over a sequence of strings and insert itself between -adjacent elements. This method can be used with any argument which obeys the -rules for sequence objects, including any new classes you might define yourself. -Similar methods exist for bytes and bytearray objects. - - -How fast are exceptions? ------------------------- - -A try/except block is extremely efficient if no exceptions are raised. Actually -catching an exception is expensive. In versions of Python prior to 2.0 it was -common to use this idiom:: - - try: - value = mydict[key] - except KeyError: - mydict[key] = getvalue(key) - value = mydict[key] - -This only made sense when you expected the dict to have the key almost all the -time. If that wasn't the case, you coded it like this:: - - if key in mydict: - value = mydict[key] - else: - value = mydict[key] = getvalue(key) - -For this specific case, you could also use ``value = dict.setdefault(key, -getvalue(key))``, but only if the ``getvalue()`` call is cheap enough because it -is evaluated in all cases. - - -Why isn't there a switch or case statement in Python? ------------------------------------------------------ - -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. - -For cases where you need to choose from a very large number of possibilities, -you can create a dictionary mapping case values to functions to call. For -example:: - - def function_1(...): - ... - - functions = {'a': function_1, - 'b': function_2, - 'c': self.method_1, ...} - - func = functions[value] - func() - -For calling methods on objects, you can simplify yet further by using the -:func:`getattr` built-in to retrieve methods with a particular name:: - - def visit_a(self, ...): - ... - ... - - def dispatch(self, value): - method_name = 'visit_' + str(value) - method = getattr(self, method_name) - method() - -It's suggested that you use a prefix for the method names, such as ``visit_`` in -this example. Without such a prefix, if values are coming from an untrusted -source, an attacker would be able to call any method on your object. - - -Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? --------------------------------------------------------------------------------------------------------- - -Answer 1: Unfortunately, the interpreter pushes at least one C stack frame for -each Python stack frame. Also, extensions can call back into Python at almost -random moments. Therefore, a complete threads implementation requires thread -support for C. - -Answer 2: Fortunately, there is `Stackless Python `_, -which has a completely redesigned interpreter loop that avoids the C stack. - - -Why can't lambda expressions contain statements? ------------------------------------------------- - -Python lambda expressions cannot contain statements because Python's syntactic -framework can't handle statements nested inside expressions. However, in -Python, this is not a serious problem. Unlike lambda forms in other languages, -where they add functionality, Python lambdas are only a shorthand notation if -you're too lazy to define a function. - -Functions are already first class objects in Python, and can be declared in a -local scope. Therefore the only advantage of using a lambda instead of a -locally-defined function is that you don't need to invent a name for the -function -- but that's just a local variable to which the function object (which -is exactly the same type of object that a lambda expression yields) is assigned! - - -Can Python be compiled to machine code, C or some other language? ------------------------------------------------------------------ - -`Cython `_ compiles a modified version of Python with -optional annotations into C extensions. `Nuitka `_ is -an up-and-coming compiler of Python into C++ code, aiming to support the full -Python language. For compiling to Java you can consider -`VOC `_. - - -How does Python manage memory? ------------------------------- - -The details of Python memory management depend on the implementation. The -standard implementation of Python, :term:`CPython`, uses reference counting to -detect inaccessible objects, and another mechanism to collect reference cycles, -periodically executing a cycle detection algorithm which looks for inaccessible -cycles and deletes the objects involved. The :mod:`gc` module provides functions -to perform a garbage collection, obtain debugging statistics, and tune the -collector's parameters. - -Other implementations (such as `Jython `_ or -`PyPy `_), however, can rely on a different mechanism -such as a full-blown garbage collector. This difference can cause some -subtle porting problems if your Python code depends on the behavior of the -reference counting implementation. - -In some Python implementations, the following code (which is fine in CPython) -will probably run out of file descriptors:: - - for file in very_long_list_of_files: - f = open(file) - c = f.read(1) - -Indeed, using CPython's reference counting and destructor scheme, each new -assignment to *f* closes the previous file. With a traditional GC, however, -those file objects will only get collected (and closed) at varying and possibly -long intervals. - -If you want to write code that will work with any Python implementation, -you should explicitly close the file or use the :keyword:`with` statement; -this will work regardless of memory management scheme:: - - for file in very_long_list_of_files: - with open(file) as f: - c = f.read(1) - - -Why doesn't CPython use a more traditional garbage collection scheme? ---------------------------------------------------------------------- - -For one thing, this is not a C standard feature and hence it's not portable. -(Yes, we know about the Boehm GC library. It has bits of assembler code for -*most* common platforms, not for all of them, and although it is mostly -transparent, it isn't completely transparent; patches are required to get -Python to work with it.) - -Traditional GC also becomes a problem when Python is embedded into other -applications. While in a standalone Python it's fine to replace the standard -malloc() and free() with versions provided by the GC library, an application -embedding Python may want to have its *own* substitute for malloc() and free(), -and may not want Python's. Right now, CPython works with anything that -implements malloc() and free() properly. - - -Why isn't all memory freed when CPython exits? ----------------------------------------------- - -Objects referenced from the global namespaces of Python modules are not always -deallocated when Python exits. This may happen if there are circular -references. There are also certain bits of memory that are allocated by the C -library that are impossible to free (e.g. a tool like Purify will complain about -these). Python is, however, aggressive about cleaning up memory on exit and -does try to destroy every single object. - -If you want to force Python to delete certain things on deallocation use the -:mod:`atexit` module to run a function that will force those deletions. - - -Why are there separate tuple and list data types? -------------------------------------------------- - -Lists and tuples, while similar in many respects, are generally used in -fundamentally different ways. Tuples can be thought of as being similar to -Pascal records or C structs; they're small collections of related data which may -be of different types which are operated on as a group. For example, a -Cartesian coordinate is appropriately represented as a tuple of two or three -numbers. - -Lists, on the other hand, are more like arrays in other languages. They tend to -hold a varying number of objects all of which have the same type and which are -operated on one-by-one. For example, ``os.listdir('.')`` returns a list of -strings representing the files in the current directory. Functions which -operate on this output would generally not break if you added another file or -two to the directory. - -Tuples are immutable, meaning that once a tuple has been created, you can't -replace any of its elements with a new value. Lists are mutable, meaning that -you can always change a list's elements. Only immutable elements can be used as -dictionary keys, and hence only tuples and not lists can be used as keys. - - -How are lists implemented in CPython? -------------------------------------- - -CPython's lists are really variable-length arrays, not Lisp-style linked lists. -The implementation uses a contiguous array of references to other objects, and -keeps a pointer to this array and the array's length in a list head structure. - -This makes indexing a list ``a[i]`` an operation whose cost is independent of -the size of the list or the value of the index. - -When items are appended or inserted, the array of references is resized. Some -cleverness is applied to improve the performance of appending items repeatedly; -when the array must be grown, some extra space is allocated so the next few -times don't require an actual resize. - - -How are dictionaries implemented in CPython? --------------------------------------------- - -CPython's dictionaries are implemented as resizable hash tables. Compared to -B-trees, this gives better performance for lookup (the most common operation by -far) under most circumstances, and the implementation is simpler. - -Dictionaries work by computing a hash code for each key stored in the dictionary -using the :func:`hash` built-in function. The hash code varies widely depending -on the key and a per-process seed; for example, "Python" could hash to --539294296 while "python", a string that differs by a single bit, could hash -to 1142331976. The hash code is then used to calculate a location in an -internal array where the value will be stored. Assuming that you're storing -keys that all have different hash values, this means that dictionaries take -constant time -- O(1), in Big-O notation -- to retrieve a key. - - -Why must dictionary keys be immutable? --------------------------------------- - -The hash table implementation of dictionaries uses a hash value calculated from -the key value to find the key. If the key were a mutable object, its value -could change, and thus its hash could also change. But since whoever changes -the key object can't tell that it was being used as a dictionary key, it can't -move the entry around in the dictionary. Then, when you try to look up the same -object in the dictionary it won't be found because its hash value is different. -If you tried to look up the old value it wouldn't be found either, because the -value of the object found in that hash bin would be different. - -If you want a dictionary indexed with a list, simply convert the list to a tuple -first; the function ``tuple(L)`` creates a tuple with the same entries as the -list ``L``. Tuples are immutable and can therefore be used as dictionary keys. - -Some unacceptable solutions that have been proposed: - -- Hash lists by their address (object ID). This doesn't work because if you - construct a new list with the same value it won't be found; e.g.:: - - mydict = {[1, 2]: '12'} - print(mydict[[1, 2]]) - - would raise a :exc:`KeyError` exception because the id of the ``[1, 2]`` used in the - second line differs from that in the first line. In other words, dictionary - keys should be compared using ``==``, not using :keyword:`is`. - -- Make a copy when using a list as a key. This doesn't work because the list, - being a mutable object, could contain a reference to itself, and then the - copying code would run into an infinite loop. - -- Allow lists as keys but tell the user not to modify them. This would allow a - class of hard-to-track bugs in programs when you forgot or modified a list by - accident. It also invalidates an important invariant of dictionaries: every - value in ``d.keys()`` is usable as a key of the dictionary. - -- Mark lists as read-only once they are used as a dictionary key. The problem - is that it's not just the top-level object that could change its value; you - could use a tuple containing a list as a key. Entering anything as a key into - a dictionary would require marking all objects reachable from there as - read-only -- and again, self-referential objects could cause an infinite loop. - -There is a trick to get around this if you need to, but use it at your own risk: -You can wrap a mutable structure inside a class instance which has both a -:meth:`__eq__` and a :meth:`__hash__` method. You must then make sure that the -hash value for all such wrapper objects that reside in a dictionary (or other -hash based structure), remain fixed while the object is in the dictionary (or -other structure). :: - - class ListWrapper: - def __init__(self, the_list): - self.the_list = the_list - - def __eq__(self, other): - return self.the_list == other.the_list - - def __hash__(self): - l = self.the_list - result = 98767 - len(l)*555 - for i, el in enumerate(l): - try: - result = result + (hash(el) % 9999999) * 1001 + i - except Exception: - result = (result % 7777777) + i * 333 - return result - -Note that the hash computation is complicated by the possibility that some -members of the list may be unhashable and also by the possibility of arithmetic -overflow. - -Furthermore it must always be the case that if ``o1 == o2`` (ie ``o1.__eq__(o2) -is True``) then ``hash(o1) == hash(o2)`` (ie, ``o1.__hash__() == o2.__hash__()``), -regardless of whether the object is in a dictionary or not. If you fail to meet -these restrictions dictionaries and other hash based structures will misbehave. - -In the case of ListWrapper, whenever the wrapper object is in a dictionary the -wrapped list must not change to avoid anomalies. Don't do this unless you are -prepared to think hard about the requirements and the consequences of not -meeting them correctly. Consider yourself warned. - - -Why doesn't list.sort() return the sorted list? ------------------------------------------------ - -In situations where performance matters, making a copy of the list just to sort -it would be wasteful. Therefore, :meth:`list.sort` sorts the list in place. In -order to remind you of that fact, it does not return the sorted list. This way, -you won't be fooled into accidentally overwriting a list when you need a sorted -copy but also need to keep the unsorted version around. - -If you want to return a new list, use the built-in :func:`sorted` function -instead. This function creates a new list from a provided iterable, sorts -it and returns it. For example, here's how to iterate over the keys of a -dictionary in sorted order:: - - for key in sorted(mydict): - ... # do whatever with mydict[key]... - - -How do you specify and enforce an interface spec in Python? ------------------------------------------------------------ - -An interface specification for a module as provided by languages such as C++ and -Java describes the prototypes for the methods and functions of the module. Many -feel that compile-time enforcement of interface specifications helps in the -construction of large programs. - -Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes -(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check -whether an instance or a class implements a particular ABC. The -:mod:`collections.abc` module defines a set of useful ABCs such as -:class:`~collections.abc.Iterable`, :class:`~collections.abc.Container`, and -:class:`~collections.abc.MutableMapping`. - -For Python, many of the advantages of interface specifications can be obtained -by an appropriate test discipline for components. - -A good test suite for a module can both provide a regression test and serve as a -module interface specification and a set of examples. Many Python modules can -be run as a script to provide a simple "self test." Even modules which use -complex external interfaces can often be tested in isolation using trivial -"stub" emulations of the external interface. The :mod:`doctest` and -:mod:`unittest` modules or third-party test frameworks can be used to construct -exhaustive test suites that exercise every line of code in a module. - -An appropriate testing discipline can help build large complex applications in -Python as well as having interface specifications would. In fact, it can be -better because an interface specification cannot test certain properties of a -program. For example, the :meth:`append` method is expected to add new elements -to the end of some internal list; an interface specification cannot test that -your :meth:`append` implementation will actually do this correctly, but it's -trivial to check this property in a test suite. - -Writing test suites is very helpful, and you might want to design your code to -make it easily tested. One increasingly popular technique, test-driven -development, calls for writing parts of the test suite first, before you write -any of the actual code. Of course Python allows you to be sloppy and not write -test cases at all. - - -Why is there no goto? ---------------------- - -You can use exceptions to provide a "structured goto" that even works across -function calls. Many feel that exceptions can conveniently emulate all -reasonable uses of the "go" or "goto" constructs of C, Fortran, and other -languages. For example:: - - class label(Exception): pass # declare a label - - try: - ... - if condition: raise label() # goto label - ... - except label: # where to goto - pass - ... - -This doesn't allow you to jump into the middle of a loop, but that's usually -considered an abuse of goto anyway. Use sparingly. - - -Why can't raw strings (r-strings) end with a backslash? -------------------------------------------------------- - -More precisely, they can't end with an odd number of backslashes: the unpaired -backslash at the end escapes the closing quote character, leaving an -unterminated string. - -Raw strings were designed to ease creating input for processors (chiefly regular -expression engines) that want to do their own backslash escape processing. Such -processors consider an unmatched trailing backslash to be an error anyway, so -raw strings disallow that. In return, they allow you to pass on the string -quote character by escaping it with a backslash. These rules work well when -r-strings are used for their intended purpose. - -If you're trying to build Windows pathnames, note that all Windows system calls -accept forward slashes too:: - - f = open("/mydir/file.txt") # works fine! - -If you're trying to build a pathname for a DOS command, try e.g. one of :: - - dir = r"\this\is\my\dos\dir" "\\" - dir = r"\this\is\my\dos\dir\ "[:-1] - dir = "\\this\\is\\my\\dos\\dir\\" - - -Why doesn't Python have a "with" statement for attribute assignments? ---------------------------------------------------------------------- - -Python has a 'with' statement that wraps the execution of a block, calling code -on the entrance and exit from the block. Some languages have a construct that -looks like this:: - - with obj: - a = 1 # equivalent to obj.a = 1 - total = total + 1 # obj.total = obj.total + 1 - -In Python, such a construct would be ambiguous. - -Other languages, such as Object Pascal, Delphi, and C++, use static types, so -it's possible to know, in an unambiguous way, what member is being assigned -to. This is the main point of static typing -- the compiler *always* knows the -scope of every variable at compile time. - -Python uses dynamic types. It is impossible to know in advance which attribute -will be referenced at runtime. Member attributes may be added or removed from -objects on the fly. This makes it impossible to know, from a simple reading, -what attribute is being referenced: a local one, a global one, or a member -attribute? - -For instance, take the following incomplete snippet:: - - def foo(a): - with a: - print(x) - -The snippet assumes that "a" must have a member attribute called "x". However, -there is nothing in Python that tells the interpreter this. What should happen -if "a" is, let us say, an integer? If there is a global variable named "x", -will it be used inside the with block? As you see, the dynamic nature of Python -makes such choices much harder. - -The primary benefit of "with" and similar language features (reduction of code -volume) can, however, easily be achieved in Python by assignment. Instead of:: - - function(args).mydict[index][index].a = 21 - function(args).mydict[index][index].b = 42 - function(args).mydict[index][index].c = 63 - -write this:: - - ref = function(args).mydict[index][index] - ref.a = 21 - ref.b = 42 - ref.c = 63 - -This also has the side-effect of increasing execution speed because name -bindings are resolved at run-time in Python, and the second version only needs -to perform the resolution once. - - -Why are colons required for the if/while/def/class statements? --------------------------------------------------------------- - -The colon is required primarily to enhance readability (one of the results of -the experimental ABC language). Consider this:: - - if a == b - print(a) - -versus :: - - if a == b: - print(a) - -Notice how the second one is slightly easier to read. Notice further how a -colon sets off the example in this FAQ answer; it's a standard usage in English. - -Another minor reason is that the colon makes it easier for editors with syntax -highlighting; they can look for colons to decide when indentation needs to be -increased instead of having to do a more elaborate parsing of the program text. - - -Why does Python allow commas at the end of lists and tuples? ------------------------------------------------------------- - -Python lets you add a trailing comma at the end of lists, tuples, and -dictionaries:: - - [1, 2, 3,] - ('a', 'b', 'c',) - d = { - "A": [1, 5], - "B": [6, 7], # last trailing comma is optional but good style - } - - -There are several reasons to allow this. - -When you have a literal value for a list, tuple, or dictionary spread across -multiple lines, it's easier to add more elements because you don't have to -remember to add a comma to the previous line. The lines can also be reordered -without creating a syntax error. - -Accidentally omitting the comma can lead to errors that are hard to diagnose. -For example:: - - x = [ - "fee", - "fie" - "foo", - "fum" - ] - -This list looks like it has four elements, but it actually contains three: -"fee", "fiefoo" and "fum". Always adding the comma avoids this source of error. - -Allowing the trailing comma may also make programmatic code generation easier. diff --git a/Doc/howto/descriptor.rst.bak b/Doc/howto/descriptor.rst.bak deleted file mode 100644 index 94a8b4e6b40b96..00000000000000 --- a/Doc/howto/descriptor.rst.bak +++ /dev/null @@ -1,1575 +0,0 @@ -.. _descriptorhowto: - -====================== -Descriptor HowTo Guide -====================== - -:Author: Raymond Hettinger -:Contact: - -.. Contents:: - - -:term:`Descriptors ` let objects customize attribute lookup, -storage, and deletion. - -This guide has four major sections: - -1) The "primer" gives a basic overview, moving gently from simple examples, - adding one feature at a time. Start here if you're new to descriptors. - -2) The second section shows a complete, practical descriptor example. If you - already know the basics, start there. - -3) The third section provides a more technical tutorial that goes into the - detailed mechanics of how descriptors work. Most people don't need this - level of detail. - -4) The last section has pure Python equivalents for built-in descriptors that - are written in C. Read this if you're curious about how functions turn - into bound methods or about the implementation of common tools like - :func:`classmethod`, :func:`staticmethod`, :func:`property`, and - :term:`__slots__`. - - -Primer -^^^^^^ - -In this primer, we start with the most basic possible example and then we'll -add new capabilities one by one. - - -Simple example: A descriptor that returns a constant ----------------------------------------------------- - -The :class:`Ten` class is a descriptor whose :meth:`__get__` method always -returns the constant ``10``: - -.. testcode:: - - class Ten: - def __get__(self, obj, objtype=None): - return 10 - -To use the descriptor, it must be stored as a class variable in another class: - -.. testcode:: - - class A: - x = 5 # Regular class attribute - y = Ten() # Descriptor instance - -An interactive session shows the difference between normal attribute lookup -and descriptor lookup: - -.. doctest:: - - >>> a = A() # Make an instance of class A - >>> a.x # Normal attribute lookup - 5 - >>> a.y # Descriptor lookup - 10 - -In the ``a.x`` attribute lookup, the dot operator finds ``'x': 5`` -in the class dictionary. In the ``a.y`` lookup, the dot operator -finds a descriptor instance, recognized by its ``__get__`` method. -Calling that method returns ``10``. - -Note that the value ``10`` is not stored in either the class dictionary or the -instance dictionary. Instead, the value ``10`` is computed on demand. - -This example shows how a simple descriptor works, but it isn't very useful. -For retrieving constants, normal attribute lookup would be better. - -In the next section, we'll create something more useful, a dynamic lookup. - - -Dynamic lookups ---------------- - -Interesting descriptors typically run computations instead of returning -constants: - -.. testcode:: - - import os - - class DirectorySize: - - def __get__(self, obj, objtype=None): - return len(os.listdir(obj.dirname)) - - class Directory: - - size = DirectorySize() # Descriptor instance - - def __init__(self, dirname): - self.dirname = dirname # Regular instance attribute - -An interactive session shows that the lookup is dynamic — it computes -different, updated answers each time:: - - >>> s = Directory('songs') - >>> g = Directory('games') - >>> s.size # The songs directory has twenty files - 20 - >>> g.size # The games directory has three files - 3 - >>> open('games/newfile').close() # Add a fourth file to the directory - >>> g.size # File count is automatically updated - 4 - -Besides showing how descriptors can run computations, this example also -reveals the purpose of the parameters to :meth:`__get__`. The *self* -parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is -either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that -lets the :meth:`__get__` method learn the target directory. The *objtype* -parameter is the class *Directory*. - - -Managed attributes ------------------- - -A popular use for descriptors is managing access to instance data. The -descriptor is assigned to a public attribute in the class dictionary while the -actual data is stored as a private attribute in the instance dictionary. The -descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when -the public attribute is accessed. - -In the following example, *age* is the public attribute and *_age* is the -private attribute. When the public attribute is accessed, the descriptor logs -the lookup or update: - -.. testcode:: - - import logging - - logging.basicConfig(level=logging.INFO) - - class LoggedAgeAccess: - - def __get__(self, obj, objtype=None): - value = obj._age - logging.info('Accessing %r giving %r', 'age', value) - return value - - def __set__(self, obj, value): - logging.info('Updating %r to %r', 'age', value) - obj._age = value - - class Person: - - age = LoggedAgeAccess() # Descriptor instance - - def __init__(self, name, age): - self.name = name # Regular instance attribute - self.age = age # Calls __set__() - - def birthday(self): - self.age += 1 # Calls both __get__() and __set__() - - -An interactive session shows that all access to the managed attribute *age* is -logged, but that the regular attribute *name* is not logged: - -.. testcode:: - :hide: - - import logging, sys - logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) - -.. doctest:: - - >>> mary = Person('Mary M', 30) # The initial age update is logged - INFO:root:Updating 'age' to 30 - >>> dave = Person('David D', 40) - INFO:root:Updating 'age' to 40 - - >>> vars(mary) # The actual data is in a private attribute - {'name': 'Mary M', '_age': 30} - >>> vars(dave) - {'name': 'David D', '_age': 40} - - >>> mary.age # Access the data and log the lookup - INFO:root:Accessing 'age' giving 30 - 30 - >>> mary.birthday() # Updates are logged as well - INFO:root:Accessing 'age' giving 30 - INFO:root:Updating 'age' to 31 - - >>> dave.name # Regular attribute lookup isn't logged - 'David D' - >>> dave.age # Only the managed attribute is logged - INFO:root:Accessing 'age' giving 40 - 40 - -One major issue with this example is that the private name *_age* is hardwired in -the *LoggedAgeAccess* class. That means that each instance can only have one -logged attribute and that its name is unchangeable. In the next example, -we'll fix that problem. - - -Customized names ----------------- - -When a class uses descriptors, it can inform each descriptor about which -variable name was used. - -In this example, the :class:`Person` class has two descriptor instances, -*name* and *age*. When the :class:`Person` class is defined, it makes a -callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can -be recorded, giving each descriptor its own *public_name* and *private_name*: - -.. testcode:: - - import logging - - logging.basicConfig(level=logging.INFO) - - class LoggedAccess: - - def __set_name__(self, owner, name): - self.public_name = name - self.private_name = '_' + name - - def __get__(self, obj, objtype=None): - value = getattr(obj, self.private_name) - logging.info('Accessing %r giving %r', self.public_name, value) - return value - - def __set__(self, obj, value): - logging.info('Updating %r to %r', self.public_name, value) - setattr(obj, self.private_name, value) - - class Person: - - name = LoggedAccess() # First descriptor instance - age = LoggedAccess() # Second descriptor instance - - def __init__(self, name, age): - self.name = name # Calls the first descriptor - self.age = age # Calls the second descriptor - - def birthday(self): - self.age += 1 - -An interactive session shows that the :class:`Person` class has called -:meth:`__set_name__` so that the field names would be recorded. Here -we call :func:`vars` to look up the descriptor without triggering it: - -.. doctest:: - - >>> vars(vars(Person)['name']) - {'public_name': 'name', 'private_name': '_name'} - >>> vars(vars(Person)['age']) - {'public_name': 'age', 'private_name': '_age'} - -The new class now logs access to both *name* and *age*: - -.. testcode:: - :hide: - - import logging, sys - logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) - -.. doctest:: - - >>> pete = Person('Peter P', 10) - INFO:root:Updating 'name' to 'Peter P' - INFO:root:Updating 'age' to 10 - >>> kate = Person('Catherine C', 20) - INFO:root:Updating 'name' to 'Catherine C' - INFO:root:Updating 'age' to 20 - -The two *Person* instances contain only the private names:: - - >>> vars(pete) - {'_name': 'Peter P', '_age': 10} - >>> vars(kate) - {'_name': 'Catherine C', '_age': 20} - - -Closing thoughts ----------------- - -A :term:`descriptor` is what we call any object that defines :meth:`__get__`, -:meth:`__set__`, or :meth:`__delete__`. - -Optionally, descriptors can have a :meth:`__set_name__` method. This is only -used in cases where a descriptor needs to know either the class where it was -created or the name of class variable it was assigned to. (This method, if -present, is called even if the class is not a descriptor.) - -Descriptors get invoked by the dot operator during attribute lookup. If a -descriptor is accessed indirectly with ``vars(some_class)[descriptor_name]``, -the descriptor instance is returned without invoking it. - -Descriptors only work when used as class variables. When put in instances, -they have no effect. - -The main motivation for descriptors is to provide a hook allowing objects -stored in class variables to control what happens during attribute lookup. - -Traditionally, the calling class controls what happens during lookup. -Descriptors invert that relationship and allow the data being looked-up to -have a say in the matter. - -Descriptors are used throughout the language. It is how functions turn into -bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, -:func:`property`, and :func:`functools.cached_property` are all implemented as -descriptors. - - -Complete Practical Example -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In this example, we create a practical and powerful tool for locating -notoriously hard to find data corruption bugs. - - -Validator class ---------------- - -A validator is a descriptor for managed attribute access. Prior to storing -any data, it verifies that the new value meets various type and range -restrictions. If those restrictions aren't met, it raises an exception to -prevent data corruption at its source. - -This :class:`Validator` class is both an :term:`abstract base class` and a -managed attribute descriptor: - -.. testcode:: - - from abc import ABC, abstractmethod - - class Validator(ABC): - - def __set_name__(self, owner, name): - self.private_name = '_' + name - - def __get__(self, obj, objtype=None): - return getattr(obj, self.private_name) - - def __set__(self, obj, value): - self.validate(value) - setattr(obj, self.private_name, value) - - @abstractmethod - def validate(self, value): - pass - -Custom validators need to inherit from :class:`Validator` and must supply a -:meth:`validate` method to test various restrictions as needed. - - -Custom validators ------------------ - -Here are three practical data validation utilities: - -1) :class:`OneOf` verifies that a value is one of a restricted set of options. - -2) :class:`Number` verifies that a value is either an :class:`int` or - :class:`float`. Optionally, it verifies that a value is between a given - minimum or maximum. - -3) :class:`String` verifies that a value is a :class:`str`. Optionally, it - validates a given minimum or maximum length. It can validate a - user-defined `predicate - `_ as well. - -.. testcode:: - - class OneOf(Validator): - - def __init__(self, *options): - self.options = set(options) - - def validate(self, value): - if value not in self.options: - raise ValueError(f'Expected {value!r} to be one of {self.options!r}') - - class Number(Validator): - - def __init__(self, minvalue=None, maxvalue=None): - self.minvalue = minvalue - self.maxvalue = maxvalue - - def validate(self, value): - if not isinstance(value, (int, float)): - raise TypeError(f'Expected {value!r} to be an int or float') - if self.minvalue is not None and value < self.minvalue: - raise ValueError( - f'Expected {value!r} to be at least {self.minvalue!r}' - ) - if self.maxvalue is not None and value > self.maxvalue: - raise ValueError( - f'Expected {value!r} to be no more than {self.maxvalue!r}' - ) - - class String(Validator): - - def __init__(self, minsize=None, maxsize=None, predicate=None): - self.minsize = minsize - self.maxsize = maxsize - self.predicate = predicate - - def validate(self, value): - if not isinstance(value, str): - raise TypeError(f'Expected {value!r} to be an str') - if self.minsize is not None and len(value) < self.minsize: - raise ValueError( - f'Expected {value!r} to be no smaller than {self.minsize!r}' - ) - if self.maxsize is not None and len(value) > self.maxsize: - raise ValueError( - f'Expected {value!r} to be no bigger than {self.maxsize!r}' - ) - if self.predicate is not None and not self.predicate(value): - raise ValueError( - f'Expected {self.predicate} to be true for {value!r}' - ) - - -Practical application ---------------------- - -Here's how the data validators can be used in a real class: - -.. testcode:: - - class Component: - - name = String(minsize=3, maxsize=10, predicate=str.isupper) - kind = OneOf('wood', 'metal', 'plastic') - quantity = Number(minvalue=0) - - def __init__(self, name, kind, quantity): - self.name = name - self.kind = kind - self.quantity = quantity - -The descriptors prevent invalid instances from being created: - -.. doctest:: - - >>> Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase - Traceback (most recent call last): - ... - ValueError: Expected to be true for 'Widget' - - >>> Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled - Traceback (most recent call last): - ... - ValueError: Expected 'metle' to be one of {'metal', 'plastic', 'wood'} - - >>> Component('WIDGET', 'metal', -5) # Blocked: -5 is negative - Traceback (most recent call last): - ... - ValueError: Expected -5 to be at least 0 - >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number - Traceback (most recent call last): - ... - TypeError: Expected 'V' to be an int or float - - >>> c = Component('WIDGET', 'metal', 5) # Allowed: The inputs are valid - - -Technical Tutorial -^^^^^^^^^^^^^^^^^^ - -What follows is a more technical tutorial for the mechanics and details of how -descriptors work. - - -Abstract --------- - -Defines descriptors, summarizes the protocol, and shows how descriptors are -called. Provides an example showing how object relational mappings work. - -Learning about descriptors not only provides access to a larger toolset, it -creates a deeper understanding of how Python works. - - -Definition and introduction ---------------------------- - -In general, a descriptor is an attribute value that has one of the methods in -the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, -and :meth:`__delete__`. If any of those methods are defined for an -attribute, it is said to be a :term:`descriptor`. - -The default behavior for attribute access is to get, set, or delete the -attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain -starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and -continuing through the method resolution order of ``type(a)``. If the -looked-up value is an object defining one of the descriptor methods, then Python -may override the default behavior and invoke the descriptor method instead. -Where this occurs in the precedence chain depends on which descriptor methods -were defined. - -Descriptors are a powerful, general purpose protocol. They are the mechanism -behind properties, methods, static methods, class methods, and -:func:`super()`. They are used throughout Python itself. Descriptors -simplify the underlying C code and offer a flexible set of new tools for -everyday Python programs. - - -Descriptor protocol -------------------- - -``descr.__get__(self, obj, type=None) -> value`` - -``descr.__set__(self, obj, value) -> None`` - -``descr.__delete__(self, obj) -> None`` - -That is all there is to it. Define any of these methods and an object is -considered a descriptor and can override default behavior upon being looked up -as an attribute. - -If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered -a data descriptor. Descriptors that only define :meth:`__get__` are called -non-data descriptors (they are often used for methods but other uses are -possible). - -Data and non-data descriptors differ in how overrides are calculated with -respect to entries in an instance's dictionary. If an instance's dictionary -has an entry with the same name as a data descriptor, the data descriptor -takes precedence. If an instance's dictionary has an entry with the same -name as a non-data descriptor, the dictionary entry takes precedence. - -To make a read-only data descriptor, define both :meth:`__get__` and -:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when -called. Defining the :meth:`__set__` method with an exception raising -placeholder is enough to make it a data descriptor. - - -Overview of descriptor invocation ---------------------------------- - -A descriptor can be called directly with ``desc.__get__(obj)`` or -``desc.__get__(None, cls)``. - -But it is more common for a descriptor to be invoked automatically from -attribute access. - -The expression ``obj.x`` looks up the attribute ``x`` in the chain of -namespaces for ``obj``. If the search finds a descriptor outside of the -instance ``__dict__``, its :meth:`__get__` method is invoked according to the -precedence rules listed below. - -The details of invocation depend on whether ``obj`` is an object, class, or -instance of super. - - -Invocation from an instance ---------------------------- - -Instance lookup scans through a chain of namespaces giving data descriptors -the highest priority, followed by instance variables, then non-data -descriptors, then class variables, and lastly :meth:`__getattr__` if it is -provided. - -If a descriptor is found for ``a.x``, then it is invoked with: -``desc.__get__(a, type(a))``. - -The logic for a dotted lookup is in :meth:`object.__getattribute__`. Here is -a pure Python equivalent: - -.. testcode:: - - def object_getattribute(obj, name): - "Emulate PyObject_GenericGetAttr() in Objects/object.c" - null = object() - objtype = type(obj) - cls_var = getattr(objtype, name, null) - descr_get = getattr(type(cls_var), '__get__', null) - if descr_get is not null: - if (hasattr(type(cls_var), '__set__') - or hasattr(type(cls_var), '__delete__')): - return descr_get(cls_var, obj, objtype) # data descriptor - if hasattr(obj, '__dict__') and name in vars(obj): - return vars(obj)[name] # instance variable - if descr_get is not null: - return descr_get(cls_var, obj, objtype) # non-data descriptor - if cls_var is not null: - return cls_var # class variable - raise AttributeError(name) - - -.. testcode:: - :hide: - - # Test the fidelity of object_getattribute() by comparing it with the - # normal object.__getattribute__(). The former will be accessed by - # square brackets and the latter by the dot operator. - - class Object: - - def __getitem__(obj, name): - try: - return object_getattribute(obj, name) - except AttributeError: - if not hasattr(type(obj), '__getattr__'): - raise - return type(obj).__getattr__(obj, name) # __getattr__ - - class DualOperator(Object): - - x = 10 - - def __init__(self, z): - self.z = z - - @property - def p2(self): - return 2 * self.x - - @property - def p3(self): - return 3 * self.x - - def m5(self, y): - return 5 * y - - def m7(self, y): - return 7 * y - - def __getattr__(self, name): - return ('getattr_hook', self, name) - - class DualOperatorWithSlots: - - __getitem__ = Object.__getitem__ - - __slots__ = ['z'] - - x = 15 - - def __init__(self, z): - self.z = z - - @property - def p2(self): - return 2 * self.x - - def m5(self, y): - return 5 * y - - def __getattr__(self, name): - return ('getattr_hook', self, name) - - -.. doctest:: - :hide: - - >>> a = DualOperator(11) - >>> vars(a).update(p3 = '_p3', m7 = '_m7') - >>> a.x == a['x'] == 10 - True - >>> a.z == a['z'] == 11 - True - >>> a.p2 == a['p2'] == 20 - True - >>> a.p3 == a['p3'] == 30 - True - >>> a.m5(100) == a.m5(100) == 500 - True - >>> a.m7 == a['m7'] == '_m7' - True - >>> a.g == a['g'] == ('getattr_hook', a, 'g') - True - - >>> b = DualOperatorWithSlots(22) - >>> b.x == b['x'] == 15 - True - >>> b.z == b['z'] == 22 - True - >>> b.p2 == b['p2'] == 30 - True - >>> b.m5(200) == b['m5'](200) == 1000 - True - >>> b.g == b['g'] == ('getattr_hook', b, 'g') - True - - -Interestingly, attribute lookup doesn't call :meth:`object.__getattribute__` -directly. Instead, both the dot operator and the :func:`getattr` function -perform attribute lookup by way of a helper function: - -.. testcode:: - - def getattr_hook(obj, name): - "Emulate slot_tp_getattr_hook() in Objects/typeobject.c" - try: - return obj.__getattribute__(name) - except AttributeError: - if not hasattr(type(obj), '__getattr__'): - raise - return type(obj).__getattr__(obj, name) # __getattr__ - -So if :meth:`__getattr__` exists, it is called whenever :meth:`__getattribute__` -raises :exc:`AttributeError` (either directly or in one of the descriptor calls). - -Also, if a user calls :meth:`object.__getattribute__` directly, the -:meth:`__getattr__` hook is bypassed entirely. - - -Invocation from a class ------------------------ - -The logic for a dotted lookup such as ``A.x`` is in -:meth:`type.__getattribute__`. The steps are similar to those for -:meth:`object.__getattribute__` but the instance dictionary lookup is replaced -by a search through the class's :term:`method resolution order`. - -If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. - -The full C implementation can be found in :c:func:`type_getattro()` and -:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`. - - -Invocation from super ---------------------- - -The logic for super's dotted lookup is in the :meth:`__getattribute__` method for -object returned by :class:`super()`. - -A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` -for the base class ``B`` immediately following ``A`` and then returns -``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned -unchanged. - -The full C implementation can be found in :c:func:`super_getattro()` in -:source:`Objects/typeobject.c`. A pure Python equivalent can be found in -`Guido's Tutorial -`_. - - -Summary of invocation logic ---------------------------- - -The mechanism for descriptors is embedded in the :meth:`__getattribute__()` -methods for :class:`object`, :class:`type`, and :func:`super`. - -The important points to remember are: - -* Descriptors are invoked by the :meth:`__getattribute__` method. - -* Classes inherit this machinery from :class:`object`, :class:`type`, or - :func:`super`. - -* Overriding :meth:`__getattribute__` prevents automatic descriptor calls - because all the descriptor logic is in that method. - -* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make - different calls to :meth:`__get__`. The first includes the instance and may - include the class. The second puts in ``None`` for the instance and always - includes the class. - -* Data descriptors always override instance dictionaries. - -* Non-data descriptors may be overridden by instance dictionaries. - - -Automatic name notification ---------------------------- - -Sometimes it is desirable for a descriptor to know what class variable name it -was assigned to. When a new class is created, the :class:`type` metaclass -scans the dictionary of the new class. If any of the entries are descriptors -and if they define :meth:`__set_name__`, that method is called with two -arguments. The *owner* is the class where the descriptor is used, and the -*name* is the class variable the descriptor was assigned to. - -The implementation details are in :c:func:`type_new()` and -:c:func:`set_names()` in :source:`Objects/typeobject.c`. - -Since the update logic is in :meth:`type.__new__`, notifications only take -place at the time of class creation. If descriptors are added to the class -afterwards, :meth:`__set_name__` will need to be called manually. - - -ORM example ------------ - -The following code is simplified skeleton showing how data descriptors could -be used to implement an `object relational mapping -`_. - -The essential idea is that the data is stored in an external database. The -Python instances only hold keys to the database's tables. Descriptors take -care of lookups or updates: - -.. testcode:: - - class Field: - - def __set_name__(self, owner, name): - self.fetch = f'SELECT {name} FROM {owner.table} WHERE {owner.key}=?;' - self.store = f'UPDATE {owner.table} SET {name}=? WHERE {owner.key}=?;' - - def __get__(self, obj, objtype=None): - return conn.execute(self.fetch, [obj.key]).fetchone()[0] - - def __set__(self, obj, value): - conn.execute(self.store, [value, obj.key]) - conn.commit() - -We can use the :class:`Field` class to define `models -`_ that describe the schema for -each table in a database: - -.. testcode:: - - class Movie: - table = 'Movies' # Table name - key = 'title' # Primary key - director = Field() - year = Field() - - def __init__(self, key): - self.key = key - - class Song: - table = 'Music' - key = 'title' - artist = Field() - year = Field() - genre = Field() - - def __init__(self, key): - self.key = key - -To use the models, first connect to the database:: - - >>> import sqlite3 - >>> conn = sqlite3.connect('entertainment.db') - -An interactive session shows how data is retrieved from the database and how -it can be updated: - -.. testsetup:: - - song_data = [ - ('Country Roads', 'John Denver', 1972), - ('Me and Bobby McGee', 'Janice Joplin', 1971), - ('Coal Miners Daughter', 'Loretta Lynn', 1970), - ] - - movie_data = [ - ('Star Wars', 'George Lucas', 1977), - ('Jaws', 'Steven Spielberg', 1975), - ('Aliens', 'James Cameron', 1986), - ] - - import sqlite3 - - conn = sqlite3.connect(':memory:') - conn.execute('CREATE TABLE Music (title text, artist text, year integer);') - conn.execute('CREATE INDEX MusicNdx ON Music (title);') - conn.executemany('INSERT INTO Music VALUES (?, ?, ?);', song_data) - conn.execute('CREATE TABLE Movies (title text, director text, year integer);') - conn.execute('CREATE INDEX MovieNdx ON Music (title);') - conn.executemany('INSERT INTO Movies VALUES (?, ?, ?);', movie_data) - conn.commit() - -.. doctest:: - - >>> Movie('Star Wars').director - 'George Lucas' - >>> jaws = Movie('Jaws') - >>> f'Released in {jaws.year} by {jaws.director}' - 'Released in 1975 by Steven Spielberg' - - >>> Song('Country Roads').artist - 'John Denver' - - >>> Movie('Star Wars').director = 'J.J. Abrams' - >>> Movie('Star Wars').director - 'J.J. Abrams' - - -Pure Python Equivalents -^^^^^^^^^^^^^^^^^^^^^^^ - -The descriptor protocol is simple and offers exciting possibilities. Several -use cases are so common that they have been prepackaged into built-in tools. -Properties, bound methods, static methods, class methods, and \_\_slots\_\_ are -all based on the descriptor protocol. - - -Properties ----------- - -Calling :func:`property` is a succinct way of building a data descriptor that -triggers a function call upon access to an attribute. Its signature is:: - - property(fget=None, fset=None, fdel=None, doc=None) -> property - -The documentation shows a typical use to define a managed attribute ``x``: - -.. testcode:: - - class C: - def getx(self): return self.__x - def setx(self, value): self.__x = value - def delx(self): del self.__x - x = property(getx, setx, delx, "I'm the 'x' property.") - -To see how :func:`property` is implemented in terms of the descriptor protocol, -here is a pure Python equivalent: - -.. testcode:: - - class Property: - "Emulate PyProperty_Type() in Objects/descrobject.c" - - def __init__(self, fget=None, fset=None, fdel=None, doc=None): - self.fget = fget - self.fset = fset - self.fdel = fdel - if doc is None and fget is not None: - doc = fget.__doc__ - self.__doc__ = doc - self._name = '' - - def __set_name__(self, owner, name): - self._name = name - - def __get__(self, obj, objtype=None): - if obj is None: - return self - if self.fget is None: - raise AttributeError(f'unreadable attribute {self._name}') - return self.fget(obj) - - def __set__(self, obj, value): - if self.fset is None: - raise AttributeError(f"can't set attribute {self._name}") - self.fset(obj, value) - - def __delete__(self, obj): - if self.fdel is None: - raise AttributeError(f"can't delete attribute {self._name}") - self.fdel(obj) - - def getter(self, fget): - prop = type(self)(fget, self.fset, self.fdel, self.__doc__) - prop._name = self._name - return prop - - def setter(self, fset): - prop = type(self)(self.fget, fset, self.fdel, self.__doc__) - prop._name = self._name - return prop - - def deleter(self, fdel): - prop = type(self)(self.fget, self.fset, fdel, self.__doc__) - prop._name = self._name - return prop - -.. testcode:: - :hide: - - # Verify the Property() emulation - - class CC: - def getx(self): - return self.__x - def setx(self, value): - self.__x = value - def delx(self): - del self.__x - x = Property(getx, setx, delx, "I'm the 'x' property.") - - # Now do it again but use the decorator style - - class CCC: - @Property - def x(self): - return self.__x - @x.setter - def x(self, value): - self.__x = value - @x.deleter - def x(self): - del self.__x - - -.. doctest:: - :hide: - - >>> cc = CC() - >>> hasattr(cc, 'x') - False - >>> cc.x = 33 - >>> cc.x - 33 - >>> del cc.x - >>> hasattr(cc, 'x') - False - - >>> ccc = CCC() - >>> hasattr(ccc, 'x') - False - >>> ccc.x = 333 - >>> ccc.x == 333 - True - >>> del ccc.x - >>> hasattr(ccc, 'x') - False - -The :func:`property` builtin helps whenever a user interface has granted -attribute access and then subsequent changes require the intervention of a -method. - -For instance, a spreadsheet class may grant access to a cell value through -``Cell('b10').value``. Subsequent improvements to the program require the cell -to be recalculated on every access; however, the programmer does not want to -affect existing client code accessing the attribute directly. The solution is -to wrap access to the value attribute in a property data descriptor: - -.. testcode:: - - class Cell: - ... - - @property - def value(self): - "Recalculate the cell before returning value" - self.recalc() - return self._value - -Either the built-in :func:`property` or our :func:`Property` equivalent would -work in this example. - - -Functions and methods ---------------------- - -Python's object oriented features are built upon a function based environment. -Using non-data descriptors, the two are merged seamlessly. - -Functions stored in class dictionaries get turned into methods when invoked. -Methods only differ from regular functions in that the object instance is -prepended to the other arguments. By convention, the instance is called -*self* but could be called *this* or any other variable name. - -Methods can be created manually with :class:`types.MethodType` which is -roughly equivalent to: - -.. testcode:: - - class MethodType: - "Emulate Py_MethodType in Objects/classobject.c" - - def __init__(self, func, obj): - self.__func__ = func - self.__self__ = obj - - def __call__(self, *args, **kwargs): - func = self.__func__ - obj = self.__self__ - return func(obj, *args, **kwargs) - -To support automatic creation of methods, functions include the -:meth:`__get__` method for binding methods during attribute access. This -means that functions are non-data descriptors that return bound methods -during dotted lookup from an instance. Here's how it works: - -.. testcode:: - - class Function: - ... - - def __get__(self, obj, objtype=None): - "Simulate func_descr_get() in Objects/funcobject.c" - if obj is None: - return self - return MethodType(self, obj) - -Running the following class in the interpreter shows how the function -descriptor works in practice: - -.. testcode:: - - class D: - def f(self, x): - return x - -The function has a :term:`qualified name` attribute to support introspection: - -.. doctest:: - - >>> D.f.__qualname__ - 'D.f' - -Accessing the function through the class dictionary does not invoke -:meth:`__get__`. Instead, it just returns the underlying function object:: - - >>> D.__dict__['f'] - - -Dotted access from a class calls :meth:`__get__` which just returns the -underlying function unchanged:: - - >>> D.f - - -The interesting behavior occurs during dotted access from an instance. The -dotted lookup calls :meth:`__get__` which returns a bound method object:: - - >>> d = D() - >>> d.f - > - -Internally, the bound method stores the underlying function and the bound -instance:: - - >>> d.f.__func__ - - - >>> d.f.__self__ - <__main__.D object at 0x1012e1f98> - -If you have ever wondered where *self* comes from in regular methods or where -*cls* comes from in class methods, this is it! - - -Static methods --------------- - -Non-data descriptors provide a simple mechanism for variations on the usual -patterns of binding functions into methods. - -To recap, functions have a :meth:`__get__` method so that they can be converted -to a method when accessed as attributes. The non-data descriptor transforms an -``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` -becomes ``f(*args)``. - -This chart summarizes the binding and its two most useful variants: - - +-----------------+----------------------+------------------+ - | Transformation | Called from an | Called from a | - | | object | class | - +=================+======================+==================+ - | function | f(obj, \*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | staticmethod | f(\*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | classmethod | f(type(obj), \*args) | f(cls, \*args) | - +-----------------+----------------------+------------------+ - -Static methods return the underlying function without changes. Calling either -``c.f`` or ``C.f`` is the equivalent of a direct lookup into -``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a -result, the function becomes identically accessible from either an object or a -class. - -Good candidates for static methods are methods that do not reference the -``self`` variable. - -For instance, a statistics package may include a container class for -experimental data. The class provides normal methods for computing the average, -mean, median, and other descriptive statistics that depend on the data. However, -there may be useful functions which are conceptually related but do not depend -on the data. For instance, ``erf(x)`` is handy conversion routine that comes up -in statistical work but does not directly depend on a particular dataset. -It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or -``Sample.erf(1.5) --> .9332``. - -Since static methods return the underlying function with no changes, the -example calls are unexciting: - -.. testcode:: - - class E: - @staticmethod - def f(x): - print(x) - -.. doctest:: - - >>> E.f(3) - 3 - >>> E().f(3) - 3 - -Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this: - -.. doctest:: - - class StaticMethod: - "Emulate PyStaticMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - - def __get__(self, obj, objtype=None): - return self.f - - -Class methods -------------- - -Unlike static methods, class methods prepend the class reference to the -argument list before calling the function. This format is the same -for whether the caller is an object or a class: - -.. testcode:: - - class F: - @classmethod - def f(cls, x): - return cls.__name__, x - -.. doctest:: - - >>> F.f(3) - ('F', 3) - >>> F().f(3) - ('F', 3) - -This behavior is useful whenever the method only needs to have a class -reference and does not rely on data stored in a specific instance. One use for -class methods is to create alternate class constructors. For example, the -classmethod :func:`dict.fromkeys` creates a new dictionary from a list of -keys. The pure Python equivalent is: - -.. testcode:: - - class Dict(dict): - @classmethod - def fromkeys(cls, iterable, value=None): - "Emulate dict_fromkeys() in Objects/dictobject.c" - d = cls() - for key in iterable: - d[key] = value - return d - -Now a new dictionary of unique keys can be constructed like this: - -.. doctest:: - - >>> d = Dict.fromkeys('abracadabra') - >>> type(d) is Dict - True - >>> d - {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} - -Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this: - -.. testcode:: - - class ClassMethod: - "Emulate PyClassMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - - def __get__(self, obj, cls=None): - if cls is None: - cls = type(obj) - if hasattr(obj, '__get__'): - return self.f.__get__(cls) - return MethodType(self.f, cls) - -.. testcode:: - :hide: - - # Verify the emulation works - class T: - @ClassMethod - def cm(cls, x, y): - return (cls, x, y) - -.. doctest:: - :hide: - - >>> T.cm(11, 22) - (, 11, 22) - - # Also call it from an instance - >>> t = T() - >>> t.cm(11, 22) - (, 11, 22) - -The code path for ``hasattr(obj, '__get__')`` was added in Python 3.9 and -makes it possible for :func:`classmethod` to support chained decorators. -For example, a classmethod and property could be chained together: - -.. testcode:: - - class G: - @classmethod - @property - def __doc__(cls): - return f'A doc for {cls.__name__!r}' - -.. doctest:: - - >>> G.__doc__ - "A doc for 'G'" - - -Member objects and __slots__ ----------------------------- - -When a class defines ``__slots__``, it replaces instance dictionaries with a -fixed-length array of slot values. From a user point of view that has -several effects: - -1. Provides immediate detection of bugs due to misspelled attribute -assignments. Only attribute names specified in ``__slots__`` are allowed: - -.. testcode:: - - class Vehicle: - __slots__ = ('id_number', 'make', 'model') - -.. doctest:: - - >>> auto = Vehicle() - >>> auto.id_nubmer = 'VYE483814LQEX' - Traceback (most recent call last): - ... - AttributeError: 'Vehicle' object has no attribute 'id_nubmer' - -2. Helps create immutable objects where descriptors manage access to private -attributes stored in ``__slots__``: - -.. testcode:: - - class Immutable: - - __slots__ = ('_dept', '_name') # Replace the instance dictionary - - def __init__(self, dept, name): - self._dept = dept # Store to private attribute - self._name = name # Store to private attribute - - @property # Read-only descriptor - def dept(self): - return self._dept - - @property - def name(self): # Read-only descriptor - return self._name - -.. doctest:: - - >>> mark = Immutable('Botany', 'Mark Watney') - >>> mark.dept - 'Botany' - >>> mark.dept = 'Space Pirate' - Traceback (most recent call last): - ... - AttributeError: can't set attribute - >>> mark.location = 'Mars' - Traceback (most recent call last): - ... - AttributeError: 'Immutable' object has no attribute 'location' - -3. Saves memory. On a 64-bit Linux build, an instance with two attributes -takes 48 bytes with ``__slots__`` and 152 bytes without. This `flyweight -design pattern `_ likely only -matters when a large number of instances are going to be created. - -4. Improves speed. Reading instance variables is 35% faster with -``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). - -5. Blocks tools like :func:`functools.cached_property` which require an -instance dictionary to function correctly: - -.. testcode:: - - from functools import cached_property - - class CP: - __slots__ = () # Eliminates the instance dict - - @cached_property # Requires an instance dict - def pi(self): - return 4 * sum((-1.0)**n / (2.0*n + 1.0) - for n in reversed(range(100_000))) - -.. doctest:: - - >>> CP().pi - Traceback (most recent call last): - ... - TypeError: No '__dict__' attribute on 'CP' instance to cache 'pi' property. - -It is not possible to create an exact drop-in pure Python version of -``__slots__`` because it requires direct access to C structures and control -over object memory allocation. However, we can build a mostly faithful -simulation where the actual C structure for slots is emulated by a private -``_slotvalues`` list. Reads and writes to that private structure are managed -by member descriptors: - -.. testcode:: - - null = object() - - class Member: - - def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' - # Also see descr_new() in Objects/descrobject.c - self.name = name - self.clsname = clsname - self.offset = offset - - def __get__(self, obj, objtype=None): - 'Emulate member_get() in Objects/descrobject.c' - # Also see PyMember_GetOne() in Python/structmember.c - value = obj._slotvalues[self.offset] - if value is null: - raise AttributeError(self.name) - return value - - def __set__(self, obj, value): - 'Emulate member_set() in Objects/descrobject.c' - obj._slotvalues[self.offset] = value - - def __delete__(self, obj): - 'Emulate member_delete() in Objects/descrobject.c' - value = obj._slotvalues[self.offset] - if value is null: - raise AttributeError(self.name) - obj._slotvalues[self.offset] = null - - def __repr__(self): - 'Emulate member_repr() in Objects/descrobject.c' - return f'' - -The :meth:`type.__new__` method takes care of adding member objects to class -variables: - -.. testcode:: - - class Type(type): - 'Simulate how the type metaclass adds member objects for slots' - - def __new__(mcls, clsname, bases, mapping): - 'Emuluate type_new() in Objects/typeobject.c' - # type_new() calls PyTypeReady() which calls add_methods() - slot_names = mapping.get('slot_names', []) - for offset, name in enumerate(slot_names): - mapping[name] = Member(name, clsname, offset) - return type.__new__(mcls, clsname, bases, mapping) - -The :meth:`object.__new__` method takes care of creating instances that have -slots instead of an instance dictionary. Here is a rough simulation in pure -Python: - -.. testcode:: - - class Object: - 'Simulate how object.__new__() allocates memory for __slots__' - - def __new__(cls, *args): - 'Emulate object_new() in Objects/typeobject.c' - inst = super().__new__(cls) - if hasattr(cls, 'slot_names'): - empty_slots = [null] * len(cls.slot_names) - object.__setattr__(inst, '_slotvalues', empty_slots) - return inst - - def __setattr__(self, name, value): - 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' - cls = type(self) - if hasattr(cls, 'slot_names') and name not in cls.slot_names: - raise AttributeError( - f'{type(self).__name__!r} object has no attribute {name!r}' - ) - super().__setattr__(name, value) - - def __delattr__(self, name): - 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' - cls = type(self) - if hasattr(cls, 'slot_names') and name not in cls.slot_names: - raise AttributeError( - f'{type(self).__name__!r} object has no attribute {name!r}' - ) - super().__delattr__(name) - -To use the simulation in a real class, just inherit from :class:`Object` and -set the :term:`metaclass` to :class:`Type`: - -.. testcode:: - - class H(Object, metaclass=Type): - 'Instance variables stored in slots' - - slot_names = ['x', 'y'] - - def __init__(self, x, y): - self.x = x - self.y = y - -At this point, the metaclass has loaded member objects for *x* and *y*:: - - >>> from pprint import pp - >>> pp(dict(vars(H))) - {'__module__': '__main__', - '__doc__': 'Instance variables stored in slots', - 'slot_names': ['x', 'y'], - '__init__': , - 'x': , - 'y': } - -.. doctest:: - :hide: - - # We test this separately because the preceding section is not - # doctestable due to the hex memory address for the __init__ function - >>> isinstance(vars(H)['x'], Member) - True - >>> isinstance(vars(H)['y'], Member) - True - -When instances are created, they have a ``slot_values`` list where the -attributes are stored: - -.. doctest:: - - >>> h = H(10, 20) - >>> vars(h) - {'_slotvalues': [10, 20]} - >>> h.x = 55 - >>> vars(h) - {'_slotvalues': [55, 20]} - -Misspelled or unassigned attributes will raise an exception: - -.. doctest:: - - >>> h.xz - Traceback (most recent call last): - ... - AttributeError: 'H' object has no attribute 'xz' - -.. doctest:: - :hide: - - # Examples for deleted attributes are not shown because this section - # is already a bit lengthy. We still test that code here. - >>> del h.x - >>> hasattr(h, 'x') - False - - # Also test the code for uninitialized slots - >>> class HU(Object, metaclass=Type): - ... slot_names = ['x', 'y'] - ... - >>> hu = HU() - >>> hasattr(hu, 'x') - False - >>> hasattr(hu, 'y') - False diff --git a/Doc/library/_thread.rst.bak b/Doc/library/_thread.rst.bak deleted file mode 100644 index bd653ab32bb9c4..00000000000000 --- a/Doc/library/_thread.rst.bak +++ /dev/null @@ -1,215 +0,0 @@ -:mod:`_thread` --- Low-level threading API -========================================== - -.. module:: _thread - :synopsis: Low-level threading API. - -.. index:: - single: light-weight processes - single: processes, light-weight - single: binary semaphores - single: semaphores, binary - --------------- - -This module provides low-level primitives for working with multiple threads -(also called :dfn:`light-weight processes` or :dfn:`tasks`) --- multiple threads of -control sharing their global data space. For synchronization, simple locks -(also called :dfn:`mutexes` or :dfn:`binary semaphores`) are provided. -The :mod:`threading` module provides an easier to use and higher-level -threading API built on top of this module. - -.. index:: - single: pthreads - pair: threads; POSIX - -.. versionchanged:: 3.7 - This module used to be optional, it is now always available. - -This module defines the following constants and functions: - -.. exception:: error - - Raised on thread-specific errors. - - .. versionchanged:: 3.3 - This is now a synonym of the built-in :exc:`RuntimeError`. - - -.. data:: LockType - - This is the type of lock objects. - - -.. function:: start_new_thread(function, args[, kwargs]) - - Start a new thread and return its identifier. The thread executes the - function *function* with the argument list *args* (which must be a tuple). - The optional *kwargs* argument specifies a dictionary of keyword arguments. - - When the function returns, the thread silently exits. - - When the function terminates with an unhandled exception, - :func:`sys.unraisablehook` is called to handle the exception. The *object* - attribute of the hook argument is *function*. By default, a stack trace is - printed and then the thread exits (but other threads continue to run). - - When the function raises a :exc:`SystemExit` exception, it is silently - ignored. - - .. versionchanged:: 3.8 - :func:`sys.unraisablehook` is now used to handle unhandled exceptions. - - -.. function:: interrupt_main() - - Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main - thread. A thread can use this function to interrupt the main thread. - - If :data:`signal.SIGINT` isn't handled by Python (it was set to - :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does - nothing. - - -.. function:: exit() - - Raise the :exc:`SystemExit` exception. When not caught, this will cause the - thread to exit silently. - -.. - function:: exit_prog(status) - - Exit all threads and report the value of the integer argument - *status* as the exit status of the entire program. - **Caveat:** code in pending :keyword:`finally` clauses, in this thread - or in other threads, is not executed. - - -.. function:: allocate_lock() - - Return a new lock object. Methods of locks are described below. The lock is - initially unlocked. - - -.. function:: get_ident() - - Return the 'thread identifier' of the current thread. This is a nonzero - integer. Its value has no direct meaning; it is intended as a magic cookie to - be used e.g. to index a dictionary of thread-specific data. Thread identifiers - may be recycled when a thread exits and another thread is created. - - -.. function:: get_native_id() - - Return the native integral Thread ID of the current thread assigned by the kernel. - This is a non-negative integer. - Its value may be used to uniquely identify this particular thread system-wide - (until the thread terminates, after which the value may be recycled by the OS). - - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. - - .. versionadded:: 3.8 - - -.. function:: stack_size([size]) - - Return the thread stack size used when creating new threads. The optional - *size* argument specifies the stack size to be used for subsequently created - threads, and must be 0 (use platform or configured default) or a positive - integer value of at least 32,768 (32 KiB). If *size* is not specified, - 0 is used. If changing the thread stack size is - unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is - invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB - is currently the minimum supported stack size value to guarantee sufficient - stack space for the interpreter itself. Note that some platforms may have - particular restrictions on values for the stack size, such as requiring a - minimum stack size > 32 KiB or requiring allocation in multiples of the system - memory page size - platform documentation should be referred to for more - information (4 KiB pages are common; using multiples of 4096 for the stack size is - the suggested approach in the absence of more specific information). - - .. availability:: Windows, systems with POSIX threads. - - -.. data:: TIMEOUT_MAX - - The maximum value allowed for the *timeout* parameter of - :meth:`Lock.acquire`. Specifying a timeout greater than this value will - raise an :exc:`OverflowError`. - - .. versionadded:: 3.2 - - -Lock objects have the following methods: - - -.. method:: lock.acquire(waitflag=1, timeout=-1) - - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). - - If the integer *waitflag* argument is present, the action depends on its - value: if it is zero, the lock is only acquired if it can be acquired - immediately without waiting, while if it is nonzero, the lock is acquired - unconditionally as above. - - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *waitflag* is zero. - - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. - - -.. method:: lock.release() - - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. - - -.. method:: lock.locked() - - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. - -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: - - import _thread - - a_lock = _thread.allocate_lock() - - with a_lock: - print("a_lock is locked while this executes") - -**Caveats:** - - .. index:: module: signal - -* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` - exception will be received by an arbitrary thread. (When the :mod:`signal` - module is available, interrupts always go to the main thread.) - -* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is - equivalent to calling :func:`_thread.exit`. - -* It is not possible to interrupt the :meth:`acquire` method on a lock --- the - :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. - -* When the main thread exits, it is system defined whether the other threads - survive. On most systems, they are killed without executing - :keyword:`try` ... :keyword:`finally` clauses or executing object - destructors. - -* When the main thread exits, it does not do any of its usual cleanup (except - that :keyword:`try` ... :keyword:`finally` clauses are honored), and the - standard I/O files are not flushed. - diff --git a/Doc/library/ast.rst.bak b/Doc/library/ast.rst.bak deleted file mode 100644 index b5e1be82691157..00000000000000 --- a/Doc/library/ast.rst.bak +++ /dev/null @@ -1,1993 +0,0 @@ -:mod:`ast` --- Abstract Syntax Trees -==================================== - -.. module:: ast - :synopsis: Abstract Syntax Tree classes and manipulation. - -.. sectionauthor:: Martin v. Löwis -.. sectionauthor:: Georg Brandl - -.. testsetup:: - - import ast - -**Source code:** :source:`Lib/ast.py` - --------------- - -The :mod:`ast` module helps Python applications to process trees of the Python -abstract syntax grammar. The abstract syntax itself might change with each -Python release; this module helps to find out programmatically what the current -grammar looks like. - -An abstract syntax tree can be generated by passing :data:`ast.PyCF_ONLY_AST` as -a flag to the :func:`compile` built-in function, or using the :func:`parse` -helper provided in this module. The result will be a tree of objects whose -classes all inherit from :class:`ast.AST`. An abstract syntax tree can be -compiled into a Python code object using the built-in :func:`compile` function. - - -.. _abstract-grammar: - -Abstract Grammar ----------------- - -The abstract grammar is currently defined as follows: - -.. literalinclude:: ../../Parser/Python.asdl - :language: asdl - - -Node classes ------------- - -.. class:: AST - - This is the base of all AST node classes. The actual node classes are - derived from the :file:`Parser/Python.asdl` file, which is reproduced - :ref:`below `. They are defined in the :mod:`_ast` C - module and re-exported in :mod:`ast`. - - There is one class defined for each left-hand side symbol in the abstract - grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, - there is one class defined for each constructor on the right-hand side; these - classes inherit from the classes for the left-hand side trees. For example, - :class:`ast.BinOp` inherits from :class:`ast.expr`. For production rules - with alternatives (aka "sums"), the left-hand side class is abstract: only - instances of specific constructor nodes are ever created. - - .. index:: single: ? (question mark); in AST grammar - .. index:: single: * (asterisk); in AST grammar - - .. attribute:: _fields - - Each concrete class has an attribute :attr:`_fields` which gives the names - of all child nodes. - - Each instance of a concrete class has one attribute for each child node, - of the type as defined in the grammar. For example, :class:`ast.BinOp` - instances have an attribute :attr:`left` of type :class:`ast.expr`. - - If these attributes are marked as optional in the grammar (using a - question mark), the value might be ``None``. If the attributes can have - zero-or-more values (marked with an asterisk), the values are represented - as Python lists. All possible attributes must be present and have valid - values when compiling an AST with :func:`compile`. - - .. attribute:: lineno - col_offset - end_lineno - end_col_offset - - Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have - :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and - :attr:`end_col_offset` attributes. The :attr:`lineno` and :attr:`end_lineno` - are the first and last line numbers of source text span (1-indexed so the - first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset` - are the corresponding UTF-8 byte offsets of the first and last tokens that - generated the node. The UTF-8 offset is recorded because the parser uses - UTF-8 internally. - - Note that the end positions are not required by the compiler and are - therefore optional. The end offset is *after* the last symbol, for example - one can get the source segment of a one-line expression node using - ``source_line[node.col_offset : node.end_col_offset]``. - - The constructor of a class :class:`ast.T` parses its arguments as follows: - - * If there are positional arguments, there must be as many as there are items - in :attr:`T._fields`; they will be assigned as attributes of these names. - * If there are keyword arguments, they will set the attributes of the same - names to the given values. - - For example, to create and populate an :class:`ast.UnaryOp` node, you could - use :: - - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Constant() - node.operand.value = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - - node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), - lineno=0, col_offset=0) - -.. versionchanged:: 3.8 - - Class :class:`ast.Constant` is now used for all constants. - -.. versionchanged:: 3.9 - - Simple indices are represented by their value, extended slices are - represented as tuples. - -.. deprecated:: 3.8 - - Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, - :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meantime, - instantiating them will return an instance of a different class. - -.. deprecated:: 3.9 - - Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still - available, but they will be removed in future Python releases. - In the meantime, instantiating them will return an instance of - a different class. - -.. note:: - The descriptions of the specific node classes displayed here - were initially adapted from the fantastic `Green Tree - Snakes `__ project and - all its contributors. - -Literals -^^^^^^^^ - -.. class:: Constant(value) - - A constant value. The ``value`` attribute of the ``Constant`` literal contains the - Python object it represents. The values represented can be simple types - such as a number, string or ``None``, but also immutable container types - (tuples and frozensets) if all of their elements are constant. - - .. doctest:: - - >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) - Expression( - body=Constant(value=123)) - - -.. class:: FormattedValue(value, conversion, format_spec) - - Node representing a single formatting field in an f-string. If the string - contains a single formatting field and nothing else the node can be - isolated otherwise it appears in :class:`JoinedStr`. - - * ``value`` is any expression node (such as a literal, a variable, or a - function call). - * ``conversion`` is an integer: - - * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting - - * ``format_spec`` is a :class:`JoinedStr` node representing the formatting - of the value, or ``None`` if no format was specified. Both - ``conversion`` and ``format_spec`` can be set at the same time. - - -.. class:: JoinedStr(values) - - An f-string, comprising a series of :class:`FormattedValue` and :class:`Constant` - nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('f"sin({a}) is {sin(a):.3}"', mode='eval'), indent=4)) - Expression( - body=JoinedStr( - values=[ - Constant(value='sin('), - FormattedValue( - value=Name(id='a', ctx=Load()), - conversion=-1), - Constant(value=') is '), - FormattedValue( - value=Call( - func=Name(id='sin', ctx=Load()), - args=[ - Name(id='a', ctx=Load())], - keywords=[]), - conversion=-1, - format_spec=JoinedStr( - values=[ - Constant(value='.3')]))])) - - -.. class:: List(elts, ctx) - Tuple(elts, ctx) - - A list or tuple. ``elts`` holds a list of nodes representing the elements. - ``ctx`` is :class:`Store` if the container is an assignment target (i.e. - ``(x,y)=something``), and :class:`Load` otherwise. - - .. doctest:: - - >>> print(ast.dump(ast.parse('[1, 2, 3]', mode='eval'), indent=4)) - Expression( - body=List( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)], - ctx=Load())) - >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) - Expression( - body=Tuple( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)], - ctx=Load())) - - -.. class:: Set(elts) - - A set. ``elts`` holds a list of nodes representing the set's elements. - - .. doctest:: - - >>> print(ast.dump(ast.parse('{1, 2, 3}', mode='eval'), indent=4)) - Expression( - body=Set( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)])) - - -.. class:: Dict(keys, values) - - A dictionary. ``keys`` and ``values`` hold lists of nodes representing the - keys and the values respectively, in matching order (what would be returned - when calling :code:`dictionary.keys()` and :code:`dictionary.values()`). - - When doing dictionary unpacking using dictionary literals the expression to be - expanded goes in the ``values`` list, with a ``None`` at the corresponding - position in ``keys``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('{"a":1, **d}', mode='eval'), indent=4)) - Expression( - body=Dict( - keys=[ - Constant(value='a'), - None], - values=[ - Constant(value=1), - Name(id='d', ctx=Load())])) - - -Variables -^^^^^^^^^ - -.. class:: Name(id, ctx) - - A variable name. ``id`` holds the name as a string, and ``ctx`` is one of - the following types. - - -.. class:: Load() - Store() - Del() - - Variable references can be used to load the value of a variable, to assign - a new value to it, or to delete it. Variable references are given a context - to distinguish these cases. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a'), indent=4)) - Module( - body=[ - Expr( - value=Name(id='a', ctx=Load()))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a = 1'), indent=4)) - Module( - body=[ - Assign( - targets=[ - Name(id='a', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('del a'), indent=4)) - Module( - body=[ - Delete( - targets=[ - Name(id='a', ctx=Del())])], - type_ignores=[]) - - -.. class:: Starred(value, ctx) - - A ``*var`` variable reference. ``value`` holds the variable, typically a - :class:`Name` node. This type must be used when building a :class:`Call` - node with ``*args``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a, *b = it'), indent=4)) - Module( - body=[ - Assign( - targets=[ - Tuple( - elts=[ - Name(id='a', ctx=Store()), - Starred( - value=Name(id='b', ctx=Store()), - ctx=Store())], - ctx=Store())], - value=Name(id='it', ctx=Load()))], - type_ignores=[]) - - -Expressions -^^^^^^^^^^^ - -.. class:: Expr(value) - - When an expression, such as a function call, appears as a statement by itself - with its return value not used or stored, it is wrapped in this container. - ``value`` holds one of the other nodes in this section, a :class:`Constant`, a - :class:`Name`, a :class:`Lambda`, a :class:`Yield` or :class:`YieldFrom` node. - - .. doctest:: - - >>> print(ast.dump(ast.parse('-a'), indent=4)) - Module( - body=[ - Expr( - value=UnaryOp( - op=USub(), - operand=Name(id='a', ctx=Load())))], - type_ignores=[]) - - -.. class:: UnaryOp(op, operand) - - A unary operation. ``op`` is the operator, and ``operand`` any expression - node. - - -.. class:: UAdd - USub - Not - Invert - - Unary operator tokens. :class:`Not` is the ``not`` keyword, :class:`Invert` - is the ``~`` operator. - - .. doctest:: - - >>> print(ast.dump(ast.parse('not x', mode='eval'), indent=4)) - Expression( - body=UnaryOp( - op=Not(), - operand=Name(id='x', ctx=Load()))) - - -.. class:: BinOp(left, op, right) - - A binary operation (like addition or division). ``op`` is the operator, and - ``left`` and ``right`` are any expression nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) - Expression( - body=BinOp( - left=Name(id='x', ctx=Load()), - op=Add(), - right=Name(id='y', ctx=Load()))) - - -.. class:: Add - Sub - Mult - Div - FloorDiv - Mod - Pow - LShift - RShift - BitOr - BitXor - BitAnd - MatMult - - Binary operator tokens. - - -.. class:: BoolOp(op, values) - - A boolean operation, 'or' or 'and'. ``op`` is :class:`Or` or :class:`And`. - ``values`` are the values involved. Consecutive operations with the same - operator, such as ``a or b or c``, are collapsed into one node with several - values. - - This doesn't include ``not``, which is a :class:`UnaryOp`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x or y', mode='eval'), indent=4)) - Expression( - body=BoolOp( - op=Or(), - values=[ - Name(id='x', ctx=Load()), - Name(id='y', ctx=Load())])) - - -.. class:: And - Or - - Boolean operator tokens. - - -.. class:: Compare(left, ops, comparators) - - A comparison of two or more values. ``left`` is the first value in the - comparison, ``ops`` the list of operators, and ``comparators`` the list - of values after the first element in the comparison. - - .. doctest:: - - >>> print(ast.dump(ast.parse('1 <= a < 10', mode='eval'), indent=4)) - Expression( - body=Compare( - left=Constant(value=1), - ops=[ - LtE(), - Lt()], - comparators=[ - Name(id='a', ctx=Load()), - Constant(value=10)])) - - -.. class:: Eq - NotEq - Lt - LtE - Gt - GtE - Is - IsNot - In - NotIn - - Comparison operator tokens. - - -.. class:: Call(func, args, keywords, starargs, kwargs) - - A function call. ``func`` is the function, which will often be a - :class:`Name` or :class:`Attribute` object. Of the arguments: - - * ``args`` holds a list of the arguments passed by position. - * ``keywords`` holds a list of :class:`keyword` objects representing - arguments passed by keyword. - - When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. ``starargs`` and ``kwargs`` are optional. - - .. doctest:: - - >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) - Expression( - body=Call( - func=Name(id='func', ctx=Load()), - args=[ - Name(id='a', ctx=Load()), - Starred( - value=Name(id='d', ctx=Load()), - ctx=Load())], - keywords=[ - keyword( - arg='b', - value=Name(id='c', ctx=Load())), - keyword( - value=Name(id='e', ctx=Load()))])) - - -.. class:: keyword(arg, value) - - A keyword argument to a function call or class definition. ``arg`` is a raw - string of the parameter name, ``value`` is a node to pass in. - - -.. class:: IfExp(test, body, orelse) - - An expression such as ``a if b else c``. Each field holds a single node, so - in the following example, all three are :class:`Name` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) - Expression( - body=IfExp( - test=Name(id='b', ctx=Load()), - body=Name(id='a', ctx=Load()), - orelse=Name(id='c', ctx=Load()))) - - -.. class:: Attribute(value, attr, ctx) - - Attribute access, e.g. ``d.keys``. ``value`` is a node, typically a - :class:`Name`. ``attr`` is a bare string giving the name of the attribute, - and ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` according to how - the attribute is acted on. - - .. doctest:: - - >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) - Expression( - body=Attribute( - value=Name(id='snake', ctx=Load()), - attr='colour', - ctx=Load())) - - -.. class:: NamedExpr(target, value) - - A named expression. This AST node is produced by the assignment expressions - operator (also known as the walrus operator). As opposed to the :class:`Assign` - node in which the first argument can be multiple nodes, in this case both - ``target`` and ``value`` must be single nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('(x := 4)', mode='eval'), indent=4)) - Expression( - body=NamedExpr( - target=Name(id='x', ctx=Store()), - value=Constant(value=4))) - - -Subscripting -~~~~~~~~~~~~ - -.. class:: Subscript(value, slice, ctx) - - A subscript, such as ``l[1]``. ``value`` is the subscripted object - (usually sequence or mapping). ``slice`` is an index, slice or key. - It can be a :class:`Tuple` and contain a :class:`Slice`. - ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` - according to the action performed with the subscript. - - .. doctest:: - - >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) - Expression( - body=Subscript( - value=Name(id='l', ctx=Load()), - slice=Tuple( - elts=[ - Slice( - lower=Constant(value=1), - upper=Constant(value=2)), - Constant(value=3)], - ctx=Load()), - ctx=Load())) - - -.. class:: Slice(lower, upper, step) - - Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``). - Can occur only inside the *slice* field of :class:`Subscript`, either - directly or as an element of :class:`Tuple`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) - Expression( - body=Subscript( - value=Name(id='l', ctx=Load()), - slice=Slice( - lower=Constant(value=1), - upper=Constant(value=2)), - ctx=Load())) - - -Comprehensions -~~~~~~~~~~~~~~ - -.. class:: ListComp(elt, generators) - SetComp(elt, generators) - GeneratorExp(elt, generators) - DictComp(key, value, generators) - - List and set comprehensions, generator expressions, and dictionary - comprehensions. ``elt`` (or ``key`` and ``value``) is a single node - representing the part that will be evaluated for each item. - - ``generators`` is a list of :class:`comprehension` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) - Expression( - body=ListComp( - elt=Name(id='x', ctx=Load()), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) - Expression( - body=DictComp( - key=Name(id='x', ctx=Load()), - value=BinOp( - left=Name(id='x', ctx=Load()), - op=Pow(), - right=Constant(value=2)), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) - Expression( - body=SetComp( - elt=Name(id='x', ctx=Load()), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - - -.. class:: comprehension(target, iter, ifs, is_async) - - One ``for`` clause in a comprehension. ``target`` is the reference to use for - each element - typically a :class:`Name` or :class:`Tuple` node. ``iter`` - is the object to iterate over. ``ifs`` is a list of test expressions: each - ``for`` clause can have multiple ``ifs``. - - ``is_async`` indicates a comprehension is asynchronous (using an - ``async for`` instead of ``for``). The value is an integer (0 or 1). - - .. doctest:: - - >>> print(ast.dump(ast.parse('[ord(c) for line in file for c in line]', mode='eval'), - ... indent=4)) # Multiple comprehensions in one. - Expression( - body=ListComp( - elt=Call( - func=Name(id='ord', ctx=Load()), - args=[ - Name(id='c', ctx=Load())], - keywords=[]), - generators=[ - comprehension( - target=Name(id='line', ctx=Store()), - iter=Name(id='file', ctx=Load()), - ifs=[], - is_async=0), - comprehension( - target=Name(id='c', ctx=Store()), - iter=Name(id='line', ctx=Load()), - ifs=[], - is_async=0)])) - - >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), - ... indent=4)) # generator comprehension - Expression( - body=GeneratorExp( - elt=BinOp( - left=Name(id='n', ctx=Load()), - op=Pow(), - right=Constant(value=2)), - generators=[ - comprehension( - target=Name(id='n', ctx=Store()), - iter=Name(id='it', ctx=Load()), - ifs=[ - Compare( - left=Name(id='n', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=5)]), - Compare( - left=Name(id='n', ctx=Load()), - ops=[ - Lt()], - comparators=[ - Constant(value=10)])], - is_async=0)])) - - >>> print(ast.dump(ast.parse('[i async for i in soc]', mode='eval'), - ... indent=4)) # Async comprehension - Expression( - body=ListComp( - elt=Name(id='i', ctx=Load()), - generators=[ - comprehension( - target=Name(id='i', ctx=Store()), - iter=Name(id='soc', ctx=Load()), - ifs=[], - is_async=1)])) - -Statements -^^^^^^^^^^ - -.. class:: Assign(targets, value, type_comment) - - An assignment. ``targets`` is a list of nodes, and ``value`` is a single node. - - Multiple nodes in ``targets`` represents assigning the same value to each. - Unpacking is represented by putting a :class:`Tuple` or :class:`List` - within ``targets``. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a = b = 1'), indent=4)) # Multiple assignment - Module( - body=[ - Assign( - targets=[ - Name(id='a', ctx=Store()), - Name(id='b', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking - Module( - body=[ - Assign( - targets=[ - Tuple( - elts=[ - Name(id='a', ctx=Store()), - Name(id='b', ctx=Store())], - ctx=Store())], - value=Name(id='c', ctx=Load()))], - type_ignores=[]) - - -.. class:: AnnAssign(target, annotation, value, simple) - - An assignment with a type annotation. ``target`` is a single node and can - be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. - ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` - node. ``value`` is a single optional node. ``simple`` is a boolean integer - set to True for a :class:`Name` node in ``target`` that do not appear in - between parenthesis and are hence pure names and not expressions. - - .. doctest:: - - >>> print(ast.dump(ast.parse('c: int'), indent=4)) - Module( - body=[ - AnnAssign( - target=Name(id='c', ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=1)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis - Module( - body=[ - AnnAssign( - target=Name(id='a', ctx=Store()), - annotation=Name(id='int', ctx=Load()), - value=Constant(value=1), - simple=0)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation - Module( - body=[ - AnnAssign( - target=Attribute( - value=Name(id='a', ctx=Load()), - attr='b', - ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation - Module( - body=[ - AnnAssign( - target=Subscript( - value=Name(id='a', ctx=Load()), - slice=Constant(value=1), - ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) - - -.. class:: AugAssign(target, op, value) - - Augmented assignment, such as ``a += 1``. In the following example, - ``target`` is a :class:`Name` node for ``x`` (with the :class:`Store` - context), ``op`` is :class:`Add`, and ``value`` is a :class:`Constant` with - value for 1. - - The ``target`` attribute connot be of class :class:`Tuple` or :class:`List`, - unlike the targets of :class:`Assign`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x += 2'), indent=4)) - Module( - body=[ - AugAssign( - target=Name(id='x', ctx=Store()), - op=Add(), - value=Constant(value=2))], - type_ignores=[]) - - -.. class:: Raise(exc, cause) - - A ``raise`` statement. ``exc`` is the exception object to be raised, normally a - :class:`Call` or :class:`Name`, or ``None`` for a standalone ``raise``. - ``cause`` is the optional part for ``y`` in ``raise x from y``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('raise x from y'), indent=4)) - Module( - body=[ - Raise( - exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))], - type_ignores=[]) - - -.. class:: Assert(test, msg) - - An assertion. ``test`` holds the condition, such as a :class:`Compare` node. - ``msg`` holds the failure message. - - .. doctest:: - - >>> print(ast.dump(ast.parse('assert x,y'), indent=4)) - Module( - body=[ - Assert( - test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))], - type_ignores=[]) - - -.. class:: Delete(targets) - - Represents a ``del`` statement. ``targets`` is a list of nodes, such as - :class:`Name`, :class:`Attribute` or :class:`Subscript` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('del x,y,z'), indent=4)) - Module( - body=[ - Delete( - targets=[ - Name(id='x', ctx=Del()), - Name(id='y', ctx=Del()), - Name(id='z', ctx=Del())])], - type_ignores=[]) - - -.. class:: Pass() - - A ``pass`` statement. - - .. doctest:: - - >>> print(ast.dump(ast.parse('pass'), indent=4)) - Module( - body=[ - Pass()], - type_ignores=[]) - - -Other statements which are only applicable inside functions or loops are -described in other sections. - -Imports -~~~~~~~ - -.. class:: Import(names) - - An import statement. ``names`` is a list of :class:`alias` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('import x,y,z'), indent=4)) - Module( - body=[ - Import( - names=[ - alias(name='x'), - alias(name='y'), - alias(name='z')])], - type_ignores=[]) - - -.. class:: ImportFrom(module, names, level) - - Represents ``from x import y``. ``module`` is a raw string of the 'from' name, - without any leading dots, or ``None`` for statements such as ``from . import foo``. - ``level`` is an integer holding the level of the relative import (0 means - absolute import). - - .. doctest:: - - >>> print(ast.dump(ast.parse('from y import x,y,z'), indent=4)) - Module( - body=[ - ImportFrom( - module='y', - names=[ - alias(name='x'), - alias(name='y'), - alias(name='z')], - level=0)], - type_ignores=[]) - - -.. class:: alias(name, asname) - - Both parameters are raw strings of the names. ``asname`` can be ``None`` if - the regular name is to be used. - - .. doctest:: - - >>> print(ast.dump(ast.parse('from ..foo.bar import a as b, c'), indent=4)) - Module( - body=[ - ImportFrom( - module='foo.bar', - names=[ - alias(name='a', asname='b'), - alias(name='c')], - level=2)], - type_ignores=[]) - -Control flow -^^^^^^^^^^^^ - -.. note:: - Optional clauses such as ``else`` are stored as an empty list if they're - not present. - -.. class:: If(test, body, orelse) - - An ``if`` statement. ``test`` holds a single node, such as a :class:`Compare` - node. ``body`` and ``orelse`` each hold a list of nodes. - - ``elif`` clauses don't have a special representation in the AST, but rather - appear as extra :class:`If` nodes within the ``orelse`` section of the - previous one. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... if x: - ... ... - ... elif y: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - If( - test=Name(id='x', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - If( - test=Name(id='y', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -.. class:: For(target, iter, body, orelse, type_comment) - - A ``for`` loop. ``target`` holds the variable(s) the loop assigns to, as a - single :class:`Name`, :class:`Tuple` or :class:`List` node. ``iter`` holds - the item to be looped over, again as a single node. ``body`` and ``orelse`` - contain lists of nodes to execute. Those in ``orelse`` are executed if the - loop finishes normally, rather than via a ``break`` statement. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... for x in y: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - For( - target=Name(id='x', ctx=Store()), - iter=Name(id='y', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: While(test, body, orelse) - - A ``while`` loop. ``test`` holds the condition, such as a :class:`Compare` - node. - - .. doctest:: - - >> print(ast.dump(ast.parse(""" - ... while x: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - While( - test=Name(id='x', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: Break - Continue - - The ``break`` and ``continue`` statements. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... for a in b: - ... if a > 5: - ... break - ... else: - ... continue - ... - ... """), indent=4)) - Module( - body=[ - For( - target=Name(id='a', ctx=Store()), - iter=Name(id='b', ctx=Load()), - body=[ - If( - test=Compare( - left=Name(id='a', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=5)]), - body=[ - Break()], - orelse=[ - Continue()])], - orelse=[])], - type_ignores=[]) - - -.. class:: Try(body, handlers, orelse, finalbody) - - ``try`` blocks. All attributes are list of nodes to execute, except for - ``handlers``, which is a list of :class:`ExceptHandler` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... try: - ... ... - ... except Exception: - ... ... - ... except OtherException as e: - ... ... - ... else: - ... ... - ... finally: - ... ... - ... """), indent=4)) - Module( - body=[ - Try( - body=[ - Expr( - value=Constant(value=Ellipsis))], - handlers=[ - ExceptHandler( - type=Name(id='Exception', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - ExceptHandler( - type=Name(id='OtherException', ctx=Load()), - name='e', - body=[ - Expr( - value=Constant(value=Ellipsis))])], - orelse=[ - Expr( - value=Constant(value=Ellipsis))], - finalbody=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: ExceptHandler(type, name, body) - - A single ``except`` clause. ``type`` is the exception type it will match, - typically a :class:`Name` node (or ``None`` for a catch-all ``except:`` clause). - ``name`` is a raw string for the name to hold the exception, or ``None`` if - the clause doesn't have ``as foo``. ``body`` is a list of nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... try: - ... a + 1 - ... except TypeError: - ... pass - ... """), indent=4)) - Module( - body=[ - Try( - body=[ - Expr( - value=BinOp( - left=Name(id='a', ctx=Load()), - op=Add(), - right=Constant(value=1)))], - handlers=[ - ExceptHandler( - type=Name(id='TypeError', ctx=Load()), - body=[ - Pass()])], - orelse=[], - finalbody=[])], - type_ignores=[]) - - -.. class:: With(items, body, type_comment) - - A ``with`` block. ``items`` is a list of :class:`withitem` nodes representing - the context managers, and ``body`` is the indented block inside the context. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - -.. class:: withitem(context_expr, optional_vars) - - A single context manager in a ``with`` block. ``context_expr`` is the context - manager, often a :class:`Call` node. ``optional_vars`` is a :class:`Name`, - :class:`Tuple` or :class:`List` for the ``as foo`` part, or ``None`` if that - isn't used. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... with a as b, c as d: - ... something(b, d) - ... """), indent=4)) - Module( - body=[ - With( - items=[ - withitem( - context_expr=Name(id='a', ctx=Load()), - optional_vars=Name(id='b', ctx=Store())), - withitem( - context_expr=Name(id='c', ctx=Load()), - optional_vars=Name(id='d', ctx=Store()))], - body=[ - Expr( - value=Call( - func=Name(id='something', ctx=Load()), - args=[ - Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())], - keywords=[]))])], - type_ignores=[]) - - -.. class:: Match(subject, cases) - - A ``match`` statement. ``subject`` holds the subject of the match (the object - that is being matched against the cases) and ``cases`` contains an iterable of - :class:`match_case` nodes with the different cases. - - -.. class:: match_case(context_expr, optional_vars) - - A single case pattern in a ``match`` statement. ``pattern`` contains the - match pattern that will be used to match the subject against. Notice that - the meaning of the :class:`AST` nodes in this attribute have a different - meaning than in other places, as they represent patterns to match against. - The ``guard`` attribute contains an expression that will be evaluated if - the pattern matches the subject. If the pattern matches and the ``guard`` condition - is truthy, the body of the case shall be executed. ``body`` contains a list - of nodes to execute if the guard is truthy. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] if x>0: - ... ... - ... case tuple(): - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - guard=Compare( - left=Name(id='x', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=0)]), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - match_case( - pattern=Call( - func=Name(id='tuple', ctx=Load()), - args=[], - keywords=[]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchAs(pattern, name) - - A match "as-pattern". The as-pattern matches whatever pattern is on its - left-hand side, but also binds the value to a name. ``pattern`` contains - the match pattern that will be used to match the subject agsinst. The ``name`` - attribute contains the name that will be binded if the pattern is successful. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] as y: - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchAs( - pattern=List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - name='y'), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -.. class:: MatchOr(patterns) - - A match "or-pattern". An or-pattern matches each of its subpatterns in turn - to the subject, until one succeeds. The or-pattern is then deemed to - succeed. If none of the subpatterns succeed the or-pattern fails. The - ``patterns`` attribute contains a list of match patterns nodes that will be - matched against the subject. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] | (y): - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchOr( - patterns=[ - List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - Name(id='y', ctx=Store())]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -Function and class definitions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment) - - A function definition. - - * ``name`` is a raw string of the function name. - * ``args`` is a :class:`arguments` node. - * ``body`` is the list of nodes inside the function. - * ``decorator_list`` is the list of decorators to be applied, stored outermost - first (i.e. the first in the list will be applied last). - * ``returns`` is the return annotation. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - -.. class:: Lambda(args, body) - - ``lambda`` is a minimal function definition that can be used inside an - expression. Unlike :class:`FunctionDef`, ``body`` holds a single node. - - .. doctest:: - - >>> print(ast.dump(ast.parse('lambda x,y: ...'), indent=4)) - Module( - body=[ - Expr( - value=Lambda( - args=arguments( - posonlyargs=[], - args=[ - arg(arg='x'), - arg(arg='y')], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=Constant(value=Ellipsis)))], - type_ignores=[]) - - -.. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) - - The arguments for a function. - - * ``posonlyargs``, ``args`` and ``kwonlyargs`` are lists of :class:`arg` nodes. - * ``vararg`` and ``kwarg`` are single :class:`arg` nodes, referring to the - ``*args, **kwargs`` parameters. - * ``kw_defaults`` is a list of default values for keyword-only arguments. If - one is ``None``, the corresponding argument is required. - * ``defaults`` is a list of default values for arguments that can be passed - positionally. If there are fewer defaults, they correspond to the last n - arguments. - - -.. class:: arg(arg, annotation, type_comment) - - A single argument in a list. ``arg`` is a raw string of the argument - name, ``annotation`` is its annotation, such as a :class:`Str` or - :class:`Name` node. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... @decorator1 - ... @decorator2 - ... def f(a: 'annotation', b=1, c=2, *d, e, f=3, **g) -> 'return annotation': - ... pass - ... """), indent=4)) - Module( - body=[ - FunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[ - arg( - arg='a', - annotation=Constant(value='annotation')), - arg(arg='b'), - arg(arg='c')], - vararg=arg(arg='d'), - kwonlyargs=[ - arg(arg='e'), - arg(arg='f')], - kw_defaults=[ - None, - Constant(value=3)], - kwarg=arg(arg='g'), - defaults=[ - Constant(value=1), - Constant(value=2)]), - body=[ - Pass()], - decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], - returns=Constant(value='return annotation'))], - type_ignores=[]) - - -.. class:: Return(value) - - A ``return`` statement. - - .. doctest:: - - >>> print(ast.dump(ast.parse('return 4'), indent=4)) - Module( - body=[ - Return( - value=Constant(value=4))], - type_ignores=[]) - - -.. class:: Yield(value) - YieldFrom(value) - - A ``yield`` or ``yield from`` expression. Because these are expressions, they - must be wrapped in a :class:`Expr` node if the value sent back is not used. - - .. doctest:: - - >>> print(ast.dump(ast.parse('yield x'), indent=4)) - Module( - body=[ - Expr( - value=Yield( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('yield from x'), indent=4)) - Module( - body=[ - Expr( - value=YieldFrom( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) - - -.. class:: Global(names) - Nonlocal(names) - - ``global`` and ``nonlocal`` statements. ``names`` is a list of raw strings. - - .. doctest:: - - >>> print(ast.dump(ast.parse('global x,y,z'), indent=4)) - Module( - body=[ - Global( - names=[ - 'x', - 'y', - 'z'])], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) - Module( - body=[ - Nonlocal( - names=[ - 'x', - 'y', - 'z'])], - type_ignores=[]) - - -.. class:: ClassDef(name, bases, keywords, starargs, kwargs, body, decorator_list) - - A class definition. - - * ``name`` is a raw string for the class name - * ``bases`` is a list of nodes for explicitly specified base classes. - * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. - Other keywords will be passed to the metaclass, as per `PEP-3115 - `_. - * ``starargs`` and ``kwargs`` are each a single node, as in a function call. - starargs will be expanded to join the list of base classes, and kwargs will - be passed to the metaclass. - * ``body`` is a list of nodes representing the code within the class - definition. - * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... @decorator1 - ... @decorator2 - ... class Foo(base1, base2, metaclass=meta): - ... pass - ... """), indent=4)) - Module( - body=[ - ClassDef( - name='Foo', - bases=[ - Name(id='base1', ctx=Load()), - Name(id='base2', ctx=Load())], - keywords=[ - keyword( - arg='metaclass', - value=Name(id='meta', ctx=Load()))], - body=[ - Pass()], - decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())])], - type_ignores=[]) - -Async and await -^^^^^^^^^^^^^^^ - -.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment) - - An ``async def`` function definition. Has the same fields as - :class:`FunctionDef`. - - -.. class:: Await(value) - - An ``await`` expression. ``value`` is what it waits for. - Only valid in the body of an :class:`AsyncFunctionDef`. - -.. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... async def f(): - ... await other_func() - ... """), indent=4)) - Module( - body=[ - AsyncFunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=[ - Expr( - value=Await( - value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[])], - type_ignores=[]) - - -.. class:: AsyncFor(target, iter, body, orelse, type_comment) - AsyncWith(items, body, type_comment) - - ``async for`` loops and ``async with`` context managers. They have the same - fields as :class:`For` and :class:`With`, respectively. Only valid in the - body of an :class:`AsyncFunctionDef`. - -.. note:: - When a string is parsed by :func:`ast.parse`, operator nodes (subclasses - of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, - :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree - will be singletons. Changes to one will be reflected in all other - occurrences of the same value (e.g. :class:`ast.Add`). - - -:mod:`ast` Helpers ------------------- - -Apart from the node classes, the :mod:`ast` module defines these utility functions -and classes for traversing abstract syntax trees: - -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) - - Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, ast.PyCF_ONLY_AST)``. - - If ``type_comments=True`` is given, the parser is modified to check - and return type comments as specified by :pep:`484` and :pep:`526`. - This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the - flags passed to :func:`compile()`. This will report syntax errors - for misplaced type comments. Without this flag, type comments will - be ignored, and the ``type_comment`` field on selected AST nodes - will always be ``None``. In addition, the locations of ``# type: - ignore`` comments will be returned as the ``type_ignores`` - attribute of :class:`Module` (otherwise it is always an empty list). - - In addition, if ``mode`` is ``'func_type'``, the input syntax is - modified to correspond to :pep:`484` "signature type comments", - e.g. ``(str, int) -> List[str]``. - - Also, setting ``feature_version`` to a tuple ``(major, minor)`` - will attempt to parse using that Python version's grammar. - Currently ``major`` must equal to ``3``. For example, setting - ``feature_version=(3, 4)`` will allow the use of ``async`` and - ``await`` as variable names. The lowest supported version is - ``(3, 4)``; the highest is ``sys.version_info[0:2]``. - - .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. - - .. versionchanged:: 3.8 - Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. - - -.. function:: unparse(ast_obj) - - Unparse an :class:`ast.AST` object and generate a string with code - that would produce an equivalent :class:`ast.AST` object if parsed - back with :func:`ast.parse`. - - .. warning:: - The produced code string will not necessarily be equal to the original - code that generated the :class:`ast.AST` object (without any compiler - optimizations, such as constant tuples/frozensets). - - .. warning:: - Trying to unparse a highly complex expression would result with - :exc:`RecursionError`. - - .. versionadded:: 3.9 - - -.. function:: literal_eval(node_or_string) - - Safely evaluate an expression node or a string containing a Python literal or - container display. The string or node provided may only consist of the - following Python literal structures: strings, bytes, numbers, tuples, lists, - dicts, sets, booleans, ``None`` and ``Ellipsis``. - - This can be used for safely evaluating strings containing Python values from - untrusted sources without the need to parse the values oneself. It is not - capable of evaluating arbitrarily complex expressions, for example involving - operators or indexing. - - .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. - - It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, - :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed - input. - - .. versionchanged:: 3.2 - Now allows bytes and set literals. - - .. versionchanged:: 3.9 - Now supports creating empty sets with ``'set()'``. - - .. versionchanged:: 3.10 - For string inputs, leading spaces and tabs are now stripped. - - -.. function:: get_docstring(node, clean=True) - - Return the docstring of the given *node* (which must be a - :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`, - or :class:`Module` node), or ``None`` if it has no docstring. - If *clean* is true, clean up the docstring's indentation with - :func:`inspect.cleandoc`. - - .. versionchanged:: 3.5 - :class:`AsyncFunctionDef` is now supported. - - -.. function:: get_source_segment(source, node, *, padded=False) - - Get source code segment of the *source* that generated *node*. - If some location information (:attr:`lineno`, :attr:`end_lineno`, - :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. - - If *padded* is ``True``, the first line of a multi-line statement will - be padded with spaces to match its original position. - - .. versionadded:: 3.8 - - -.. function:: fix_missing_locations(node) - - When you compile a node tree with :func:`compile`, the compiler expects - :attr:`lineno` and :attr:`col_offset` attributes for every node that supports - them. This is rather tedious to fill in for generated nodes, so this helper - adds these attributes recursively where not already set, by setting them to - the values of the parent node. It works recursively starting at *node*. - - -.. function:: increment_lineno(node, n=1) - - Increment the line number and end line number of each node in the tree - starting at *node* by *n*. This is useful to "move code" to a different - location in a file. - - -.. function:: copy_location(new_node, old_node) - - Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, - and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, - and return *new_node*. - - -.. function:: iter_fields(node) - - Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` - that is present on *node*. - - -.. function:: iter_child_nodes(node) - - Yield all direct child nodes of *node*, that is, all fields that are nodes - and all items of fields that are lists of nodes. - - -.. function:: walk(node) - - Recursively yield all descendant nodes in the tree starting at *node* - (including *node* itself), in no specified order. This is useful if you only - want to modify nodes in place and don't care about the context. - - -.. class:: NodeVisitor() - - A node visitor base class that walks the abstract syntax tree and calls a - visitor function for every node found. This function may return a value - which is forwarded by the :meth:`visit` method. - - This class is meant to be subclassed, with the subclass adding visitor - methods. - - .. method:: visit(node) - - Visit a node. The default implementation calls the method called - :samp:`self.visit_{classname}` where *classname* is the name of the node - class, or :meth:`generic_visit` if that method doesn't exist. - - .. method:: generic_visit(node) - - This visitor calls :meth:`visit` on all children of the node. - - Note that child nodes of nodes that have a custom visitor method won't be - visited unless the visitor calls :meth:`generic_visit` or visits them - itself. - - Don't use the :class:`NodeVisitor` if you want to apply changes to nodes - during traversal. For this a special visitor exists - (:class:`NodeTransformer`) that allows modifications. - - .. deprecated:: 3.8 - - Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, - :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated - now and will not be called in future Python versions. Add the - :meth:`visit_Constant` method to handle all constant nodes. - - -.. class:: NodeTransformer() - - A :class:`NodeVisitor` subclass that walks the abstract syntax tree and - allows modification of nodes. - - The :class:`NodeTransformer` will walk the AST and use the return value of - the visitor methods to replace or remove the old node. If the return value - of the visitor method is ``None``, the node will be removed from its - location, otherwise it is replaced with the return value. The return value - may be the original node in which case no replacement takes place. - - Here is an example transformer that rewrites all occurrences of name lookups - (``foo``) to ``data['foo']``:: - - class RewriteName(NodeTransformer): - - def visit_Name(self, node): - return Subscript( - value=Name(id='data', ctx=Load()), - slice=Constant(value=node.id), - ctx=node.ctx - ) - - Keep in mind that if the node you're operating on has child nodes you must - either transform the child nodes yourself or call the :meth:`generic_visit` - method for the node first. - - For nodes that were part of a collection of statements (that applies to all - statement nodes), the visitor may also return a list of nodes rather than - just a single node. - - If :class:`NodeTransformer` introduces new nodes (that weren't part of - original tree) without giving them location information (such as - :attr:`lineno`), :func:`fix_missing_locations` should be called with - the new sub-tree to recalculate the location information:: - - tree = ast.parse('foo', mode='eval') - new_tree = fix_missing_locations(RewriteName().visit(tree)) - - Usually you use the transformer like this:: - - node = YourTransformer().visit(node) - - -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) - - Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. If *annotate_fields* is true (by default), - the returned string will show the names and the values for fields. - If *annotate_fields* is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to true. - - If *indent* is a non-negative integer or string, then the tree will be - pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the single line representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. - - .. versionchanged:: 3.9 - Added the *indent* option. - - -.. _ast-compiler-flags: - -Compiler Flags --------------- - -The following flags may be passed to :func:`compile` in order to change -effects on the compilation of a program: - -.. data:: PyCF_ALLOW_TOP_LEVEL_AWAIT - - Enables support for top-level ``await``, ``async for``, ``async with`` - and async comprehensions. - - .. versionadded:: 3.8 - -.. data:: PyCF_ONLY_AST - - Generates and returns an abstract syntax tree instead of returning a - compiled code object. - -.. data:: PyCF_TYPE_COMMENTS - - Enables support for :pep:`484` and :pep:`526` style type comments - (``# type: ``, ``# type: ignore ``). - - .. versionadded:: 3.8 - - -.. _ast-cli: - -Command-Line Usage ------------------- - -.. versionadded:: 3.9 - -The :mod:`ast` module can be executed as a script from the command line. -It is as simple as: - -.. code-block:: sh - - python -m ast [-m ] [-a] [infile] - -The following options are accepted: - -.. program:: ast - -.. cmdoption:: -h, --help - - Show the help message and exit. - -.. cmdoption:: -m - --mode - - Specify what kind of code must be compiled, like the *mode* argument - in :func:`parse`. - -.. cmdoption:: --no-type-comments - - Don't parse type comments. - -.. cmdoption:: -a, --include-attributes - - Include attributes such as line numbers and column offsets. - -.. cmdoption:: -i - --indent - - Indentation of nodes in AST (number of spaces). - -If :file:`infile` is specified its contents are parsed to AST and dumped -to stdout. Otherwise, the content is read from stdin. - - -.. seealso:: - - `Green Tree Snakes `_, an external - documentation resource, has good details on working with Python ASTs. - - `ASTTokens `_ - annotates Python ASTs with the positions of tokens and text in the source - code that generated them. This is helpful for tools that make source code - transformations. - - `leoAst.py `_ unifies the - token-based and parse-tree-based views of python programs by inserting - two-way links between tokens and ast nodes. - - `LibCST `_ parses code as a Concrete Syntax - Tree that looks like an ast tree and keeps all formatting details. It's - useful for building automated refactoring (codemod) applications and - linters. - - `Parso `_ is a Python parser that supports - error recovery and round-trip parsing for different Python versions (in - multiple Python versions). Parso is also able to list multiple syntax errors - in your python file. diff --git a/Doc/library/asyncio-api-index.rst.bak b/Doc/library/asyncio-api-index.rst.bak deleted file mode 100644 index 047e5bbc58ccad..00000000000000 --- a/Doc/library/asyncio-api-index.rst.bak +++ /dev/null @@ -1,221 +0,0 @@ -.. currentmodule:: asyncio - - -==================== -High-level API Index -==================== - -This page lists all high-level async/await enabled asyncio APIs. - - -Tasks -===== - -Utilities to run asyncio programs, create Tasks, and -await on multiple things with timeouts. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :func:`run` - - Create event loop, run a coroutine, close the loop. - - * - :func:`create_task` - - Start an asyncio Task. - - * - ``await`` :func:`sleep` - - Sleep for a number of seconds. - - * - ``await`` :func:`gather` - - Schedule and wait for things concurrently. - - * - ``await`` :func:`wait_for` - - Run with a timeout. - - * - ``await`` :func:`shield` - - Shield from cancellation. - - * - ``await`` :func:`wait` - - Monitor for completion. - - * - :func:`current_task` - - Return the current Task. - - * - :func:`all_tasks` - - Return all tasks for an event loop. - - * - :class:`Task` - - Task object. - - * - :func:`to_thread` - - Asychronously run a function in a separate OS thread. - - * - :func:`run_coroutine_threadsafe` - - Schedule a coroutine from another OS thread. - - * - ``for in`` :func:`as_completed` - - Monitor for completion with a ``for`` loop. - - -.. rubric:: Examples - -* :ref:`Using asyncio.gather() to run things in parallel - `. - -* :ref:`Using asyncio.wait_for() to enforce a timeout - `. - -* :ref:`Cancellation `. - -* :ref:`Using asyncio.sleep() `. - -* See also the main :ref:`Tasks documentation page `. - - -Queues -====== - -Queues should be used to distribute work amongst multiple asyncio Tasks, -implement connection pools, and pub/sub patterns. - - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :class:`Queue` - - A FIFO queue. - - * - :class:`PriorityQueue` - - A priority queue. - - * - :class:`LifoQueue` - - A LIFO queue. - - -.. rubric:: Examples - -* :ref:`Using asyncio.Queue to distribute workload between several - Tasks `. - -* See also the :ref:`Queues documentation page `. - - -Subprocesses -============ - -Utilities to spawn subprocesses and run shell commands. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :func:`create_subprocess_exec` - - Create a subprocess. - - * - ``await`` :func:`create_subprocess_shell` - - Run a shell command. - - -.. rubric:: Examples - -* :ref:`Executing a shell command `. - -* See also the :ref:`subprocess APIs ` - documentation. - - -Streams -======= - -High-level APIs to work with network IO. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :func:`open_connection` - - Establish a TCP connection. - - * - ``await`` :func:`open_unix_connection` - - Establish a Unix socket connection. - - * - ``await`` :func:`start_server` - - Start a TCP server. - - * - ``await`` :func:`start_unix_server` - - Start a Unix socket server. - - * - :class:`StreamReader` - - High-level async/await object to receive network data. - - * - :class:`StreamWriter` - - High-level async/await object to send network data. - - -.. rubric:: Examples - -* :ref:`Example TCP client `. - -* See also the :ref:`streams APIs ` - documentation. - - -Synchronization -=============== - -Threading-like synchronization primitives that can be used in Tasks. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :class:`Lock` - - A mutex lock. - - * - :class:`Event` - - An event object. - - * - :class:`Condition` - - A condition object. - - * - :class:`Semaphore` - - A semaphore. - - * - :class:`BoundedSemaphore` - - A bounded semaphore. - - -.. rubric:: Examples - -* :ref:`Using asyncio.Event `. - -* See also the documentation of asyncio - :ref:`synchronization primitives `. - - -Exceptions -========== - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - - * - :exc:`asyncio.TimeoutError` - - Raised on timeout by functions like :func:`wait_for`. - Keep in mind that ``asyncio.TimeoutError`` is **unrelated** - to the built-in :exc:`TimeoutError` exception. - - * - :exc:`asyncio.CancelledError` - - Raised when a Task is cancelled. See also :meth:`Task.cancel`. - - -.. rubric:: Examples - -* :ref:`Handling CancelledError to run code on cancellation request - `. - -* See also the full list of - :ref:`asyncio-specific exceptions `. diff --git a/Doc/library/collections.rst.bak b/Doc/library/collections.rst.bak deleted file mode 100644 index aa0acfaae45a5f..00000000000000 --- a/Doc/library/collections.rst.bak +++ /dev/null @@ -1,1278 +0,0 @@ -:mod:`collections` --- Container datatypes -========================================== - -.. module:: collections - :synopsis: Container datatypes - -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - -**Source code:** :source:`Lib/collections/__init__.py` - -.. testsetup:: * - - from collections import * - import itertools - __name__ = '' - --------------- - -This module implements specialized container datatypes providing alternatives to -Python's general purpose built-in containers, :class:`dict`, :class:`list`, -:class:`set`, and :class:`tuple`. - -===================== ==================================================================== -:func:`namedtuple` factory function for creating tuple subclasses with named fields -:class:`deque` list-like container with fast appends and pops on either end -:class:`ChainMap` dict-like class for creating a single view of multiple mappings -:class:`Counter` dict subclass for counting hashable objects -:class:`OrderedDict` dict subclass that remembers the order entries were added -:class:`defaultdict` dict subclass that calls a factory function to supply missing values -:class:`UserDict` wrapper around dictionary objects for easier dict subclassing -:class:`UserList` wrapper around list objects for easier list subclassing -:class:`UserString` wrapper around string objects for easier string subclassing -===================== ==================================================================== - - -:class:`ChainMap` objects -------------------------- - -.. versionadded:: 3.3 - -A :class:`ChainMap` class is provided for quickly linking a number of mappings -so they can be treated as a single unit. It is often much faster than creating -a new dictionary and running multiple :meth:`~dict.update` calls. - -The class can be used to simulate nested scopes and is useful in templating. - -.. class:: ChainMap(*maps) - - A :class:`ChainMap` groups multiple dicts or other mappings together to - create a single, updateable view. If no *maps* are specified, a single empty - dictionary is provided so that a new chain always has at least one mapping. - - The underlying mappings are stored in a list. That list is public and can - be accessed or updated using the *maps* attribute. There is no other state. - - Lookups search the underlying mappings successively until a key is found. In - contrast, writes, updates, and deletions only operate on the first mapping. - - A :class:`ChainMap` incorporates the underlying mappings by reference. So, if - one of the underlying mappings gets updated, those changes will be reflected - in :class:`ChainMap`. - - All of the usual dictionary methods are supported. In addition, there is a - *maps* attribute, a method for creating new subcontexts, and a property for - accessing all but the first mapping: - - .. attribute:: maps - - A user updateable list of mappings. The list is ordered from - first-searched to last-searched. It is the only stored state and can - be modified to change which mappings are searched. The list should - always contain at least one mapping. - - .. method:: new_child(m=None) - - Returns a new :class:`ChainMap` containing a new map followed by - all of the maps in the current instance. If ``m`` is specified, - it becomes the new map at the front of the list of mappings; if not - specified, an empty dict is used, so that a call to ``d.new_child()`` - is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for - creating subcontexts that can be updated without altering values in any - of the parent mappings. - - .. versionchanged:: 3.4 - The optional ``m`` parameter was added. - - .. attribute:: parents - - Property returning a new :class:`ChainMap` containing all of the maps in - the current instance except the first one. This is useful for skipping - the first map in the search. Use cases are similar to those for the - :keyword:`nonlocal` keyword used in :term:`nested scopes `. The use cases also parallel those for the built-in - :func:`super` function. A reference to ``d.parents`` is equivalent to: - ``ChainMap(*d.maps[1:])``. - - Note, the iteration order of a :class:`ChainMap()` is determined by - scanning the mappings last to first:: - - >>> baseline = {'music': 'bach', 'art': 'rembrandt'} - >>> adjustments = {'art': 'van gogh', 'opera': 'carmen'} - >>> list(ChainMap(adjustments, baseline)) - ['music', 'art', 'opera'] - - This gives the same ordering as a series of :meth:`dict.update` calls - starting with the last mapping:: - - >>> combined = baseline.copy() - >>> combined.update(adjustments) - >>> list(combined) - ['music', 'art', 'opera'] - - .. versionchanged:: 3.9 - Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. - -.. seealso:: - - * The `MultiContext class - `_ - in the Enthought `CodeTools package - `_ has options to support - writing to any mapping in the chain. - - * Django's `Context class - `_ - for templating is a read-only chain of mappings. It also features - pushing and popping of contexts similar to the - :meth:`~collections.ChainMap.new_child` method and the - :attr:`~collections.ChainMap.parents` property. - - * The `Nested Contexts recipe - `_ has options to control - whether writes and other mutations apply only to the first mapping or to - any mapping in the chain. - - * A `greatly simplified read-only version of Chainmap - `_. - - -:class:`ChainMap` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with chained maps. - - -Example of simulating Python's internal lookup chain:: - - import builtins - pylookup = ChainMap(locals(), globals(), vars(builtins)) - -Example of letting user specified command-line arguments take precedence over -environment variables which in turn take precedence over default values:: - - import os, argparse - - defaults = {'color': 'red', 'user': 'guest'} - - parser = argparse.ArgumentParser() - parser.add_argument('-u', '--user') - parser.add_argument('-c', '--color') - namespace = parser.parse_args() - command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} - - combined = ChainMap(command_line_args, os.environ, defaults) - print(combined['color']) - print(combined['user']) - -Example patterns for using the :class:`ChainMap` class to simulate nested -contexts:: - - c = ChainMap() # Create root context - d = c.new_child() # Create nested child context - e = c.new_child() # Child of c, independent from d - e.maps[0] # Current context dictionary -- like Python's locals() - e.maps[-1] # Root context -- like Python's globals() - e.parents # Enclosing context chain -- like Python's nonlocals - - d['x'] = 1 # Set value in current context - d['x'] # Get first key in the chain of contexts - del d['x'] # Delete from current context - list(d) # All nested values - k in d # Check all nested values - len(d) # Number of nested values - d.items() # All nested items - dict(d) # Flatten into a regular dictionary - -The :class:`ChainMap` class only makes updates (writes and deletions) to the -first mapping in the chain while lookups will search the full chain. However, -if deep writes and deletions are desired, it is easy to make a subclass that -updates keys found deeper in the chain:: - - class DeepChainMap(ChainMap): - 'Variant of ChainMap that allows direct updates to inner scopes' - - def __setitem__(self, key, value): - for mapping in self.maps: - if key in mapping: - mapping[key] = value - return - self.maps[0][key] = value - - def __delitem__(self, key): - for mapping in self.maps: - if key in mapping: - del mapping[key] - return - raise KeyError(key) - - >>> d = DeepChainMap({'zebra': 'black'}, {'elephant': 'blue'}, {'lion': 'yellow'}) - >>> d['lion'] = 'orange' # update an existing key two levels down - >>> d['snake'] = 'red' # new keys get added to the topmost dict - >>> del d['elephant'] # remove an existing key one level down - >>> d # display result - DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) - - -:class:`Counter` objects ------------------------- - -A counter tool is provided to support convenient and rapid tallies. -For example:: - - >>> # Tally occurrences of words in a list - >>> cnt = Counter() - >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: - ... cnt[word] += 1 - >>> cnt - Counter({'blue': 3, 'red': 2, 'green': 1}) - - >>> # Find the ten most common words in Hamlet - >>> import re - >>> words = re.findall(r'\w+', open('hamlet.txt').read().lower()) - >>> Counter(words).most_common(10) - [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), - ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] - -.. class:: Counter([iterable-or-mapping]) - - A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. - It is a collection where elements are stored as dictionary keys - and their counts are stored as dictionary values. Counts are allowed to be - any integer value including zero or negative counts. The :class:`Counter` - class is similar to bags or multisets in other languages. - - Elements are counted from an *iterable* or initialized from another - *mapping* (or counter): - - >>> c = Counter() # a new, empty counter - >>> c = Counter('gallahad') # a new counter from an iterable - >>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping - >>> c = Counter(cats=4, dogs=8) # a new counter from keyword args - - Counter objects have a dictionary interface except that they return a zero - count for missing items instead of raising a :exc:`KeyError`: - - >>> c = Counter(['eggs', 'ham']) - >>> c['bacon'] # count of a missing element is zero - 0 - - Setting a count to zero does not remove an element from a counter. - Use ``del`` to remove it entirely: - - >>> c['sausage'] = 0 # counter entry with a zero count - >>> del c['sausage'] # del actually removes the entry - - .. versionadded:: 3.1 - - .. versionchanged:: 3.7 As a :class:`dict` subclass, :class:`Counter` - Inherited the capability to remember insertion order. Math operations - on *Counter* objects also preserve order. Results are ordered - according to when an element is first encountered in the left operand - and then by the order encountered in the right operand. - - Counter objects support three methods beyond those available for all - dictionaries: - - .. method:: elements() - - Return an iterator over elements repeating each as many times as its - count. Elements are returned in the order first encountered. If an - element's count is less than one, :meth:`elements` will ignore it. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> sorted(c.elements()) - ['a', 'a', 'a', 'a', 'b', 'b'] - - .. method:: most_common([n]) - - Return a list of the *n* most common elements and their counts from the - most common to the least. If *n* is omitted or ``None``, - :meth:`most_common` returns *all* elements in the counter. - Elements with equal counts are ordered in the order first encountered: - - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('b', 2), ('r', 2)] - - .. method:: subtract([iterable-or-mapping]) - - Elements are subtracted from an *iterable* or from another *mapping* - (or counter). Like :meth:`dict.update` but subtracts counts instead - of replacing them. Both inputs and outputs may be zero or negative. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> d = Counter(a=1, b=2, c=3, d=4) - >>> c.subtract(d) - >>> c - Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6}) - - .. versionadded:: 3.2 - - The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. - - .. method:: fromkeys(iterable) - - This class method is not implemented for :class:`Counter` objects. - - .. method:: update([iterable-or-mapping]) - - Elements are counted from an *iterable* or added-in from another - *mapping* (or counter). Like :meth:`dict.update` but adds counts - instead of replacing them. Also, the *iterable* is expected to be a - sequence of elements, not a sequence of ``(key, value)`` pairs. - -Counters support rich comparison operators for equality, subset, and -superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. -All of those tests treat missing elements as having zero counts so that -``Counter(a=1) == Counter(a=1, b=0)`` returns true. - -.. versionadded:: 3.10 - Rich comparison operations we were added - -.. versionchanged:: 3.10 - In equality tests, missing elements are treated as having zero counts. - Formerly, ``Counter(a=3)`` and ``Counter(a=3, b=0)`` were considered - distinct. - -Common patterns for working with :class:`Counter` objects:: - - sum(c.values()) # total of all counts - c.clear() # reset all counts - list(c) # list unique elements - set(c) # convert to a set - dict(c) # convert to a regular dictionary - c.items() # convert to a list of (elem, cnt) pairs - Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs - c.most_common()[:-n-1:-1] # n least common elements - +c # remove zero and negative counts - -Several mathematical operations are provided for combining :class:`Counter` -objects to produce multisets (counters that have counts greater than zero). -Addition and subtraction combine counters by adding or subtracting the counts -of corresponding elements. Intersection and union return the minimum and -maximum of corresponding counts. Each operation can accept inputs with signed -counts, but the output will exclude results with counts of zero or less. - - >>> c = Counter(a=3, b=1) - >>> d = Counter(a=1, b=2) - >>> c + d # add two counters together: c[x] + d[x] - Counter({'a': 4, 'b': 3}) - >>> c - d # subtract (keeping only positive counts) - Counter({'a': 2}) - >>> c & d # intersection: min(c[x], d[x]) # doctest: +SKIP - Counter({'a': 1, 'b': 1}) - >>> c | d # union: max(c[x], d[x]) - Counter({'a': 3, 'b': 2}) - -Unary addition and subtraction are shortcuts for adding an empty counter -or subtracting from an empty counter. - - >>> c = Counter(a=2, b=-4) - >>> +c - Counter({'a': 2}) - >>> -c - Counter({'b': 4}) - -.. versionadded:: 3.3 - Added support for unary plus, unary minus, and in-place multiset operations. - -.. note:: - - Counters were primarily designed to work with positive integers to represent - running counts; however, care was taken to not unnecessarily preclude use - cases needing other types or negative values. To help with those use cases, - this section documents the minimum range and type restrictions. - - * The :class:`Counter` class itself is a dictionary subclass with no - restrictions on its keys and values. The values are intended to be numbers - representing counts, but you *could* store anything in the value field. - - * The :meth:`~Counter.most_common` method requires only that the values be orderable. - - * For in-place operations such as ``c[key] += 1``, the value type need only - support addition and subtraction. So fractions, floats, and decimals would - work and negative values are supported. The same is also true for - :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values - for both inputs and outputs. - - * The multiset methods are designed only for use cases with positive values. - The inputs may be negative or zero, but only outputs with positive values - are created. There are no type restrictions, but the value type needs to - support addition, subtraction, and comparison. - - * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and - negative counts. - -.. seealso:: - - * `Bag class `_ - in Smalltalk. - - * Wikipedia entry for `Multisets `_. - - * `C++ multisets `_ - tutorial with examples. - - * For mathematical operations on multisets and their use cases, see - *Knuth, Donald. The Art of Computer Programming Volume II, - Section 4.6.3, Exercise 19*. - - * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`:: - - map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC - - -:class:`deque` objects ----------------------- - -.. class:: deque([iterable, [maxlen]]) - - Returns a new deque object initialized left-to-right (using :meth:`append`) with - data from *iterable*. If *iterable* is not specified, the new deque is empty. - - Deques are a generalization of stacks and queues (the name is pronounced "deck" - and is short for "double-ended queue"). Deques support thread-safe, memory - efficient appends and pops from either side of the deque with approximately the - same O(1) performance in either direction. - - Though :class:`list` objects support similar operations, they are optimized for - fast fixed-length operations and incur O(n) memory movement costs for - ``pop(0)`` and ``insert(0, v)`` operations which change both the size and - position of the underlying data representation. - - - If *maxlen* is not specified or is ``None``, deques may grow to an - arbitrary length. Otherwise, the deque is bounded to the specified maximum - length. Once a bounded length deque is full, when new items are added, a - corresponding number of items are discarded from the opposite end. Bounded - length deques provide functionality similar to the ``tail`` filter in - Unix. They are also useful for tracking transactions and other pools of data - where only the most recent activity is of interest. - - - Deque objects support the following methods: - - .. method:: append(x) - - Add *x* to the right side of the deque. - - - .. method:: appendleft(x) - - Add *x* to the left side of the deque. - - - .. method:: clear() - - Remove all elements from the deque leaving it with length 0. - - - .. method:: copy() - - Create a shallow copy of the deque. - - .. versionadded:: 3.5 - - - .. method:: count(x) - - Count the number of deque elements equal to *x*. - - .. versionadded:: 3.2 - - - .. method:: extend(iterable) - - Extend the right side of the deque by appending elements from the iterable - argument. - - - .. method:: extendleft(iterable) - - Extend the left side of the deque by appending elements from *iterable*. - Note, the series of left appends results in reversing the order of - elements in the iterable argument. - - - .. method:: index(x[, start[, stop]]) - - Return the position of *x* in the deque (at or after index *start* - and before index *stop*). Returns the first match or raises - :exc:`ValueError` if not found. - - .. versionadded:: 3.5 - - - .. method:: insert(i, x) - - Insert *x* into the deque at position *i*. - - If the insertion would cause a bounded deque to grow beyond *maxlen*, - an :exc:`IndexError` is raised. - - .. versionadded:: 3.5 - - - .. method:: pop() - - Remove and return an element from the right side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: popleft() - - Remove and return an element from the left side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: remove(value) - - Remove the first occurrence of *value*. If not found, raises a - :exc:`ValueError`. - - - .. method:: reverse() - - Reverse the elements of the deque in-place and then return ``None``. - - .. versionadded:: 3.2 - - - .. method:: rotate(n=1) - - Rotate the deque *n* steps to the right. If *n* is negative, rotate - to the left. - - When the deque is not empty, rotating one step to the right is equivalent - to ``d.appendleft(d.pop())``, and rotating one step to the left is - equivalent to ``d.append(d.popleft())``. - - - Deque objects also provide one read-only attribute: - - .. attribute:: maxlen - - Maximum size of a deque or ``None`` if unbounded. - - .. versionadded:: 3.1 - - -In addition to the above, deques support iteration, pickling, ``len(d)``, -``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with -the :keyword:`in` operator, and subscript references such as ``d[0]`` to access -the first element. Indexed access is O(1) at both ends but slows to O(n) in -the middle. For fast random access, use lists instead. - -Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, -and ``__imul__()``. - -Example: - -.. doctest:: - - >>> from collections import deque - >>> d = deque('ghi') # make a new deque with three items - >>> for elem in d: # iterate over the deque's elements - ... print(elem.upper()) - G - H - I - - >>> d.append('j') # add a new entry to the right side - >>> d.appendleft('f') # add a new entry to the left side - >>> d # show the representation of the deque - deque(['f', 'g', 'h', 'i', 'j']) - - >>> d.pop() # return and remove the rightmost item - 'j' - >>> d.popleft() # return and remove the leftmost item - 'f' - >>> list(d) # list the contents of the deque - ['g', 'h', 'i'] - >>> d[0] # peek at leftmost item - 'g' - >>> d[-1] # peek at rightmost item - 'i' - - >>> list(reversed(d)) # list the contents of a deque in reverse - ['i', 'h', 'g'] - >>> 'h' in d # search the deque - True - >>> d.extend('jkl') # add multiple elements at once - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - >>> d.rotate(1) # right rotation - >>> d - deque(['l', 'g', 'h', 'i', 'j', 'k']) - >>> d.rotate(-1) # left rotation - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - - >>> deque(reversed(d)) # make a new deque in reverse order - deque(['l', 'k', 'j', 'i', 'h', 'g']) - >>> d.clear() # empty the deque - >>> d.pop() # cannot pop from an empty deque - Traceback (most recent call last): - File "", line 1, in -toplevel- - d.pop() - IndexError: pop from an empty deque - - >>> d.extendleft('abc') # extendleft() reverses the input order - >>> d - deque(['c', 'b', 'a']) - - -:class:`deque` Recipes -^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with deques. - -Bounded length deques provide functionality similar to the ``tail`` filter -in Unix:: - - def tail(filename, n=10): - 'Return the last n lines of a file' - with open(filename) as f: - return deque(f, n) - -Another approach to using deques is to maintain a sequence of recently -added elements by appending to the right and popping to the left:: - - def moving_average(iterable, n=3): - # moving_average([40, 30, 50, 46, 39, 44]) --> 40.0 42.0 45.0 43.0 - # http://en.wikipedia.org/wiki/Moving_average - it = iter(iterable) - d = deque(itertools.islice(it, n-1)) - d.appendleft(0) - s = sum(d) - for elem in it: - s += elem - d.popleft() - d.append(elem) - yield s / n - -A `round-robin scheduler -`_ can be implemented with -input iterators stored in a :class:`deque`. Values are yielded from the active -iterator in position zero. If that iterator is exhausted, it can be removed -with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with -the :meth:`~deque.rotate` method:: - - def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" - iterators = deque(map(iter, iterables)) - while iterators: - try: - while True: - yield next(iterators[0]) - iterators.rotate(-1) - except StopIteration: - # Remove an exhausted iterator. - iterators.popleft() - -The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and -deletion. For example, a pure Python implementation of ``del d[n]`` relies on -the ``rotate()`` method to position elements to be popped:: - - def delete_nth(d, n): - d.rotate(-n) - d.popleft() - d.rotate(n) - -To implement :class:`deque` slicing, use a similar approach applying -:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove -old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then -reverse the rotation. -With minor variations on that approach, it is easy to implement Forth style -stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, -``rot``, and ``roll``. - - -:class:`defaultdict` objects ----------------------------- - -.. class:: defaultdict([default_factory[, ...]]) - - Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the - built-in :class:`dict` class. It overrides one method and adds one writable - instance variable. The remaining functionality is the same as for the - :class:`dict` class and is not documented here. - - The first argument provides the initial value for the :attr:`default_factory` - attribute; it defaults to ``None``. All remaining arguments are treated the same - as if they were passed to the :class:`dict` constructor, including keyword - arguments. - - - :class:`defaultdict` objects support the following method in addition to the - standard :class:`dict` operations: - - .. method:: __missing__(key) - - If the :attr:`default_factory` attribute is ``None``, this raises a - :exc:`KeyError` exception with the *key* as argument. - - If :attr:`default_factory` is not ``None``, it is called without arguments - to provide a default value for the given *key*, this value is inserted in - the dictionary for the *key*, and returned. - - If calling :attr:`default_factory` raises an exception this exception is - propagated unchanged. - - This method is called by the :meth:`__getitem__` method of the - :class:`dict` class when the requested key is not found; whatever it - returns or raises is then returned or raised by :meth:`__getitem__`. - - Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`__getitem__`. This means that :meth:`get` will, like normal - dictionaries, return ``None`` as a default rather than using - :attr:`default_factory`. - - - :class:`defaultdict` objects support the following instance variable: - - - .. attribute:: default_factory - - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. - - .. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in - :pep:`584`. - - -:class:`defaultdict` Examples -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a -sequence of key-value pairs into a dictionary of lists: - - >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] - >>> d = defaultdict(list) - >>> for k, v in s: - ... d[k].append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -When each key is encountered for the first time, it is not already in the -mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`list.append` -operation then attaches the value to the new list. When keys are encountered -again, the look-up proceeds normally (returning the list for that key) and the -:meth:`list.append` operation adds another value to the list. This technique is -simpler and faster than an equivalent technique using :meth:`dict.setdefault`: - - >>> d = {} - >>> for k, v in s: - ... d.setdefault(k, []).append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the -:class:`defaultdict` useful for counting (like a bag or multiset in other -languages): - - >>> s = 'mississippi' - >>> d = defaultdict(int) - >>> for k in s: - ... d[k] += 1 - ... - >>> sorted(d.items()) - [('i', 4), ('m', 1), ('p', 2), ('s', 4)] - -When a letter is first encountered, it is missing from the mapping, so the -:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of -zero. The increment operation then builds up the count for each letter. - -The function :func:`int` which always returns zero is just a special case of -constant functions. A faster and more flexible way to create constant functions -is to use a lambda function which can supply any constant value (not just -zero): - - >>> def constant_factory(value): - ... return lambda: value - >>> d = defaultdict(constant_factory('')) - >>> d.update(name='John', action='ran') - >>> '%(name)s %(action)s to %(object)s' % d - 'John ran to ' - -Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the -:class:`defaultdict` useful for building a dictionary of sets: - - >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] - >>> d = defaultdict(set) - >>> for k, v in s: - ... d[k].add(v) - ... - >>> sorted(d.items()) - [('blue', {2, 4}), ('red', {1, 3})] - - -:func:`namedtuple` Factory Function for Tuples with Named Fields ----------------------------------------------------------------- - -Named tuples assign meaning to each position in a tuple and allow for more readable, -self-documenting code. They can be used wherever regular tuples are used, and -they add the ability to access fields by name instead of position index. - -.. function:: namedtuple(typename, field_names, *, rename=False, defaults=None, module=None) - - Returns a new tuple subclass named *typename*. The new subclass is used to - create tuple-like objects that have fields accessible by attribute lookup as - well as being indexable and iterable. Instances of the subclass also have a - helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` - method which lists the tuple contents in a ``name=value`` format. - - The *field_names* are a sequence of strings such as ``['x', 'y']``. - Alternatively, *field_names* can be a single string with each fieldname - separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. - - Any valid Python identifier may be used for a fieldname except for names - starting with an underscore. Valid identifiers consist of letters, digits, - and underscores but do not start with a digit or underscore and cannot be - a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, - or *raise*. - - If *rename* is true, invalid fieldnames are automatically replaced - with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is - converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword - ``def`` and the duplicate fieldname ``abc``. - - *defaults* can be ``None`` or an :term:`iterable` of default values. - Since fields with a default value must come after any fields without a - default, the *defaults* are applied to the rightmost parameters. For - example, if the fieldnames are ``['x', 'y', 'z']`` and the defaults are - ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to - ``1``, and ``z`` will default to ``2``. - - If *module* is defined, the ``__module__`` attribute of the named tuple is - set to that value. - - Named tuple instances do not have per-instance dictionaries, so they are - lightweight and require no more memory than regular tuples. - - To support pickling, the named tuple class should be assigned to a variable - that matches *typename*. - - .. versionchanged:: 3.1 - Added support for *rename*. - - .. versionchanged:: 3.6 - The *verbose* and *rename* parameters became - :ref:`keyword-only arguments `. - - .. versionchanged:: 3.6 - Added the *module* parameter. - - .. versionchanged:: 3.7 - Removed the *verbose* parameter and the :attr:`_source` attribute. - - .. versionchanged:: 3.7 - Added the *defaults* parameter and the :attr:`_field_defaults` - attribute. - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> # Basic example - >>> Point = namedtuple('Point', ['x', 'y']) - >>> p = Point(11, y=22) # instantiate with positional or keyword arguments - >>> p[0] + p[1] # indexable like the plain tuple (11, 22) - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessible by name - 33 - >>> p # readable __repr__ with a name=value style - Point(x=11, y=22) - -Named tuples are especially useful for assigning field names to result tuples returned -by the :mod:`csv` or :mod:`sqlite3` modules:: - - EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') - - import csv - for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): - print(emp.name, emp.title) - - import sqlite3 - conn = sqlite3.connect('/companydata') - cursor = conn.cursor() - cursor.execute('SELECT name, age, title, department, paygrade FROM employees') - for emp in map(EmployeeRecord._make, cursor.fetchall()): - print(emp.name, emp.title) - -In addition to the methods inherited from tuples, named tuples support -three additional methods and two attributes. To prevent conflicts with -field names, the method and attribute names start with an underscore. - -.. classmethod:: somenamedtuple._make(iterable) - - Class method that makes a new instance from an existing sequence or iterable. - - .. doctest:: - - >>> t = [11, 22] - >>> Point._make(t) - Point(x=11, y=22) - -.. method:: somenamedtuple._asdict() - - Return a new :class:`dict` which maps field names to their corresponding - values: - - .. doctest:: - - >>> p = Point(x=11, y=22) - >>> p._asdict() - {'x': 11, 'y': 22} - - .. versionchanged:: 3.1 - Returns an :class:`OrderedDict` instead of a regular :class:`dict`. - - .. versionchanged:: 3.8 - Returns a regular :class:`dict` instead of an :class:`OrderedDict`. - As of Python 3.7, regular dicts are guaranteed to be ordered. If the - extra features of :class:`OrderedDict` are required, the suggested - remediation is to cast the result to the desired type: - ``OrderedDict(nt._asdict())``. - -.. method:: somenamedtuple._replace(**kwargs) - - Return a new instance of the named tuple replacing specified fields with new - values:: - - >>> p = Point(x=11, y=22) - >>> p._replace(x=33) - Point(x=33, y=22) - - >>> for partnum, record in inventory.items(): - ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) - -.. attribute:: somenamedtuple._fields - - Tuple of strings listing the field names. Useful for introspection - and for creating new named tuple types from existing named tuples. - - .. doctest:: - - >>> p._fields # view the field names - ('x', 'y') - - >>> Color = namedtuple('Color', 'red green blue') - >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) - >>> Pixel(11, 22, 128, 255, 0) - Pixel(x=11, y=22, red=128, green=255, blue=0) - -.. attribute:: somenamedtuple._field_defaults - - Dictionary mapping field names to default values. - - .. doctest:: - - >>> Account = namedtuple('Account', ['type', 'balance'], defaults=[0]) - >>> Account._field_defaults - {'balance': 0} - >>> Account('premium') - Account(type='premium', balance=0) - -To retrieve a field whose name is stored in a string, use the :func:`getattr` -function: - - >>> getattr(p, 'x') - 11 - -To convert a dictionary to a named tuple, use the double-star-operator -(as described in :ref:`tut-unpacking-arguments`): - - >>> d = {'x': 11, 'y': 22} - >>> Point(**d) - Point(x=11, y=22) - -Since a named tuple is a regular Python class, it is easy to add or change -functionality with a subclass. Here is how to add a calculated field and -a fixed-width print format: - -.. doctest:: - - >>> class Point(namedtuple('Point', ['x', 'y'])): - ... __slots__ = () - ... @property - ... def hypot(self): - ... return (self.x ** 2 + self.y ** 2) ** 0.5 - ... def __str__(self): - ... return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) - - >>> for p in Point(3, 4), Point(14, 5/7): - ... print(p) - Point: x= 3.000 y= 4.000 hypot= 5.000 - Point: x=14.000 y= 0.714 hypot=14.018 - -The subclass shown above sets ``__slots__`` to an empty tuple. This helps -keep memory requirements low by preventing the creation of instance dictionaries. - -Subclassing is not useful for adding new, stored fields. Instead, simply -create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: - - >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) - -Docstrings can be customized by making direct assignments to the ``__doc__`` -fields: - - >>> Book = namedtuple('Book', ['id', 'title', 'authors']) - >>> Book.__doc__ += ': Hardcover book in active collection' - >>> Book.id.__doc__ = '13-digit ISBN' - >>> Book.title.__doc__ = 'Title of first printing' - >>> Book.authors.__doc__ = 'List of authors sorted by last name' - -.. versionchanged:: 3.5 - Property docstrings became writeable. - -.. seealso:: - - * See :class:`typing.NamedTuple` for a way to add type hints for named - tuples. It also provides an elegant notation using the :keyword:`class` - keyword:: - - class Component(NamedTuple): - part_number: int - weight: float - description: Optional[str] = None - - * See :meth:`types.SimpleNamespace` for a mutable namespace based on an - underlying dictionary instead of a tuple. - - * The :mod:`dataclasses` module provides a decorator and functions for - automatically adding generated special methods to user-defined classes. - - -:class:`OrderedDict` objects ----------------------------- - -Ordered dictionaries are just like regular dictionaries but have some extra -capabilities relating to ordering operations. They have become less -important now that the built-in :class:`dict` class gained the ability -to remember insertion order (this new behavior became guaranteed in -Python 3.7). - -Some differences from :class:`dict` still remain: - -* The regular :class:`dict` was designed to be very good at mapping - operations. Tracking insertion order was secondary. - -* The :class:`OrderedDict` was designed to be good at reordering operations. - Space efficiency, iteration speed, and the performance of update - operations were secondary. - -* Algorithmically, :class:`OrderedDict` can handle frequent reordering - operations better than :class:`dict`. This makes it suitable for tracking - recent accesses (for example in an `LRU cache - `_). - -* The equality operation for :class:`OrderedDict` checks for matching order. - -* The :meth:`popitem` method of :class:`OrderedDict` has a different - signature. It accepts an optional argument to specify which item is popped. - -* :class:`OrderedDict` has a :meth:`move_to_end` method to - efficiently reposition an element to an endpoint. - -* Until Python 3.8, :class:`dict` lacked a :meth:`__reversed__` method. - - -.. class:: OrderedDict([items]) - - Return an instance of a :class:`dict` subclass that has methods - specialized for rearranging dictionary order. - - .. versionadded:: 3.1 - - .. method:: popitem(last=True) - - The :meth:`popitem` method for ordered dictionaries returns and removes a - (key, value) pair. The pairs are returned in - :abbr:`LIFO (last-in, first-out)` order if *last* is true - or :abbr:`FIFO (first-in, first-out)` order if false. - - .. method:: move_to_end(key, last=True) - - Move an existing *key* to either end of an ordered dictionary. The item - is moved to the right end if *last* is true (the default) or to the - beginning if *last* is false. Raises :exc:`KeyError` if the *key* does - not exist:: - - >>> d = OrderedDict.fromkeys('abcde') - >>> d.move_to_end('b') - >>> ''.join(d.keys()) - 'acdeb' - >>> d.move_to_end('b', last=False) - >>> ''.join(d.keys()) - 'bacde' - - .. versionadded:: 3.2 - -In addition to the usual mapping methods, ordered dictionaries also support -reverse iteration using :func:`reversed`. - -Equality tests between :class:`OrderedDict` objects are order-sensitive -and are implemented as ``list(od1.items())==list(od2.items())``. -Equality tests between :class:`OrderedDict` objects and other -:class:`~collections.abc.Mapping` objects are order-insensitive like regular -dictionaries. This allows :class:`OrderedDict` objects to be substituted -anywhere a regular dictionary is used. - -.. versionchanged:: 3.5 - The items, keys, and values :term:`views ` - of :class:`OrderedDict` now support reverse iteration using :func:`reversed`. - -.. versionchanged:: 3.6 - With the acceptance of :pep:`468`, order is retained for keyword arguments - passed to the :class:`OrderedDict` constructor and its :meth:`update` - method. - -.. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. - - -:class:`OrderedDict` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is straightforward to create an ordered dictionary variant -that remembers the order the keys were *last* inserted. -If a new entry overwrites an existing entry, the -original insertion position is changed and moved to the end:: - - class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' - - def __setitem__(self, key, value): - super().__setitem__(key, value) - self.move_to_end(key) - -An :class:`OrderedDict` would also be useful for implementing -variants of :func:`functools.lru_cache`:: - - class LRU(OrderedDict): - 'Limit size, evicting the least recently looked-up key when full' - - def __init__(self, maxsize=128, /, *args, **kwds): - self.maxsize = maxsize - super().__init__(*args, **kwds) - - def __getitem__(self, key): - value = super().__getitem__(key) - self.move_to_end(key) - return value - - def __setitem__(self, key, value): - if key in self: - self.move_to_end(key) - super().__setitem__(key, value) - if len(self) > self.maxsize: - oldest = next(iter(self)) - del self[oldest] - - -:class:`UserDict` objects -------------------------- - -The class, :class:`UserDict` acts as a wrapper around dictionary objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`dict`; however, this class can be easier -to work with because the underlying dictionary is accessible as an -attribute. - -.. class:: UserDict([initialdata]) - - Class that simulates a dictionary. The instance's contents are kept in a - regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it be used for other purposes. - - In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: - - .. attribute:: data - - A real dictionary used to store the contents of the :class:`UserDict` - class. - - - -:class:`UserList` objects -------------------------- - -This class acts as a wrapper around list objects. It is a useful base class -for your own list-like classes which can inherit from them and override -existing methods or add new ones. In this way, one can add new behaviors to -lists. - -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`list`; however, this class can be easier -to work with because the underlying list is accessible as an attribute. - -.. class:: UserList([list]) - - Class that simulates a list. The instance's contents are kept in a regular - list, which is accessible via the :attr:`data` attribute of :class:`UserList` - instances. The instance's contents are initially set to a copy of *list*, - defaulting to the empty list ``[]``. *list* can be any iterable, for - example a real Python list or a :class:`UserList` object. - - In addition to supporting the methods and operations of mutable sequences, - :class:`UserList` instances provide the following attribute: - - .. attribute:: data - - A real :class:`list` object used to store the contents of the - :class:`UserList` class. - -**Subclassing requirements:** Subclasses of :class:`UserList` are expected to -offer a constructor which can be called with either no arguments or one -argument. List operations which return a new sequence attempt to create an -instance of the actual implementation class. To do so, it assumes that the -constructor can be called with a single parameter, which is a sequence object -used as a data source. - -If a derived class does not wish to comply with this requirement, all of the -special methods supported by this class will need to be overridden; please -consult the sources for information about the methods which need to be provided -in that case. - -:class:`UserString` objects ---------------------------- - -The class, :class:`UserString` acts as a wrapper around string objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`str`; however, this class can be easier -to work with because the underlying string is accessible as an -attribute. - -.. class:: UserString(seq) - - Class that simulates a string object. The instance's - content is kept in a regular string object, which is accessible via the - :attr:`data` attribute of :class:`UserString` instances. The instance's - contents are initially set to a copy of *seq*. The *seq* argument can - be any object which can be converted into a string using the built-in - :func:`str` function. - - In addition to supporting the methods and operations of strings, - :class:`UserString` instances provide the following attribute: - - .. attribute:: data - - A real :class:`str` object used to store the contents of the - :class:`UserString` class. - - .. versionchanged:: 3.5 - New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, - ``format_map``, ``isprintable``, and ``maketrans``. diff --git a/Doc/library/dataclasses.rst.bak b/Doc/library/dataclasses.rst.bak deleted file mode 100644 index e706f7fcc566d8..00000000000000 --- a/Doc/library/dataclasses.rst.bak +++ /dev/null @@ -1,595 +0,0 @@ -:mod:`dataclasses` --- Data Classes -=================================== - -.. module:: dataclasses - :synopsis: Generate special methods on user-defined classes. - -.. moduleauthor:: Eric V. Smith -.. sectionauthor:: Eric V. Smith - -**Source code:** :source:`Lib/dataclasses.py` - --------------- - -This module provides a decorator and functions for automatically -adding generated :term:`special method`\s such as :meth:`__init__` and -:meth:`__repr__` to user-defined classes. It was originally described -in :pep:`557`. - -The member variables to use in these generated methods are defined -using :pep:`526` type annotations. For example this code:: - - from dataclasses import dataclass - - @dataclass - class InventoryItem: - """Class for keeping track of an item in inventory.""" - name: str - unit_price: float - quantity_on_hand: int = 0 - - def total_cost(self) -> float: - return self.unit_price * self.quantity_on_hand - -Will add, among other things, a :meth:`__init__` that looks like:: - - def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0): - self.name = name - self.unit_price = unit_price - self.quantity_on_hand = quantity_on_hand - -Note that this method is automatically added to the class: it is not -directly specified in the ``InventoryItem`` definition shown above. - -.. versionadded:: 3.7 - -Module-level decorators, classes, and functions ------------------------------------------------ - -.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) - - This function is a :term:`decorator` that is used to add generated - :term:`special method`\s to classes, as described below. - - The :func:`dataclass` decorator examines the class to find - ``field``\s. A ``field`` is defined as class variable that has a - :term:`type annotation `. With two - exceptions described below, nothing in :func:`dataclass` - examines the type specified in the variable annotation. - - The order of the fields in all of the generated methods is the - order in which they appear in the class definition. - - The :func:`dataclass` decorator will add various "dunder" methods to - the class, described below. If any of the added methods already - exist on the class, the behavior depends on the parameter, as documented - below. The decorator returns the same class that is called on; no new - class is created. - - If :func:`dataclass` is used just as a simple decorator with no parameters, - it acts as if it has the default values documented in this - signature. That is, these three uses of :func:`dataclass` are - equivalent:: - - @dataclass - class C: - ... - - @dataclass() - class C: - ... - - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) - class C: - ... - - The parameters to :func:`dataclass` are: - - - ``init``: If true (the default), a :meth:`__init__` method will be - generated. - - If the class already defines :meth:`__init__`, this parameter is - ignored. - - - ``repr``: If true (the default), a :meth:`__repr__` method will be - generated. The generated repr string will have the class name and - the name and repr of each field, in the order they are defined in - the class. Fields that are marked as being excluded from the repr - are not included. For example: - ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. - - If the class already defines :meth:`__repr__`, this parameter is - ignored. - - - ``eq``: If true (the default), an :meth:`__eq__` method will be - generated. This method compares the class as if it were a tuple - of its fields, in order. Both instances in the comparison must - be of the identical type. - - If the class already defines :meth:`__eq__`, this parameter is - ignored. - - - ``order``: If true (the default is ``False``), :meth:`__lt__`, - :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` methods will be - generated. These compare the class as if it were a tuple of its - fields, in order. Both instances in the comparison must be of the - identical type. If ``order`` is true and ``eq`` is false, a - :exc:`ValueError` is raised. - - If the class already defines any of :meth:`__lt__`, - :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then - :exc:`TypeError` is raised. - - - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method - is generated according to how ``eq`` and ``frozen`` are set. - - :meth:`__hash__` is used by built-in :meth:`hash()`, and when objects are - added to hashed collections such as dictionaries and sets. Having a - :meth:`__hash__` implies that instances of the class are immutable. - Mutability is a complicated property that depends on the programmer's - intent, the existence and behavior of :meth:`__eq__`, and the values of - the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. - - By default, :func:`dataclass` will not implicitly add a :meth:`__hash__` - method unless it is safe to do so. Neither will it add or change an - existing explicitly defined :meth:`__hash__` method. Setting the class - attribute ``__hash__ = None`` has a specific meaning to Python, as - described in the :meth:`__hash__` documentation. - - If :meth:`__hash__` is not explicit defined, or if it is set to ``None``, - then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. - Although not recommended, you can force :func:`dataclass` to create a - :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case - if your class is logically immutable but can nonetheless be mutated. - This is a specialized use case and should be considered carefully. - - Here are the rules governing implicit creation of a :meth:`__hash__` - method. Note that you cannot both have an explicit :meth:`__hash__` - method in your dataclass and set ``unsafe_hash=True``; this will result - in a :exc:`TypeError`. - - If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will - generate a :meth:`__hash__` method for you. If ``eq`` is true and - ``frozen`` is false, :meth:`__hash__` will be set to ``None``, marking it - unhashable (which it is, since it is mutable). If ``eq`` is false, - :meth:`__hash__` will be left untouched meaning the :meth:`__hash__` - method of the superclass will be used (if the superclass is - :class:`object`, this means it will fall back to id-based hashing). - - - ``frozen``: If true (the default is ``False``), assigning to fields will - generate an exception. This emulates read-only frozen instances. If - :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then - :exc:`TypeError` is raised. See the discussion below. - - ``field``\s may optionally specify a default value, using normal - Python syntax:: - - @dataclass - class C: - a: int # 'a' has no default value - b: int = 0 # assign a default value for 'b' - - In this example, both ``a`` and ``b`` will be included in the added - :meth:`__init__` method, which will be defined as:: - - def __init__(self, a: int, b: int = 0): - - :exc:`TypeError` will be raised if a field without a default value - follows a field with a default value. This is true either when this - occurs in a single class, or as a result of class inheritance. - -.. function:: field(*, default=MISSING, default_factory=MISSING, repr=True, hash=None, init=True, compare=True, metadata=None) - - For common and simple use cases, no other functionality is - required. There are, however, some dataclass features that - require additional per-field information. To satisfy this need for - additional information, you can replace the default field value - with a call to the provided :func:`field` function. For example:: - - @dataclass - class C: - mylist: list[int] = field(default_factory=list) - - c = C() - c.mylist += [1, 2, 3] - - As shown above, the ``MISSING`` value is a sentinel object used to - detect if the ``default`` and ``default_factory`` parameters are - provided. This sentinel is used because ``None`` is a valid value - for ``default``. No code should directly use the ``MISSING`` - value. - - The parameters to :func:`field` are: - - - ``default``: If provided, this will be the default value for this - field. This is needed because the :meth:`field` call itself - replaces the normal position of the default value. - - - ``default_factory``: If provided, it must be a zero-argument - callable that will be called when a default value is needed for - this field. Among other purposes, this can be used to specify - fields with mutable default values, as discussed below. It is an - error to specify both ``default`` and ``default_factory``. - - - ``init``: If true (the default), this field is included as a - parameter to the generated :meth:`__init__` method. - - - ``repr``: If true (the default), this field is included in the - string returned by the generated :meth:`__repr__` method. - - - ``compare``: If true (the default), this field is included in the - generated equality and comparison methods (:meth:`__eq__`, - :meth:`__gt__`, et al.). - - - ``hash``: This can be a bool or ``None``. If true, this field is - included in the generated :meth:`__hash__` method. If ``None`` (the - default), use the value of ``compare``: this would normally be - the expected behavior. A field should be considered in the hash - if it's used for comparisons. Setting this value to anything - other than ``None`` is discouraged. - - One possible reason to set ``hash=False`` but ``compare=True`` - would be if a field is expensive to compute a hash value for, - that field is needed for equality testing, and there are other - fields that contribute to the type's hash value. Even if a field - is excluded from the hash, it will still be used for comparisons. - - - ``metadata``: This can be a mapping or None. None is treated as - an empty dict. This value is wrapped in - :func:`~types.MappingProxyType` to make it read-only, and exposed - on the :class:`Field` object. It is not used at all by Data - Classes, and is provided as a third-party extension mechanism. - Multiple third-parties can each have their own key, to use as a - namespace in the metadata. - - If the default value of a field is specified by a call to - :func:`field()`, then the class attribute for this field will be - replaced by the specified ``default`` value. If no ``default`` is - provided, then the class attribute will be deleted. The intent is - that after the :func:`dataclass` decorator runs, the class - attributes will all contain the default values for the fields, just - as if the default value itself were specified. For example, - after:: - - @dataclass - class C: - x: int - y: int = field(repr=False) - z: int = field(repr=False, default=10) - t: int = 20 - - The class attribute ``C.z`` will be ``10``, the class attribute - ``C.t`` will be ``20``, and the class attributes ``C.x`` and - ``C.y`` will not be set. - -.. class:: Field - - :class:`Field` objects describe each defined field. These objects - are created internally, and are returned by the :func:`fields` - module-level method (see below). Users should never instantiate a - :class:`Field` object directly. Its documented attributes are: - - - ``name``: The name of the field. - - - ``type``: The type of the field. - - - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, - ``compare``, and ``metadata`` have the identical meaning and - values as they do in the :func:`field` declaration. - - Other attributes may exist, but they are private and must not be - inspected or relied on. - -.. function:: fields(class_or_instance) - - Returns a tuple of :class:`Field` objects that define the fields for this - dataclass. Accepts either a dataclass, or an instance of a dataclass. - Raises :exc:`TypeError` if not passed a dataclass or instance of one. - Does not return pseudo-fields which are ``ClassVar`` or ``InitVar``. - -.. function:: asdict(instance, *, dict_factory=dict) - - Converts the dataclass ``instance`` to a dict (by using the - factory function ``dict_factory``). Each dataclass is converted - to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. For example:: - - @dataclass - class Point: - x: int - y: int - - @dataclass - class C: - mylist: list[Point] - - p = Point(10, 20) - assert asdict(p) == {'x': 10, 'y': 20} - - c = C([Point(0, 0), Point(10, 4)]) - assert asdict(c) == {'mylist': [{'x': 0, 'y': 0}, {'x': 10, 'y': 4}]} - - Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. - -.. function:: astuple(instance, *, tuple_factory=tuple) - - Converts the dataclass ``instance`` to a tuple (by using the - factory function ``tuple_factory``). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. - - Continuing from the previous example:: - - assert astuple(p) == (10, 20) - assert astuple(c) == ([(0, 0), (10, 4)],) - - Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. - -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) - - Creates a new dataclass with name ``cls_name``, fields as defined - in ``fields``, base classes as given in ``bases``, and initialized - with a namespace as given in ``namespace``. ``fields`` is an - iterable whose elements are each either ``name``, ``(name, type)``, - or ``(name, type, Field)``. If just ``name`` is supplied, - ``typing.Any`` is used for ``type``. The values of ``init``, - ``repr``, ``eq``, ``order``, ``unsafe_hash``, and ``frozen`` have - the same meaning as they do in :func:`dataclass`. - - This function is not strictly required, because any Python - mechanism for creating a new class with ``__annotations__`` can - then apply the :func:`dataclass` function to convert that class to - a dataclass. This function is provided as a convenience. For - example:: - - C = make_dataclass('C', - [('x', int), - 'y', - ('z', int, field(default=5))], - namespace={'add_one': lambda self: self.x + 1}) - - Is equivalent to:: - - @dataclass - class C: - x: int - y: 'typing.Any' - z: int = 5 - - def add_one(self): - return self.x + 1 - -.. function:: replace(instance, /, **changes) - - Creates a new object of the same type of ``instance``, replacing - fields with values from ``changes``. If ``instance`` is not a Data - Class, raises :exc:`TypeError`. If values in ``changes`` do not - specify fields, raises :exc:`TypeError`. - - The newly returned object is created by calling the :meth:`__init__` - method of the dataclass. This ensures that - :meth:`__post_init__`, if present, is also called. - - Init-only variables without default values, if any exist, must be - specified on the call to :func:`replace` so that they can be passed to - :meth:`__init__` and :meth:`__post_init__`. - - It is an error for ``changes`` to contain any fields that are - defined as having ``init=False``. A :exc:`ValueError` will be raised - in this case. - - Be forewarned about how ``init=False`` fields work during a call to - :func:`replace`. They are not copied from the source object, but - rather are initialized in :meth:`__post_init__`, if they're - initialized at all. It is expected that ``init=False`` fields will - be rarely and judiciously used. If they are used, it might be wise - to have alternate class constructors, or perhaps a custom - ``replace()`` (or similarly named) method which handles instance - copying. - -.. function:: is_dataclass(class_or_instance) - - Return ``True`` if its parameter is a dataclass or an instance of one, - otherwise return ``False``. - - If you need to know if a class is an instance of a dataclass (and - not a dataclass itself), then add a further check for ``not - isinstance(obj, type)``:: - - def is_dataclass_instance(obj): - return is_dataclass(obj) and not isinstance(obj, type) - -Post-init processing --------------------- - -The generated :meth:`__init__` code will call a method named -:meth:`__post_init__`, if :meth:`__post_init__` is defined on the -class. It will normally be called as ``self.__post_init__()``. -However, if any ``InitVar`` fields are defined, they will also be -passed to :meth:`__post_init__` in the order they were defined in the -class. If no :meth:`__init__` method is generated, then -:meth:`__post_init__` will not automatically be called. - -Among other uses, this allows for initializing field values that -depend on one or more other fields. For example:: - - @dataclass - class C: - a: float - b: float - c: float = field(init=False) - - def __post_init__(self): - self.c = self.a + self.b - -See the section below on init-only variables for ways to pass -parameters to :meth:`__post_init__`. Also see the warning about how -:func:`replace` handles ``init=False`` fields. - -Class variables ---------------- - -One of two places where :func:`dataclass` actually inspects the type -of a field is to determine if a field is a class variable as defined -in :pep:`526`. It does this by checking if the type of the field is -``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded -from consideration as a field and is ignored by the dataclass -mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the -module-level :func:`fields` function. - -Init-only variables -------------------- - -The other place where :func:`dataclass` inspects a type annotation is to -determine if a field is an init-only variable. It does this by seeing -if the type of a field is of type ``dataclasses.InitVar``. If a field -is an ``InitVar``, it is considered a pseudo-field called an init-only -field. As it is not a true field, it is not returned by the -module-level :func:`fields` function. Init-only fields are added as -parameters to the generated :meth:`__init__` method, and are passed to -the optional :meth:`__post_init__` method. They are not otherwise used -by dataclasses. - -For example, suppose a field will be initialized from a database, if a -value is not provided when creating the class:: - - @dataclass - class C: - i: int - j: int = None - database: InitVar[DatabaseType] = None - - def __post_init__(self, database): - if self.j is None and database is not None: - self.j = database.lookup('j') - - c = C(10, database=my_database) - -In this case, :func:`fields` will return :class:`Field` objects for ``i`` and -``j``, but not for ``database``. - -Frozen instances ----------------- - -It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :meth:`dataclass` decorator you can -emulate immutability. In that case, dataclasses will add -:meth:`__setattr__` and :meth:`__delattr__` methods to the class. These -methods will raise a :exc:`FrozenInstanceError` when invoked. - -There is a tiny performance penalty when using ``frozen=True``: -:meth:`__init__` cannot use simple assignment to initialize fields, and -must use :meth:`object.__setattr__`. - -Inheritance ------------ - -When the dataclass is being created by the :meth:`dataclass` decorator, -it looks through all of the class's base classes in reverse MRO (that -is, starting at :class:`object`) and, for each dataclass that it finds, -adds the fields from that base class to an ordered mapping of fields. -After all of the base class fields are added, it adds its own fields -to the ordered mapping. All of the generated methods will use this -combined, calculated ordered mapping of fields. Because the fields -are in insertion order, derived classes override base classes. An -example:: - - @dataclass - class Base: - x: Any = 15.0 - y: int = 0 - - @dataclass - class C(Base): - z: int = 10 - x: int = 15 - -The final list of fields is, in order, ``x``, ``y``, ``z``. The final -type of ``x`` is ``int``, as specified in class ``C``. - -The generated :meth:`__init__` method for ``C`` will look like:: - - def __init__(self, x: int = 15, y: int = 0, z: int = 10): - -Default factory functions -------------------------- - - If a :func:`field` specifies a ``default_factory``, it is called with - zero arguments when a default value for the field is needed. For - example, to create a new instance of a list, use:: - - mylist: list = field(default_factory=list) - - If a field is excluded from :meth:`__init__` (using ``init=False``) - and the field also specifies ``default_factory``, then the default - factory function will always be called from the generated - :meth:`__init__` function. This happens because there is no other - way to give the field an initial value. - -Mutable default values ----------------------- - - Python stores default member variable values in class attributes. - Consider this example, not using dataclasses:: - - class C: - x = [] - def add(self, element): - self.x.append(element) - - o1 = C() - o2 = C() - o1.add(1) - o2.add(2) - assert o1.x == [1, 2] - assert o1.x is o2.x - - Note that the two instances of class ``C`` share the same class - variable ``x``, as expected. - - Using dataclasses, *if* this code was valid:: - - @dataclass - class D: - x: List = [] - def add(self, element): - self.x += element - - it would generate code similar to:: - - class D: - x = [] - def __init__(self, x=x): - self.x = x - def add(self, element): - self.x += element - - assert D().x is D().x - - This has the same issue as the original example using class ``C``. - That is, two instances of class ``D`` that do not specify a value for - ``x`` when creating a class instance will share the same copy of - ``x``. Because dataclasses just use normal Python class creation - they also share this behavior. There is no general way for Data - Classes to detect this condition. Instead, dataclasses will raise a - :exc:`TypeError` if it detects a default parameter of type ``list``, - ``dict``, or ``set``. This is a partial solution, but it does protect - against many common errors. - - Using default factory functions is a way to create new instances of - mutable types as default values for fields:: - - @dataclass - class D: - x: list = field(default_factory=list) - - assert D().x is not D().x - -Exceptions ----------- - -.. exception:: FrozenInstanceError - - Raised when an implicitly defined :meth:`__setattr__` or - :meth:`__delattr__` is called on a dataclass which was defined with - ``frozen=True``. diff --git a/Doc/library/gc.rst.bak b/Doc/library/gc.rst.bak deleted file mode 100644 index a3d201d5055c8e..00000000000000 --- a/Doc/library/gc.rst.bak +++ /dev/null @@ -1,322 +0,0 @@ -:mod:`gc` --- Garbage Collector interface -========================================= - -.. module:: gc - :synopsis: Interface to the cycle-detecting garbage collector. - -.. moduleauthor:: Neil Schemenauer -.. sectionauthor:: Neil Schemenauer - --------------- - -This module provides an interface to the optional garbage collector. It -provides the ability to disable the collector, tune the collection frequency, -and set debugging options. It also provides access to unreachable objects that -the collector found but cannot free. Since the collector supplements the -reference counting already used in Python, you can disable the collector if you -are sure your program does not create reference cycles. Automatic collection -can be disabled by calling ``gc.disable()``. To debug a leaking program call -``gc.set_debug(gc.DEBUG_LEAK)``. Notice that this includes -``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in -gc.garbage for inspection. - -The :mod:`gc` module provides the following functions: - - -.. function:: enable() - - Enable automatic garbage collection. - - -.. function:: disable() - - Disable automatic garbage collection. - - -.. function:: isenabled() - - Return ``True`` if automatic collection is enabled. - - -.. function:: collect(generation=2) - - With no arguments, run a full collection. The optional argument *generation* - may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The number of - unreachable objects found is returned. - - The free lists maintained for a number of built-in types are cleared - whenever a full collection or collection of the highest generation (2) - is run. Not all items in some free lists may be freed due to the - particular implementation, in particular :class:`float`. - - -.. function:: set_debug(flags) - - Set the garbage collection debugging flags. Debugging information will be - written to ``sys.stderr``. See below for a list of debugging flags which can be - combined using bit operations to control debugging. - - -.. function:: get_debug() - - Return the debugging flags currently set. - - -.. function:: get_objects(generation=None) - - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not None, return only the objects tracked by - the collector that are in that generation. - - .. versionchanged:: 3.8 - New *generation* parameter. - -.. function:: get_stats() - - Return a list of three per-generation dictionaries containing collection - statistics since interpreter start. The number of keys may change - in the future, but currently each dictionary will contain the following - items: - - * ``collections`` is the number of times this generation was collected; - - * ``collected`` is the total number of objects collected inside this - generation; - - * ``uncollectable`` is the total number of objects which were found - to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. - - .. versionadded:: 3.4 - - -.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) - - Set the garbage collection thresholds (the collection frequency). Setting - *threshold0* to zero disables collection. - - The GC classifies objects into three generations depending on how many - collection sweeps they have survived. New objects are placed in the youngest - generation (generation ``0``). If an object survives a collection it is moved - into the next older generation. Since generation ``2`` is the oldest - generation, objects in that generation remain there after a collection. In - order to decide when to run, the collector keeps track of the number object - allocations and deallocations since the last collection. When the number of - allocations minus the number of deallocations exceeds *threshold0*, collection - starts. Initially only generation ``0`` is examined. If generation ``0`` has - been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. - With the third generation, things are a bit more complicated, - see `Collecting the oldest generation `_ for more information. - - -.. function:: get_count() - - Return the current collection counts as a tuple of ``(count0, count1, - count2)``. - - -.. function:: get_threshold() - - Return the current collection thresholds as a tuple of ``(threshold0, - threshold1, threshold2)``. - - -.. function:: get_referrers(*objs) - - Return the list of objects that directly refer to any of objs. This function - will only locate those containers which support garbage collection; extension - types which do refer to other objects but do not support garbage collection will - not be found. - - Note that objects which have already been dereferenced, but which live in cycles - and have not yet been collected by the garbage collector can be listed among the - resulting referrers. To get only currently live objects, call :func:`collect` - before calling :func:`get_referrers`. - - .. warning:: - Care must be taken when using objects returned by :func:`get_referrers` because - some of them could still be under construction and hence in a temporarily - invalid state. Avoid using :func:`get_referrers` for any purpose other than - debugging. - - -.. function:: get_referents(*objs) - - Return a list of objects directly referred to by any of the arguments. The - referents returned are those objects visited by the arguments' C-level - :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually - directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects - that support garbage collection, and are only required to visit objects that may - be involved in a cycle. So, for example, if an integer is directly reachable - from an argument, that integer object may or may not appear in the result list. - - -.. function:: is_tracked(obj) - - Returns ``True`` if the object is currently tracked by the garbage collector, - ``False`` otherwise. As a general rule, instances of atomic types aren't - tracked and instances of non-atomic types (containers, user-defined - objects...) are. However, some type-specific optimizations can be present - in order to suppress the garbage collector footprint of simple instances - (e.g. dicts containing only atomic keys and values):: - - >>> gc.is_tracked(0) - False - >>> gc.is_tracked("a") - False - >>> gc.is_tracked([]) - True - >>> gc.is_tracked({}) - False - >>> gc.is_tracked({"a": 1}) - False - >>> gc.is_tracked({"a": []}) - True - - .. versionadded:: 3.1 - - -.. function:: is_finalized(obj) - - Returns ``True`` if the given object has been finalized by the - garbage collector, ``False`` otherwise. :: - - >>> x = None - >>> class Lazarus: - ... def __del__(self): - ... global x - ... x = self - ... - >>> lazarus = Lazarus() - >>> gc.is_finalized(lazarus) - False - >>> del lazarus - >>> gc.is_finalized(x) - True - - .. versionadded:: 3.9 - - -.. function:: freeze() - - Freeze all the objects tracked by gc - move them to a permanent generation - and ignore all the future collections. This can be used before a POSIX - fork() call to make the gc copy-on-write friendly or to speed up collection. - Also collection before a POSIX fork() call may free pages for future - allocation which can cause copy-on-write too so it's advised to disable gc - in parent process and freeze before fork and enable gc in child process. - - .. versionadded:: 3.7 - - -.. function:: unfreeze() - - Unfreeze the objects in the permanent generation, put them back into the - oldest generation. - - .. versionadded:: 3.7 - - -.. function:: get_freeze_count() - - Return the number of objects in the permanent generation. - - .. versionadded:: 3.7 - - -The following variables are provided for read-only access (you can mutate the -values but should not rebind them): - -.. data:: garbage - - A list of objects which the collector found to be unreachable but could - not be freed (uncollectable objects). Starting with Python 3.4, this - list should be empty most of the time, except when using instances of - C extension types with a non-``NULL`` ``tp_del`` slot. - - If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be - added to this list rather than freed. - - .. versionchanged:: 3.2 - If this list is non-empty at :term:`interpreter shutdown`, a - :exc:`ResourceWarning` is emitted, which is silent by default. If - :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects - are printed. - - .. versionchanged:: 3.4 - Following :pep:`442`, objects with a :meth:`__del__` method don't end - up in :attr:`gc.garbage` anymore. - -.. data:: callbacks - - A list of callbacks that will be invoked by the garbage collector before and - after collection. The callbacks will be called with two arguments, - *phase* and *info*. - - *phase* can be one of two values: - - "start": The garbage collection is about to start. - - "stop": The garbage collection has finished. - - *info* is a dict providing more information for the callback. The following - keys are currently defined: - - "generation": The oldest generation being collected. - - "collected": When *phase* is "stop", the number of objects - successfully collected. - - "uncollectable": When *phase* is "stop", the number of objects - that could not be collected and were put in :data:`garbage`. - - Applications can add their own callbacks to this list. The primary - use cases are: - - Gathering statistics about garbage collection, such as how often - various generations are collected, and how long the collection - takes. - - Allowing applications to identify and clear their own uncollectable - types when they appear in :data:`garbage`. - - .. versionadded:: 3.3 - - -The following constants are provided for use with :func:`set_debug`: - - -.. data:: DEBUG_STATS - - Print statistics during collection. This information can be useful when tuning - the collection frequency. - - -.. data:: DEBUG_COLLECTABLE - - Print information on collectable objects found. - - -.. data:: DEBUG_UNCOLLECTABLE - - Print information of uncollectable objects found (objects which are not - reachable but cannot be freed by the collector). These objects will be added - to the ``garbage`` list. - - .. versionchanged:: 3.2 - Also print the contents of the :data:`garbage` list at - :term:`interpreter shutdown`, if it isn't empty. - -.. data:: DEBUG_SAVEALL - - When set, all unreachable objects found will be appended to *garbage* rather - than being freed. This can be useful for debugging a leaking program. - - -.. data:: DEBUG_LEAK - - The debugging flags necessary for the collector to print information about a - leaking program (equal to ``DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | - DEBUG_SAVEALL``). diff --git a/Doc/library/importlib.metadata.rst.bak b/Doc/library/importlib.metadata.rst.bak deleted file mode 100644 index 7f154ea02cc4f2..00000000000000 --- a/Doc/library/importlib.metadata.rst.bak +++ /dev/null @@ -1,262 +0,0 @@ -.. _using: - -================================= - Using :mod:`!importlib.metadata` -================================= - -.. note:: - This functionality is provisional and may deviate from the usual - version semantics of the standard library. - -``importlib.metadata`` is a library that provides for access to installed -package metadata. Built in part on Python's import system, this library -intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources` in Python 3.7 -and newer (backported as `importlib_resources`_ for older versions of -Python), this can eliminate the need to use the older and less efficient -``pkg_resources`` package. - -By "installed package" we generally mean a third-party package installed into -Python's ``site-packages`` directory via tools such as `pip -`_. Specifically, -it means a package with either a discoverable ``dist-info`` or ``egg-info`` -directory, and metadata defined by :pep:`566` or its older specifications. -By default, package metadata can live on the file system or in zip archives on -:data:`sys.path`. Through an extension mechanism, the metadata can live almost -anywhere. - - -Overview -======== - -Let's say you wanted to get the version string for a package you've installed -using ``pip``. We start by creating a virtual environment and installing -something into it: - -.. code-block:: shell-session - - $ python3 -m venv example - $ source example/bin/activate - (example) $ pip install wheel - -You can get the version string for ``wheel`` by running the following: - -.. code-block:: pycon - - (example) $ python - >>> from importlib.metadata import version # doctest: +SKIP - >>> version('wheel') # doctest: +SKIP - '0.32.3' - -You can also get the set of entry points keyed by group, such as -``console_scripts``, ``distutils.commands`` and others. Each group contains a -sequence of :ref:`EntryPoint ` objects. - -You can get the :ref:`metadata for a distribution `:: - - >>> list(metadata('wheel')) # doctest: +SKIP - ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] - -You can also get a :ref:`distribution's version number `, list its -:ref:`constituent files `, and get a list of the distribution's -:ref:`requirements`. - - -Functional API -============== - -This package provides the following functionality via its public API. - - -.. _entry-points: - -Entry points ------------- - -The ``entry_points()`` function returns a dictionary of all entry points, -keyed by group. Entry points are represented by ``EntryPoint`` instances; -each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. There are also ``.module``, -``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute:: - - >>> eps = entry_points() # doctest: +SKIP - >>> list(eps) # doctest: +SKIP - ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] - >>> scripts = eps['console_scripts'] # doctest: +SKIP - >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP - >>> wheel # doctest: +SKIP - EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') - >>> wheel.module # doctest: +SKIP - 'wheel.cli' - >>> wheel.attr # doctest: +SKIP - 'main' - >>> wheel.extras # doctest: +SKIP - [] - >>> main = wheel.load() # doctest: +SKIP - >>> main # doctest: +SKIP - - -The ``group`` and ``name`` are arbitrary values defined by the package author -and usually a client will wish to resolve all entry points for a particular -group. Read `the setuptools docs -`_ -for more information on entry points, their definition, and usage. - - -.. _metadata: - -Distribution metadata ---------------------- - -Every distribution includes some metadata, which you can extract using the -``metadata()`` function:: - - >>> wheel_metadata = metadata('wheel') # doctest: +SKIP - -The keys of the returned data structure, a ``PackageMetadata``, -name the metadata keywords, and -the values are returned unparsed from the distribution metadata:: - - >>> wheel_metadata['Requires-Python'] # doctest: +SKIP - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - - -.. _version: - -Distribution versions ---------------------- - -The ``version()`` function is the quickest way to get a distribution's version -number, as a string:: - - >>> version('wheel') # doctest: +SKIP - '0.32.3' - - -.. _files: - -Distribution files ------------------- - -You can also get the full set of files contained within a distribution. The -``files()`` function takes a distribution package name and returns all of the -files installed by this distribution. Each file object returned is a -``PackagePath``, a :class:`pathlib.Path` derived object with additional ``dist``, -``size``, and ``hash`` properties as indicated by the metadata. For example:: - - >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP - >>> util # doctest: +SKIP - PackagePath('wheel/util.py') - >>> util.size # doctest: +SKIP - 859 - >>> util.dist # doctest: +SKIP - - >>> util.hash # doctest: +SKIP - - -Once you have the file, you can also read its contents:: - - >>> print(util.read_text()) # doctest: +SKIP - import base64 - import sys - ... - def as_bytes(s): - if isinstance(s, text_type): - return s.encode('utf-8') - return s - -In the case where the metadata file listing files -(RECORD or SOURCES.txt) is missing, ``files()`` will -return ``None``. The caller may wish to wrap calls to -``files()`` in `always_iterable -`_ -or otherwise guard against this condition if the target -distribution is not known to have the metadata present. - -.. _requirements: - -Distribution requirements -------------------------- - -To get the full set of requirements for a distribution, use the ``requires()`` -function:: - - >>> requires('wheel') # doctest: +SKIP - ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] - - -Distributions -============= - -While the above API is the most common and convenient usage, you can get all -of that information from the ``Distribution`` class. A ``Distribution`` is an -abstract object that represents the metadata for a Python package. You can -get the ``Distribution`` instance:: - - >>> from importlib.metadata import distribution # doctest: +SKIP - >>> dist = distribution('wheel') # doctest: +SKIP - -Thus, an alternative way to get the version number is through the -``Distribution`` instance:: - - >>> dist.version # doctest: +SKIP - '0.32.3' - -There are all kinds of additional metadata available on the ``Distribution`` -instance:: - - >>> dist.metadata['Requires-Python'] # doctest: +SKIP - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> dist.metadata['License'] # doctest: +SKIP - 'MIT' - -The full set of available metadata is not described here. See :pep:`566` -for additional details. - - -Extending the search algorithm -============================== - -Because package metadata is not available through :data:`sys.path` searches, or -package loaders directly, the metadata for a package is found through import -system :ref:`finders `. To find a distribution package's metadata, -``importlib.metadata`` queries the list of :term:`meta path finders ` on -:data:`sys.meta_path`. - -The default ``PathFinder`` for Python includes a hook that calls into -``importlib.metadata.MetadataPathFinder`` for finding distributions -loaded from typical file-system-based paths. - -The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the -interface expected of finders by Python's import system. -``importlib.metadata`` extends this protocol by looking for an optional -``find_distributions`` callable on the finders from -:data:`sys.meta_path` and presents this extended interface as the -``DistributionFinder`` abstract base class, which defines this abstract -method:: - - @abc.abstractmethod - def find_distributions(context=DistributionFinder.Context()): - """Return an iterable of all Distribution instances capable of - loading the metadata for packages for the indicated ``context``. - """ - -The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` -properties indicating the path to search and name to match and may -supply other relevant context. - -What this means in practice is that to support finding distribution package -metadata in locations other than the file system, subclass -``Distribution`` and implement the abstract methods. Then from -a custom finder, return instances of this derived ``Distribution`` in the -``find_distributions()`` method. - - -.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points -.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api -.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html - - -.. rubric:: Footnotes diff --git a/Doc/library/logging.rst.bak b/Doc/library/logging.rst.bak deleted file mode 100644 index 431a5849fa9bff..00000000000000 --- a/Doc/library/logging.rst.bak +++ /dev/null @@ -1,1350 +0,0 @@ -:mod:`logging` --- Logging facility for Python -============================================== - -.. module:: logging - :synopsis: Flexible event logging system for applications. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -**Source code:** :source:`Lib/logging/__init__.py` - -.. index:: pair: Errors; logging - -.. sidebar:: Important - - This page contains the API reference information. For tutorial - information and discussion of more advanced topics, see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Logging Cookbook ` - --------------- - -This module defines functions and classes which implement a flexible event -logging system for applications and libraries. - -The key benefit of having the logging API provided by a standard library module -is that all Python modules can participate in logging, so your application log -can include your own messages integrated with messages from third-party -modules. - -The module provides a lot of functionality and flexibility. If you are -unfamiliar with logging, the best way to get to grips with it is to see the -tutorials (see the links on the right). - -The basic classes defined by the module, together with their functions, are -listed below. - -* Loggers expose the interface that application code directly uses. -* Handlers send the log records (created by loggers) to the appropriate - destination. -* Filters provide a finer grained facility for determining which log records - to output. -* Formatters specify the layout of log records in the final output. - - -.. _logger: - -Logger Objects --------------- - -Loggers have the following attributes and methods. Note that Loggers should -*NEVER* be instantiated directly, but always through the module-level function -``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same -name will always return a reference to the same Logger object. - -The ``name`` is potentially a period-separated hierarchical value, like -``foo.bar.baz`` (though it could also be just plain ``foo``, for example). -Loggers that are further down in the hierarchical list are children of loggers -higher up in the list. For example, given a logger with a name of ``foo``, -loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. - - -.. class:: Logger - - .. attribute:: Logger.propagate - - If this attribute evaluates to true, events logged to this logger will be - passed to the handlers of higher level (ancestor) loggers, in addition to - any handlers attached to this logger. Messages are passed directly to the - ancestor loggers' handlers - neither the level nor filters of the ancestor - loggers in question are considered. - - If this evaluates to false, logging messages are not passed to the handlers - of ancestor loggers. - - The constructor sets this attribute to ``True``. - - .. note:: If you attach a handler to a logger *and* one or more of its - ancestors, it may emit the same record multiple times. In general, you - should not need to attach a handler to more than one logger - if you just - attach it to the appropriate logger which is highest in the logger - hierarchy, then it will see all events logged by all descendant loggers, - provided that their propagate setting is left set to ``True``. A common - scenario is to attach handlers only to the root logger, and to let - propagation take care of the rest. - - .. method:: Logger.setLevel(level) - - Sets the threshold for this logger to *level*. Logging messages which are less - severe than *level* will be ignored; logging messages which have severity *level* - or higher will be emitted by whichever handler or handlers service this logger, - unless a handler's level has been set to a higher severity level than *level*. - - When a logger is created, the level is set to :const:`NOTSET` (which causes - all messages to be processed when the logger is the root logger, or delegation - to the parent when the logger is a non-root logger). Note that the root logger - is created with level :const:`WARNING`. - - The term 'delegation to the parent' means that if a logger has a level of - NOTSET, its chain of ancestor loggers is traversed until either an ancestor with - a level other than NOTSET is found, or the root is reached. - - If an ancestor is found with a level other than NOTSET, then that ancestor's - level is treated as the effective level of the logger where the ancestor search - began, and is used to determine how a logging event is handled. - - If the root is reached, and it has a level of NOTSET, then all messages will be - processed. Otherwise, the root's level will be used as the effective level. - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. Note, however, that levels are internally stored - as integers, and methods such as e.g. :meth:`getEffectiveLevel` and - :meth:`isEnabledFor` will return/expect to be passed integers. - - - .. method:: Logger.isEnabledFor(level) - - Indicates if a message of severity *level* would be processed by this logger. - This method checks first the module-level level set by - ``logging.disable(level)`` and then the logger's effective level as determined - by :meth:`getEffectiveLevel`. - - - .. method:: Logger.getEffectiveLevel() - - Indicates the effective level for this logger. If a value other than - :const:`NOTSET` has been set using :meth:`setLevel`, it is returned. Otherwise, - the hierarchy is traversed towards the root until a value other than - :const:`NOTSET` is found, and that value is returned. The value returned is - an integer, typically one of :const:`logging.DEBUG`, :const:`logging.INFO` - etc. - - - .. method:: Logger.getChild(suffix) - - Returns a logger which is a descendant to this logger, as determined by the suffix. - Thus, ``logging.getLogger('abc').getChild('def.ghi')`` would return the same - logger as would be returned by ``logging.getLogger('abc.def.ghi')``. This is a - convenience method, useful when the parent logger is named using e.g. ``__name__`` - rather than a literal string. - - .. versionadded:: 3.2 - - - .. method:: Logger.debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on this logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - No % formatting operation is performed on *msg* when no *args* are supplied. - - There are four keyword arguments in *kwargs* which are inspected: - *exc_info*, *stack_info*, *stacklevel* and *extra*. - - If *exc_info* does not evaluate as false, it causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *stacklevel*, which defaults to ``1``. - If greater than 1, the corresponding number of stack frames are skipped - when computing the line number and function name set in the :class:`LogRecord` - created for the logging event. This can be used in logging helpers so that - the function name, filename and line number recorded are not the information - for the helper function/method, but rather its caller. The name of this - parameter mirrors the equivalent one in the :mod:`warnings` module. - - The fourth keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the :class:`LogRecord` - created for the logging event with user-defined attributes. These custom - attributes can then be used as you like. For example, they could be - incorporated into logged messages. For example:: - - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logger = logging.getLogger('tcpserver') - logger.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the :class:`LogRecord`. If these are missing, the message will - not be logged because a string formatting exception will occur. So in this case, - you always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. - - .. versionchanged:: 3.5 - The *exc_info* parameter can now accept exception instances. - - .. versionchanged:: 3.8 - The *stacklevel* parameter was added. - - - .. method:: Logger.info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on this logger. The arguments are - interpreted as for :meth:`debug`. - - .. note:: There is an obsolete method ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - .. method:: Logger.error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.log(level, msg, *args, **kwargs) - - Logs a message with integer level *level* on this logger. The other arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. Exception info is added to the logging - message. This method should only be called from an exception handler. - - - .. method:: Logger.addFilter(filter) - - Adds the specified filter *filter* to this logger. - - - .. method:: Logger.removeFilter(filter) - - Removes the specified filter *filter* from this logger. - - - .. method:: Logger.filter(record) - - Apply this logger's filters to the record and return ``True`` if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be processed (passed to handlers). If one returns a false value, no - further processing of the record occurs. - - - .. method:: Logger.addHandler(hdlr) - - Adds the specified handler *hdlr* to this logger. - - - .. method:: Logger.removeHandler(hdlr) - - Removes the specified handler *hdlr* from this logger. - - - .. method:: Logger.findCaller(stack_info=False, stacklevel=1) - - Finds the caller's source filename and line number. Returns the filename, line - number, function name and stack information as a 4-element tuple. The stack - information is returned as ``None`` unless *stack_info* is ``True``. - - The *stacklevel* parameter is passed from code calling the :meth:`debug` - and other APIs. If greater than 1, the excess is used to skip stack frames - before determining the values to be returned. This will generally be useful - when calling logging APIs from helper/wrapper code, so that the information - in the event log refers not to the helper/wrapper code, but to the code that - calls it. - - - .. method:: Logger.handle(record) - - Handles a record by passing it to all handlers associated with this logger and - its ancestors (until a false value of *propagate* is found). This method is used - for unpickled records received from a socket, as well as those created locally. - Logger-level filtering is applied using :meth:`~Logger.filter`. - - - .. method:: Logger.makeRecord(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) - - This is a factory method which can be overridden in subclasses to create - specialized :class:`LogRecord` instances. - - .. method:: Logger.hasHandlers() - - Checks to see if this logger has any handlers configured. This is done by - looking for handlers in this logger and its parents in the logger hierarchy. - Returns ``True`` if a handler was found, else ``False``. The method stops searching - up the hierarchy whenever a logger with the 'propagate' attribute set to - false is found - that will be the last logger which is checked for the - existence of handlers. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.7 - Loggers can now be pickled and unpickled. - -.. _levels: - -Logging Levels --------------- - -The numeric values of logging levels are given in the following table. These are -primarily of interest if you want to define your own levels, and need them to -have specific values relative to the predefined levels. If you define a level -with the same numeric value, it overwrites the predefined value; the predefined -name is lost. - -+--------------+---------------+ -| Level | Numeric value | -+==============+===============+ -| ``CRITICAL`` | 50 | -+--------------+---------------+ -| ``ERROR`` | 40 | -+--------------+---------------+ -| ``WARNING`` | 30 | -+--------------+---------------+ -| ``INFO`` | 20 | -+--------------+---------------+ -| ``DEBUG`` | 10 | -+--------------+---------------+ -| ``NOTSET`` | 0 | -+--------------+---------------+ - - -.. _handler: - -Handler Objects ---------------- - -Handlers have the following attributes and methods. Note that :class:`Handler` -is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`__init__` method in subclasses needs to call -:meth:`Handler.__init__`. - -.. class:: Handler - - .. method:: Handler.__init__(level=NOTSET) - - Initializes the :class:`Handler` instance by setting its level, setting the list - of filters to the empty list and creating a lock (using :meth:`createLock`) for - serializing access to an I/O mechanism. - - - .. method:: Handler.createLock() - - Initializes a thread lock which can be used to serialize access to underlying - I/O functionality which may not be threadsafe. - - - .. method:: Handler.acquire() - - Acquires the thread lock created with :meth:`createLock`. - - - .. method:: Handler.release() - - Releases the thread lock acquired with :meth:`acquire`. - - - .. method:: Handler.setLevel(level) - - Sets the threshold for this handler to *level*. Logging messages which are - less severe than *level* will be ignored. When a handler is created, the - level is set to :const:`NOTSET` (which causes all messages to be - processed). - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. - - - .. method:: Handler.setFormatter(fmt) - - Sets the :class:`Formatter` for this handler to *fmt*. - - - .. method:: Handler.addFilter(filter) - - Adds the specified filter *filter* to this handler. - - - .. method:: Handler.removeFilter(filter) - - Removes the specified filter *filter* from this handler. - - - .. method:: Handler.filter(record) - - Apply this handler's filters to the record and return ``True`` if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be emitted. If one returns a false value, the handler will not emit the - record. - - - .. method:: Handler.flush() - - Ensure all logging output has been flushed. This version does nothing and is - intended to be implemented by subclasses. - - - .. method:: Handler.close() - - Tidy up any resources used by the handler. This version does no output but - removes the handler from an internal list of handlers which is closed when - :func:`shutdown` is called. Subclasses should ensure that this gets called - from overridden :meth:`close` methods. - - - .. method:: Handler.handle(record) - - Conditionally emits the specified logging record, depending on filters which may - have been added to the handler. Wraps the actual emission of the record with - acquisition/release of the I/O thread lock. - - - .. method:: Handler.handleError(record) - - This method should be called from handlers when an exception is encountered - during an :meth:`emit` call. If the module-level attribute - ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is - what is mostly wanted for a logging system - most users will not care about - errors in the logging system, they are more interested in application - errors. You could, however, replace this with a custom handler if you wish. - The specified record is the one which was being processed when the exception - occurred. (The default value of ``raiseExceptions`` is ``True``, as that is - more useful during development). - - - .. method:: Handler.format(record) - - Do formatting for a record - if a formatter is set, use it. Otherwise, use the - default formatter for the module. - - - .. method:: Handler.emit(record) - - Do whatever it takes to actually log the specified logging record. This version - is intended to be implemented by subclasses and so raises a - :exc:`NotImplementedError`. - -For a list of handlers included as standard, see :mod:`logging.handlers`. - -.. _formatter-objects: - -Formatter Objects ------------------ - -.. currentmodule:: logging - -:class:`Formatter` objects have the following attributes and methods. They are -responsible for converting a :class:`LogRecord` to (usually) a string which can -be interpreted by either a human or an external system. The base -:class:`Formatter` allows a formatting string to be specified. If none is -supplied, the default value of ``'%(message)s'`` is used, which just includes -the message in the logging call. To have additional items of information in the -formatted output (such as a timestamp), keep reading. - -A Formatter can be initialized with a format string which makes use of knowledge -of the :class:`LogRecord` attributes - such as the default value mentioned above -making use of the fact that the user's message and arguments are pre-formatted -into a :class:`LogRecord`'s *message* attribute. This format string contains -standard Python %-style mapping keys. See section :ref:`old-string-formatting` -for more information on string formatting. - -The useful mapping keys in a :class:`LogRecord` are given in the section on -:ref:`logrecord-attributes`. - - -.. class:: Formatter(fmt=None, datefmt=None, style='%', validate=True, *, defaults=None) - - Returns a new instance of the :class:`Formatter` class. The instance is - initialized with a format string for the message as a whole, as well as a - format string for the date/time portion of a message. If no *fmt* is - specified, ``'%(message)s'`` is used. If no *datefmt* is specified, a format - is used which is described in the :meth:`formatTime` documentation. - - The *style* parameter can be one of '%', '{' or '$' and determines how - the format string will be merged with its data: using one of %-formatting, - :meth:`str.format` or :class:`string.Template`. This only applies to the - format string *fmt* (e.g. ``'%(message)s'`` or ``{message}``), not to the - actual log messages passed to ``Logger.debug`` etc; see - :ref:`formatting-styles` for more information on using {- and $-formatting - for log messages. - - The *defaults* parameter can be a dictionary with default values to use in - custom fields. For example: - ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` - - .. versionchanged:: 3.2 - The *style* parameter was added. - - .. versionchanged:: 3.8 - The *validate* parameter was added. Incorrect or mismatched style and fmt - will raise a ``ValueError``. - For example: ``logging.Formatter('%(asctime)s - %(message)s', style='{')``. - - .. versionchanged:: 3.10 - The *defaults* parameter was added. - - .. method:: format(record) - - The record's attribute dictionary is used as the operand to a string - formatting operation. Returns the resulting string. Before formatting the - dictionary, a couple of preparatory steps are carried out. The *message* - attribute of the record is computed using *msg* % *args*. If the - formatting string contains ``'(asctime)'``, :meth:`formatTime` is called - to format the event time. If there is exception information, it is - formatted using :meth:`formatException` and appended to the message. Note - that the formatted exception information is cached in attribute - *exc_text*. This is useful because the exception information can be - pickled and sent across the wire, but you should be careful if you have - more than one :class:`Formatter` subclass which customizes the formatting - of exception information. In this case, you will have to clear the cached - value (by setting the *exc_text* attribute to ``None``) after a formatter - has done its formatting, so that the next formatter to handle the event - doesn't use the cached value, but recalculates it afresh. - - If stack information is available, it's appended after the exception - information, using :meth:`formatStack` to transform it if necessary. - - - .. method:: formatTime(record, datefmt=None) - - This method should be called from :meth:`format` by a formatter which - wants to make use of a formatted time. This method can be overridden in - formatters to provide for any specific requirement, but the basic behavior - is as follows: if *datefmt* (a string) is specified, it is used with - :func:`time.strftime` to format the creation time of the - record. Otherwise, the format '%Y-%m-%d %H:%M:%S,uuu' is used, where the - uuu part is a millisecond value and the other letters are as per the - :func:`time.strftime` documentation. An example time in this format is - ``2003-01-23 00:29:50,411``. The resulting string is returned. - - This function uses a user-configurable function to convert the creation - time to a tuple. By default, :func:`time.localtime` is used; to change - this for a particular formatter instance, set the ``converter`` attribute - to a function with the same signature as :func:`time.localtime` or - :func:`time.gmtime`. To change it for all formatters, for example if you - want all logging times to be shown in GMT, set the ``converter`` - attribute in the ``Formatter`` class. - - .. versionchanged:: 3.3 - Previously, the default format was hard-coded as in this example: - ``2010-09-06 22:38:15,292`` where the part before the comma is - handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the - part after the comma is a millisecond value. Because strptime does not - have a format placeholder for milliseconds, the millisecond value is - appended using another format string, ``'%s,%03d'`` --- and both of these - format strings have been hardcoded into this method. With the change, - these strings are defined as class-level attributes which can be - overridden at the instance level when desired. The names of the - attributes are ``default_time_format`` (for the strptime format string) - and ``default_msec_format`` (for appending the millisecond value). - - .. versionchanged:: 3.9 - The ``default_msec_format`` can be ``None``. - - .. method:: formatException(exc_info) - - Formats the specified exception information (a standard exception tuple as - returned by :func:`sys.exc_info`) as a string. This default implementation - just uses :func:`traceback.print_exception`. The resulting string is - returned. - - .. method:: formatStack(stack_info) - - Formats the specified stack information (a string as returned by - :func:`traceback.print_stack`, but with the last newline removed) as a - string. This default implementation just returns the input value. - -.. _filter: - -Filter Objects --------------- - -``Filters`` can be used by ``Handlers`` and ``Loggers`` for more sophisticated -filtering than is provided by levels. The base filter class only allows events -which are below a certain point in the logger hierarchy. For example, a filter -initialized with 'A.B' will allow events logged by loggers 'A.B', 'A.B.C', -'A.B.C.D', 'A.B.D' etc. but not 'A.BB', 'B.A.B' etc. If initialized with the -empty string, all events are passed. - - -.. class:: Filter(name='') - - Returns an instance of the :class:`Filter` class. If *name* is specified, it - names a logger which, together with its children, will have its events allowed - through the filter. If *name* is the empty string, allows every event. - - - .. method:: filter(record) - - Is the specified record to be logged? Returns zero for no, nonzero for - yes. If deemed appropriate, the record may be modified in-place by this - method. - -Note that filters attached to handlers are consulted before an event is -emitted by the handler, whereas filters attached to loggers are consulted -whenever an event is logged (using :meth:`debug`, :meth:`info`, -etc.), before sending an event to handlers. This means that events which have -been generated by descendant loggers will not be filtered by a logger's filter -setting, unless the filter has also been applied to those descendant loggers. - -You don't actually need to subclass ``Filter``: you can pass any instance -which has a ``filter`` method with the same semantics. - -.. versionchanged:: 3.2 - You don't need to create specialized ``Filter`` classes, or use other - classes with a ``filter`` method: you can use a function (or other - callable) as a filter. The filtering logic will check to see if the filter - object has a ``filter`` attribute: if it does, it's assumed to be a - ``Filter`` and its :meth:`~Filter.filter` method is called. Otherwise, it's - assumed to be a callable and called with the record as the single - parameter. The returned value should conform to that returned by - :meth:`~Filter.filter`. - -Although filters are used primarily to filter records based on more -sophisticated criteria than levels, they get to see every record which is -processed by the handler or logger they're attached to: this can be useful if -you want to do things like counting how many records were processed by a -particular logger or handler, or adding, changing or removing attributes in -the :class:`LogRecord` being processed. Obviously changing the LogRecord needs -to be done with some care, but it does allow the injection of contextual -information into logs (see :ref:`filters-contextual`). - -.. _log-record: - -LogRecord Objects ------------------ - -:class:`LogRecord` instances are created automatically by the :class:`Logger` -every time something is logged, and can be created manually via -:func:`makeLogRecord` (for example, from a pickled event received over the -wire). - - -.. class:: LogRecord(name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None) - - Contains all the information pertinent to the event being logged. - - The primary information is passed in :attr:`msg` and :attr:`args`, which - are combined using ``msg % args`` to create the :attr:`message` field of the - record. - - :param name: The name of the logger used to log the event represented by - this LogRecord. Note that this name will always have this - value, even though it may be emitted by a handler attached to - a different (ancestor) logger. - :param level: The numeric level of the logging event (one of DEBUG, INFO etc.) - Note that this is converted to *two* attributes of the LogRecord: - ``levelno`` for the numeric value and ``levelname`` for the - corresponding level name. - :param pathname: The full pathname of the source file where the logging call - was made. - :param lineno: The line number in the source file where the logging call was - made. - :param msg: The event description message, possibly a format string with - placeholders for variable data. - :param args: Variable data to merge into the *msg* argument to obtain the - event description. - :param exc_info: An exception tuple with the current exception information, - or ``None`` if no exception information is available. - :param func: The name of the function or method from which the logging call - was invoked. - :param sinfo: A text string representing stack information from the base of - the stack in the current thread, up to the logging call. - - .. method:: getMessage() - - Returns the message for this :class:`LogRecord` instance after merging any - user-supplied arguments with the message. If the user-supplied message - argument to the logging call is not a string, :func:`str` is called on it to - convert it to a string. This allows use of user-defined classes as - messages, whose ``__str__`` method can return the actual format string to - be used. - - .. versionchanged:: 3.2 - The creation of a :class:`LogRecord` has been made more configurable by - providing a factory which is used to create the record. The factory can be - set using :func:`getLogRecordFactory` and :func:`setLogRecordFactory` - (see this for the factory's signature). - - This functionality can be used to inject your own values into a - :class:`LogRecord` at creation time. You can use the following pattern:: - - old_factory = logging.getLogRecordFactory() - - def record_factory(*args, **kwargs): - record = old_factory(*args, **kwargs) - record.custom_attribute = 0xdecafbad - return record - - logging.setLogRecordFactory(record_factory) - - With this pattern, multiple factories could be chained, and as long - as they don't overwrite each other's attributes or unintentionally - overwrite the standard attributes listed above, there should be no - surprises. - - -.. _logrecord-attributes: - -LogRecord attributes --------------------- - -The LogRecord has a number of attributes, most of which are derived from the -parameters to the constructor. (Note that the names do not always correspond -exactly between the LogRecord constructor parameters and the LogRecord -attributes.) These attributes can be used to merge data from the record into -the format string. The following table lists (in alphabetical order) the -attribute names, their meanings and the corresponding placeholder in a %-style -format string. - -If you are using {}-formatting (:func:`str.format`), you can use -``{attrname}`` as the placeholder in the format string. If you are using -$-formatting (:class:`string.Template`), use the form ``${attrname}``. In -both cases, of course, replace ``attrname`` with the actual attribute name -you want to use. - -In the case of {}-formatting, you can specify formatting flags by placing them -after the attribute name, separated from it with a colon. For example: a -placeholder of ``{msecs:03d}`` would format a millisecond value of ``4`` as -``004``. Refer to the :meth:`str.format` documentation for full details on -the options available to you. - -+----------------+-------------------------+-----------------------------------------------+ -| Attribute name | Format | Description | -+================+=========================+===============================================+ -| args | You shouldn't need to | The tuple of arguments merged into ``msg`` to | -| | format this yourself. | produce ``message``, or a dict whose values | -| | | are used for the merge (when there is only one| -| | | argument, and it is a dictionary). | -+----------------+-------------------------+-----------------------------------------------+ -| asctime | ``%(asctime)s`` | Human-readable time when the | -| | | :class:`LogRecord` was created. By default | -| | | this is of the form '2003-07-08 16:49:45,896' | -| | | (the numbers after the comma are millisecond | -| | | portion of the time). | -+----------------+-------------------------+-----------------------------------------------+ -| created | ``%(created)f`` | Time when the :class:`LogRecord` was created | -| | | (as returned by :func:`time.time`). | -+----------------+-------------------------+-----------------------------------------------+ -| exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | -| | format this yourself. | if no exception has occurred, ``None``. | -+----------------+-------------------------+-----------------------------------------------+ -| filename | ``%(filename)s`` | Filename portion of ``pathname``. | -+----------------+-------------------------+-----------------------------------------------+ -| funcName | ``%(funcName)s`` | Name of function containing the logging call. | -+----------------+-------------------------+-----------------------------------------------+ -| levelname | ``%(levelname)s`` | Text logging level for the message | -| | | (``'DEBUG'``, ``'INFO'``, ``'WARNING'``, | -| | | ``'ERROR'``, ``'CRITICAL'``). | -+----------------+-------------------------+-----------------------------------------------+ -| levelno | ``%(levelno)s`` | Numeric logging level for the message | -| | | (:const:`DEBUG`, :const:`INFO`, | -| | | :const:`WARNING`, :const:`ERROR`, | -| | | :const:`CRITICAL`). | -+----------------+-------------------------+-----------------------------------------------+ -| lineno | ``%(lineno)d`` | Source line number where the logging call was | -| | | issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| message | ``%(message)s`` | The logged message, computed as ``msg % | -| | | args``. This is set when | -| | | :meth:`Formatter.format` is invoked. | -+----------------+-------------------------+-----------------------------------------------+ -| module | ``%(module)s`` | Module (name portion of ``filename``). | -+----------------+-------------------------+-----------------------------------------------+ -| msecs | ``%(msecs)d`` | Millisecond portion of the time when the | -| | | :class:`LogRecord` was created. | -+----------------+-------------------------+-----------------------------------------------+ -| msg | You shouldn't need to | The format string passed in the original | -| | format this yourself. | logging call. Merged with ``args`` to | -| | | produce ``message``, or an arbitrary object | -| | | (see :ref:`arbitrary-object-messages`). | -+----------------+-------------------------+-----------------------------------------------+ -| name | ``%(name)s`` | Name of the logger used to log the call. | -+----------------+-------------------------+-----------------------------------------------+ -| pathname | ``%(pathname)s`` | Full pathname of the source file where the | -| | | logging call was issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| process | ``%(process)d`` | Process ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| processName | ``%(processName)s`` | Process name (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| relativeCreated| ``%(relativeCreated)d`` | Time in milliseconds when the LogRecord was | -| | | created, relative to the time the logging | -| | | module was loaded. | -+----------------+-------------------------+-----------------------------------------------+ -| stack_info | You shouldn't need to | Stack frame information (where available) | -| | format this yourself. | from the bottom of the stack in the current | -| | | thread, up to and including the stack frame | -| | | of the logging call which resulted in the | -| | | creation of this record. | -+----------------+-------------------------+-----------------------------------------------+ -| thread | ``%(thread)d`` | Thread ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| threadName | ``%(threadName)s`` | Thread name (if available). | -+----------------+-------------------------+-----------------------------------------------+ - -.. versionchanged:: 3.1 - *processName* was added. - - -.. _logger-adapter: - -LoggerAdapter Objects ---------------------- - -:class:`LoggerAdapter` instances are used to conveniently pass contextual -information into logging calls. For a usage example, see the section on -:ref:`adding contextual information to your logging output `. - -.. class:: LoggerAdapter(logger, extra) - - Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance and a dict-like object. - - .. method:: process(msg, kwargs) - - Modifies the message and/or keyword arguments passed to a logging call in - order to insert contextual information. This implementation takes the object - passed as *extra* to the constructor and adds it to *kwargs* using key - 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the - (possibly modified) versions of the arguments passed in. - -In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their -counterparts in :class:`Logger`, so you can use the two types of instances -interchangeably. - -.. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. - - -Thread Safety -------------- - -The logging module is intended to be thread-safe without any special work -needing to be done by its clients. It achieves this though using threading -locks; there is one lock to serialize access to the module's shared data, and -each handler also creates a lock to serialize access to its underlying I/O. - -If you are implementing asynchronous signal handlers using the :mod:`signal` -module, you may not be able to use logging from within such handlers. This is -because lock implementations in the :mod:`threading` module are not always -re-entrant, and so cannot be invoked from such signal handlers. - - -Module-Level Functions ----------------------- - -In addition to the classes described above, there are a number of module-level -functions. - - -.. function:: getLogger(name=None) - - Return a logger with the specified name or, if name is ``None``, return a - logger which is the root logger of the hierarchy. If specified, the name is - typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. - Choice of these names is entirely up to the developer who is using logging. - - All calls to this function with a given name return the same logger instance. - This means that logger instances never need to be passed between different parts - of an application. - - -.. function:: getLoggerClass() - - Return either the standard :class:`Logger` class, or the last class passed to - :func:`setLoggerClass`. This function may be called from within a new class - definition, to ensure that installing a customized :class:`Logger` class will - not undo customizations already applied by other code. For example:: - - class MyLogger(logging.getLoggerClass()): - # ... override behaviour here - - -.. function:: getLogRecordFactory() - - Return a callable which is used to create a :class:`LogRecord`. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`setLogRecordFactory`, - to allow developers more control over how the :class:`LogRecord` - representing a logging event is constructed. - - See :func:`setLogRecordFactory` for more information about the how the - factory is called. - -.. function:: debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like: - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. - -.. function:: info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. - - .. note:: There is an obsolete function ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - -.. function:: error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. - - -.. function:: exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging - message. This function should only be called from an exception handler. - -.. function:: log(level, msg, *args, **kwargs) - - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. - - .. note:: The above module-level convenience functions, which delegate to the - root logger, call :func:`basicConfig` to ensure that at least one handler - is available. Because of this, they should *not* be used in threads, - in versions of Python earlier than 2.7.1 and 3.2, unless at least one - handler has been added to the root logger *before* the threads are - started. In earlier versions of Python, due to a thread safety shortcoming - in :func:`basicConfig`, this can (under rare circumstances) lead to - handlers being added multiple times to the root logger, which can in turn - lead to multiple messages for the same event. - -.. function:: disable(level=CRITICAL) - - Provides an overriding level *level* for all loggers which takes precedence over - the logger's own level. When the need arises to temporarily throttle logging - output down across the whole application, this function can be useful. Its - effect is to disable all logging calls of severity *level* and below, so that - if you call it with a value of INFO, then all INFO and DEBUG events would be - discarded, whereas those of severity WARNING and above would be processed - according to the logger's effective level. If - ``logging.disable(logging.NOTSET)`` is called, it effectively removes this - overriding level, so that logging output again depends on the effective - levels of individual loggers. - - Note that if you have defined any custom logging level higher than - ``CRITICAL`` (this is not recommended), you won't be able to rely on the - default value for the *level* parameter, but will have to explicitly supply a - suitable value. - - .. versionchanged:: 3.7 - The *level* parameter was defaulted to level ``CRITICAL``. See - :issue:`28524` for more information about this change. - -.. function:: addLevelName(level, levelName) - - Associates level *level* with text *levelName* in an internal dictionary, which is - used to map numeric levels to a textual representation, for example when a - :class:`Formatter` formats a message. This function can also be used to define - your own levels. The only constraints are that all levels used must be - registered using this function, levels should be positive integers and they - should increase in increasing order of severity. - - .. note:: If you are thinking of defining your own levels, please see the - section on :ref:`custom-levels`. - -.. function:: getLevelName(level) - - Returns the textual representation of logging level *level*. If the level is one - of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, - :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you - have associated levels with names using :func:`addLevelName` then the name you - have associated with *level* is returned. If a numeric value corresponding to one - of the defined levels is passed in, the corresponding string representation is - returned. Otherwise, the string 'Level %s' % level is returned. - - .. note:: Levels are internally integers (as they need to be compared in the - logging logic). This function is used to convert between an integer level - and the level name displayed in the formatted log output by means of the - ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). - - .. versionchanged:: 3.4 - In Python versions earlier than 3.4, this function could also be passed a - text level, and would return the corresponding numeric value of the level. - This undocumented behaviour was considered a mistake, and was removed in - Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. - -.. function:: makeLogRecord(attrdict) - - Creates and returns a new :class:`LogRecord` instance whose attributes are - defined by *attrdict*. This function is useful for taking a pickled - :class:`LogRecord` attribute dictionary, sent over a socket, and reconstituting - it as a :class:`LogRecord` instance at the receiving end. - - -.. function:: basicConfig(**kwargs) - - Does basic configuration for the logging system by creating a - :class:`StreamHandler` with a default :class:`Formatter` and adding it to the - root logger. The functions :func:`debug`, :func:`info`, :func:`warning`, - :func:`error` and :func:`critical` will call :func:`basicConfig` automatically - if no handlers are defined for the root logger. - - This function does nothing if the root logger already has handlers - configured, unless the keyword argument *force* is set to ``True``. - - .. note:: This function should be called from the main thread - before other threads are started. In versions of Python prior to - 2.7.1 and 3.2, if this function is called from multiple threads, - it is possible (in rare circumstances) that a handler will be added - to the root logger more than once, leading to unexpected results - such as messages being duplicated in the log. - - The following keyword arguments are supported. - - .. tabularcolumns:: |l|L| - - +--------------+---------------------------------------------+ - | Format | Description | - +==============+=============================================+ - | *filename* | Specifies that a :class:`FileHandler` be | - | | created, using the specified filename, | - | | rather than a :class:`StreamHandler`. | - +--------------+---------------------------------------------+ - | *filemode* | If *filename* is specified, open the file | - | | in this :ref:`mode `. Defaults | - | | to ``'a'``. | - +--------------+---------------------------------------------+ - | *format* | Use the specified format string for the | - | | handler. Defaults to attributes | - | | ``levelname``, ``name`` and ``message`` | - | | separated by colons. | - +--------------+---------------------------------------------+ - | *datefmt* | Use the specified date/time format, as | - | | accepted by :func:`time.strftime`. | - +--------------+---------------------------------------------+ - | *style* | If *format* is specified, use this style | - | | for the format string. One of ``'%'``, | - | | ``'{'`` or ``'$'`` for :ref:`printf-style | - | | `, | - | | :meth:`str.format` or | - | | :class:`string.Template` respectively. | - | | Defaults to ``'%'``. | - +--------------+---------------------------------------------+ - | *level* | Set the root logger level to the specified | - | | :ref:`level `. | - +--------------+---------------------------------------------+ - | *stream* | Use the specified stream to initialize the | - | | :class:`StreamHandler`. Note that this | - | | argument is incompatible with *filename* - | - | | if both are present, a ``ValueError`` is | - | | raised. | - +--------------+---------------------------------------------+ - | *handlers* | If specified, this should be an iterable of | - | | already created handlers to add to the root | - | | logger. Any handlers which don't already | - | | have a formatter set will be assigned the | - | | default formatter created in this function. | - | | Note that this argument is incompatible | - | | with *filename* or *stream* - if both | - | | are present, a ``ValueError`` is raised. | - +--------------+---------------------------------------------+ - | *force* | If this keyword argument is specified as | - | | true, any existing handlers attached to the | - | | root logger are removed and closed, before | - | | carrying out the configuration as specified | - | | by the other arguments. | - +--------------+---------------------------------------------+ - | *encoding* | If this keyword argument is specified along | - | | with *filename*, its value is used when the | - | | :class:`FileHandler` is created, and thus | - | | used when opening the output file. | - +--------------+---------------------------------------------+ - | *errors* | If this keyword argument is specified along | - | | with *filename*, its value is used when the | - | | :class:`FileHandler` is created, and thus | - | | used when opening the output file. If not | - | | specified, the value 'backslashreplace' is | - | | used. Note that if ``None`` is specified, | - | | it will be passed as such to :func:`open`, | - | | which means that it will be treated the | - | | same as passing 'errors'. | - +--------------+---------------------------------------------+ - - .. versionchanged:: 3.2 - The *style* argument was added. - - .. versionchanged:: 3.3 - The *handlers* argument was added. Additional checks were added to - catch situations where incompatible arguments are specified (e.g. - *handlers* together with *stream* or *filename*, or *stream* - together with *filename*). - - .. versionchanged:: 3.8 - The *force* argument was added. - - .. versionchanged:: 3.9 - The *encoding* and *errors* arguments were added. - -.. function:: shutdown() - - Informs the logging system to perform an orderly shutdown by flushing and - closing all handlers. This should be called at application exit and no - further use of the logging system should be made after this call. - - When the logging module is imported, it registers this function as an exit - handler (see :mod:`atexit`), so normally there's no need to do that - manually. - - -.. function:: setLoggerClass(klass) - - Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`__init__` such that only a name argument is - required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This - function is typically called before any loggers are instantiated by applications - which need to use custom logger behavior. After this call, as at any other - time, do not instantiate loggers directly using the subclass: continue to use - the :func:`logging.getLogger` API to get your loggers. - - -.. function:: setLogRecordFactory(factory) - - Set a callable which is used to create a :class:`LogRecord`. - - :param factory: The factory callable to be used to instantiate a log record. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`getLogRecordFactory`, to - allow developers more control over how the :class:`LogRecord` representing - a logging event is constructed. - - The factory has the following signature: - - ``factory(name, level, fn, lno, msg, args, exc_info, func=None, sinfo=None, **kwargs)`` - - :name: The logger name. - :level: The logging level (numeric). - :fn: The full pathname of the file where the logging call was made. - :lno: The line number in the file where the logging call was made. - :msg: The logging message. - :args: The arguments for the logging message. - :exc_info: An exception tuple, or ``None``. - :func: The name of the function or method which invoked the logging - call. - :sinfo: A stack traceback such as is provided by - :func:`traceback.print_stack`, showing the call hierarchy. - :kwargs: Additional keyword arguments. - - -Module-Level Attributes ------------------------ - -.. attribute:: lastResort - - A "handler of last resort" is available through this attribute. This - is a :class:`StreamHandler` writing to ``sys.stderr`` with a level of - ``WARNING``, and is used to handle logging events in the absence of any - logging configuration. The end result is to just print the message to - ``sys.stderr``. This replaces the earlier error message saying that - "no handlers could be found for logger XYZ". If you need the earlier - behaviour for some reason, ``lastResort`` can be set to ``None``. - - .. versionadded:: 3.2 - -Integration with the warnings module ------------------------------------- - -The :func:`captureWarnings` function can be used to integrate :mod:`logging` -with the :mod:`warnings` module. - -.. function:: captureWarnings(capture) - - This function is used to turn the capture of warnings by logging on and - off. - - If *capture* is ``True``, warnings issued by the :mod:`warnings` module will - be redirected to the logging system. Specifically, a warning will be - formatted using :func:`warnings.formatwarning` and the resulting string - logged to a logger named ``'py.warnings'`` with a severity of :const:`WARNING`. - - If *capture* is ``False``, the redirection of warnings to the logging system - will stop, and warnings will be redirected to their original destinations - (i.e. those in effect before ``captureWarnings(True)`` was called). - - -.. seealso:: - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :pep:`282` - A Logging System - The proposal which described this feature for inclusion in the Python standard - library. - - `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the - package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard - library. diff --git a/Doc/library/readline.rst.bak b/Doc/library/readline.rst.bak deleted file mode 100644 index 3ff64885f7fced..00000000000000 --- a/Doc/library/readline.rst.bak +++ /dev/null @@ -1,361 +0,0 @@ -:mod:`readline` --- GNU readline interface -========================================== - -.. module:: readline - :platform: Unix - :synopsis: GNU readline support for Python. - -.. sectionauthor:: Skip Montanaro - --------------- - -The :mod:`readline` module defines a number of functions to facilitate -completion and reading/writing of history files from the Python interpreter. -This module can be used directly, or via the :mod:`rlcompleter` module, which -supports completion of Python identifiers at the interactive prompt. Settings -made using this module affect the behaviour of both the interpreter's -interactive prompt and the prompts offered by the built-in :func:`input` -function. - -Readline keybindings may be configured via an initialization file, typically -``.inputrc`` in your home directory. See `Readline Init File -`_ -in the GNU Readline manual for information about the format and -allowable constructs of that file, and the capabilities of the -Readline library in general. - -.. note:: - - The underlying Readline library API may be implemented by - the ``libedit`` library instead of GNU readline. - On macOS the :mod:`readline` module detects which library is being used - at run time. - - The configuration file for ``libedit`` is different from that - of GNU readline. If you programmatically load configuration strings - you can check for the text "libedit" in :const:`readline.__doc__` - to differentiate between GNU readline and libedit. - - If you use *editline*/``libedit`` readline emulation on macOS, the - initialization file located in your home directory is named - ``.editrc``. For example, the following content in ``~/.editrc`` will - turn ON *vi* keybindings and TAB completion:: - - python:bind -v - python:bind ^I rl_complete - - -Init file ---------- - -The following functions relate to the init file and user configuration: - - -.. function:: parse_and_bind(string) - - Execute the init line provided in the *string* argument. This calls - :c:func:`rl_parse_and_bind` in the underlying library. - - -.. function:: read_init_file([filename]) - - Execute a readline initialization file. The default filename is the last filename - used. This calls :c:func:`rl_read_init_file` in the underlying library. - - -Line buffer ------------ - -The following functions operate on the line buffer: - - -.. function:: get_line_buffer() - - Return the current contents of the line buffer (:c:data:`rl_line_buffer` - in the underlying library). - - -.. function:: insert_text(string) - - Insert text into the line buffer at the cursor position. This calls - :c:func:`rl_insert_text` in the underlying library, but ignores - the return value. - - -.. function:: redisplay() - - Change what's displayed on the screen to reflect the current contents of the - line buffer. This calls :c:func:`rl_redisplay` in the underlying library. - - -History file ------------- - -The following functions operate on a history file: - - -.. function:: read_history_file([filename]) - - Load a readline history file, and append it to the history list. - The default filename is :file:`~/.history`. This calls - :c:func:`read_history` in the underlying library. - - -.. function:: write_history_file([filename]) - - Save the history list to a readline history file, overwriting any - existing file. The default filename is :file:`~/.history`. This calls - :c:func:`write_history` in the underlying library. - - -.. function:: append_history_file(nelements[, filename]) - - Append the last *nelements* items of history to a file. The default filename is - :file:`~/.history`. The file must already exist. This calls - :c:func:`append_history` in the underlying library. This function - only exists if Python was compiled for a version of the library - that supports it. - - .. versionadded:: 3.5 - - -.. function:: get_history_length() - set_history_length(length) - - Set or return the desired number of lines to save in the history file. - The :func:`write_history_file` function uses this value to truncate - the history file, by calling :c:func:`history_truncate_file` in - the underlying library. Negative values imply - unlimited history file size. - - -History list ------------- - -The following functions operate on a global history list: - - -.. function:: clear_history() - - Clear the current history. This calls :c:func:`clear_history` in the - underlying library. The Python function only exists if Python was - compiled for a version of the library that supports it. - - -.. function:: get_current_history_length() - - Return the number of items currently in the history. (This is different from - :func:`get_history_length`, which returns the maximum number of lines that will - be written to a history file.) - - -.. function:: get_history_item(index) - - Return the current contents of history item at *index*. The item index - is one-based. This calls :c:func:`history_get` in the underlying library. - - -.. function:: remove_history_item(pos) - - Remove history item specified by its position from the history. - The position is zero-based. This calls :c:func:`remove_history` in - the underlying library. - - -.. function:: replace_history_item(pos, line) - - Replace history item specified by its position with *line*. - The position is zero-based. This calls :c:func:`replace_history_entry` - in the underlying library. - - -.. function:: add_history(line) - - Append *line* to the history buffer, as if it was the last line typed. - This calls :c:func:`add_history` in the underlying library. - - -.. function:: set_auto_history(enabled) - - Enable or disable automatic calls to :c:func:`add_history` when reading - input via readline. The *enabled* argument should be a Boolean value - that when true, enables auto history, and that when false, disables - auto history. - - .. versionadded:: 3.6 - - .. impl-detail:: - Auto history is enabled by default, and changes to this do not persist - across multiple sessions. - - -Startup hooks -------------- - - -.. function:: set_startup_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_startup_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any function - already installed is removed. The hook is called with no - arguments just before readline prints the first prompt. - - -.. function:: set_pre_input_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_pre_input_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any - function already installed is removed. The hook is called - with no arguments after the first prompt has been printed and just before - readline starts reading input characters. This function only exists - if Python was compiled for a version of the library that supports it. - - -Completion ----------- - -The following functions relate to implementing a custom word completion -function. This is typically operated by the Tab key, and can suggest and -automatically complete a word being typed. By default, Readline is set up -to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used -with a custom completer, a different set of word delimiters should be set. - - -.. function:: set_completer([function]) - - Set or remove the completer function. If *function* is specified, it will be - used as the new completer function; if omitted or ``None``, any completer - function already installed is removed. The completer function is called as - ``function(text, state)``, for *state* in ``0``, ``1``, ``2``, ..., until it - returns a non-string value. It should return the next possible completion - starting with *text*. - - The installed completer function is invoked by the *entry_func* callback - passed to :c:func:`rl_completion_matches` in the underlying library. - The *text* string comes from the first parameter to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. - - -.. function:: get_completer() - - Get the completer function, or ``None`` if no completer function has been set. - - -.. function:: get_completion_type() - - Get the type of completion being attempted. This returns the - :c:data:`rl_completion_type` variable in the underlying library as - an integer. - - -.. function:: get_begidx() - get_endidx() - - Get the beginning or ending index of the completion scope. - These indexes are the *start* and *end* arguments passed to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. The values may be different in the same - input editing scenario based on the underlying C readline implemtation. - Ex: libedit is known to behave differently than libreadline. - - -.. function:: set_completer_delims(string) - get_completer_delims() - - Set or get the word delimiters for completion. These determine the - start of the word to be considered for completion (the completion scope). - These functions access the :c:data:`rl_completer_word_break_characters` - variable in the underlying library. - - -.. function:: set_completion_display_matches_hook([function]) - - Set or remove the completion display function. If *function* is - specified, it will be used as the new completion display function; - if omitted or ``None``, any completion display function already - installed is removed. This sets or clears the - :c:data:`rl_completion_display_matches_hook` callback in the - underlying library. The completion display function is called as - ``function(substitution, [matches], longest_match_length)`` once - each time matches need to be displayed. - - -.. _readline-example: - -Example -------- - -The following example demonstrates how to use the :mod:`readline` module's -history reading and writing functions to automatically load and save a history -file named :file:`.python_history` from the user's home directory. The code -below would normally be executed automatically during interactive sessions -from the user's :envvar:`PYTHONSTARTUP` file. :: - - import atexit - import os - import readline - - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - try: - readline.read_history_file(histfile) - # default history len is -1 (infinite), which may grow unruly - readline.set_history_length(1000) - except FileNotFoundError: - pass - - atexit.register(readline.write_history_file, histfile) - -This code is actually automatically run when Python is run in -:ref:`interactive mode ` (see :ref:`rlcompleter-config`). - -The following example achieves the same goal but supports concurrent interactive -sessions, by only appending the new history. :: - - import atexit - import os - import readline - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - - try: - readline.read_history_file(histfile) - h_len = readline.get_current_history_length() - except FileNotFoundError: - open(histfile, 'wb').close() - h_len = 0 - - def save(prev_h_len, histfile): - new_h_len = readline.get_current_history_length() - readline.set_history_length(1000) - readline.append_history_file(new_h_len - prev_h_len, histfile) - atexit.register(save, h_len, histfile) - -The following example extends the :class:`code.InteractiveConsole` class to -support history save/restore. :: - - import atexit - import code - import os - import readline - - class HistoryConsole(code.InteractiveConsole): - def __init__(self, locals=None, filename="", - histfile=os.path.expanduser("~/.console-history")): - code.InteractiveConsole.__init__(self, locals, filename) - self.init_history(histfile) - - def init_history(self, histfile): - readline.parse_and_bind("tab: complete") - if hasattr(readline, "read_history_file"): - try: - readline.read_history_file(histfile) - except FileNotFoundError: - pass - atexit.register(self.save_history, histfile) - - def save_history(self, histfile): - readline.set_history_length(1000) - readline.write_history_file(histfile) diff --git a/Doc/library/sqlite3.rst.bak b/Doc/library/sqlite3.rst.bak deleted file mode 100644 index 172ce6c6bb03ba..00000000000000 --- a/Doc/library/sqlite3.rst.bak +++ /dev/null @@ -1,1094 +0,0 @@ -:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases -============================================================ - -.. module:: sqlite3 - :synopsis: A DB-API 2.0 implementation using SQLite 3.x. - -.. sectionauthor:: Gerhard Häring - -**Source code:** :source:`Lib/sqlite3/` - --------------- - -SQLite is a C library that provides a lightweight disk-based database that -doesn't require a separate server process and allows accessing the database -using a nonstandard variant of the SQL query language. Some applications can use -SQLite for internal data storage. It's also possible to prototype an -application using SQLite and then port the code to a larger database such as -PostgreSQL or Oracle. - -The sqlite3 module was written by Gerhard Häring. It provides a SQL interface -compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.7.15 or newer. - -To use the module, you must first create a :class:`Connection` object that -represents the database. Here the data will be stored in the -:file:`example.db` file:: - - import sqlite3 - con = sqlite3.connect('example.db') - -You can also supply the special name ``:memory:`` to create a database in RAM. - -Once you have a :class:`Connection`, you can create a :class:`Cursor` object -and call its :meth:`~Cursor.execute` method to perform SQL commands:: - - cur = con.cursor() - - # Create table - cur.execute('''CREATE TABLE stocks - (date text, trans text, symbol text, qty real, price real)''') - - # Insert a row of data - cur.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") - - # Save (commit) the changes - con.commit() - - # We can also close the connection if we are done with it. - # Just be sure any changes have been committed or they will be lost. - con.close() - -The data you've saved is persistent and is available in subsequent sessions:: - - import sqlite3 - con = sqlite3.connect('example.db') - cur = con.cursor() - -Usually your SQL operations will need to use values from Python variables. You -shouldn't assemble your query using Python's string operations because doing so -is insecure; it makes your program vulnerable to an SQL injection attack -(see https://xkcd.com/327/ for humorous example of what can go wrong). - -Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder -wherever you want to use a value, and then provide a tuple of values as the -second argument to the cursor's :meth:`~Cursor.execute` method. (Other database -modules may use a different placeholder, such as ``%s`` or ``:1``.) For -example:: - - # Never do this -- insecure! - symbol = 'RHAT' - cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) - - # Do this instead - t = ('RHAT',) - cur.execute('SELECT * FROM stocks WHERE symbol=?', t) - print(cur.fetchone()) - - # Larger example that inserts many records at a time - purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), - ('2006-04-06', 'SELL', 'IBM', 500, 53.00), - ] - cur.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) - -To retrieve data after executing a SELECT statement, you can either treat the -cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to -retrieve a single matching row, or call :meth:`~Cursor.fetchall` to get a list of the -matching rows. - -This example uses the iterator form:: - - >>> for row in cur.execute('SELECT * FROM stocks ORDER BY price'): - print(row) - - ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) - ('2006-03-28', 'BUY', 'IBM', 1000, 45.0) - ('2006-04-06', 'SELL', 'IBM', 500, 53.0) - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) - - -.. seealso:: - - https://www.sqlite.org - The SQLite web page; the documentation describes the syntax and the - available data types for the supported SQL dialect. - - https://www.w3schools.com/sql/ - Tutorial, reference and examples for learning SQL syntax. - - :pep:`249` - Database API Specification 2.0 - PEP written by Marc-André Lemburg. - - -.. _sqlite3-module-contents: - -Module functions and constants ------------------------------- - - -.. data:: version - - The version number of this module, as a string. This is not the version of - the SQLite library. - - -.. data:: version_info - - The version number of this module, as a tuple of integers. This is not the - version of the SQLite library. - - -.. data:: sqlite_version - - The version number of the run-time SQLite library, as a string. - - -.. data:: sqlite_version_info - - The version number of the run-time SQLite library, as a tuple of integers. - - -.. data:: PARSE_DECLTYPES - - This constant is meant to be used with the *detect_types* parameter of the - :func:`connect` function. - - Setting it makes the :mod:`sqlite3` module parse the declared type for each - column it returns. It will parse out the first word of the declared type, - i. e. for "integer primary key", it will parse out "integer", or for - "number(10)" it will parse out "number". Then for that column, it will look - into the converters dictionary and use the converter function registered for - that type there. - - -.. data:: PARSE_COLNAMES - - This constant is meant to be used with the *detect_types* parameter of the - :func:`connect` function. - - Setting this makes the SQLite interface parse the column name for each column it - returns. It will look for a string formed [mytype] in there, and then decide - that 'mytype' is the type of the column. It will try to find an entry of - 'mytype' in the converters dictionary and then use the converter function found - there to return the value. The column name found in :attr:`Cursor.description` - does not include the type, i. e. if you use something like - ``'as "Expiration date [datetime]"'`` in your SQL, then we will parse out - everything until the first ``'['`` for the column name and strip - the preceeding space: the column name would simply be "Expiration date". - - -.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) - - Opens a connection to the SQLite database file *database*. By default returns a - :class:`Connection` object, unless a custom *factory* is given. - - *database* is a :term:`path-like object` giving the pathname (absolute or - relative to the current working directory) of the database file to be opened. - You can use ``":memory:"`` to open a database connection to a database that - resides in RAM instead of on disk. - - When a database is accessed by multiple connections, and one of the processes - modifies the database, the SQLite database is locked until that transaction is - committed. The *timeout* parameter specifies how long the connection should wait - for the lock to go away until raising an exception. The default for the timeout - parameter is 5.0 (five seconds). - - For the *isolation_level* parameter, please see the - :attr:`~Connection.isolation_level` property of :class:`Connection` objects. - - SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If - you want to use other types you must add support for them yourself. The - *detect_types* parameter and the using custom **converters** registered with the - module-level :func:`register_converter` function allow you to easily do that. - - *detect_types* defaults to 0 (i. e. off, no type detection), you can set it to - any combination of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to turn - type detection on. Due to SQLite behaviour, types can't be detected for generated - fields (for example ``max(data)``), even when *detect_types* parameter is set. In - such case, the returned type is :class:`str`. - - By default, *check_same_thread* is :const:`True` and only the creating thread may - use the connection. If set :const:`False`, the returned connection may be shared - across multiple threads. When using multiple threads with the same connection - writing operations should be serialized by the user to avoid data corruption. - - By default, the :mod:`sqlite3` module uses its :class:`Connection` class for the - connect call. You can, however, subclass the :class:`Connection` class and make - :func:`connect` use your class instead by providing your class for the *factory* - parameter. - - Consult the section :ref:`sqlite3-types` of this manual for details. - - The :mod:`sqlite3` module internally uses a statement cache to avoid SQL parsing - overhead. If you want to explicitly set the number of statements that are cached - for the connection, you can set the *cached_statements* parameter. The currently - implemented default is to cache 100 statements. - - If *uri* is true, *database* is interpreted as a URI. This allows you - to specify options. For example, to open a database in read-only mode - you can use:: - - db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) - - More information about this feature, including a list of recognized options, can - be found in the `SQLite URI documentation `_. - - .. audit-event:: sqlite3.connect database sqlite3.connect - - .. versionchanged:: 3.4 - Added the *uri* parameter. - - .. versionchanged:: 3.7 - *database* can now also be a :term:`path-like object`, not only a string. - - -.. function:: register_converter(typename, callable) - - Registers a callable to convert a bytestring from the database into a custom - Python type. The callable will be invoked for all database values that are of - the type *typename*. Confer the parameter *detect_types* of the :func:`connect` - function for how the type detection works. Note that *typename* and the name of - the type in your query are matched in case-insensitive manner. - - -.. function:: register_adapter(type, callable) - - Registers a callable to convert the custom Python type *type* into one of - SQLite's supported types. The callable *callable* accepts as single parameter - the Python value, and must return a value of the following types: int, - float, str or bytes. - - -.. function:: complete_statement(sql) - - Returns :const:`True` if the string *sql* contains one or more complete SQL - statements terminated by semicolons. It does not verify that the SQL is - syntactically correct, only that there are no unclosed string literals and the - statement is terminated by a semicolon. - - This can be used to build a shell for SQLite, as in the following example: - - - .. literalinclude:: ../includes/sqlite3/complete_statement.py - - -.. function:: enable_callback_tracebacks(flag) - - By default you will not get any tracebacks in user-defined functions, - aggregates, converters, authorizer callbacks etc. If you want to debug them, - you can call this function with *flag* set to ``True``. Afterwards, you will - get tracebacks from callbacks on ``sys.stderr``. Use :const:`False` to - disable the feature again. - - -.. _sqlite3-connection-objects: - -Connection Objects ------------------- - -.. class:: Connection - - A SQLite database connection has the following attributes and methods: - - .. attribute:: isolation_level - - Get or set the current default isolation level. :const:`None` for autocommit mode or - one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section - :ref:`sqlite3-controlling-transactions` for a more detailed explanation. - - .. attribute:: in_transaction - - :const:`True` if a transaction is active (there are uncommitted changes), - :const:`False` otherwise. Read-only attribute. - - .. versionadded:: 3.2 - - .. method:: cursor(factory=Cursor) - - The cursor method accepts a single optional parameter *factory*. If - supplied, this must be a callable returning an instance of :class:`Cursor` - or its subclasses. - - .. method:: commit() - - This method commits the current transaction. If you don't call this method, - anything you did since the last call to ``commit()`` is not visible from - other database connections. If you wonder why you don't see the data you've - written to the database, please check you didn't forget to call this method. - - .. method:: rollback() - - This method rolls back any changes to the database since the last call to - :meth:`commit`. - - .. method:: close() - - This closes the database connection. Note that this does not automatically - call :meth:`commit`. If you just close your database connection without - calling :meth:`commit` first, your changes will be lost! - - .. method:: execute(sql[, parameters]) - - This is a nonstandard shortcut that creates a cursor object by calling - the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.execute` method with the *parameters* given, and returns - the cursor. - - .. method:: executemany(sql[, parameters]) - - This is a nonstandard shortcut that creates a cursor object by - calling the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.executemany` method with the *parameters* given, and - returns the cursor. - - .. method:: executescript(sql_script) - - This is a nonstandard shortcut that creates a cursor object by - calling the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.executescript` method with the given *sql_script*, and - returns the cursor. - - .. method:: create_function(name, num_params, func, *, deterministic=False) - - Creates a user-defined function that you can later use from within SQL - statements under the function name *name*. *num_params* is the number of - parameters the function accepts (if *num_params* is -1, the function may - take any number of arguments), and *func* is a Python callable that is - called as the SQL function. If *deterministic* is true, the created function - is marked as `deterministic `_, which - allows SQLite to perform additional optimizations. This flag is supported by - SQLite 3.8.3 or higher, :exc:`NotSupportedError` will be raised if used - with older versions. - - The function can return any of the types supported by SQLite: bytes, str, int, - float and ``None``. - - .. versionchanged:: 3.8 - The *deterministic* parameter was added. - - Example: - - .. literalinclude:: ../includes/sqlite3/md5func.py - - - .. method:: create_aggregate(name, num_params, aggregate_class) - - Creates a user-defined aggregate function. - - The aggregate class must implement a ``step`` method, which accepts the number - of parameters *num_params* (if *num_params* is -1, the function may take - any number of arguments), and a ``finalize`` method which will return the - final result of the aggregate. - - The ``finalize`` method can return any of the types supported by SQLite: - bytes, str, int, float and ``None``. - - Example: - - .. literalinclude:: ../includes/sqlite3/mysumaggr.py - - - .. method:: create_collation(name, callable) - - Creates a collation with the specified *name* and *callable*. The callable will - be passed two string arguments. It should return -1 if the first is ordered - lower than the second, 0 if they are ordered equal and 1 if the first is ordered - higher than the second. Note that this controls sorting (ORDER BY in SQL) so - your comparisons don't affect other SQL operations. - - Note that the callable will get its parameters as Python bytestrings, which will - normally be encoded in UTF-8. - - The following example shows a custom collation that sorts "the wrong way": - - .. literalinclude:: ../includes/sqlite3/collation_reverse.py - - To remove a collation, call ``create_collation`` with ``None`` as callable:: - - con.create_collation("reverse", None) - - - .. method:: interrupt() - - You can call this method from a different thread to abort any queries that might - be executing on the connection. The query will then abort and the caller will - get an exception. - - - .. method:: set_authorizer(authorizer_callback) - - This routine registers a callback. The callback is invoked for each attempt to - access a column of a table in the database. The callback should return - :const:`SQLITE_OK` if access is allowed, :const:`SQLITE_DENY` if the entire SQL - statement should be aborted with an error and :const:`SQLITE_IGNORE` if the - column should be treated as a NULL value. These constants are available in the - :mod:`sqlite3` module. - - The first argument to the callback signifies what kind of operation is to be - authorized. The second and third argument will be arguments or :const:`None` - depending on the first argument. The 4th argument is the name of the database - ("main", "temp", etc.) if applicable. The 5th argument is the name of the - inner-most trigger or view that is responsible for the access attempt or - :const:`None` if this access attempt is directly from input SQL code. - - Please consult the SQLite documentation about the possible values for the first - argument and the meaning of the second and third argument depending on the first - one. All necessary constants are available in the :mod:`sqlite3` module. - - - .. method:: set_progress_handler(handler, n) - - This routine registers a callback. The callback is invoked for every *n* - instructions of the SQLite virtual machine. This is useful if you want to - get called from SQLite during long-running operations, for example to update - a GUI. - - If you want to clear any previously installed progress handler, call the - method with :const:`None` for *handler*. - - Returning a non-zero value from the handler function will terminate the - currently executing query and cause it to raise an :exc:`OperationalError` - exception. - - - .. method:: set_trace_callback(trace_callback) - - Registers *trace_callback* to be called for each SQL statement that is - actually executed by the SQLite backend. - - The only argument passed to the callback is the statement (as string) that - is being executed. The return value of the callback is ignored. Note that - the backend does not only run statements passed to the :meth:`Cursor.execute` - methods. Other sources include the transaction management of the Python - module and the execution of triggers defined in the current database. - - Passing :const:`None` as *trace_callback* will disable the trace callback. - - .. versionadded:: 3.3 - - - .. method:: enable_load_extension(enabled) - - This routine allows/disallows the SQLite engine to load SQLite extensions - from shared libraries. SQLite extensions can define new functions, - aggregates or whole new virtual table implementations. One well-known - extension is the fulltext-search extension distributed with SQLite. - - Loadable extensions are disabled by default. See [#f1]_. - - .. versionadded:: 3.2 - - .. literalinclude:: ../includes/sqlite3/load_extension.py - - .. method:: load_extension(path) - - This routine loads a SQLite extension from a shared library. You have to - enable extension loading with :meth:`enable_load_extension` before you can - use this routine. - - Loadable extensions are disabled by default. See [#f1]_. - - .. versionadded:: 3.2 - - .. attribute:: row_factory - - You can change this attribute to a callable that accepts the cursor and the - original row as a tuple and will return the real result row. This way, you can - implement more advanced ways of returning results, such as returning an object - that can also access columns by name. - - Example: - - .. literalinclude:: ../includes/sqlite3/row_factory.py - - If returning a tuple doesn't suffice and you want name-based access to - columns, you should consider setting :attr:`row_factory` to the - highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both - index-based and case-insensitive name-based access to columns with almost no - memory overhead. It will probably be better than your own custom - dictionary-based approach or even a db_row based solution. - - .. XXX what's a db_row-based solution? - - - .. attribute:: text_factory - - Using this attribute you can control what objects are returned for the ``TEXT`` - data type. By default, this attribute is set to :class:`str` and the - :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to - return bytestrings instead, you can set it to :class:`bytes`. - - You can also set it to any other callable that accepts a single bytestring - parameter and returns the resulting object. - - See the following example code for illustration: - - .. literalinclude:: ../includes/sqlite3/text_factory.py - - - .. attribute:: total_changes - - Returns the total number of database rows that have been modified, inserted, or - deleted since the database connection was opened. - - - .. method:: iterdump - - Returns an iterator to dump the database in an SQL text format. Useful when - saving an in-memory database for later restoration. This function provides - the same capabilities as the :kbd:`.dump` command in the :program:`sqlite3` - shell. - - Example:: - - # Convert file existing_db.db to SQL dump file dump.sql - import sqlite3 - - con = sqlite3.connect('existing_db.db') - with open('dump.sql', 'w') as f: - for line in con.iterdump(): - f.write('%s\n' % line) - con.close() - - - .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) - - This method makes a backup of a SQLite database even while it's being accessed - by other clients, or concurrently by the same connection. The copy will be - written into the mandatory argument *target*, that must be another - :class:`Connection` instance. - - By default, or when *pages* is either ``0`` or a negative integer, the entire - database is copied in a single step; otherwise the method performs a loop - copying up to *pages* pages at a time. - - If *progress* is specified, it must either be ``None`` or a callable object that - will be executed at each iteration with three integer arguments, respectively - the *status* of the last iteration, the *remaining* number of pages still to be - copied and the *total* number of pages. - - The *name* argument specifies the database name that will be copied: it must be - a string containing either ``"main"``, the default, to indicate the main - database, ``"temp"`` to indicate the temporary database or the name specified - after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached - database. - - The *sleep* argument specifies the number of seconds to sleep by between - successive attempts to backup remaining pages, can be specified either as an - integer or a floating point value. - - Example 1, copy an existing database into another:: - - import sqlite3 - - def progress(status, remaining, total): - print(f'Copied {total-remaining} of {total} pages...') - - con = sqlite3.connect('existing_db.db') - bck = sqlite3.connect('backup.db') - with bck: - con.backup(bck, pages=1, progress=progress) - bck.close() - con.close() - - Example 2, copy an existing database into a transient copy:: - - import sqlite3 - - source = sqlite3.connect('existing_db.db') - dest = sqlite3.connect(':memory:') - source.backup(dest) - - .. versionadded:: 3.7 - - -.. _sqlite3-cursor-objects: - -Cursor Objects --------------- - -.. class:: Cursor - - A :class:`Cursor` instance has the following attributes and methods. - - .. index:: single: ? (question mark); in SQL statements - .. index:: single: : (colon); in SQL statements - - .. method:: execute(sql[, parameters]) - - Executes an SQL statement. The SQL statement may be parameterized (i. e. - placeholders instead of SQL literals). The :mod:`sqlite3` module supports two - kinds of placeholders: question marks (qmark style) and named placeholders - (named style). - - Here's an example of both styles: - - .. literalinclude:: ../includes/sqlite3/execute_1.py - - :meth:`execute` will only execute a single SQL statement. If you try to execute - more than one statement with it, it will raise a :exc:`.Warning`. Use - :meth:`executescript` if you want to execute multiple SQL statements with one - call. - - - .. method:: executemany(sql, seq_of_parameters) - - Executes an SQL command against all parameter sequences or mappings found in - the sequence *seq_of_parameters*. The :mod:`sqlite3` module also allows - using an :term:`iterator` yielding parameters instead of a sequence. - - .. literalinclude:: ../includes/sqlite3/executemany_1.py - - Here's a shorter example using a :term:`generator`: - - .. literalinclude:: ../includes/sqlite3/executemany_2.py - - - .. method:: executescript(sql_script) - - This is a nonstandard convenience method for executing multiple SQL statements - at once. It issues a ``COMMIT`` statement first, then executes the SQL script it - gets as a parameter. - - *sql_script* can be an instance of :class:`str`. - - Example: - - .. literalinclude:: ../includes/sqlite3/executescript.py - - - .. method:: fetchone() - - Fetches the next row of a query result set, returning a single sequence, - or :const:`None` when no more data is available. - - - .. method:: fetchmany(size=cursor.arraysize) - - Fetches the next set of rows of a query result, returning a list. An empty - list is returned when no more rows are available. - - The number of rows to fetch per call is specified by the *size* parameter. - If it is not given, the cursor's arraysize determines the number of rows - to be fetched. The method should try to fetch as many rows as indicated by - the size parameter. If this is not possible due to the specified number of - rows not being available, fewer rows may be returned. - - Note there are performance considerations involved with the *size* parameter. - For optimal performance, it is usually best to use the arraysize attribute. - If the *size* parameter is used, then it is best for it to retain the same - value from one :meth:`fetchmany` call to the next. - - .. method:: fetchall() - - Fetches all (remaining) rows of a query result, returning a list. Note that - the cursor's arraysize attribute can affect the performance of this operation. - An empty list is returned when no rows are available. - - .. method:: close() - - Close the cursor now (rather than whenever ``__del__`` is called). - - The cursor will be unusable from this point forward; a :exc:`ProgrammingError` - exception will be raised if any operation is attempted with the cursor. - - .. attribute:: rowcount - - Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this - attribute, the database engine's own support for the determination of "rows - affected"/"rows selected" is quirky. - - For :meth:`executemany` statements, the number of modifications are summed up - into :attr:`rowcount`. - - As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in - case no ``executeXX()`` has been performed on the cursor or the rowcount of the - last operation is not determinable by the interface". This includes ``SELECT`` - statements because we cannot determine the number of rows a query produced - until all rows were fetched. - - .. attribute:: lastrowid - - This read-only attribute provides the rowid of the last modified row. It is - only set if you issued an ``INSERT`` or a ``REPLACE`` statement using the - :meth:`execute` method. For operations other than ``INSERT`` or - ``REPLACE`` or when :meth:`executemany` is called, :attr:`lastrowid` is - set to :const:`None`. - - If the ``INSERT`` or ``REPLACE`` statement failed to insert the previous - successful rowid is returned. - - .. versionchanged:: 3.6 - Added support for the ``REPLACE`` statement. - - .. attribute:: arraysize - - Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. - The default value is 1 which means a single row would be fetched per call. - - .. attribute:: description - - This read-only attribute provides the column names of the last query. To - remain compatible with the Python DB API, it returns a 7-tuple for each - column where the last six items of each tuple are :const:`None`. - - It is set for ``SELECT`` statements without any matching rows as well. - - .. attribute:: connection - - This read-only attribute provides the SQLite database :class:`Connection` - used by the :class:`Cursor` object. A :class:`Cursor` object created by - calling :meth:`con.cursor() ` will have a - :attr:`connection` attribute that refers to *con*:: - - >>> con = sqlite3.connect(":memory:") - >>> cur = con.cursor() - >>> cur.connection == con - True - -.. _sqlite3-row-objects: - -Row Objects ------------ - -.. class:: Row - - A :class:`Row` instance serves as a highly optimized - :attr:`~Connection.row_factory` for :class:`Connection` objects. - It tries to mimic a tuple in most of its features. - - It supports mapping access by column name and index, iteration, - representation, equality testing and :func:`len`. - - If two :class:`Row` objects have exactly the same columns and their - members are equal, they compare equal. - - .. method:: keys - - This method returns a list of column names. Immediately after a query, - it is the first member of each tuple in :attr:`Cursor.description`. - - .. versionchanged:: 3.5 - Added support of slicing. - -Let's assume we initialize a table as in the example given above:: - - con = sqlite3.connect(":memory:") - cur = con.cursor() - cur.execute('''create table stocks - (date text, trans text, symbol text, - qty real, price real)''') - cur.execute("""insert into stocks - values ('2006-01-05','BUY','RHAT',100,35.14)""") - con.commit() - cur.close() - -Now we plug :class:`Row` in:: - - >>> con.row_factory = sqlite3.Row - >>> cur = con.cursor() - >>> cur.execute('select * from stocks') - - >>> r = cur.fetchone() - >>> type(r) - - >>> tuple(r) - ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14) - >>> len(r) - 5 - >>> r[2] - 'RHAT' - >>> r.keys() - ['date', 'trans', 'symbol', 'qty', 'price'] - >>> r['qty'] - 100.0 - >>> for member in r: - ... print(member) - ... - 2006-01-05 - BUY - RHAT - 100.0 - 35.14 - - -.. _sqlite3-exceptions: - -Exceptions ----------- - -.. exception:: Warning - - A subclass of :exc:`Exception`. - -.. exception:: Error - - The base class of the other exceptions in this module. It is a subclass - of :exc:`Exception`. - -.. exception:: DatabaseError - - Exception raised for errors that are related to the database. - -.. exception:: IntegrityError - - Exception raised when the relational integrity of the database is affected, - e.g. a foreign key check fails. It is a subclass of :exc:`DatabaseError`. - -.. exception:: ProgrammingError - - Exception raised for programming errors, e.g. table not found or already - exists, syntax error in the SQL statement, wrong number of parameters - specified, etc. It is a subclass of :exc:`DatabaseError`. - -.. exception:: OperationalError - - Exception raised for errors that are related to the database's operation - and not necessarily under the control of the programmer, e.g. an unexpected - disconnect occurs, the data source name is not found, a transaction could - not be processed, etc. It is a subclass of :exc:`DatabaseError`. - -.. exception:: NotSupportedError - - Exception raised in case a method or database API was used which is not - supported by the database, e.g. calling the :meth:`~Connection.rollback` - method on a connection that does not support transaction or has - transactions turned off. It is a subclass of :exc:`DatabaseError`. - - -.. _sqlite3-types: - -SQLite and Python types ------------------------ - - -Introduction -^^^^^^^^^^^^ - -SQLite natively supports the following types: ``NULL``, ``INTEGER``, -``REAL``, ``TEXT``, ``BLOB``. - -The following Python types can thus be sent to SQLite without any problem: - -+-------------------------------+-------------+ -| Python type | SQLite type | -+===============================+=============+ -| :const:`None` | ``NULL`` | -+-------------------------------+-------------+ -| :class:`int` | ``INTEGER`` | -+-------------------------------+-------------+ -| :class:`float` | ``REAL`` | -+-------------------------------+-------------+ -| :class:`str` | ``TEXT`` | -+-------------------------------+-------------+ -| :class:`bytes` | ``BLOB`` | -+-------------------------------+-------------+ - - -This is how SQLite types are converted to Python types by default: - -+-------------+----------------------------------------------+ -| SQLite type | Python type | -+=============+==============================================+ -| ``NULL`` | :const:`None` | -+-------------+----------------------------------------------+ -| ``INTEGER`` | :class:`int` | -+-------------+----------------------------------------------+ -| ``REAL`` | :class:`float` | -+-------------+----------------------------------------------+ -| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | -| | :class:`str` by default | -+-------------+----------------------------------------------+ -| ``BLOB`` | :class:`bytes` | -+-------------+----------------------------------------------+ - -The type system of the :mod:`sqlite3` module is extensible in two ways: you can -store additional Python types in a SQLite database via object adaptation, and -you can let the :mod:`sqlite3` module convert SQLite types to different Python -types via converters. - - -Using adapters to store additional Python types in SQLite databases -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As described before, SQLite supports only a limited set of types natively. To -use other Python types with SQLite, you must **adapt** them to one of the -sqlite3 module's supported types for SQLite: one of NoneType, int, float, -str, bytes. - -There are two ways to enable the :mod:`sqlite3` module to adapt a custom Python -type to one of the supported ones. - - -Letting your object adapt itself -"""""""""""""""""""""""""""""""" - -This is a good approach if you write the class yourself. Let's suppose you have -a class like this:: - - class Point: - def __init__(self, x, y): - self.x, self.y = x, y - -Now you want to store the point in a single SQLite column. First you'll have to -choose one of the supported types to be used for representing the point. -Let's just use str and separate the coordinates using a semicolon. Then you need -to give your class a method ``__conform__(self, protocol)`` which must return -the converted value. The parameter *protocol* will be :class:`PrepareProtocol`. - -.. literalinclude:: ../includes/sqlite3/adapter_point_1.py - - -Registering an adapter callable -""""""""""""""""""""""""""""""" - -The other possibility is to create a function that converts the type to the -string representation and register the function with :meth:`register_adapter`. - -.. literalinclude:: ../includes/sqlite3/adapter_point_2.py - -The :mod:`sqlite3` module has two default adapters for Python's built-in -:class:`datetime.date` and :class:`datetime.datetime` types. Now let's suppose -we want to store :class:`datetime.datetime` objects not in ISO representation, -but as a Unix timestamp. - -.. literalinclude:: ../includes/sqlite3/adapter_datetime.py - - -Converting SQLite values to custom Python types -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Writing an adapter lets you send custom Python types to SQLite. But to make it -really useful we need to make the Python to SQLite to Python roundtrip work. - -Enter converters. - -Let's go back to the :class:`Point` class. We stored the x and y coordinates -separated via semicolons as strings in SQLite. - -First, we'll define a converter function that accepts the string as a parameter -and constructs a :class:`Point` object from it. - -.. note:: - - Converter functions **always** get called with a :class:`bytes` object, no - matter under which data type you sent the value to SQLite. - -:: - - def convert_point(s): - x, y = map(float, s.split(b";")) - return Point(x, y) - -Now you need to make the :mod:`sqlite3` module know that what you select from -the database is actually a point. There are two ways of doing this: - -* Implicitly via the declared type - -* Explicitly via the column name - -Both ways are described in section :ref:`sqlite3-module-contents`, in the entries -for the constants :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES`. - -The following example illustrates both approaches. - -.. literalinclude:: ../includes/sqlite3/converter_point.py - - -Default adapters and converters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are default adapters for the date and datetime types in the datetime -module. They will be sent as ISO dates/ISO timestamps to SQLite. - -The default converters are registered under the name "date" for -:class:`datetime.date` and under the name "timestamp" for -:class:`datetime.datetime`. - -This way, you can use date/timestamps from Python without any additional -fiddling in most cases. The format of the adapters is also compatible with the -experimental SQLite date/time functions. - -The following example demonstrates this. - -.. literalinclude:: ../includes/sqlite3/pysqlite_datetime.py - -If a timestamp stored in SQLite has a fractional part longer than 6 -numbers, its value will be truncated to microsecond precision by the -timestamp converter. - - -.. _sqlite3-controlling-transactions: - -Controlling Transactions ------------------------- - -The underlying ``sqlite3`` library operates in ``autocommit`` mode by default, -but the Python :mod:`sqlite3` module by default does not. - -``autocommit`` mode means that statements that modify the database take effect -immediately. A ``BEGIN`` or ``SAVEPOINT`` statement disables ``autocommit`` -mode, and a ``COMMIT``, a ``ROLLBACK``, or a ``RELEASE`` that ends the -outermost transaction, turns ``autocommit`` mode back on. - -The Python :mod:`sqlite3` module by default issues a ``BEGIN`` statement -implicitly before a Data Modification Language (DML) statement (i.e. -``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). - -You can control which kind of ``BEGIN`` statements :mod:`sqlite3` implicitly -executes via the *isolation_level* parameter to the :func:`connect` -call, or via the :attr:`isolation_level` property of connections. -If you specify no *isolation_level*, a plain ``BEGIN`` is used, which is -equivalent to specifying ``DEFERRED``. Other possible values are ``IMMEDIATE`` -and ``EXCLUSIVE``. - -You can disable the :mod:`sqlite3` module's implicit transaction management by -setting :attr:`isolation_level` to ``None``. This will leave the underlying -``sqlite3`` library operating in ``autocommit`` mode. You can then completely -control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, -``SAVEPOINT``, and ``RELEASE`` statements in your code. - -.. versionchanged:: 3.6 - :mod:`sqlite3` used to implicitly commit an open transaction before DDL - statements. This is no longer the case. - - -Using :mod:`sqlite3` efficiently --------------------------------- - - -Using shortcut methods -^^^^^^^^^^^^^^^^^^^^^^ - -Using the nonstandard :meth:`execute`, :meth:`executemany` and -:meth:`executescript` methods of the :class:`Connection` object, your code can -be written more concisely because you don't have to create the (often -superfluous) :class:`Cursor` objects explicitly. Instead, the :class:`Cursor` -objects are created implicitly and these shortcut methods return the cursor -objects. This way, you can execute a ``SELECT`` statement and iterate over it -directly using only a single call on the :class:`Connection` object. - -.. literalinclude:: ../includes/sqlite3/shortcut_methods.py - - -Accessing columns by name instead of by index -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -One useful feature of the :mod:`sqlite3` module is the built-in -:class:`sqlite3.Row` class designed to be used as a row factory. - -Rows wrapped with this class can be accessed both by index (like tuples) and -case-insensitively by name: - -.. literalinclude:: ../includes/sqlite3/rowclass.py - - -Using the connection as a context manager -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Connection objects can be used as context managers -that automatically commit or rollback transactions. In the event of an -exception, the transaction is rolled back; otherwise, the transaction is -committed: - -.. literalinclude:: ../includes/sqlite3/ctx_manager.py - - -.. rubric:: Footnotes - -.. [#f1] The sqlite3 module is not built with loadable extension support by - default, because some platforms (notably Mac OS X) have SQLite - libraries which are compiled without this feature. To get loadable - extension support, you must pass --enable-loadable-sqlite-extensions to - configure. diff --git a/Doc/library/statistics.rst.bak b/Doc/library/statistics.rst.bak deleted file mode 100644 index 6b6d3154a28810..00000000000000 --- a/Doc/library/statistics.rst.bak +++ /dev/null @@ -1,879 +0,0 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= - -.. module:: statistics - :synopsis: Mathematical statistics functions - -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/statistics.py` - -.. testsetup:: * - - from statistics import * - __name__ = '' - --------------- - -This module provides functions for calculating mathematical statistics of -numeric (:class:`~numbers.Real`-valued) data. - -The module is not intended to be a competitor to third-party libraries such -as `NumPy `_, `SciPy `_, or -proprietary full-featured statistics packages aimed at professional -statisticians such as Minitab, SAS and Matlab. It is aimed at the level of -graphing and scientific calculators. - -Unless explicitly noted, these functions support :class:`int`, -:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. -Behaviour with other types (whether in the numeric tower or not) is -currently unsupported. Collections with a mix of types are also undefined -and implementation-dependent. If your input data consists of mixed types, -you may be able to use :func:`map` to ensure a consistent result, for -example: ``map(float, input_data)``. - -Averages and measures of central location ------------------------------------------ - -These functions calculate an average or typical value from a population -or sample. - -======================= =============================================================== -:func:`mean` Arithmetic mean ("average") of data. -:func:`fmean` Fast, floating point arithmetic mean. -:func:`geometric_mean` Geometric mean of data. -:func:`harmonic_mean` Harmonic mean of data. -:func:`median` Median (middle value) of data. -:func:`median_low` Low median of data. -:func:`median_high` High median of data. -:func:`median_grouped` Median, or 50th percentile, of grouped data. -:func:`mode` Single mode (most common value) of discrete or nominal data. -:func:`multimode` List of modes (most common values) of discrete or nomimal data. -:func:`quantiles` Divide data into intervals with equal probability. -======================= =============================================================== - -Measures of spread ------------------- - -These functions calculate a measure of how much the population or sample -tends to deviate from the typical or average values. - -======================= ============================================= -:func:`pstdev` Population standard deviation of data. -:func:`pvariance` Population variance of data. -:func:`stdev` Sample standard deviation of data. -:func:`variance` Sample variance of data. -======================= ============================================= - - -Function details ----------------- - -Note: The functions do not require the data given to them to be sorted. -However, for reading convenience, most of the examples show sorted sequences. - -.. function:: mean(data) - - Return the sample arithmetic mean of *data* which can be a sequence or iterable. - - The arithmetic mean is the sum of the data divided by the number of data - points. It is commonly called "the average", although it is only one of many - different mathematical averages. It is a measure of the central location of - the data. - - If *data* is empty, :exc:`StatisticsError` will be raised. - - Some examples of use: - - .. doctest:: - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - >>> mean([-1.0, 2.5, 3.25, 5.75]) - 2.625 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - .. note:: - - The mean is strongly affected by outliers and is not a robust estimator - for central location: the mean is not necessarily a typical example of - the data points. For more robust measures of central location, see - :func:`median` and :func:`mode`. - - The sample mean gives an unbiased estimate of the true population mean, - so that when taken on average over all the possible samples, - ``mean(sample)`` converges on the true mean of the entire population. If - *data* represents the entire population rather than a sample, then - ``mean(data)`` is equivalent to calculating the true population mean μ. - - -.. function:: fmean(data) - - Convert *data* to floats and compute the arithmetic mean. - - This runs faster than the :func:`mean` function and it always returns a - :class:`float`. The *data* may be a sequence or iterable. If the input - dataset is empty, raises a :exc:`StatisticsError`. - - .. doctest:: - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - - .. versionadded:: 3.8 - - -.. function:: geometric_mean(data) - - Convert *data* to floats and compute the geometric mean. - - The geometric mean indicates the central tendency or typical value of the - *data* using the product of the values (as opposed to the arithmetic mean - which uses their sum). - - Raises a :exc:`StatisticsError` if the input dataset is empty, - if it contains a zero, or if it contains a negative value. - The *data* may be a sequence or iterable. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - .. doctest:: - - >>> round(geometric_mean([54, 24, 36]), 1) - 36.0 - - .. versionadded:: 3.8 - - -.. function:: harmonic_mean(data, weights=None) - - Return the harmonic mean of *data*, a sequence or iterable of - real-valued numbers. If *weights* is omitted or *None*, then - equal weighting is assumed. - - The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the - reciprocals of the data. For example, the harmonic mean of three values *a*, - *b* and *c* will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the - values is zero, the result will be zero. - - The harmonic mean is a type of average, a measure of the central - location of the data. It is often appropriate when averaging - ratios or rates, for example speeds. - - Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. - What is the average speed? - - .. doctest:: - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - .. doctest:: - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 - - :exc:`StatisticsError` is raised if *data* is empty, any element - is less than zero, or if the weighted sum isn't positive. - - The current algorithm has an early-out when it encounters a zero - in the input. This means that the subsequent inputs are not tested - for validity. (This behavior may change in the future.) - - .. versionadded:: 3.6 - - .. versionchanged:: 3.10 - Added support for *weights*. - -.. function:: median(data) - - Return the median (middle value) of numeric data, using the common "mean of - middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. - *data* can be a sequence or iterable. - - The median is a robust measure of central location and is less affected by - the presence of outliers. When the number of data points is odd, the - middle data point is returned: - - .. doctest:: - - >>> median([1, 3, 5]) - 3 - - When the number of data points is even, the median is interpolated by taking - the average of the two middle values: - - .. doctest:: - - >>> median([1, 3, 5, 7]) - 4.0 - - This is suited for when your data is discrete, and you don't mind that the - median may not be an actual data point. - - If the data is ordinal (supports order operations) but not numeric (doesn't - support addition), consider using :func:`median_low` or :func:`median_high` - instead. - -.. function:: median_low(data) - - Return the low median of numeric data. If *data* is empty, - :exc:`StatisticsError` is raised. *data* can be a sequence or iterable. - - The low median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the smaller of - the two middle values is returned. - - .. doctest:: - - >>> median_low([1, 3, 5]) - 3 - >>> median_low([1, 3, 5, 7]) - 3 - - Use the low median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_high(data) - - Return the high median of data. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. - - The high median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the larger of - the two middle values is returned. - - .. doctest:: - - >>> median_high([1, 3, 5]) - 3 - >>> median_high([1, 3, 5, 7]) - 5 - - Use the high median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_grouped(data, interval=1) - - Return the median of grouped continuous data, calculated as the 50th - percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. - - .. doctest:: - - >>> median_grouped([52, 52, 53, 54]) - 52.5 - - In the following example, the data are rounded, so that each value represents - the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 - is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data - given, the middle value falls somewhere in the class 3.5--4.5, and - interpolation is used to estimate it: - - .. doctest:: - - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - - Optional argument *interval* represents the class interval, and defaults - to 1. Changing the class interval naturally will change the interpolation: - - .. doctest:: - - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 - - This function does not check whether the data points are at least - *interval* apart. - - .. impl-detail:: - - Under some circumstances, :func:`median_grouped` may coerce data points to - floats. This behaviour is likely to change in the future. - - .. seealso:: - - * "Statistics for the Behavioral Sciences", Frederick J Gravetter and - Larry B Wallnau (8th Edition). - - * The `SSMEDIAN - `_ - function in the Gnome Gnumeric spreadsheet, including `this discussion - `_. - - -.. function:: mode(data) - - Return the single most common data point from discrete or nominal *data*. - The mode (when it exists) is the most typical value and serves as a - measure of central location. - - If there are multiple modes with the same frequency, returns the first one - encountered in the *data*. If the smallest or largest of those is - desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``. - If the input *data* is empty, :exc:`StatisticsError` is raised. - - ``mode`` assumes discrete data and returns a single value. This is the - standard treatment of the mode as commonly taught in schools: - - .. doctest:: - - >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) - 3 - - The mode is unique in that it is the only statistic in this package that - also applies to nominal (non-numeric) data: - - .. doctest:: - - >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) - 'red' - - .. versionchanged:: 3.8 - Now handles multimodal datasets by returning the first mode encountered. - Formerly, it raised :exc:`StatisticsError` when more than one mode was - found. - - -.. function:: multimode(data) - - Return a list of the most frequently occurring values in the order they - were first encountered in the *data*. Will return more than one result if - there are multiple modes or an empty list if the *data* is empty: - - .. doctest:: - - >>> multimode('aabbbbccddddeeffffgg') - ['b', 'd', 'f'] - >>> multimode('') - [] - - .. versionadded:: 3.8 - - -.. function:: pstdev(data, mu=None) - - Return the population standard deviation (the square root of the population - variance). See :func:`pvariance` for arguments and other details. - - .. doctest:: - - >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 0.986893273527251 - - -.. function:: pvariance(data, mu=None) - - Return the population variance of *data*, a non-empty sequence or iterable - of real-valued numbers. Variance, or second moment about the mean, is a - measure of the variability (spread or dispersion) of data. A large - variance indicates that the data is spread out; a small variance indicates - it is clustered closely around the mean. - - If the optional second argument *mu* is given, it is typically the mean of - the *data*. It can also be used to compute the second moment around a - point that is not the mean. If it is missing or ``None`` (the default), - the arithmetic mean is automatically calculated. - - Use this function to calculate the variance from the entire population. To - estimate the variance from a sample, the :func:`variance` function is usually - a better choice. - - Raises :exc:`StatisticsError` if *data* is empty. - - Examples: - - .. doctest:: - - >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] - >>> pvariance(data) - 1.25 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *mu* to avoid recalculation: - - .. doctest:: - - >>> mu = mean(data) - >>> pvariance(data, mu) - 1.25 - - Decimals and Fractions are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('24.815') - - >>> from fractions import Fraction as F - >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) - Fraction(13, 72) - - .. note:: - - When called with the entire population, this gives the population variance - σ². When called on a sample instead, this is the biased sample variance - s², also known as variance with N degrees of freedom. - - If you somehow know the true population mean μ, you may use this - function to calculate the variance of a sample, giving the known - population mean as the second argument. Provided the data points are a - random sample of the population, the result will be an unbiased estimate - of the population variance. - - -.. function:: stdev(data, xbar=None) - - Return the sample standard deviation (the square root of the sample - variance). See :func:`variance` for arguments and other details. - - .. doctest:: - - >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 1.0810874155219827 - - -.. function:: variance(data, xbar=None) - - Return the sample variance of *data*, an iterable of at least two real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. - - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. - - Use this function when your data is a sample from a population. To calculate - the variance from the entire population, see :func:`pvariance`. - - Raises :exc:`StatisticsError` if *data* has fewer than two values. - - Examples: - - .. doctest:: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: - - .. doctest:: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not attempt to verify that you have passed the actual mean - as *xbar*. Using arbitrary values for *xbar* can lead to invalid or - impossible results. - - Decimal and Fraction values are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) - - .. note:: - - This is the sample variance s² with Bessel's correction, also known as - variance with N-1 degrees of freedom. Provided that the data points are - representative (e.g. independent and identically distributed), the result - should be an unbiased estimate of the true population variance. - - If you somehow know the actual population mean μ you should pass it to the - :func:`pvariance` function as the *mu* parameter to get the variance of a - sample. - -.. function:: quantiles(data, *, n=4, method='exclusive') - - Divide *data* into *n* continuous intervals with equal probability. - Returns a list of ``n - 1`` cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. Set - *n* to 100 for percentiles which gives the 99 cuts points that separate - *data* into 100 equal sized groups. Raises :exc:`StatisticsError` if *n* - is not least 1. - - The *data* can be any iterable containing sample data. For meaningful - results, the number of data points in *data* should be larger than *n*. - Raises :exc:`StatisticsError` if there are not at least two data points. - - The cut points are linearly interpolated from the - two nearest data points. For example, if a cut point falls one-third - of the distance between two sample values, ``100`` and ``112``, the - cut-point will evaluate to ``104``. - - The *method* for computing quantiles can be varied depending on - whether the *data* includes or excludes the lowest and - highest possible values from the population. - - The default *method* is "exclusive" and is used for data sampled from - a population that can have more extreme values than found in the - samples. The portion of the population falling below the *i-th* of - *m* sorted data points is computed as ``i / (m + 1)``. Given nine - sample values, the method sorts them and assigns the following - percentiles: 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%. - - Setting the *method* to "inclusive" is used for describing population - data or for samples that are known to include the most extreme values - from the population. The minimum value in *data* is treated as the 0th - percentile and the maximum value is treated as the 100th percentile. - The portion of the population falling below the *i-th* of *m* sorted - data points is computed as ``(i - 1) / (m - 1)``. Given 11 sample - values, the method sorts them and assigns the following percentiles: - 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%. - - .. doctest:: - - # Decile cut points for empirically sampled data - >>> data = [105, 129, 87, 86, 111, 111, 89, 81, 108, 92, 110, - ... 100, 75, 105, 103, 109, 76, 119, 99, 91, 103, 129, - ... 106, 101, 84, 111, 74, 87, 86, 103, 103, 106, 86, - ... 111, 75, 87, 102, 121, 111, 88, 89, 101, 106, 95, - ... 103, 107, 101, 81, 109, 104] - >>> [round(q, 1) for q in quantiles(data, n=10)] - [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] - - .. versionadded:: 3.8 - - -Exceptions ----------- - -A single exception is defined: - -.. exception:: StatisticsError - - Subclass of :exc:`ValueError` for statistics-related exceptions. - - -:class:`NormalDist` objects ---------------------------- - -:class:`NormalDist` is a tool for creating and manipulating normal -distributions of a `random variable -`_. It is a -class that treats the mean and standard deviation of data -measurements as a single entity. - -Normal distributions arise from the `Central Limit Theorem -`_ and have a wide range -of applications in statistics. - -.. class:: NormalDist(mu=0.0, sigma=1.0) - - Returns a new *NormalDist* object where *mu* represents the `arithmetic - mean `_ and *sigma* - represents the `standard deviation - `_. - - If *sigma* is negative, raises :exc:`StatisticsError`. - - .. attribute:: mean - - A read-only property for the `arithmetic mean - `_ of a normal - distribution. - - .. attribute:: median - - A read-only property for the `median - `_ of a normal - distribution. - - .. attribute:: mode - - A read-only property for the `mode - `_ of a normal - distribution. - - .. attribute:: stdev - - A read-only property for the `standard deviation - `_ of a normal - distribution. - - .. attribute:: variance - - A read-only property for the `variance - `_ of a normal - distribution. Equal to the square of the standard deviation. - - .. classmethod:: NormalDist.from_samples(data) - - Makes a normal distribution instance with *mu* and *sigma* parameters - estimated from the *data* using :func:`fmean` and :func:`stdev`. - - The *data* can be any :term:`iterable` and should consist of values - that can be converted to type :class:`float`. If *data* does not - contain at least two elements, raises :exc:`StatisticsError` because it - takes at least one point to estimate a central value and at least two - points to estimate dispersion. - - .. method:: NormalDist.samples(n, *, seed=None) - - Generates *n* random samples for a given mean and standard deviation. - Returns a :class:`list` of :class:`float` values. - - If *seed* is given, creates a new instance of the underlying random - number generator. This is useful for creating reproducible results, - even in a multi-threading context. - - .. method:: NormalDist.pdf(x) - - Using a `probability density function (pdf) - `_, compute - the relative likelihood that a random variable *X* will be near the - given value *x*. Mathematically, it is the limit of the ratio ``P(x <= - X < x+dx) / dx`` as *dx* approaches zero. - - The relative likelihood is computed as the probability of a sample - occurring in a narrow range divided by the width of the range (hence - the word "density"). Since the likelihood is relative to other points, - its value can be greater than `1.0`. - - .. method:: NormalDist.cdf(x) - - Using a `cumulative distribution function (cdf) - `_, - compute the probability that a random variable *X* will be less than or - equal to *x*. Mathematically, it is written ``P(X <= x)``. - - .. method:: NormalDist.inv_cdf(p) - - Compute the inverse cumulative distribution function, also known as the - `quantile function `_ - or the `percent-point - `_ - function. Mathematically, it is written ``x : P(X <= x) = p``. - - Finds the value *x* of the random variable *X* such that the - probability of the variable being less than or equal to that value - equals the given probability *p*. - - .. method:: NormalDist.overlap(other) - - Measures the agreement between two normal probability distributions. - Returns a value between 0.0 and 1.0 giving `the overlapping area for - the two probability density functions - `_. - - .. method:: NormalDist.quantiles(n=4) - - Divide the normal distribution into *n* continuous intervals with - equal probability. Returns a list of (n - 1) cut points separating - the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate the normal distribution into 100 equal sized groups. - - .. method:: NormalDist.zscore(x) - - Compute the - `Standard Score `_ - describing *x* in terms of the number of standard deviations - above or below the mean of the normal distribution: - ``(x - mean) / stdev``. - - .. versionadded:: 3.9 - - Instances of :class:`NormalDist` support addition, subtraction, - multiplication and division by a constant. These operations - are used for translation and scaling. For example: - - .. doctest:: - - >>> temperature_february = NormalDist(5, 2.5) # Celsius - >>> temperature_february * (9/5) + 32 # Fahrenheit - NormalDist(mu=41.0, sigma=4.5) - - Dividing a constant by an instance of :class:`NormalDist` is not supported - because the result wouldn't be normally distributed. - - Since normal distributions arise from additive effects of independent - variables, it is possible to `add and subtract two independent normally - distributed random variables - `_ - represented as instances of :class:`NormalDist`. For example: - - .. doctest:: - - >>> birth_weights = NormalDist.from_samples([2.5, 3.1, 2.1, 2.4, 2.7, 3.5]) - >>> drug_effects = NormalDist(0.4, 0.15) - >>> combined = birth_weights + drug_effects - >>> round(combined.mean, 1) - 3.1 - >>> round(combined.stdev, 1) - 0.5 - - .. versionadded:: 3.8 - - -:class:`NormalDist` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`NormalDist` readily solves classic probability problems. - -For example, given `historical data for SAT exams -`_ showing -that scores are normally distributed with a mean of 1060 and a standard -deviation of 195, determine the percentage of students with test scores -between 1100 and 1200, after rounding to the nearest whole number: - -.. doctest:: - - >>> sat = NormalDist(1060, 195) - >>> fraction = sat.cdf(1200 + 0.5) - sat.cdf(1100 - 0.5) - >>> round(fraction * 100.0, 1) - 18.4 - -Find the `quartiles `_ and `deciles -`_ for the SAT scores: - -.. doctest:: - - >>> list(map(round, sat.quantiles())) - [928, 1060, 1192] - >>> list(map(round, sat.quantiles(n=10))) - [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] - -To estimate the distribution for a model than isn't easy to solve -analytically, :class:`NormalDist` can generate input samples for a `Monte -Carlo simulation `_: - -.. doctest:: - - >>> def model(x, y, z): - ... return (3*x + 7*x*y - 5*y) / (11 * z) - ... - >>> n = 100_000 - >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728) - >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471) - >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453) - >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP - [1.4591308524824727, 1.8035946855390597, 2.175091447274739] - -Normal distributions can be used to approximate `Binomial -distributions `_ -when the sample size is large and when the probability of a successful -trial is near 50%. - -For example, an open source conference has 750 attendees and two rooms with a -500 person capacity. There is a talk about Python and another about Ruby. -In previous conferences, 65% of the attendees preferred to listen to Python -talks. Assuming the population preferences haven't changed, what is the -probability that the Python room will stay within its capacity limits? - -.. doctest:: - - >>> n = 750 # Sample size - >>> p = 0.65 # Preference for Python - >>> q = 1.0 - p # Preference for Ruby - >>> k = 500 # Room capacity - - >>> # Approximation using the cumulative normal distribution - >>> from math import sqrt - >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) - 0.8402 - - >>> # Solution using the cumulative binomial distribution - >>> from math import comb, fsum - >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) - 0.8402 - - >>> # Approximation using a simulation - >>> from random import seed, choices - >>> seed(8675309) - >>> def trial(): - ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') - >>> mean(trial() <= k for i in range(10_000)) - 0.8398 - -Normal distributions commonly arise in machine learning problems. - -Wikipedia has a `nice example of a Naive Bayesian Classifier -`_. -The challenge is to predict a person's gender from measurements of normally -distributed features including height, weight, and foot size. - -We're given a training dataset with measurements for eight people. The -measurements are assumed to be normally distributed, so we summarize the data -with :class:`NormalDist`: - -.. doctest:: - - >>> height_male = NormalDist.from_samples([6, 5.92, 5.58, 5.92]) - >>> height_female = NormalDist.from_samples([5, 5.5, 5.42, 5.75]) - >>> weight_male = NormalDist.from_samples([180, 190, 170, 165]) - >>> weight_female = NormalDist.from_samples([100, 150, 130, 150]) - >>> foot_size_male = NormalDist.from_samples([12, 11, 12, 10]) - >>> foot_size_female = NormalDist.from_samples([6, 8, 7, 9]) - -Next, we encounter a new person whose feature measurements are known but whose -gender is unknown: - -.. doctest:: - - >>> ht = 6.0 # height - >>> wt = 130 # weight - >>> fs = 8 # foot size - -Starting with a 50% `prior probability -`_ of being male or female, -we compute the posterior as the prior times the product of likelihoods for the -feature measurements given the gender: - -.. doctest:: - - >>> prior_male = 0.5 - >>> prior_female = 0.5 - >>> posterior_male = (prior_male * height_male.pdf(ht) * - ... weight_male.pdf(wt) * foot_size_male.pdf(fs)) - - >>> posterior_female = (prior_female * height_female.pdf(ht) * - ... weight_female.pdf(wt) * foot_size_female.pdf(fs)) - -The final prediction goes to the largest posterior. This is known as the -`maximum a posteriori -`_ or MAP: - -.. doctest:: - - >>> 'male' if posterior_male > posterior_female else 'female' - 'female' - - -.. - # This modelines must appear within the last ten lines of the file. - kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/tempfile.rst.bak b/Doc/library/tempfile.rst.bak deleted file mode 100644 index 2b8a35e2651e73..00000000000000 --- a/Doc/library/tempfile.rst.bak +++ /dev/null @@ -1,374 +0,0 @@ -:mod:`tempfile` --- Generate temporary files and directories -============================================================ - -.. module:: tempfile - :synopsis: Generate temporary files and directories. - -.. sectionauthor:: Zack Weinberg - -**Source code:** :source:`Lib/tempfile.py` - -.. index:: - pair: temporary; file name - pair: temporary; file - --------------- - -This module creates temporary files and directories. It works on all -supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, -:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level -interfaces which provide automatic cleanup and can be used as -context managers. :func:`mkstemp` and -:func:`mkdtemp` are lower-level functions which require manual cleanup. - -All the user-callable functions and constructors take additional arguments which -allow direct control over the location and name of temporary files and -directories. Files names used by this module include a string of -random characters which allows those files to be securely created in -shared temporary directories. -To maintain backward compatibility, the argument order is somewhat odd; it -is recommended to use keyword arguments for clarity. - -The module defines the following user-callable items: - -.. function:: TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) - - Return a :term:`file-like object` that can be used as a temporary storage area. - The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon - as it is closed (including an implicit close when the object is garbage - collected). Under Unix, the directory entry for the file is either not created at all or is removed - immediately after the file is created. Other platforms do not support - this; your code should not rely on a temporary file created using this - function having or not having a visible name in the file system. - - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or - destruction of the file object the temporary file will be removed - from the filesystem. - - The *mode* parameter defaults to ``'w+b'`` so that the file created can - be read and written without being closed. Binary mode is used so that it - behaves consistently on all platforms without regard for the data that is - stored. *buffering*, *encoding*, *errors* and *newline* are interpreted as for - :func:`open`. - - The *dir*, *prefix* and *suffix* parameters have the same meaning and - defaults as with :func:`mkstemp`. - - The returned object is a true file object on POSIX platforms. On other - platforms, it is a file-like object whose :attr:`!file` attribute is the - underlying true file object. - - The :py:data:`os.O_TMPFILE` flag is used if it is available and works - (Linux-specific, requires Linux kernel 3.11 or later). - - .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile - - .. versionchanged:: 3.5 - - The :py:data:`os.O_TMPFILE` flag is now used if available. - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) - - This function operates exactly as :func:`TemporaryFile` does, except that - the file is guaranteed to have a visible name in the file system (on - Unix, the directory entry is not unlinked). That name can be retrieved - from the :attr:`name` attribute of the returned - file-like object. Whether the name can be - used to open the file a second time, while the named temporary file is - still open, varies across platforms (it can be so used on Unix; it cannot - on Windows NT or later). If *delete* is true (the default), the file is - deleted as soon as it is closed. - The returned object is always a file-like object whose :attr:`!file` - attribute is the underlying true file object. This file-like object can - be used in a :keyword:`with` statement, just like a normal file. - - .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) - - This function operates exactly as :func:`TemporaryFile` does, except that - data is spooled in memory until the file size exceeds *max_size*, or - until the file's :func:`fileno` method is called, at which point the - contents are written to disk and operation proceeds as with - :func:`TemporaryFile`. - - The resulting file has one additional method, :func:`rollover`, which - causes the file to roll over to an on-disk file regardless of its size. - - The returned object is a file-like object whose :attr:`_file` attribute - is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object - (depending on whether binary or text *mode* was specified) or a true file - object, depending on whether :func:`rollover` has been called. This - file-like object can be used in a :keyword:`with` statement, just like - a normal file. - - .. versionchanged:: 3.3 - the truncate method now accepts a ``size`` argument. - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) - - This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or destruction - of the temporary directory object the newly created temporary directory - and all its contents are removed from the filesystem. - - The directory name can be retrieved from the :attr:`name` attribute of the - returned object. When the returned object is used as a context manager, the - :attr:`name` will be assigned to the target of the :keyword:`!as` clause in - the :keyword:`with` statement, if there is one. - - The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. - - .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory - - .. versionadded:: 3.2 - - -.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) - - Creates a temporary file in the most secure manner possible. There are - no race conditions in the file's creation, assuming that the platform - properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The - file is readable and writable only by the creating user ID. If the - platform uses permission bits to indicate whether a file is executable, - the file is executable by no one. The file descriptor is not inherited - by child processes. - - Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible - for deleting the temporary file when done with it. - - If *suffix* is not ``None``, the file name will end with that suffix, - otherwise there will be no suffix. :func:`mkstemp` does not put a dot - between the file name and the suffix; if you need one, put it at the - beginning of *suffix*. - - If *prefix* is not ``None``, the file name will begin with that prefix; - otherwise, a default prefix is used. The default is the return value of - :func:`gettempprefix` or :func:`gettempprefixb`, as appropriate. - - If *dir* is not ``None``, the file will be created in that directory; - otherwise, a default directory is used. The default directory is chosen - from a platform-dependent list, but the user of the application can - control the directory location by setting the *TMPDIR*, *TEMP* or *TMP* - environment variables. There is thus no guarantee that the generated - filename will have any nice properties, such as not requiring quoting - when passed to external commands via ``os.popen()``. - - If any of *suffix*, *prefix*, and *dir* are not - ``None``, they must be the same type. - If they are bytes, the returned name will be bytes instead of str. - If you want to force a bytes return value with otherwise default behavior, - pass ``suffix=b''``. - - If *text* is specified and true, the file is opened in text mode. - Otherwise, (the default) the file is opened in binary mode. - - :func:`mkstemp` returns a tuple containing an OS-level handle to an open - file (as would be returned by :func:`os.open`) and the absolute pathname - of that file, in that order. - - .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - .. versionchanged:: 3.6 - The *dir* parameter now accepts a :term:`path-like object`. - - -.. function:: mkdtemp(suffix=None, prefix=None, dir=None) - - Creates a temporary directory in the most secure manner possible. There - are no race conditions in the directory's creation. The directory is - readable, writable, and searchable only by the creating user ID. - - The user of :func:`mkdtemp` is responsible for deleting the temporary - directory and its contents when done with it. - - The *prefix*, *suffix*, and *dir* arguments are the same as for - :func:`mkstemp`. - - :func:`mkdtemp` returns the absolute pathname of the new directory. - - .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - .. versionchanged:: 3.6 - The *dir* parameter now accepts a :term:`path-like object`. - - -.. function:: gettempdir() - - Return the name of the directory used for temporary files. This - defines the default value for the *dir* argument to all functions - in this module. - - Python searches a standard list of directories to find one which - the calling user can create files in. The list is: - - #. The directory named by the :envvar:`TMPDIR` environment variable. - - #. The directory named by the :envvar:`TEMP` environment variable. - - #. The directory named by the :envvar:`TMP` environment variable. - - #. A platform-specific location: - - * On Windows, the directories :file:`C:\\TEMP`, :file:`C:\\TMP`, - :file:`\\TEMP`, and :file:`\\TMP`, in that order. - - * On all other platforms, the directories :file:`/tmp`, :file:`/var/tmp`, and - :file:`/usr/tmp`, in that order. - - #. As a last resort, the current working directory. - - The result of this search is cached, see the description of - :data:`tempdir` below. - - .. versionchanged:: 3.10 - - Always returns a str. Previously it would return any :data:`tempdir` - value regardless of type so long as it was not ``None``. - -.. function:: gettempdirb() - - Same as :func:`gettempdir` but the return value is in bytes. - - .. versionadded:: 3.5 - -.. function:: gettempprefix() - - Return the filename prefix used to create temporary files. This does not - contain the directory component. - -.. function:: gettempprefixb() - - Same as :func:`gettempprefix` but the return value is in bytes. - - .. versionadded:: 3.5 - -The module uses a global variable to store the name of the directory -used for temporary files returned by :func:`gettempdir`. It can be -set directly to override the selection process, but this is discouraged. -All functions in this module take a *dir* argument which can be used -to specify the directory. This is the recommended approach that does -not surprise other unsuspecting code by changing global API behavior. - -.. data:: tempdir - - When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to the functions defined in this - module, including its type, bytes or str. It cannot be a - :term:`path-like object`. - - If ``tempdir`` is ``None`` (the default) at any call to any of the above - functions except :func:`gettempprefix` it is initialized following the - algorithm described in :func:`gettempdir`. - - .. note:: - - Beware that if you set ``tempdir`` to a bytes value, there is a - nasty side effect: The global default return type of - :func:`mkstemp` and :func:`mkdtemp` changes to bytes when no - explicit ``prefix``, ``suffix``, or ``dir`` arguments of type - str are supplied. Please do not write code expecting or - depending on this. This awkward behavior is maintained for - compatibility with the historcal implementation. - -.. _tempfile-examples: - -Examples --------- - -Here are some examples of typical usage of the :mod:`tempfile` module:: - - >>> import tempfile - - # create a temporary file and write some data to it - >>> fp = tempfile.TemporaryFile() - >>> fp.write(b'Hello world!') - # read data from file - >>> fp.seek(0) - >>> fp.read() - b'Hello world!' - # close the file, it will be removed - >>> fp.close() - - # create a temporary file using a context manager - >>> with tempfile.TemporaryFile() as fp: - ... fp.write(b'Hello world!') - ... fp.seek(0) - ... fp.read() - b'Hello world!' - >>> - # file is now closed and removed - - # create a temporary directory using the context manager - >>> with tempfile.TemporaryDirectory() as tmpdirname: - ... print('created temporary directory', tmpdirname) - >>> - # directory and contents have been removed - - -Deprecated functions and variables ----------------------------------- - -A historical way to create temporary files was to first generate a -file name with the :func:`mktemp` function and then create a file -using this name. Unfortunately this is not secure, because a different -process may create a file with this name in the time between the call -to :func:`mktemp` and the subsequent attempt to create the file by the -first process. The solution is to combine the two steps and create the -file immediately. This approach is used by :func:`mkstemp` and the -other functions described above. - -.. function:: mktemp(suffix='', prefix='tmp', dir=None) - - .. deprecated:: 2.3 - Use :func:`mkstemp` instead. - - Return an absolute pathname of a file that did not exist at the time the - call is made. The *prefix*, *suffix*, and *dir* arguments are similar - to those of :func:`mkstemp`, except that bytes file names, ``suffix=None`` - and ``prefix=None`` are not supported. - - .. warning:: - - Use of this function may introduce a security hole in your program. By - the time you get around to doing anything with the file name it returns, - someone else may have beaten you to the punch. :func:`mktemp` usage can - be replaced easily with :func:`NamedTemporaryFile`, passing it the - ``delete=False`` parameter:: - - >>> f = NamedTemporaryFile(delete=False) - >>> f.name - '/tmp/tmptjujjt' - >>> f.write(b"Hello World!\n") - 13 - >>> f.close() - >>> os.unlink(f.name) - >>> os.path.exists(f.name) - False diff --git a/Doc/library/venv.rst.bak b/Doc/library/venv.rst.bak deleted file mode 100644 index 5d4a36481f1dcc..00000000000000 --- a/Doc/library/venv.rst.bak +++ /dev/null @@ -1,496 +0,0 @@ -:mod:`venv` --- Creation of virtual environments -================================================ - -.. module:: venv - :synopsis: Creation of virtual environments. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/venv/` - -.. index:: pair: Environments; virtual - --------------- - -The :mod:`venv` module provides support for creating lightweight "virtual -environments" with their own site directories, optionally isolated from system -site directories. Each virtual environment has its own Python binary (which -matches the version of the binary that was used to create this environment) and -can have its own independent set of installed Python packages in its site -directories. - -See :pep:`405` for more information about Python virtual environments. - -.. seealso:: - - `Python Packaging User Guide: Creating and using virtual environments - `__ - - -Creating virtual environments ------------------------------ - -.. include:: /using/venv-create.inc - - -.. _venv-def: - -.. note:: A virtual environment is a Python environment such that the Python - interpreter, libraries and scripts installed into it are isolated from those - installed in other virtual environments, and (by default) any libraries - installed in a "system" Python, i.e., one which is installed as part of your - operating system. - - A virtual environment is a directory tree which contains Python executable - files and other files which indicate that it is a virtual environment. - - Common installation tools such as setuptools_ and pip_ work as - expected with virtual environments. In other words, when a virtual - environment is active, they install Python packages into the virtual - environment without needing to be told to do so explicitly. - - When a virtual environment is active (i.e., the virtual environment's Python - interpreter is running), the attributes :attr:`sys.prefix` and - :attr:`sys.exec_prefix` point to the base directory of the virtual - environment, whereas :attr:`sys.base_prefix` and - :attr:`sys.base_exec_prefix` point to the non-virtual environment Python - installation which was used to create the virtual environment. If a virtual - environment is not active, then :attr:`sys.prefix` is the same as - :attr:`sys.base_prefix` and :attr:`sys.exec_prefix` is the same as - :attr:`sys.base_exec_prefix` (they all point to a non-virtual environment - Python installation). - - When a virtual environment is active, any options that change the - installation path will be ignored from all :mod:`distutils` configuration - files to prevent projects being inadvertently installed outside of the - virtual environment. - - When working in a command shell, users can make a virtual environment active - by running an ``activate`` script in the virtual environment's executables - directory (the precise filename and command to use the file is - shell-dependent), which prepends the virtual environment's directory for - executables to the ``PATH`` environment variable for the running shell. There - should be no need in other circumstances to activate a virtual - environment; scripts installed into virtual environments have a "shebang" - line which points to the virtual environment's Python interpreter. This means - that the script will run with that interpreter regardless of the value of - ``PATH``. On Windows, "shebang" line processing is supported if you have the - Python Launcher for Windows installed (this was added to Python in 3.3 - see - :pep:`397` for more details). Thus, double-clicking an installed script in a - Windows Explorer window should run the script with the correct interpreter - without there needing to be any reference to its virtual environment in - ``PATH``. - - -.. _venv-api: - -API ---- - -.. highlight:: python - -The high-level method described above makes use of a simple API which provides -mechanisms for third-party virtual environment creators to customize environment -creation according to their needs, the :class:`EnvBuilder` class. - -.. class:: EnvBuilder(system_site_packages=False, clear=False, \ - symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False) - - The :class:`EnvBuilder` class accepts the following keyword arguments on - instantiation: - - * ``system_site_packages`` -- a Boolean value indicating that the system Python - site-packages should be available to the environment (defaults to ``False``). - - * ``clear`` -- a Boolean value which, if true, will delete the contents of - any existing target directory, before creating the environment. - - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the - Python binary rather than copying. - - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing - environment with the running Python - for use when that Python has been - upgraded in-place (defaults to ``False``). - - * ``with_pip`` -- a Boolean value which, if true, ensures pip is - installed in the virtual environment. This uses :mod:`ensurepip` with - the ``--default-pip`` option. - - * ``prompt`` -- a String to be used after virtual environment is activated - (defaults to ``None`` which means directory name of the environment would - be used). If the special string ``"."`` is provided, the basename of the - current directory is used as the prompt. - - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - .. versionadded:: 3.6 - Added the ``prompt`` parameter - - .. versionadded:: 3.9 - Added the ``upgrade_deps`` parameter - - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: - - .. method:: create(env_dir) - - Create a virtual environment by specifying the target directory - (absolute or relative to the current directory) which is to contain the - virtual environment. The ``create`` method will either create the - environment in the specified directory, or raise an appropriate - exception. - - The ``create`` method of the :class:`EnvBuilder` class illustrates the - hooks available for subclass customization:: - - def create(self, env_dir): - """ - Create a virtualized Python environment in a directory. - env_dir is the target directory to create an environment in. - """ - env_dir = os.path.abspath(env_dir) - context = self.ensure_directories(env_dir) - self.create_configuration(context) - self.setup_python(context) - self.setup_scripts(context) - self.post_setup(context) - - Each of the methods :meth:`ensure_directories`, - :meth:`create_configuration`, :meth:`setup_python`, - :meth:`setup_scripts` and :meth:`post_setup` can be overridden. - - .. method:: ensure_directories(env_dir) - - Creates the environment directory and all necessary directories, and - returns a context object. This is just a holder for attributes (such as - paths), for use by the other methods. The directories are allowed to - exist already, as long as either ``clear`` or ``upgrade`` were - specified to allow operating on an existing environment directory. - - .. method:: create_configuration(context) - - Creates the ``pyvenv.cfg`` configuration file in the environment. - - .. method:: setup_python(context) - - Creates a copy or symlink to the Python executable in the environment. - On POSIX systems, if a specific executable ``python3.x`` was used, - symlinks to ``python`` and ``python3`` will be created pointing to that - executable, unless files with those names already exist. - - .. method:: setup_scripts(context) - - Installs activation scripts appropriate to the platform into the virtual - environment. - - .. method:: upgrade_dependencies(context) - - Upgrades the core venv dependency packages (currently ``pip`` and - ``setuptools``) in the environment. This is done by shelling out to the - ``pip`` executable in the environment. - - .. versionadded:: 3.9 - - .. method:: post_setup(context) - - A placeholder method which can be overridden in third party - implementations to pre-install packages in the virtual environment or - perform other post-creation steps. - - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - - .. method:: install_scripts(context, path) - - *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the - directory corresponding to :data:`os.name` are copied after some text - replacement of placeholders: - - * ``__VENV_DIR__`` is replaced with the absolute path of the environment - directory. - - * ``__VENV_NAME__`` is replaced with the environment name (final path - segment of environment directory). - - * ``__VENV_PROMPT__`` is replaced with the prompt (the environment - name surrounded by parentheses and with a following space) - - * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory - (either ``bin`` or ``Scripts``). - - * ``__VENV_PYTHON__`` is replaced with the absolute path of the - environment's executable. - - The directories are allowed to exist (for when an existing environment - is being upgraded). - -There is also a module-level convenience function: - -.. function:: create(env_dir, system_site_packages=False, clear=False, \ - symlinks=False, with_pip=False, prompt=None, \ - upgrade_deps=False) - - Create an :class:`EnvBuilder` with the given keyword arguments, and call its - :meth:`~EnvBuilder.create` method with the *env_dir* argument. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - .. versionchanged:: 3.6 - Added the ``prompt`` parameter - - .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter - -An example of extending ``EnvBuilder`` --------------------------------------- - -The following script shows how to extend :class:`EnvBuilder` by implementing a -subclass which installs setuptools and pip into a created virtual environment:: - - import os - import os.path - from subprocess import Popen, PIPE - import sys - from threading import Thread - from urllib.parse import urlparse - from urllib.request import urlretrieve - import venv - - class ExtendedEnvBuilder(venv.EnvBuilder): - """ - This builder installs setuptools and pip so that you can pip or - easy_install other packages into the created virtual environment. - - :param nodist: If true, setuptools and pip are not installed into the - created virtual environment. - :param nopip: If true, pip is not installed into the created - virtual environment. - :param progress: If setuptools or pip are installed, the progress of the - installation can be monitored by passing a progress - callable. If specified, it is called with two - arguments: a string indicating some progress, and a - context indicating where the string is coming from. - The context argument can have one of three values: - 'main', indicating that it is called from virtualize() - itself, and 'stdout' and 'stderr', which are obtained - by reading lines from the output streams of a subprocess - which is used to install the app. - - If a callable is not specified, default progress - information is output to sys.stderr. - """ - - def __init__(self, *args, **kwargs): - self.nodist = kwargs.pop('nodist', False) - self.nopip = kwargs.pop('nopip', False) - self.progress = kwargs.pop('progress', None) - self.verbose = kwargs.pop('verbose', False) - super().__init__(*args, **kwargs) - - def post_setup(self, context): - """ - Set up any packages which need to be pre-installed into the - virtual environment being created. - - :param context: The information for the virtual environment - creation request being processed. - """ - os.environ['VIRTUAL_ENV'] = context.env_dir - if not self.nodist: - self.install_setuptools(context) - # Can't install pip without setuptools - if not self.nopip and not self.nodist: - self.install_pip(context) - - def reader(self, stream, context): - """ - Read lines from a subprocess' output stream and either pass to a progress - callable (if specified) or write progress information to sys.stderr. - """ - progress = self.progress - while True: - s = stream.readline() - if not s: - break - if progress is not None: - progress(s, context) - else: - if not self.verbose: - sys.stderr.write('.') - else: - sys.stderr.write(s.decode('utf-8')) - sys.stderr.flush() - stream.close() - - def install_script(self, context, name, url): - _, _, path, _, _, _ = urlparse(url) - fn = os.path.split(path)[-1] - binpath = context.bin_path - distpath = os.path.join(binpath, fn) - # Download script into the virtual environment's binaries folder - urlretrieve(url, distpath) - progress = self.progress - if self.verbose: - term = '\n' - else: - term = '' - if progress is not None: - progress('Installing %s ...%s' % (name, term), 'main') - else: - sys.stderr.write('Installing %s ...%s' % (name, term)) - sys.stderr.flush() - # Install in the virtual environment - args = [context.env_exe, fn] - p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath) - t1 = Thread(target=self.reader, args=(p.stdout, 'stdout')) - t1.start() - t2 = Thread(target=self.reader, args=(p.stderr, 'stderr')) - t2.start() - p.wait() - t1.join() - t2.join() - if progress is not None: - progress('done.', 'main') - else: - sys.stderr.write('done.\n') - # Clean up - no longer needed - os.unlink(distpath) - - def install_setuptools(self, context): - """ - Install setuptools in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' - self.install_script(context, 'setuptools', url) - # clear up the setuptools archive which gets downloaded - pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') - files = filter(pred, os.listdir(context.bin_path)) - for f in files: - f = os.path.join(context.bin_path, f) - os.unlink(f) - - def install_pip(self, context): - """ - Install pip in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' - self.install_script(context, 'pip', url) - - def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with ' - 'Python 3.3 or later') - else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory in which to create the - 'virtual environment.') - parser.add_argument('--no-setuptools', default=False, - action='store_true', dest='nodist', - help="Don't install setuptools or pip in the " - "virtual environment.") - parser.add_argument('--no-pip', default=False, - action='store_true', dest='nopip', - help="Don't install pip in the virtual " - "environment.") - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - parser.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'virtual environment ' - 'directory if it already ' - 'exists, before virtual ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the virtual ' - 'environment directory to ' - 'use this version of ' - 'Python, assuming Python ' - 'has been upgraded ' - 'in-place.') - parser.add_argument('--verbose', default=False, action='store_true', - dest='verbose', help='Display the output ' - 'from the scripts which ' - 'install setuptools and pip.') - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = ExtendedEnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - nodist=options.nodist, - nopip=options.nopip, - verbose=options.verbose) - for d in options.dirs: - builder.create(d) - - if __name__ == '__main__': - rc = 1 - try: - main() - rc = 0 - except Exception as e: - print('Error: %s' % e, file=sys.stderr) - sys.exit(rc) - - -This script is also available for download `online -`_. - - -.. _setuptools: https://pypi.org/project/setuptools/ -.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/xml.sax.handler.rst.bak b/Doc/library/xml.sax.handler.rst.bak deleted file mode 100644 index 3746a58c9b9558..00000000000000 --- a/Doc/library/xml.sax.handler.rst.bak +++ /dev/null @@ -1,463 +0,0 @@ -:mod:`xml.sax.handler` --- Base classes for SAX handlers -======================================================== - -.. module:: xml.sax.handler - :synopsis: Base classes for SAX event handlers. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/handler.py` - --------------- - -The SAX API defines five kinds of handlers: content handlers, DTD handlers, -error handlers, entity resolvers and lexical handlers. Applications normally -only need to implement those interfaces whose events they are interested in; -they can implement the interfaces in a single object or in multiple objects. -Handler implementations should inherit from the base classes provided in the -module :mod:`xml.sax.handler`, so that all methods get default implementations. - - -.. class:: ContentHandler - - This is the main callback interface in SAX, and the one most important to - applications. The order of events in this interface mirrors the order of the - information in the document. - - -.. class:: DTDHandler - - Handle DTD events. - - This interface specifies only those DTD events required for basic parsing - (unparsed entities and attributes). - - -.. class:: EntityResolver - - Basic interface for resolving entities. If you create an object implementing - this interface, then register the object with your Parser, the parser will call - the method in your object to resolve all external entities. - - -.. class:: ErrorHandler - - Interface used by the parser to present error and warning messages to the - application. The methods of this object control whether errors are immediately - converted to exceptions or are handled in some other way. - - -.. class:: LexicalHandler - - Interface used by the parser to represent low freqency events which may not - be of interest to many applications. - -In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants -for the feature and property names. - - -.. data:: feature_namespaces - - | value: ``"http://xml.org/sax/features/namespaces"`` - | true: Perform Namespace processing. - | false: Optionally do not perform Namespace processing (implies - namespace-prefixes; default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_namespace_prefixes - - | value: ``"http://xml.org/sax/features/namespace-prefixes"`` - | true: Report the original prefixed names and attributes used for Namespace - declarations. - | false: Do not report attributes used for Namespace declarations, and - optionally do not report original prefixed names (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_string_interning - - | value: ``"http://xml.org/sax/features/string-interning"`` - | true: All element names, prefixes, attribute names, Namespace URIs, and - local names are interned using the built-in intern function. - | false: Names are not necessarily interned, although they may be (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_validation - - | value: ``"http://xml.org/sax/features/validation"`` - | true: Report all validation errors (implies external-general-entities and - external-parameter-entities). - | false: Do not report validation errors. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_ges - - | value: ``"http://xml.org/sax/features/external-general-entities"`` - | true: Include all external general (text) entities. - | false: Do not include external general entities. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_pes - - | value: ``"http://xml.org/sax/features/external-parameter-entities"`` - | true: Include all external parameter entities, including the external DTD - subset. - | false: Do not include any external parameter entities, even the external - DTD subset. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: all_features - - List of all features. - - -.. data:: property_lexical_handler - - | value: ``"http://xml.org/sax/properties/lexical-handler"`` - | data type: xml.sax.handler.LexicalHandler (not supported in Python 2) - | description: An optional extension handler for lexical events like - comments. - | access: read/write - - -.. data:: property_declaration_handler - - | value: ``"http://xml.org/sax/properties/declaration-handler"`` - | data type: xml.sax.sax2lib.DeclHandler (not supported in Python 2) - | description: An optional extension handler for DTD-related events other - than notations and unparsed entities. - | access: read/write - - -.. data:: property_dom_node - - | value: ``"http://xml.org/sax/properties/dom-node"`` - | data type: org.w3c.dom.Node (not supported in Python 2) - | description: When parsing, the current DOM node being visited if this is - a DOM iterator; when not parsing, the root DOM node for iteration. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: property_xml_string - - | value: ``"http://xml.org/sax/properties/xml-string"`` - | data type: String - | description: The literal string of characters that was the source for the - current event. - | access: read-only - - -.. data:: all_properties - - List of all known property names. - - -.. _content-handler-objects: - -ContentHandler Objects ----------------------- - -Users are expected to subclass :class:`ContentHandler` to support their -application. The following methods are called by the parser on the appropriate -events in the input document: - - -.. method:: ContentHandler.setDocumentLocator(locator) - - Called by the parser to give the application a locator for locating the origin - of document events. - - SAX parsers are strongly encouraged (though not absolutely required) to supply a - locator: if it does so, it must supply the locator to the application by - invoking this method before invoking any of the other methods in the - DocumentHandler interface. - - The locator allows the application to determine the end position of any - document-related event, even if the parser is not reporting an error. Typically, - the application will use this information for reporting its own errors (such as - character content that does not match an application's business rules). The - information returned by the locator is probably not sufficient for use with a - search engine. - - Note that the locator will return correct information only during the invocation - of the events in this interface. The application should not attempt to use it at - any other time. - - -.. method:: ContentHandler.startDocument() - - Receive notification of the beginning of a document. - - The SAX parser will invoke this method only once, before any other methods in - this interface or in DTDHandler (except for :meth:`setDocumentLocator`). - - -.. method:: ContentHandler.endDocument() - - Receive notification of the end of a document. - - The SAX parser will invoke this method only once, and it will be the last method - invoked during the parse. The parser shall not invoke this method until it has - either abandoned parsing (because of an unrecoverable error) or reached the end - of input. - - -.. method:: ContentHandler.startPrefixMapping(prefix, uri) - - Begin the scope of a prefix-URI Namespace mapping. - - The information from this event is not necessary for normal Namespace - processing: the SAX XML reader will automatically replace prefixes for element - and attribute names when the ``feature_namespaces`` feature is enabled (the - default). - - There are cases, however, when applications need to use prefixes in character - data or in attribute values, where they cannot safely be expanded automatically; - the :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events supply the - information to the application to expand prefixes in those contexts itself, if - necessary. - - .. XXX This is not really the default, is it? MvL - - Note that :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events are not - guaranteed to be properly nested relative to each-other: all - :meth:`startPrefixMapping` events will occur before the corresponding - :meth:`startElement` event, and all :meth:`endPrefixMapping` events will occur - after the corresponding :meth:`endElement` event, but their order is not - guaranteed. - - -.. method:: ContentHandler.endPrefixMapping(prefix) - - End the scope of a prefix-URI mapping. - - See :meth:`startPrefixMapping` for details. This event will always occur after - the corresponding :meth:`endElement` event, but the order of - :meth:`endPrefixMapping` events is not otherwise guaranteed. - - -.. method:: ContentHandler.startElement(name, attrs) - - Signals the start of an element in non-namespace mode. - - The *name* parameter contains the raw XML 1.0 name of the element type as a - string and the *attrs* parameter holds an object of the - :class:`~xml.sax.xmlreader.Attributes` - interface (see :ref:`attributes-objects`) containing the attributes of - the element. The object passed as *attrs* may be re-used by the parser; holding - on to a reference to it is not a reliable way to keep a copy of the attributes. - To keep a copy of the attributes, use the :meth:`copy` method of the *attrs* - object. - - -.. method:: ContentHandler.endElement(name) - - Signals the end of an element in non-namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElement` event. - - -.. method:: ContentHandler.startElementNS(name, qname, attrs) - - Signals the start of an element in namespace mode. - - The *name* parameter contains the name of the element type as a ``(uri, - localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in - the source document, and the *attrs* parameter holds an instance of the - :class:`~xml.sax.xmlreader.AttributesNS` interface (see - :ref:`attributes-ns-objects`) - containing the attributes of the element. If no namespace is associated with - the element, the *uri* component of *name* will be ``None``. The object passed - as *attrs* may be re-used by the parser; holding on to a reference to it is not - a reliable way to keep a copy of the attributes. To keep a copy of the - attributes, use the :meth:`copy` method of the *attrs* object. - - Parsers may set the *qname* parameter to ``None``, unless the - ``feature_namespace_prefixes`` feature is activated. - - -.. method:: ContentHandler.endElementNS(name, qname) - - Signals the end of an element in namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElementNS` method, likewise the *qname* parameter. - - -.. method:: ContentHandler.characters(content) - - Receive notification of character data. - - The Parser will call this method to report each chunk of character data. SAX - parsers may return all contiguous character data in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity so that the Locator provides useful - information. - - *content* may be a string or bytes instance; the ``expat`` reader module - always produces strings. - - .. note:: - - The earlier SAX 1 interface provided by the Python XML Special Interest Group - used a more Java-like interface for this method. Since most parsers used from - Python did not take advantage of the older interface, the simpler signature was - chosen to replace it. To convert old code to the new interface, use *content* - instead of slicing content with the old *offset* and *length* parameters. - - -.. method:: ContentHandler.ignorableWhitespace(whitespace) - - Receive notification of ignorable whitespace in element content. - - Validating Parsers must use this method to report each chunk of ignorable - whitespace (see the W3C XML 1.0 recommendation, section 2.10): non-validating - parsers may also use this method if they are capable of parsing and using - content models. - - SAX parsers may return all contiguous whitespace in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity, so that the Locator provides useful - information. - - -.. method:: ContentHandler.processingInstruction(target, data) - - Receive notification of a processing instruction. - - The Parser will invoke this method once for each processing instruction found: - note that processing instructions may occur before or after the main document - element. - - A SAX parser should never report an XML declaration (XML 1.0, section 2.8) or a - text declaration (XML 1.0, section 4.3.1) using this method. - - -.. method:: ContentHandler.skippedEntity(name) - - Receive notification of a skipped entity. - - The Parser will invoke this method once for each entity skipped. Non-validating - processors may skip entities if they have not seen the declarations (because, - for example, the entity was declared in an external DTD subset). All processors - may skip external entities, depending on the values of the - ``feature_external_ges`` and the ``feature_external_pes`` properties. - - -.. _dtd-handler-objects: - -DTDHandler Objects ------------------- - -:class:`DTDHandler` instances provide the following methods: - - -.. method:: DTDHandler.notationDecl(name, publicId, systemId) - - Handle a notation declaration event. - - -.. method:: DTDHandler.unparsedEntityDecl(name, publicId, systemId, ndata) - - Handle an unparsed entity declaration event. - - -.. _entity-resolver-objects: - -EntityResolver Objects ----------------------- - - -.. method:: EntityResolver.resolveEntity(publicId, systemId) - - Resolve the system identifier of an entity and return either the system - identifier to read from as a string, or an InputSource to read from. The default - implementation returns *systemId*. - - -.. _sax-error-handler: - -ErrorHandler Objects --------------------- - -Objects with this interface are used to receive error and warning information -from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that -implements this interface, then register the object with your -:class:`~xml.sax.xmlreader.XMLReader`, the parser -will call the methods in your object to report all warnings and errors. There -are three levels of errors available: warnings, (possibly) recoverable errors, -and unrecoverable errors. All methods take a :exc:`SAXParseException` as the -only parameter. Errors and warnings may be converted to an exception by raising -the passed-in exception object. - - -.. method:: ErrorHandler.error(exception) - - Called when the parser encounters a recoverable error. If this method does not - raise an exception, parsing may continue, but further document information - should not be expected by the application. Allowing the parser to continue may - allow additional errors to be discovered in the input document. - - -.. method:: ErrorHandler.fatalError(exception) - - Called when the parser encounters an error it cannot recover from; parsing is - expected to terminate when this method returns. - - -.. method:: ErrorHandler.warning(exception) - - Called when the parser presents minor warning information to the application. - Parsing is expected to continue when this method returns, and document - information will continue to be passed to the application. Raising an exception - in this method will cause parsing to end. - - -.. _lexical-handler-objects: - -LexicalHandler Objects ----------------------- -Optional SAX2 handler for lexical events. - -This handler is used to obtain lexical information about an XML -document. Lexical information includes information describing the -document encoding used and XML comments embedded in the document, as -well as section boundaries for the DTD and for any CDATA sections. -The lexical handlers are used in the same manner as content handlers. - -Set the LexicalHandler of an XMLReader by using the setProperty method -with the property identifier -``'http://xml.org/sax/properties/lexical-handler'``. - - -.. method:: LexicalHandler.comment(content) - - Reports a comment anywhere in the document (including the DTD and - outside the document element). - -.. method:: LexicalHandler.startDTD(name, public_id, system_id) - - Reports the start of the DTD declarations if the document has an - associated DTD. - -.. method:: LexicalHandler.endDTD() - - Reports the end of DTD declaration. - -.. method:: LexicalHandler.startCDATA() - - Reports the start of a CDATA marked section. - - The contents of the CDATA marked section will be reported through - the characters handler. - -.. method:: LexicalHandler.endCDATA() - - Reports the end of a CDATA marked section. diff --git a/Doc/library/zipimport.rst.bak b/Doc/library/zipimport.rst.bak deleted file mode 100644 index 62e1e47e980e17..00000000000000 --- a/Doc/library/zipimport.rst.bak +++ /dev/null @@ -1,209 +0,0 @@ -:mod:`zipimport` --- Import modules from Zip archives -===================================================== - -.. module:: zipimport - :synopsis: Support for importing Python modules from ZIP archives. - -.. moduleauthor:: Just van Rossum - -**Source code:** :source:`Lib/zipimport.py` - --------------- - -This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not -needed to use the :mod:`zipimport` module explicitly; it is automatically used -by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths -to ZIP archives. - -Typically, :data:`sys.path` is a list of directory names as strings. This module -also allows an item of :data:`sys.path` to be a string naming a ZIP file archive. -The ZIP archive can contain a subdirectory structure to support package imports, -and a path within the archive can be specified to only import from a -subdirectory. For example, the path :file:`example.zip/lib/` would only -import from the :file:`lib/` subdirectory within the archive. - -Any files may be present in the ZIP archive, but only files :file:`.py` and -:file:`.pyc` are available for import. ZIP import of dynamic modules -(:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains -:file:`.py` files, Python will not attempt to modify the archive by adding the -corresponding :file:`.pyc` file, meaning that if a ZIP archive -doesn't contain :file:`.pyc` files, importing may be rather slow. - -.. versionchanged:: 3.8 - Previously, ZIP archives with an archive comment were not supported. - -.. seealso:: - - `PKZIP Application Note `_ - Documentation on the ZIP file format by Phil Katz, the creator of the format and - algorithms used. - - :pep:`273` - Import Modules from Zip Archives - Written by James C. Ahlstrom, who also provided an implementation. Python 2.3 - follows the specification in :pep:`273`, but uses an implementation written by Just - van Rossum that uses the import hooks described in :pep:`302`. - - :mod:`importlib` - The implementation of the import machinery - Package providing the relevant protocols for all importers to - implement. - - -This module defines an exception: - -.. exception:: ZipImportError - - Exception raised by zipimporter objects. It's a subclass of :exc:`ImportError`, - so it can be caught as :exc:`ImportError`, too. - - -.. _zipimporter-objects: - -zipimporter Objects -------------------- - -:class:`zipimporter` is the class for importing ZIP files. - -.. class:: zipimporter(archivepath) - - Create a new zipimporter instance. *archivepath* must be a path to a ZIP - file, or to a specific path within a ZIP file. For example, an *archivepath* - of :file:`foo/bar.zip/lib` will look for modules in the :file:`lib` directory - inside the ZIP file :file:`foo/bar.zip` (provided that it exists). - - :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP - archive. - - .. method:: create_module(spec) - - Implementation of :meth:`importlib.abc.Loader.create_module` that returns - :const:`None` to explicitly request the default semantics. - - .. versionadded:: 3.10 - - - .. method:: exec_module(module) - - Implementation of :meth:`importlib.abc.Loader.exec_module`. - - .. versionadded:: 3.10 - - - .. method:: find_loader(fullname, path=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_module(fullname, path=None) - - Search for a module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. It returns the zipimporter instance itself - if the module was found, or :const:`None` if it wasn't. The optional - *path* argument is ignored---it's there for compatibility with the - importer protocol. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_spec(fullname, target=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. - - .. versionadded:: 3.10 - - - .. method:: get_code(fullname) - - Return the code object for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be imported. - - - .. method:: get_data(pathname) - - Return the data associated with *pathname*. Raise :exc:`OSError` if the - file wasn't found. - - .. versionchanged:: 3.3 - :exc:`IOError` used to be raised instead of :exc:`OSError`. - - - .. method:: get_filename(fullname) - - Return the value ``__file__`` would be set to if the specified module - was imported. Raise :exc:`ZipImportError` if the module couldn't be - imported. - - .. versionadded:: 3.1 - - - .. method:: get_source(fullname) - - Return the source code for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be found, return - :const:`None` if the archive does contain the module, but has no source - for it. - - - .. method:: is_package(fullname) - - Return ``True`` if the module specified by *fullname* is a package. Raise - :exc:`ZipImportError` if the module couldn't be found. - - - .. method:: load_module(fullname) - - Load the module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. Returns the imported module on success, - raises :exc:`ZipImportError` on failure. - - .. deprecated:: 3.10 - - Use :meth:`exec_module` instead. - - .. attribute:: archive - - The file name of the importer's associated ZIP file, without a possible - subpath. - - - .. attribute:: prefix - - The subpath within the ZIP file where modules are searched. This is the - empty string for zipimporter objects which point to the root of the ZIP - file. - - The :attr:`archive` and :attr:`prefix` attributes, when combined with a - slash, equal the original *archivepath* argument given to the - :class:`zipimporter` constructor. - - -.. _zipimport-examples: - -Examples --------- - -Here is an example that imports a module from a ZIP archive - note that the -:mod:`zipimport` module is not explicitly used. - -.. code-block:: shell-session - - $ unzip -l example.zip - Archive: example.zip - Length Date Time Name - -------- ---- ---- ---- - 8467 11-26-02 22:30 jwzthreading.py - -------- ------- - 8467 1 file - $ ./python - Python 2.3 (#1, Aug 1 2003, 19:54:32) - >>> import sys - >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path - >>> import jwzthreading - >>> jwzthreading.__file__ - 'example.zip/jwzthreading.py' diff --git a/Doc/whatsnew/3.10.rst.bak b/Doc/whatsnew/3.10.rst.bak deleted file mode 100644 index c4c282e5a04eae..00000000000000 --- a/Doc/whatsnew/3.10.rst.bak +++ /dev/null @@ -1,1347 +0,0 @@ -**************************** - What's New In Python 3.10 -**************************** - -:Release: |release| -:Date: |today| - -.. Rules for maintenance: - - * Anyone can add text to this document. Do not spend very much time - on the wording of your changes, because your text will probably - get rewritten to some degree. - - * The maintainer will go through Misc/NEWS periodically and add - changes; it's therefore more important to add your changes to - Misc/NEWS than to this file. - - * This is not a complete list of every single change; completeness - is the purpose of Misc/NEWS. Some changes I consider too small - or esoteric to include. If such a change is added to the text, - I'll just remove it. (This is another reason you shouldn't spend - too much time on writing your addition.) - - * If you want to draw your new text to the attention of the - maintainer, add 'XXX' to the beginning of the paragraph or - section. - - * It's OK to just add a fragmentary note about a change. For - example: "XXX Describe the transmogrify() function added to the - socket module." The maintainer will research the change and - write the necessary text. - - * You can comment out your additions if you like, but it's not - necessary (especially when a final release is some months away). - - * Credit the author of a patch or bugfix. Just the name is - sufficient; the e-mail address isn't necessary. - - * It's helpful to add the bug/patch number as a comment: - - XXX Describe the transmogrify() function added to the socket - module. - (Contributed by P.Y. Developer in :issue:`12345`.) - - This saves the maintainer the effort of going through the Mercurial log - when researching a change. - -This article explains the new features in Python 3.10, compared to 3.9. - -For full details, see the :ref:`changelog `. - -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.10 moves towards release, - so it's worth checking back even after reading earlier versions. - - -Summary -- Release highlights -============================= - -.. This section singles out the most important changes in Python 3.10. - Brevity is key. - - -.. PEP-sized items next. - - - -New Features -============ - -.. _whatsnew310-pep563: - -Parenthesized context managers ------------------------------- - -Using enclosing parentheses for continuation across multiple lines -in context managers is now supported. This allows formatting a long -collection of context managers in multiple lines in a similar way -as it was previously possible with import statements. For instance, -all these examples are now valid: - -.. code-block:: python - - with (CtxManager() as example): - ... - - with ( - CtxManager1(), - CtxManager2() - ): - ... - - with (CtxManager1() as example, - CtxManager2()): - ... - - with (CtxManager1(), - CtxManager2() as example): - ... - - with ( - CtxManager1() as example1, - CtxManager2() as example2 - ): - ... - -it is also possible to use a trailing comma at the end of the -enclosed group: - -.. code-block:: python - - with ( - CtxManager1() as example1, - CtxManager2() as example2, - CtxManager3() as example3, - ): - ... - -This new syntax uses the non LL(1) capacities of the new parser. -Check :pep:`617` for more details. - -(Contributed by Guido van Rossum, Pablo Galindo and Lysandros Nikolaou -in :issue:`12782` and :issue:`40334`.) - - -Better error messages in the parser ------------------------------------ - -When parsing code that contains unclosed parentheses or brackets the interpreter -now includes the location of the unclosed bracket of parentheses instead of displaying -*SyntaxError: unexpected EOF while parsing* or pointing to some incorrect location. -For instance, consider the following code (notice the unclosed '{'): - -.. code-block:: python - - expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, - 38: 4, 39: 4, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 54: 6, - some_other_code = foo() - -previous versions of the interpreter reported confusing places as the location of -the syntax error: - -.. code-block:: text - - File "example.py", line 3 - some_other_code = foo() - ^ - SyntaxError: invalid syntax - -but in Python3.10 a more informative error is emitted: - -.. code-block:: text - - File "example.py", line 1 - expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, - ^ - SyntaxError: '{' was never closed - - -In a similar way, errors involving unclosed string literals (single and triple -quoted) now point to the start of the string instead of reporting EOF/EOL. - -These improvements are inspired by previous work in the PyPy interpreter. - -(Contributed by Pablo Galindo in :issue:`42864` and Batuhan Taskaya in -:issue:`40176`.) - - -PEP 634: Structural Pattern Matching ------------------------------------- - -Structural pattern matching has been added in the form of a *match statement* -and *case statements* of patterns with associated actions. Patterns -consist of sequences, mappings, primitive data types as well as class instances. -Pattern matching enables programs to extract information from complex data types, -branch on the structure of data, and apply specific actions based on different -forms of data. - -Syntax and operations -~~~~~~~~~~~~~~~~~~~~~ - -The generic syntax of pattern matching is:: - - match subject: - case : - - case : - - case : - - case _: - - -A match statement takes an expression and compares its value to successive -patterns given as one or more case blocks. Specifically, pattern matching -operates by: - - 1. using data with type and shape (the ``subject``) - 2. evaluating the ``subject`` in the ``match`` statement - 3. comparing the subject with each pattern in a ``case`` statement - from top to bottom until a match is confirmed. - 4. executing the action associated with the pattern of the confirmed - match - 5. If an exact match is not confirmed, the last case, a wildcard ``_``, - if provided, will be used as the matching case. If an exact match is - not confirmed and a wildcard case does not exists, the entire match - block is a no-op. - -Declarative approach -~~~~~~~~~~~~~~~~~~~~ - -Readers may be aware of pattern matching through the simple example of matching -a subject (data object) to a literal (pattern) with the switch statement found -in C, Java or JavaScript (and many other languages). Often the switch statement -is used for comparison of an object/expression with case statements containing -literals. - -More powerful examples of pattern matching can be found in languages, such as -Scala and Elixir. With structural pattern matching, the approach is "declarative" and -explicitly states the conditions (the patterns) for data to match. - -While an "imperative" series of instructions using nested "if" statements -could be used to accomplish something similar to structural pattern matching, -it is less clear than the "declarative" approach. Instead the "declarative" -approach states the conditions to meet for a match and is more readable through -its explicit patterns. While structural pattern matching can be used in its -simplest form comparing a variable to a literal in a case statement, its -true value for Python lies in its handling of the subject's type and shape. - -Simple pattern: match to a literal -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Let's look at this example as pattern matching in its simplest form: a value, -the subject, being matched to several literals, the patterns. In the example -below, ``status`` is the subject of the match statement. The patterns are -each of the case statements, where literals represent request status codes. -The associated action to the case is executed after a match:: - - def http_error(status): - match status: - case 400: - return "Bad request" - case 404: - return "Not found" - case 418: - return "I'm a teapot" - case _: - return "Something's wrong with the Internet" - -If the above function is passed a ``status`` of 418, "I'm a teapot" is returned. -If the above function is passed a ``status`` of 500, the case statement with -``_`` will match as a wildcard, and "Something's wrong with the Internet" is -returned. -Note the last block: the variable name, ``_``, acts as a *wildcard* and insures -the subject will always match. The use of ``_`` is optional. - -You can combine several literals in a single pattern using ``|`` ("or"):: - - case 401 | 403 | 404: - return "Not allowed" - -Behavior without the wildcard -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If we modify the above example by removing the last case block, the example -becomes:: - - def http_error(status): - match status: - case 400: - return "Bad request" - case 404: - return "Not found" - case 418: - return "I'm a teapot" - -Without the use of ``_`` in a case statement, a match may not exist. If no -match exists, the behavior is a no-op. For example, if ``status`` of 500 is -passed, a no-op occurs. - -Patterns with a literal and variable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Patterns can look like unpacking assignments, and a pattern may be used to bind -variables. In this example, a data point can be unpacked to its x-coordinate -and y-coordinate:: - - # point is an (x, y) tuple - match point: - case (0, 0): - print("Origin") - case (0, y): - print(f"Y={y}") - case (x, 0): - print(f"X={x}") - case (x, y): - print(f"X={x}, Y={y}") - case _: - raise ValueError("Not a point") - -The first pattern has two literals, ``(0, 0)``, and may be thought of as an -extension of the literal pattern shown above. The next two patterns combine a -literal and a variable, and the variable *binds* a value from the subject -(``point``). The fourth pattern captures two values, which makes it -conceptually similar to the unpacking assignment ``(x, y) = point``. - -Patterns and classes -~~~~~~~~~~~~~~~~~~~~ - -If you are using classes to structure your data, you can use as a pattern -the class name followed by an argument list resembling a constructor. This -pattern has the ability to capture class attributes into variables:: - - class Point: - x: int - y: int - - def location(point): - match point: - case Point(x=0, y=0): - print("Origin is the point's location.") - case Point(x=0, y=y): - print(f"Y={y} and the point is on the y-axis.") - case Point(x=x, y=0): - print(f"X={x} and the point is on the x-axis.") - case Point(): - print("The point is located somewhere else on the plane.") - case _: - print("Not a point") - -Patterns with positional parameters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can use positional parameters with some builtin classes that provide an -ordering for their attributes (e.g. dataclasses). You can also define a specific -position for attributes in patterns by setting the ``__match_args__`` special -attribute in your classes. If it's set to ("x", "y"), the following patterns -are all equivalent (and all bind the ``y`` attribute to the ``var`` variable):: - - Point(1, var) - Point(1, y=var) - Point(x=1, y=var) - Point(y=var, x=1) - -Nested patterns -~~~~~~~~~~~~~~~ - -Patterns can be arbitrarily nested. For example, if our data is a short -list of points, it could be matched like this:: - - match points: - case []: - print("No points in the list.") - case [Point(0, 0)]: - print("The origin is the only point in the list.") - case [Point(x, y)]: - print(f"A single point {x}, {y} is in the list.") - case [Point(0, y1), Point(0, y2)]: - print(f"Two points on the Y axis at {y1}, {y2} are in the list.") - case _: - print("Something else is found in the list.") - -Complex patterns and the wildcard -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To this point, the examples have used ``_`` alone in the last case statement. -A wildcard can be used in more complex patterns, such as ``('error', code, _)``. -For example:: - - match test_variable: - case ('warning', code, 40): - print("A warning has been received.") - case ('error', code, _): - print(f"An error {code} occured.") - -In the above case, ``test_variable`` will match for ('error', code, 100) and -('error', code, 800). - -Guard -~~~~~ - -We can add an ``if`` clause to a pattern, known as a "guard". If the -guard is false, ``match`` goes on to try the next case block. Note -that value capture happens before the guard is evaluated:: - - match point: - case Point(x, y) if x == y: - print(f"The point is located on the diagonal Y=X at {x}.") - case Point(x, y): - print(f"Point is not on the diagonal.") - -Other Key Features -~~~~~~~~~~~~~~~~~~ - -Several other key features: - -- Like unpacking assignments, tuple and list patterns have exactly the - same meaning and actually match arbitrary sequences. Technically, - the subject must be an instance of ``collections.abc.Sequence``. - Therefore, an important exception is that patterns don't match iterators. - Also, to prevent a common mistake, sequence patterns don't match strings. - -- Sequence patterns support wildcards: ``[x, y, *rest]`` and ``(x, y, - *rest)`` work similar to wildcards in unpacking assignments. The - name after ``*`` may also be ``_``, so ``(x, y, *_)`` matches a sequence - of at least two items without binding the remaining items. - -- Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the - ``"bandwidth"`` and ``"latency"`` values from a dict. Unlike sequence - patterns, extra keys are ignored. A wildcard ``**rest`` is also - supported. (But ``**_`` would be redundant, so it not allowed.) - -- Subpatterns may be captured using the ``as`` keyword:: - - case (Point(x1, y1), Point(x2, y2) as p2): ... - - This binds x1, y1, x2, y2 like you would expect without the ``as`` clause, - and p2 to the entire second item of the subject. - -- Most literals are compared by equality. However, the singletons ``True``, - ``False`` and ``None`` are compared by identity. - -- Named constants may be used in patterns. These named constants must be - dotted names to prevent the constant from being interpreted as a capture - variable:: - - from enum import Enum - class Color(Enum): - RED = 0 - GREEN = 1 - BLUE = 2 - - match color: - case Color.RED: - print("I see red!") - case Color.GREEN: - print("Grass is green") - case Color.BLUE: - print("I'm feeling the blues :(") - -For the full specification see :pep:`634`. Motivation and rationale -are in :pep:`635`, and a longer tutorial is in :pep:`636`. - - -New Features Related to Type Annotations -======================================== - -This section covers major changes affecting :pep:`484` type annotations and -the :mod:`typing` module. - - -PEP 563: Postponed Evaluation of Annotations Becomes Default ------------------------------------------------------------- - -In Python 3.7, postponed evaluation of annotations was added, -to be enabled with a ``from __future__ import annotations`` -directive. In 3.10 this became the default behavior, even -without that future directive. With this being default, all -annotations stored in :attr:`__annotations__` will be strings. -If needed, annotations can be resolved at runtime using -:func:`typing.get_type_hints`. See :pep:`563` for a full -description. Also, the :func:`inspect.signature` will try to -resolve types from now on, and when it fails it will fall back to -showing the string annotations. (Contributed by Batuhan Taskaya -in :issue:`38605`.) - - -PEP 604: New Type Union Operator --------------------------------- - -A new type union operator was introduced which enables the syntax ``X | Y``. -This provides a cleaner way of expressing 'either type X or type Y' instead of -using :data:`typing.Union`, especially in type hints (annotations). - -In previous versions of Python, to apply a type hint for functions accepting -arguments of multiple types, :data:`typing.Union` was used:: - - def square(number: Union[int, float]) -> Union[int, float]: - return number ** 2 - - -Type hints can now be written in a more succinct manner:: - - def square(number: int | float) -> int | float: - return number ** 2 - - -This new syntax is also accepted as the second argument to :func:`isinstance` -and :func:`issubclass`:: - - >>> isinstance(1, int | str) - True - -See :ref:`types-union` and :pep:`604` for more details. - -(Contributed by Maggie Moss and Philippe Prados in :issue:`41428`.) - - -PEP 612: Parameter Specification Variables ------------------------------------------- - -Two new options to improve the information provided to static type checkers for -:pep:`484`\ 's ``Callable`` have been added to the :mod:`typing` module. - -The first is the parameter specification variable. They are used to forward the -parameter types of one callable to another callable -- a pattern commonly -found in higher order functions and decorators. Examples of usage can be found -in :class:`typing.ParamSpec`. Previously, there was no easy way to type annotate -dependency of parameter types in such a precise manner. - -The second option is the new ``Concatenate`` operator. It's used in conjunction -with parameter specification variables to type annotate a higher order callable -which adds or removes parameters of another callable. Examples of usage can -be found in :class:`typing.Concatenate`. - -See :class:`typing.Callable`, :class:`typing.ParamSpec`, -:class:`typing.Concatenate` and :pep:`612` for more details. - -(Contributed by Ken Jin in :issue:`41559`.) - - -PEP 613: TypeAlias Annotation ------------------------------ - -:pep:`484` introduced the concept of type aliases, only requiring them to be -top-level unannotated assignments. This simplicity sometimes made it difficult -for type checkers to distinguish between type aliases and ordinary assignments, -especially when forward references or invalid types were involved. Compare:: - - StrCache = 'Cache[str]' # a type alias - LOG_PREFIX = 'LOG[DEBUG]' # a module constant - -Now the :mod:`typing` module has a special annotation :data:`TypeAlias` to -declare type aliases more explicitly:: - - StrCache: TypeAlias = 'Cache[str]' # a type alias - LOG_PREFIX = 'LOG[DEBUG]' # a module constant - -See :pep:`613` for more details. - -(Contributed by Mikhail Golubev in :issue:`41923`.) - - -Other Language Changes -====================== - -* The :class:`int` type has a new method :meth:`int.bit_count`, returning the - number of ones in the binary expansion of a given integer, also known - as the population count. (Contributed by Niklas Fiekas in :issue:`29882`.) - -* The views returned by :meth:`dict.keys`, :meth:`dict.values` and - :meth:`dict.items` now all have a ``mapping`` attribute that gives a - :class:`types.MappingProxyType` object wrapping the original - dictionary. (Contributed by Dennis Sweeney in :issue:`40890`.) - -* :pep:`618`: The :func:`zip` function now has an optional ``strict`` flag, used - to require that all the iterables have an equal length. - -* Builtin and extension functions that take integer arguments no longer accept - :class:`~decimal.Decimal`\ s, :class:`~fractions.Fraction`\ s and other - objects that can be converted to integers only with a loss (e.g. that have - the :meth:`~object.__int__` method but do not have the - :meth:`~object.__index__` method). - (Contributed by Serhiy Storchaka in :issue:`37999`.) - -* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will - correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. - (Contributed by Alex Shkop in :issue:`38302`.) - -* Assignment expressions can now be used unparenthesized within set literals - and set comprehensions, as well as in sequence indexes (but not slices). - -* Functions have a new ``__builtins__`` attribute which is used to look for - builtin symbols when a function is executed, instead of looking into - ``__globals__['__builtins__']``. The attribute is initialized from - ``__globals__["__builtins__"]`` if it exists, else from the current builtins. - (Contributed by Mark Shannon in :issue:`42990`.) - - -New Modules -=========== - -* None yet. - - -Improved Modules -================ - -argparse --------- - -Misleading phrase "optional arguments" was replaced with "options" in argparse help. Some tests might require adaptation if they rely on exact output match. -(Contributed by Raymond Hettinger in :issue:`9694`.) - -base64 ------- - -Add :func:`base64.b32hexencode` and :func:`base64.b32hexdecode` to support the -Base32 Encoding with Extended Hex Alphabet. - -codecs ------- - -Add a :func:`codecs.unregister` function to unregister a codec search function. -(Contributed by Hai Shi in :issue:`41842`.) - -collections.abc ---------------- - -The ``__args__`` of the :ref:`parameterized generic ` for -:class:`collections.abc.Callable` are now consistent with :data:`typing.Callable`. -:class:`collections.abc.Callable` generic now flattens type parameters, similar -to what :data:`typing.Callable` currently does. This means that -``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of -``(int, str, str)``; previously this was ``([int, str], str)``. To allow this -change, :class:`types.GenericAlias` can now be subclassed, and a subclass will -be returned when subscripting the :class:`collections.abc.Callable` type. Note -that a :exc:`TypeError` may be raised for invalid forms of parameterizing -:class:`collections.abc.Callable` which may have passed silently in Python 3.9. -(Contributed by Ken Jin in :issue:`42195`.) - -contextlib ----------- - -Add a :func:`contextlib.aclosing` context manager to safely close async generators -and objects representing asynchronously released resources. -(Contributed by Joongi Kim and John Belmonte in :issue:`41229`.) - -Add asynchronous context manager support to :func:`contextlib.nullcontext`. -(Contributed by Tom Gringauz in :issue:`41543`.) - -curses ------- - -The extended color functions added in ncurses 6.1 will be used transparently -by :func:`curses.color_content`, :func:`curses.init_color`, -:func:`curses.init_pair`, and :func:`curses.pair_content`. A new function, -:func:`curses.has_extended_color_support`, indicates whether extended color -support is provided by the underlying ncurses library. -(Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) - -The ``BUTTON5_*`` constants are now exposed in the :mod:`curses` module if -they are provided by the underlying curses library. -(Contributed by Zackery Spytz in :issue:`39273`.) - -.. _distutils-deprecated: - -distutils ---------- - -The entire ``distutils`` package is deprecated, to be removed in Python -3.12. Its functionality for specifying package builds has already been -completely replaced by third-party packages ``setuptools`` and -``packaging``, and most other commonly used APIs are available elsewhere -in the standard library (such as :mod:`platform`, :mod:`shutil`, -:mod:`subprocess` or :mod:`sysconfig`). There are no plans to migrate -any other functionality from ``distutils``, and applications that are -using other functions should plan to make private copies of the code. -Refer to :pep:`632` for discussion. - -The ``bdist_wininst`` command deprecated in Python 3.8 has been removed. -The ``bdist_wheel`` command is now recommended to distribute binary packages -on Windows. -(Contributed by Victor Stinner in :issue:`42802`.) - -doctest -------- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -encodings ---------- - -:func:`encodings.normalize_encoding` now ignores non-ASCII characters. -(Contributed by Hai Shi in :issue:`39337`.) - -gc --- - -Added audit hooks for :func:`gc.get_objects`, :func:`gc.get_referrers` and -:func:`gc.get_referents`. (Contributed by Pablo Galindo in :issue:`43439`.) - -glob ----- - -Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and -:func:`~glob.iglob` which allow to specify the root directory for searching. -(Contributed by Serhiy Storchaka in :issue:`38144`.) - -importlib.metadata ------------------- - -Feature parity with ``importlib_metadata`` 3.7. - -:func:`importlib.metadata.entry_points` now provides a nicer experience -for selecting entry points by group and name through a new -:class:`importlib.metadata.EntryPoints` class. - -Added :func:`importlib.metadata.packages_distributions` for resolving -top-level Python modules and packages to their -:class:`importlib.metadata.Distribution`. - -inspect -------- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -Added *globalns* and *localns* parameters in :func:`~inspect.signature` and -:meth:`inspect.Signature.from_callable` to retrieve the annotations in given -local and global namespaces. -(Contributed by Batuhan Taskaya in :issue:`41960`.) - -linecache ---------- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -os --- - -Added :func:`os.cpu_count()` support for VxWorks RTOS. -(Contributed by Peixing Xin in :issue:`41440`.) - -Added a new function :func:`os.eventfd` and related helpers to wrap the -``eventfd2`` syscall on Linux. -(Contributed by Christian Heimes in :issue:`41001`.) - -Added :func:`os.splice()` that allows to move data between two file -descriptors without copying between kernel address space and user -address space, where one of the file descriptors must refer to a -pipe. (Contributed by Pablo Galindo in :issue:`41625`.) - -Added :data:`~os.O_EVTONLY`, :data:`~os.O_FSYNC`, :data:`~os.O_SYMLINK` -and :data:`~os.O_NOFOLLOW_ANY` for macOS. -(Contributed by Dong-hee Na in :issue:`43106`.) - -pathlib -------- - -Added slice support to :attr:`PurePath.parents `. -(Contributed by Joshua Cannon in :issue:`35498`) - -Added negative indexing support to :attr:`PurePath.parents -`. -(Contributed by Yaroslav Pankovych in :issue:`21041`) - -platform --------- - -Added :func:`platform.freedesktop_os_release()` to retrieve operation system -identification from `freedesktop.org os-release -`_ standard file. -(Contributed by Christian Heimes in :issue:`28468`) - -py_compile ----------- - -Added ``--quiet`` option to command-line interface of :mod:`py_compile`. -(Contributed by Gregory Schevchenko in :issue:`38731`.) - -pyclbr ------- - -Added an ``end_lineno`` attribute to the ``Function`` and ``Class`` -objects in the tree returned by :func:`pyclbr.readline` and -:func:`pyclbr.readline_ex`. It matches the existing (start) ``lineno``. -(Contributed by Aviral Srivastava in :issue:`38307`.) - -shelve ------- - -The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default -instead of :mod:`pickle` protocol ``3`` when creating shelves. -(Contributed by Zackery Spytz in :issue:`34204`.) - -site ----- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -socket ------- - -The exception :exc:`socket.timeout` is now an alias of :exc:`TimeoutError`. -(Contributed by Christian Heimes in :issue:`42413`.) - -sys ---- - -Add :data:`sys.orig_argv` attribute: the list of the original command line -arguments passed to the Python executable. -(Contributed by Victor Stinner in :issue:`23427`.) - -Add :data:`sys.stdlib_module_names`, containing the list of the standard library -module names. -(Contributed by Victor Stinner in :issue:`42955`.) - -_thread -------- - -:func:`_thread.interrupt_main` now takes an optional signal number to -simulate (the default is still :data:`signal.SIGINT`). -(Contributed by Antoine Pitrou in :issue:`43356`.) - -threading ---------- - -Added :func:`threading.gettrace` and :func:`threading.getprofile` to -retrieve the functions set by :func:`threading.settrace` and -:func:`threading.setprofile` respectively. -(Contributed by Mario Corchero in :issue:`42251`.) - -Add :data:`threading.__excepthook__` to allow retrieving the original value -of :func:`threading.excepthook` in case it is set to a broken or a different -value. -(Contributed by Mario Corchero in :issue:`42308`.) - -traceback ---------- - -The :func:`~traceback.format_exception`, -:func:`~traceback.format_exception_only`, and -:func:`~traceback.print_exception` functions can now take an exception object -as a positional-only argument. -(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) - -types ------ - -Reintroduced the :data:`types.EllipsisType`, :data:`types.NoneType` -and :data:`types.NotImplementedType` classes, providing a new set -of types readily interpretable by type checkers. -(Contributed by Bas van Beek in :issue:`41810`.) - -typing ------- - -For major changes, see `New Features Related to Type Annotations`_. - -The behavior of :class:`typing.Literal` was changed to conform with :pep:`586` -and to match the behavior of static type checkers specified in the PEP. - -1. ``Literal`` now de-duplicates parameters. -2. Equality comparisons between ``Literal`` objects are now order independent. -3. ``Literal`` comparisons now respects types. For example, - ``Literal[0] == Literal[False]`` previously evaluated to ``True``. It is - now ``False``. To support this change, the internally used type cache now - supports differentiating types. -4. ``Literal`` objects will now raise a :exc:`TypeError` exception during - equality comparisons if one of their parameters are not :term:`immutable`. - Note that declaring ``Literal`` with mutable parameters will not throw - an error:: - - >>> from typing import Literal - >>> Literal[{0}] - >>> Literal[{0}] == Literal[{False}] - Traceback (most recent call last): - File "", line 1, in - TypeError: unhashable type: 'set' - -(Contributed by Yurii Karabas in :issue:`42345`.) - -unittest --------- - -Add new method :meth:`~unittest.TestCase.assertNoLogs` to complement the -existing :meth:`~unittest.TestCase.assertLogs`. (Contributed by Kit Yan Choi -in :issue:`39385`.) - -urllib.parse ------------- - -Python versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as -query parameter separators in :func:`urllib.parse.parse_qs` and -:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with -newer W3C recommendations, this has been changed to allow only a single -separator key, with ``&`` as the default. This change also affects -:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected -functions internally. For more details, please see their respective -documentation. -(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) - -xml ---- - -Add a :class:`~xml.sax.handler.LexicalHandler` class to the -:mod:`xml.sax.handler` module. -(Contributed by Jonathan Gossage and Zackery Spytz in :issue:`35018`.) - -zipimport ---------- -Add methods related to :pep:`451`: :meth:`~zipimport.zipimporter.find_spec`, -:meth:`zipimport.zipimporter.create_module`, and -:meth:`zipimport.zipimporter.exec_module`. -(Contributed by Brett Cannon in :issue:`42131`. - - -Optimizations -============= - -* Constructors :func:`str`, :func:`bytes` and :func:`bytearray` are now faster - (around 30--40% for small objects). - (Contributed by Serhiy Storchaka in :issue:`41334`.) - -* The :mod:`runpy` module now imports fewer modules. - The ``python3 -m module-name`` command startup time is 1.3x faster in - average. - (Contributed by Victor Stinner in :issue:`41006`.) - -* The ``LOAD_ATTR`` instruction now uses new "per opcode cache" mechanism. It - is about 36% faster now for regular attributes and 44% faster for slots. - (Contributed by Pablo Galindo and Yury Selivanov in :issue:`42093` and Guido - van Rossum in :issue:`42927`, based on ideas implemented originally in PyPy - and MicroPython.) - -* When building Python with ``--enable-optimizations`` now - ``-fno-semantic-interposition`` is added to both the compile and link line. - This speeds builds of the Python interpreter created with ``--enable-shared`` - with ``gcc`` by up to 30%. See `this article - `_ - for more details. (Contributed by Victor Stinner and Pablo Galindo in - :issue:`38980`.) - - -* Function parameters and their annotations are no longer computed at runtime, - but rather at compilation time. They are stored as a tuple of strings at the - bytecode level. It is now around 2 times faster to create a function with - parameter annotations. (Contributed by Yurii Karabas and Inada Naoki - in :issue:`42202`) - -* Substring search functions such as ``str1 in str2`` and ``str2.find(str1)`` - now sometimes use Crochemore & Perrin's "Two-Way" string searching - algorithm to avoid quadratic behavior on long strings. (Contributed - by Dennis Sweeney in :issue:`41972`) - -Deprecated -========== - -* Starting in this release, there will be a concerted effort to begin - cleaning up old import semantics that were kept for Python 2.7 - compatibility. Specifically, - :meth:`~importlib.abc.PathEntryFinder.find_loader`/:meth:`~importlib.abc.Finder.find_module` - (superseded by :meth:`~importlib.abc.Finder.find_spec`), - :meth:`~importlib.abc.Loader.load_module` - (superseded by :meth:`~importlib.abc.Loader.exec_module`), - :meth:`~importlib.abc.Loader.module_repr` (which the import system - takes care of for you), the ``__package__`` attribute - (superseded by ``__spec__.parent``), the ``__loader__`` attribute - (superseded by ``__spec__.loader``), and the ``__cached__`` attribute - (superseded by ``__spec__.cached``) will slowly be removed (as well - as other classes and methods in :mod:`importlib`). - :exc:`ImportWarning` and/or :exc:`DeprecationWarning` will be raised - as appropriate to help identify code which needs updating during - this transition. - -* The entire ``distutils`` namespace is deprecated, to be removed in - Python 3.12. Refer to the :ref:`module changes ` - section for more information. - -* Non-integer arguments to :func:`random.randrange` are deprecated. - The :exc:`ValueError` is deprecated in favor of a :exc:`TypeError`. - (Contributed by Serhiy Storchaka and Raymond Hettinger in :issue:`37319`.) - -* The various ``load_module()`` methods of :mod:`importlib` have been - documented as deprecated since Python 3.6, but will now also trigger - a :exc:`DeprecationWarning`. Use - :meth:`~importlib.abc.Loader.exec_module` instead. - (Contributed by Brett Cannon in :issue:`26131`.) - -* :meth:`zimport.zipimporter.load_module` has been deprecated in - preference for :meth:`~zipimport.zipimporter.exec_module`. - (Contributed by Brett Cannon in :issue:`26131`.) - -* The use of :meth:`~importlib.abc.Loader.load_module` by the import - system now triggers an :exc:`ImportWarning` as - :meth:`~importlib.abc.Loader.exec_module` is preferred. - (Contributed by Brett Cannon in :issue:`26131`.) - -* ``sqlite3.OptimizedUnicode`` has been undocumented and obsolete since Python - 3.3, when it was made an alias to :class:`str`. It is now deprecated, - scheduled for removal in Python 3.12. - (Contributed by Erlend E. Aasland in :issue:`42264`.) - -* The undocumented built-in function ``sqlite3.enable_shared_cache`` is now - deprecated, scheduled for removal in Python 3.12. Its use is strongly - discouraged by the SQLite3 documentation. See `the SQLite3 docs - `_ for more details. - If shared cache must be used, open the database in URI mode using the - ``cache=shared`` query parameter. - (Contributed by Erlend E. Aasland in :issue:`24464`.) - - -Removed -======= - -* Removed special methods ``__int__``, ``__float__``, ``__floordiv__``, - ``__mod__``, ``__divmod__``, ``__rfloordiv__``, ``__rmod__`` and - ``__rdivmod__`` of the :class:`complex` class. They always raised - a :exc:`TypeError`. - (Contributed by Serhiy Storchaka in :issue:`41974`.) - -* The ``ParserBase.error()`` method from the private and undocumented ``_markupbase`` - module has been removed. :class:`html.parser.HTMLParser` is the only subclass of - ``ParserBase`` and its ``error()`` implementation has already been removed in - Python 3.5. - (Contributed by Berker Peksag in :issue:`31844`.) - -* Removed the ``unicodedata.ucnhash_CAPI`` attribute which was an internal - PyCapsule object. The related private ``_PyUnicode_Name_CAPI`` structure was - moved to the internal C API. - (Contributed by Victor Stinner in :issue:`42157`.) - -* Removed the ``parser`` module, which was deprecated in 3.9 due to the - switch to the new PEG parser, as well as all the C source and header files - that were only being used by the old parser, including ``node.h``, ``parser.h``, - ``graminit.h`` and ``grammar.h``. - -* Removed the Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, - :c:func:`PyParser_SimpleParseStringFlagsFilename`, - :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` - that were deprecated in 3.9 due to the switch to the new PEG parser. - -* Removed the ``formatter`` module, which was deprecated in Python 3.4. - It is somewhat obsolete, little used, and not tested. It was originally - scheduled to be removed in Python 3.6, but such removals were delayed until - after Python 2.7 EOL. Existing users should copy whatever classes they use - into their code. - (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) - -* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless - now due to the _warnings module was converted to a builtin module in 2.6. - (Contributed by Hai Shi in :issue:`42599`.) - -* Remove deprecated aliases to :ref:`collections-abstract-base-classes` from - the :mod:`collections` module. - (Contributed by Victor Stinner in :issue:`37324`.) - -* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's - :doc:`high-level API <../library/asyncio-api-index>` following deprecation - in Python 3.8. The motivation behind this change is multifold: - - 1. This simplifies the high-level API. - 2. The functions in the high-level API have been implicitly getting the - current thread's running event loop since Python 3.7. There isn't a need to - pass the event loop to the API in most normal use cases. - 3. Event loop passing is error-prone especially when dealing with loops - running in different threads. - - Note that the low-level API will still accept ``loop``. - See `Changes in the Python API`_ for examples of how to replace existing code. - - (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley - in :issue:`42392`.) - - -Porting to Python 3.10 -====================== - -This section lists previously described changes and other bugfixes -that may require changes to your code. - - -Changes in the Python API -------------------------- - -* The *etype* parameters of the :func:`~traceback.format_exception`, - :func:`~traceback.format_exception_only`, and - :func:`~traceback.print_exception` functions in the :mod:`traceback` module - have been renamed to *exc*. - (Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) - -* :mod:`atexit`: At Python exit, if a callback registered with - :func:`atexit.register` fails, its exception is now logged. Previously, only - some exceptions were logged, and the last exception was always silently - ignored. - (Contributed by Victor Stinner in :issue:`42639`.) - -* :class:`collections.abc.Callable` generic now flattens type parameters, similar - to what :data:`typing.Callable` currently does. This means that - ``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of - ``(int, str, str)``; previously this was ``([int, str], str)``. Code which - accesses the arguments via :func:`typing.get_args` or ``__args__`` need to account - for this change. Furthermore, :exc:`TypeError` may be raised for invalid forms - of parameterizing :class:`collections.abc.Callable` which may have passed - silently in Python 3.9. - (Contributed by Ken Jin in :issue:`42195`.) - -* :meth:`socket.htons` and :meth:`socket.ntohs` now raise :exc:`OverflowError` - instead of :exc:`DeprecationWarning` if the given parameter will not fit in - a 16-bit unsigned integer. - (Contributed by Erlend E. Aasland in :issue:`42393`.) - -* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's - :doc:`high-level API <../library/asyncio-api-index>` following deprecation - in Python 3.8. - - A coroutine that currently look like this:: - - async def foo(loop): - await asyncio.sleep(1, loop=loop) - - Should be replaced with this:: - - async def foo(): - await asyncio.sleep(1) - - If ``foo()`` was specifically designed *not* to run in the current thread's - running event loop (e.g. running in another thread's event loop), consider - using :func:`asyncio.run_coroutine_threadsafe` instead. - - (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley - in :issue:`42392`.) - -* The :data:`types.FunctionType` constructor now inherits the current builtins - if the *globals* dictionary has no ``"__builtins__"`` key, rather than using - ``{"None": None}`` as builtins: same behavior as :func:`eval` and - :func:`exec` functions. Defining a function with ``def function(...): ...`` - in Python is not affected, globals cannot be overriden with this syntax: it - also inherits the current builtins. - (Contributed by Victor Stinner in :issue:`42990`.) - -CPython bytecode changes -======================== - -* The ``MAKE_FUNCTION`` instruction accepts tuple of strings as annotations - instead of dictionary. - (Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`) - -Build Changes -============= - -* The C99 functions :c:func:`snprintf` and :c:func:`vsnprintf` are now required - to build Python. - (Contributed by Victor Stinner in :issue:`36020`.) - -* :mod:`sqlite3` requires SQLite 3.7.15 or higher. (Contributed by Sergey Fedoseev - and Erlend E. Aasland :issue:`40744` and :issue:`40810`.) - -* The :mod:`atexit` module must now always be built as a built-in module. - (Contributed by Victor Stinner in :issue:`42639`.) - -* Added ``--disable-test-modules`` option to the ``configure`` script: - don't build nor install test modules. - (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) - -* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If - specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` - wheel packages in this directory: if both are present, these wheel packages - are used instead of ensurepip bundled wheel packages. - - Some Linux distribution packaging policies recommend against bundling - dependencies. For example, Fedora installs wheel packages in the - ``/usr/share/python-wheels/`` directory and don't install the - ``ensurepip._bundled`` package. - - (Contributed by Victor Stinner in :issue:`42856`.) - -* Add a new configure ``--without-static-libpython`` option to not build the - ``libpythonMAJOR.MINOR.a`` static library and not install the ``python.o`` - object file. - - (Contributed by Victor Stinner in :issue:`43103`.) - -* The ``configure`` script now uses the ``pkg-config`` utility, if available, - to detect the location of Tcl/Tk headers and libraries. As before, those - locations can be explicitly specified with the ``--with-tcltk-includes`` - and ``--with-tcltk-libs`` configuration options. - (Contributed by Manolis Stamatogiannakis in :issue:`42603`.) - - -C API Changes -============= - -New Features ------------- - -* The result of :c:func:`PyNumber_Index` now always has exact type :class:`int`. - Previously, the result could have been an instance of a subclass of ``int``. - (Contributed by Serhiy Storchaka in :issue:`40792`.) - -* Add a new :c:member:`~PyConfig.orig_argv` member to the :c:type:`PyConfig` - structure: the list of the original command line arguments passed to the - Python executable. - (Contributed by Victor Stinner in :issue:`23427`.) - -* The :c:func:`PyDateTime_DATE_GET_TZINFO` and - :c:func:`PyDateTime_TIME_GET_TZINFO` macros have been added for accessing - the ``tzinfo`` attributes of :class:`datetime.datetime` and - :class:`datetime.time` objects. - (Contributed by Zackery Spytz in :issue:`30155`.) - -* Add a :c:func:`PyCodec_Unregister` function to unregister a codec - search function. - (Contributed by Hai Shi in :issue:`41842`.) - -* The :c:func:`PyIter_Send` function was added to allow - sending value into iterator without raising ``StopIteration`` exception. - (Contributed by Vladimir Matveev in :issue:`41756`.) - -* Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API. - (Contributed by Alex Gaynor in :issue:`41784`.) - -* Added :c:func:`PyModule_AddObjectRef` function: similar to - :c:func:`PyModule_AddObject` but don't steal a reference to the value on - success. - (Contributed by Victor Stinner in :issue:`1635741`.) - -* Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the - reference count of an object and return the object. - (Contributed by Victor Stinner in :issue:`42262`.) - -* The :c:func:`PyType_FromSpecWithBases` and :c:func:`PyType_FromModuleAndSpec` - functions now accept a single class as the *bases* argument. - (Contributed by Serhiy Storchaka in :issue:`42423`.) - -* The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc`` - slot. - (Contributed by Hai Shi in :issue:`41832`.) - -* The :c:func:`PyType_GetSlot` function can accept static types. - (Contributed by Hai Shi and Petr Viktorin in :issue:`41073`.) - -* Add a new :c:func:`PySet_CheckExact` function to the C-API to check if an - object is an instance of :class:`set` but not an instance of a subtype. - (Contributed by Pablo Galindo in :issue:`43277`.) - -* Added :c:func:`PyErr_SetInterruptEx` which allows passing a signal number - to simulate. - (Contributed by Antoine Pitrou in :issue:`43356`.) - - -Porting to Python 3.10 ----------------------- - -* The ``PY_SSIZE_T_CLEAN`` macro must now be defined to use - :c:func:`PyArg_ParseTuple` and :c:func:`Py_BuildValue` formats which use - ``#``: ``es#``, ``et#``, ``s#``, ``u#``, ``y#``, ``z#``, ``U#`` and ``Z#``. - See :ref:`Parsing arguments and building values - ` and the :pep:`353`. - (Contributed by Victor Stinner in :issue:`40943`.) - -* Since :c:func:`Py_REFCNT()` is changed to the inline static function, - ``Py_REFCNT(obj) = new_refcnt`` must be replaced with ``Py_SET_REFCNT(obj, new_refcnt)``: - see :c:func:`Py_SET_REFCNT()` (available since Python 3.9). For backward - compatibility, this macro can be used:: - - #if PY_VERSION_HEX < 0x030900A4 - # define Py_SET_REFCNT(obj, refcnt) ((Py_REFCNT(obj) = (refcnt)), (void)0) - #endif - - (Contributed by Victor Stinner in :issue:`39573`.) - -* Calling :c:func:`PyDict_GetItem` without :term:`GIL` held had been allowed - for historical reason. It is no longer allowed. - (Contributed by Victor Stinner in :issue:`40839`.) - -* ``PyUnicode_FromUnicode(NULL, size)`` and ``PyUnicode_FromStringAndSize(NULL, size)`` - raise ``DeprecationWarning`` now. Use :c:func:`PyUnicode_New` to allocate - Unicode object without initial data. - (Contributed by Inada Naoki in :issue:`36346`.) - -* The private ``_PyUnicode_Name_CAPI`` structure of the PyCapsule API - ``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. - (Contributed by Victor Stinner in :issue:`42157`.) - -* :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and - :c:func:`Py_GetProgramName` functions now return ``NULL`` if called before - :c:func:`Py_Initialize` (before Python is initialized). Use the new - :ref:`Python Initialization Configuration API ` to get the - :ref:`Python Path Configuration. `. - (Contributed by Victor Stinner in :issue:`42260`.) - -* :c:func:`PyList_SET_ITEM`, :c:func:`PyTuple_SET_ITEM` and - :c:func:`PyCell_SET` macros can no longer be used as l-value or r-value. - For example, ``x = PyList_SET_ITEM(a, b, c)`` and - ``PyList_SET_ITEM(a, b, c) = x`` now fail with a compiler error. It prevents - bugs like ``if (PyList_SET_ITEM (a, b, c) < 0) ...`` test. - (Contributed by Zackery Spytz and Victor Stinner in :issue:`30459`.) - -* The non-limited API files ``odictobject.h``, ``parser_interface.h``, - ``picklebufobject.h``, ``pyarena.h``, ``pyctype.h``, ``pydebug.h``, - ``pyfpe.h``, and ``pytime.h`` have been moved to the ``Include/cpython`` - directory. These files must not be included directly, as they are already - included in ``Python.h``: :ref:`Include Files `. If they have - been included directly, consider including ``Python.h`` instead. - (Contributed by Nicholas Sim in :issue:`35134`) - -Deprecated ----------- - -* The ``PyUnicode_InternImmortal()`` function is now deprecated - and will be removed in Python 3.12: use :c:func:`PyUnicode_InternInPlace` - instead. - (Contributed by Victor Stinner in :issue:`41692`.) - -Removed -------- - -* ``PyObject_AsCharBuffer()``, ``PyObject_AsReadBuffer()``, ``PyObject_CheckReadBuffer()``, - and ``PyObject_AsWriteBuffer()`` are removed. Please migrate to new buffer protocol; - :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release`. - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``Py_UNICODE_str*`` functions manipulating ``Py_UNICODE*`` strings. - (Contributed by Inada Naoki in :issue:`41123`.) - - * ``Py_UNICODE_strlen``: use :c:func:`PyUnicode_GetLength` or - :c:macro:`PyUnicode_GET_LENGTH` - * ``Py_UNICODE_strcat``: use :c:func:`PyUnicode_CopyCharacters` or - :c:func:`PyUnicode_FromFormat` - * ``Py_UNICODE_strcpy``, ``Py_UNICODE_strncpy``: use - :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` - * ``Py_UNICODE_strcmp``: use :c:func:`PyUnicode_Compare` - * ``Py_UNICODE_strncmp``: use :c:func:`PyUnicode_Tailmatch` - * ``Py_UNICODE_strchr``, ``Py_UNICODE_strrchr``: use - :c:func:`PyUnicode_FindChar` - -* Removed ``PyUnicode_GetMax()``. Please migrate to new (:pep:`393`) APIs. - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``PyLong_FromUnicode()``. Please migrate to :c:func:`PyLong_FromUnicodeObject`. - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``PyUnicode_AsUnicodeCopy()``. Please use :c:func:`PyUnicode_AsUCS4Copy` or - :c:func:`PyUnicode_AsWideCharString` - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``_Py_CheckRecursionLimit`` variable: it has been replaced by - ``ceval.recursion_limit`` of the :c:type:`PyInterpreterState` structure. - (Contributed by Victor Stinner in :issue:`41834`.) - -* Removed undocumented macros ``Py_ALLOW_RECURSION`` and - ``Py_END_ALLOW_RECURSION`` and the ``recursion_critical`` field of the - :c:type:`PyInterpreterState` structure. - (Contributed by Serhiy Storchaka in :issue:`41936`.) - -* Removed the undocumented ``PyOS_InitInterrupts()`` function. Initializing - Python already implicitly installs signal handlers: see - :c:member:`PyConfig.install_signal_handlers`. - (Contributed by Victor Stinner in :issue:`41713`.) From c2e438aed7f5f4fe44dc050402124be30eb2e62b Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 15:22:46 +0000 Subject: [PATCH 09/93] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst diff --git a/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst new file mode 100644 index 00000000000000..eb89caa3976db4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst @@ -0,0 +1,2 @@ +Add an Barrier object in synchronization primitives of asyncio Lib +in order to be consistant with Barrier from *threading* and *multiprocessing* libs \ No newline at end of file From 97598f3d785035a4bba21bbd08441b1b4bd924c0 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 2 Apr 2021 14:57:51 +0200 Subject: [PATCH 10/93] Update 2021-03-31-15-22-45.bpo-43352.nSjMuE.rst --- .../next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst index eb89caa3976db4..e53ba28b64099f 100644 --- a/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst +++ b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst @@ -1,2 +1 @@ -Add an Barrier object in synchronization primitives of asyncio Lib -in order to be consistant with Barrier from *threading* and *multiprocessing* libs \ No newline at end of file +Add an Barrier object in synchronization primitives of *asyncio* Lib in order to be consistant with Barrier from *threading* and *multiprocessing* libs* From dfea4c2f96f3be0930f6bae59309e9d9ba701325 Mon Sep 17 00:00:00 2001 From: Duprat Date: Sun, 11 Apr 2021 15:36:40 +0200 Subject: [PATCH 11/93] Update documentation about add of Barrier object Add Barrier object as a clone of threading.Barrier --- Doc/library/asyncio-api-index.rst | 3 +++ Doc/library/asyncio-sync.rst | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index f558724d4a3ff6..3465a297120920 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -186,6 +186,9 @@ Threading-like synchronization primitives that can be used in Tasks. * - :class:`BoundedSemaphore` - A bounded semaphore. + * - :class:`Barrier` + - A barrier object. + .. rubric:: Examples diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index d12630afc6a326..ac6e84d070850e 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -28,6 +28,7 @@ asyncio has the following basic synchronization primitives: * :class:`Condition` * :class:`Semaphore` * :class:`BoundedSemaphore` +* :class:`Barrier` --------- @@ -325,6 +326,22 @@ BoundedSemaphore a :exc:`ValueError` in :meth:`~Semaphore.release` if it increases the internal counter above the initial *value*. + +Barrier +======= + +.. class:: Barrier(parties, action=None) + + A barrier object. Not thread-safe. + + Create a barrier object for *parties* number of tasks. An *action*, when + provided, is a callable to be called by one of the tasks when they are + released. + + A barrier object: a clone of :class:`threading.Barrier`. + + .. versionadded:: 3.10 + --------- From 44fe563315422d1147109b2f31afb7421d54bd0b Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 30 Apr 2021 14:47:28 +0200 Subject: [PATCH 12/93] Update asyncio-sync.rst Describe when action is called --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index ac6e84d070850e..f7ff876cf61dd0 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -335,7 +335,7 @@ Barrier A barrier object. Not thread-safe. Create a barrier object for *parties* number of tasks. An *action*, when - provided, is a callable to be called by one of the tasks when they are + provided, is a callable to be called by the last waiting task when they are released. A barrier object: a clone of :class:`threading.Barrier`. From e23c860978092fc07712f6b85a65334a453807ee Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 13:54:00 +0200 Subject: [PATCH 13/93] Update locks.py first feedbacks from Emmanuel Arias --- Lib/asyncio/locks.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index e7f2561468e8cf..57d998768147ca 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -1,6 +1,7 @@ """Synchronization primitives.""" -__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') +__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', + 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') import collections @@ -431,9 +432,10 @@ def release(self): # since the previous cycle. In addition, a 'resetting' state exists which is # similar to 'draining' except that tasks leave with a BrokenBarrierError, # and a 'broken' state in which all tasks get the exception. - +# +# Asyncio equivalent to threading.Barrier class Barrier(mixins._LoopBoundMixin): - """Asynchronous equivalent to threading.Barrier + """Asyncio equivalent to threading.Barrier Implements a Barrier. Useful for synchronizing a fixed number of tasks at known synchronization @@ -495,10 +497,9 @@ async def _block (self): # It is draining or resetting, wait until done await self._blocking.wait() - #see if the barrier is in a broken state + # see if the barrier is in a broken state if self._state < 0: raise BrokenBarrierError - assert self._state == 0, repr(self) # Optionally run the 'action' and release the tasks waiting # in the barrier. @@ -512,9 +513,7 @@ def _release(self): self._waiting.set() except: #an exception during the _action handler. Break and reraise - self._state = -2 - self._blocking.clear() - self._waiting.set() + self.abort() raise # Wait in the barrier until we are released. Raise an exception @@ -524,7 +523,6 @@ async def _wait(self): # no timeout so if self._state < 0: # resetting or broken raise BrokenBarrierError - assert self._state == 1, repr(self) # If we are the last tasks to exit the barrier, signal any tasks # waiting for the barrier to drain. From 6a51dc68c3efa2cc53f1f6431871da8bdf342c9b Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 13:56:05 +0200 Subject: [PATCH 14/93] Update test_locks.py Remove "import functools" --- Lib/test/test_asyncio/test_locks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index ec90698b6155aa..85f0a4d4d4428a 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -6,7 +6,6 @@ import asyncio from test.test_asyncio import utils as test_utils -import functools STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' From 04f72a37f5d7bcec228a73d23e3fdc2a0f746933 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 16:55:53 +0200 Subject: [PATCH 15/93] Update asyncio-sync.rst Suggestion of Emmanuel Arias: Add sample and description about Barrier Class --- Doc/library/asyncio-sync.rst | 124 +++++++++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 6 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index f7ff876cf61dd0..0eab76810948b8 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -330,17 +330,129 @@ BoundedSemaphore Barrier ======= -.. class:: Barrier(parties, action=None) + .. versionadded:: 3.10 A barrier object. Not thread-safe. - Create a barrier object for *parties* number of tasks. An *action*, when - provided, is a callable to be called by the last waiting task when they are - released. + This class, a clone of :class:`threading.Barrier`, provides a simple synchronization + primitive for use by a fixed number of tasks that need to wait for each other. + Each of the task tries to pass the barrier by calling the :meth:`~Barrier.wait` method + and will block until all of the tasks have made their :meth:`~Barrier.wait` calls. + At this point, the tasks are released simultaneously. - A barrier object: a clone of :class:`threading.Barrier`. + The barrier can be reused any number of times for the same number of tasks. - .. versionadded:: 3.10 + .. _asyncio_example_barrier: + + Example:: + + + import asyncio + import os + import random + + DELAYS = (1.0, 4.0, 2.0, 0.5) + + def job_done(): + msg = "ACTION: All files are removed" + print(f'{os.linesep}{msg:-^80s}{os.linesep}') + + async def remove_file(barrier, filename): + print(f'Remove "{filename}" ...') + + # remove filename + await asyncio.sleep(random.choice(DELAYS)) # simulate operation + print(f'\t"{filename}" is removed') + + return await barrier.wait() + + async def remove_folder(barrier, folder): + print(f'Waiting for remove all files from folder "{folder}" ...') + + # wait for all removed files + await barrier.wait() + + # remove folder + await asyncio.sleep(1.0) # simulates operation + print(f'Folder "{folder}" is now removed ...') + + return -1 + + async def main(folder, files, nb_files): + # Create a Barrier object with parties equal to ``nb_files+1`` + # corresponding to the number of all removing items, + # and an action as a print message. + barrier = asyncio.Barrier(nb_files+1, action=job_done) + + # Add remove_file task for each filename. + tasks = [asyncio.create_task(remove_file(barrier, f)) for f in files] + + # Add remove_folder task for the folder. + tasks.append(asyncio.create_task(remove_folder(barrier, folder))) + + # Wait until folder+filenames are removed. + await asyncio.gather(*tasks) + + asyncio.run(main("Datas", [f"file{i+1:02d}.txt" for i in range(10)], 10)) + + +.. class:: Barrier(parties, action=None) + + Create a barrier object for *parties* number of tasks. A callable *action*, when + provided, is called once at the begining of draining. + + .. coroutinemethod:: wait() + + Pass the barrier. When all the tasks party to the barrier have called + this function, they are all released simultaneously. + + The return value is an integer in the range 0 to *parties* -- 1, different + for each task. This can be used to select a task to do some special + housekeeping, e.g.:: + + ... + file_position = await barrier.wait() + if file_position == 0: + # Only one tasks needs to print this + print(f"I have the position #{file_position}") + + If an *action* was provided to the constructor, the last calling task of :meth:`wait` method will + have called it prior to being released. Should this call raise an error, + the barrier is put into the broken state. + + This method may raise a :class:`BrokenBarrierError` exception if the + barrier is broken or reset while a task is waiting. + + .. method:: reset() + + Return the barrier to the default, empty state. Any tasks waiting on it + will receive the :class:`BrokenBarrierError` exception. + + If a barrier is broken it may be better to just leave it and create a new one. + + .. method:: abort() + + Put the barrier into a broken state. This causes any active or future + calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. + Use this for example if one of the taks needs to abort, to avoid infinite waiting tasks. + + .. attribute:: parties + + The number of tasks required to pass the barrier. + + .. attribute:: n_waiting + + The number of tasks currently waiting in the barrier. + + .. attribute:: broken + + A boolean that is ``True`` if the barrier is in the broken state. + + +.. exception:: BrokenBarrierError + + This exception, a subclass of :exc:`RuntimeError`, is raised when the + :class:`Barrier` object is reset or broken. --------- From 7a0045324fd1221ff06baf3bea188fcbfa75525a Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 21:28:14 +0200 Subject: [PATCH 16/93] Update asyncio-sync.rst bpo-43352 - Add a sample and description for Barrier Object Minor updates as remove crlf --- Doc/library/asyncio-sync.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 0eab76810948b8..263d43d6cd854f 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -346,7 +346,6 @@ Barrier Example:: - import asyncio import os import random @@ -395,7 +394,6 @@ Barrier asyncio.run(main("Datas", [f"file{i+1:02d}.txt" for i in range(10)], 10)) - .. class:: Barrier(parties, action=None) Create a barrier object for *parties* number of tasks. A callable *action*, when From 3517e33f71e8b25ac3d0c3f2c30960f6a48f26d0 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 18 May 2021 08:35:21 +0200 Subject: [PATCH 17/93] Update asyncio-sync.rst bpo-43352 Suppress ` in a comment of the example --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 263d43d6cd854f..2cc437c0abfe7c 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -378,7 +378,7 @@ Barrier return -1 async def main(folder, files, nb_files): - # Create a Barrier object with parties equal to ``nb_files+1`` + # Create a Barrier object with parties equal to nb_files+1 # corresponding to the number of all removing items, # and an action as a print message. barrier = asyncio.Barrier(nb_files+1, action=job_done) From c56e7c6eae8ee13784ea6850b9ed13f6ce5df209 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 15:55:25 +0200 Subject: [PATCH 18/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a class BarrierTests to test Barrier object Run i17 tests ncluding one about Barrier does not accept loop parameter --- Lib/test/test_asyncio/test_locks.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 85f0a4d4d4428a..dbe36d6da5362e 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -11,7 +11,7 @@ r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # new line dedicated to repr barrier + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -958,6 +958,14 @@ def cancel_coros(self): task.cancel() test_utils.run_briefly(self.loop) + def test_barrier_doesnt_accept_loop_parameter(self): + with self.assertRaisesRegex( + TypeError, + rf'As of 3.10, the \*loop\* parameter was removed from ' + rf'Barrier\(\) since it is no longer necessary' + ): + asyncio.Barrier(1, loop=self.loop) + def test_repr(self): async def coro(): try: @@ -967,7 +975,7 @@ async def coro(): else: return await asyncio.sleep(1, True) - self.N = 1011001 + self.N = 999 barrier = asyncio.Barrier(self.N) self.assertTrue("[unset," in repr(barrier)) self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) From 8715b3497be17817b7e17ae0a6966fdc3f059a7b Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 16:05:55 +0200 Subject: [PATCH 19/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add the Barrier object in Synchronization section Add a link to a Barrier sample in Examples sub-section Add the BrokenBarrierError Exception in Exception section --- Doc/library/asyncio-api-index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index 3465a297120920..33c2b119dfb84a 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -194,6 +194,8 @@ Threading-like synchronization primitives that can be used in Tasks. * :ref:`Using asyncio.Event `. +* :ref:`Using asyncio.Barrier `. + * See also the documentation of asyncio :ref:`synchronization primitives `. @@ -214,6 +216,9 @@ Exceptions * - :exc:`asyncio.CancelledError` - Raised when a Task is cancelled. See also :meth:`Task.cancel`. + * - :exc:`asyncio.BrokenBarrierError` + - Raised when a Barrier is broken. See also :meth:`Barrier.wait`. + .. rubric:: Examples From d15b2a6f70a466963e2ce759193c5184bdf1e45f Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 16:09:01 +0200 Subject: [PATCH 20/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a complete documentation inspired from threading.Barrier Add a sample to show how use a Barrier Object --- Doc/library/asyncio-sync.rst | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 2cc437c0abfe7c..63cfcd47f34316 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -334,7 +334,7 @@ Barrier A barrier object. Not thread-safe. - This class, a clone of :class:`threading.Barrier`, provides a simple synchronization + This class, as a clone of :class:`threading.Barrier`, provides a simple synchronization primitive for use by a fixed number of tasks that need to wait for each other. Each of the task tries to pass the barrier by calling the :meth:`~Barrier.wait` method and will block until all of the tasks have made their :meth:`~Barrier.wait` calls. @@ -351,29 +351,31 @@ Barrier import random DELAYS = (1.0, 4.0, 2.0, 0.5) + NB = 10 def job_done(): - msg = "ACTION: All files are removed" + msg = 'ACTION: All files removed' print(f'{os.linesep}{msg:-^80s}{os.linesep}') async def remove_file(barrier, filename): - print(f'Remove "{filename}" ...') + print(f'Remove {filename!r} ...') - # remove filename - await asyncio.sleep(random.choice(DELAYS)) # simulate operation - print(f'\t"{filename}" is removed') + # remove filename (simulation) + await asyncio.sleep(random.choice(DELAYS)) + print(f'\tFile {filename!r} is removed') + # wait for others files return await barrier.wait() async def remove_folder(barrier, folder): - print(f'Waiting for remove all files from folder "{folder}" ...') + print(f'Waiting for remove all files from folder {folder!r} ...') # wait for all removed files await barrier.wait() - # remove folder - await asyncio.sleep(1.0) # simulates operation - print(f'Folder "{folder}" is now removed ...') + # remove folder (simulation) + await asyncio.sleep(1.0) + print(f'Folder {folder!r} is now removed ...') return -1 @@ -392,12 +394,12 @@ Barrier # Wait until folder+filenames are removed. await asyncio.gather(*tasks) - asyncio.run(main("Datas", [f"file{i+1:02d}.txt" for i in range(10)], 10)) + asyncio.run(main("Datas", [f'file{i+1:02d}.txt' for i in range(NB)], NB)) .. class:: Barrier(parties, action=None) Create a barrier object for *parties* number of tasks. A callable *action*, when - provided, is called once at the begining of draining. + provided, is called once just before the release phasis (draining state). .. coroutinemethod:: wait() @@ -414,9 +416,9 @@ Barrier # Only one tasks needs to print this print(f"I have the position #{file_position}") - If an *action* was provided to the constructor, the last calling task of :meth:`wait` method will - have called it prior to being released. Should this call raise an error, - the barrier is put into the broken state. + If an *action* was provided to the constructor, the last calling task of + :meth:`wait` method will have called it prior to being released. + Should this call raise an error, the barrier is put into the broken state. This method may raise a :class:`BrokenBarrierError` exception if the barrier is broken or reset while a task is waiting. @@ -432,7 +434,8 @@ Barrier Put the barrier into a broken state. This causes any active or future calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. - Use this for example if one of the taks needs to abort, to avoid infinite waiting tasks. + Use this for example if one of the taks needs to abort, to avoid infinite + waiting tasks. .. attribute:: parties From ed1b361ab23b61fff3aaf14a60ea69d353ca71ad Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 16:14:58 +0200 Subject: [PATCH 21/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a Barrier object as a clone from thraeding Barrier Update of __all__ variable to include this new class and the related BrokenBarrierError exception --- Lib/asyncio/locks.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 57d998768147ca..eec4dafed5a903 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -445,33 +445,37 @@ class Barrier(mixins._LoopBoundMixin): def __init__(self, parties, action=None, *, loop=mixins._marker): """Create a barrier, initialised to 'parties' tasks. - 'action' is a callable which, when supplied, will be called by one of - the tasks after they have all entered the barrier and just prior to - releasing them all. + 'action' is a callable which, when supplied, will be called by + the last task calling the wait() method, + just prior to releasing them all. """ super().__init__(loop=loop) if parties < 1: raise ValueError('parties must be > 0') self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block new tasks while waiting tasks are draining or broken + self._blocking = Event() # used block new tasks while waiting + # tasks are draining or broken self._action = action self._parties = parties - self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken + self._state = 0 # 0 filling, 1, draining, + # -1 resetting, -2 broken self._count = 0 # count waiting tasks def __repr__(self): res = super().__repr__() _wait = 'set' if self._waiting.is_set() else 'unset' _block = 'set' if self._blocking.is_set() else 'unset' - extra = f'{_wait}, count:{self._count}/{self._parties}, {_block}, state:{self._state}' + extra = f'{_wait}, count:{self._count}/{self._parties}' + extra += f', {_block}, state:{self._state}' return f'<{res[1:-1]} [{extra}]>' async def wait(self): """Wait for the barrier. When the specified number of tasks have started waiting, they are all - simultaneously awoken. If an 'action' was provided for the barrier, one - of the tasks will have executed that callback prior to returning. + simultaneously awoken. If an 'action' was provided for the barrier, the + last task calling this method will have executed that callback prior to + returning. Returns an individual index number from 0 to 'parties-1'. """ await self._block() # Block while the barrier drains or resets. @@ -520,6 +524,7 @@ def _release(self): # if the barrier is reset or broken. async def _wait(self): await self._waiting.wait() + # no timeout so if self._state < 0: # resetting or broken raise BrokenBarrierError From 7504a8fd13bcdd728f7022a88ecb182f5b62b838 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 21:42:40 +0200 Subject: [PATCH 22/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a class BarrierTests to test Barrier object Run 17 tests including one about Barrier does not accept loop parameter --- Lib/test/test_asyncio/test_locks.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index dbe36d6da5362e..b0d93ad55b7703 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -69,16 +69,6 @@ def test_lock_doesnt_accept_loop_parameter(self): ): cls(loop=self.loop) - # Barrier object has a positional paramater - # so tests alone - cls = asyncio.Barrier - with self.assertRaisesRegex( - TypeError, - rf'As of 3.10, the \*loop\* parameter was removed from ' - rf'{cls.__name__}\(\) since it is no longer necessary' - ): - cls(1, loop=self.loop) - def test_lock_by_with_statement(self): loop = asyncio.new_event_loop() # don't use TestLoop quirks self.set_event_loop(loop) From e3b23c2d816c478aabc1d0bfdfc3f07611dbe6af Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 15 Mar 2021 16:55:49 +0100 Subject: [PATCH 23/93] Initial commit Add a Barrier object to asyncio synchronized primitives --- Lib/asyncio/locks.py | 173 +++++++++++++- Lib/test/test_asyncio/test_locks.py | 356 +++++++++++++++++++++++++++- 2 files changed, 527 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index a7453fb1c77287..e5fcf9648de5d9 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -1,6 +1,6 @@ """Synchronization primitives.""" -__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore') +__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') import collections @@ -418,3 +418,174 @@ def release(self): if self._value >= self._bound_value: raise ValueError('BoundedSemaphore released too many times') super().release() + + +# A barrier class. Inspired in part by the pthread_barrier_* api and +# the CyclicBarrier class from Java. See +# http://sourceware.org/pthreads-win32/manual/pthread_barrier_init.html and +# http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/ +# CyclicBarrier.html +# for information. +# We maintain two main states, 'filling' and 'draining' enabling the barrier +# to be cyclic. Tasks are not allowed into it until it has fully drained +# since the previous cycle. In addition, a 'resetting' state exists which is +# similar to 'draining' except that tasks leave with a BrokenBarrierError, +# and a 'broken' state in which all tasks get the exception. + +class Barrier(mixins._LoopBoundMixin): + """Asynchronous equivalent to threading.Barrier + + Implements a Barrier. + Useful for synchronizing a fixed number of tasks at known synchronization + points. Tasks block on 'wait()' and are simultaneously awoken once they + have all made that call. + """ + + def __init__(self, parties, action=None, *, loop=mixins._marker): + """Create a barrier, initialised to 'parties' tasks. + 'action' is a callable which, when supplied, will be called by one of + the tasks after they have all entered the barrier and just prior to + releasing them all. + """ + super().__init__(loop=loop) + if parties < 1: + raise ValueError('parties must be > 0') + + self._waiting = Event() # used notify all waiting tasks + self._blocking = Event() # used block tasks while wainting tasks are draining or broken + self._action = action + self._parties = parties + self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken + self._count = 0 # count waiting tasks + + def __repr__(self): + res = super().__repr__() + _wait = 'set' if self._waiting.is_set() else 'unset' + _block = 'set' if self._blocking.is_set() else 'unset' + extra = f'{_wait}, count:{self._count}/{self._parties}, {_block}, state:{self._state}' + return f'<{res[1:-1]} [{extra}]>' + + async def wait(self): + """Wait for the barrier. + When the specified number of tasks have started waiting, they are all + simultaneously awoken. If an 'action' was provided for the barrier, one + of the tasks will have executed that callback prior to returning. + Returns an individual index number from 0 to 'parties-1'. + """ + await self._block() # Block while the barrier drains or resets. + index = self._count + self._count += 1 + try: + if index + 1 == self._parties: + # We release the barrier + self._release() + else: + # We wait until someone releases us + await self._wait() + return index + finally: + self._count -= 1 + # Wake up any tasks waiting for barrier to drain. + self._exit() + + # Block until the barrier is ready for us, or raise an exception + # if it is broken. + async def _block (self): + if self._state in (-1, 1): + # It is draining or resetting, wait until done + await self._blocking.wait() + + #see if the barrier is in a broken state + if self._state < 0: + raise BrokenBarrierError + assert self._state == 0, repr(self) + + # Optionally run the 'action' and release the tasks waiting + # in the barrier. + def _release(self): + try: + if self._action: + self._action() + # enter draining state + self._state = 1 + self._blocking.clear() + self._waiting.set() + except: + #an exception during the _action handler. Break and reraise + self._state = -2 + self._blocking.clear() + self._waiting.set() + raise + + # Wait in the barrier until we are released. Raise an exception + # if the barrier is reset or broken. + async def _wait(self): + await self._waiting.wait() + # no timeout so + if self._state < 0: + raise BrokenBarrierError + assert self._state == 1, repr(self) + + # If we are the last tasks to exit the barrier, signal any tasks + # waiting for the barrier to drain. + def _exit(self): + if self._count == 0: + if self._state == 1: + self._state = 0 + elif self._state == -1: + self._state = 0 + self._waiting.clear() + self._blocking.set() + + # async def reset(self): + def reset(self): + """Reset the barrier to the initial state. + Any tasks currently waiting will get the BrokenBarrier exception + raised. + """ + if self._count > 0: + if self._state in (0, 1): + #reset the barrier, waking up tasks + self._state = -1 + elif self._state == -2: + #was broken, set it to reset state + #which clears when the last tasks exits + self._state = -1 + self._waiting.set() + self._blocking.clear() + else: + self._state = 0 + + + # async def abort(self): + def abort(self): + """Place the barrier into a 'broken' state. + Useful in case of error. Any currently waiting tasks and tasks + attempting to 'wait()' will have BrokenBarrierError raised. + """ + self._state = -2 + self._waiting.set() + self._blocking.clear() + + @property + def parties(self): + """Return the number of tasks required to trip the barrier.""" + return self._parties + + @property + def n_waiting(self): + """Return the number of tasks currently waiting at the barrier.""" + # We don't need synchronization here since this is an ephemeral result + # anyway. It returns the correct value in the steady state. + if self._state == 0: + return self._count + return 0 + + @property + def broken(self): + """Return True if the barrier is in a broken state.""" + return self._state == -2 + +# exception raised by the Barrier class +class BrokenBarrierError(RuntimeError): + pass \ No newline at end of file diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 6194cd06176dac..824535a18cb168 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -6,11 +6,13 @@ import asyncio from test.test_asyncio import utils as test_utils +import functools STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' - r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' + r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?(, count:\d+/\d+)?(, state:^-?\d)?' + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier object r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -68,6 +70,16 @@ def test_lock_doesnt_accept_loop_parameter(self): ): cls(loop=self.loop) + # Barrier object has a positional paramater + # so check alone + cls = asyncio.Barrier + with self.assertRaisesRegex( + TypeError, + rf'As of 3.10, the \*loop\* parameter was removed from ' + rf'{cls.__name__}\(\) since it is no longer necessary' + ): + cls(1, loop=self.loop) + def test_lock_by_with_statement(self): loop = asyncio.new_event_loop() # don't use TestLoop quirks self.set_event_loop(loop) @@ -928,5 +940,347 @@ def test_release_no_waiters(self): self.assertFalse(sem.locked()) +class BarrierTests(test_utils.TestCase): + """ + Tests for Barrier objects. + """ + def setUp(self): + super().setUp() + self.loop = self.new_test_loop() + self.N = 5 + + async def run_coros(self, n, coro): + tasks = [self.loop.create_task(coro()) for _ in range(n)] + res = await asyncio.gather(*tasks) + return res, tasks + + def test_repr(self): + async def coro(): + try: + i = await barrier.wait() + except: + ... + else: + return await asyncio.sleep(1, True) + + self.N = 999 + barrier = asyncio.Barrier(self.N) + self.assertTrue("[unset," in repr(barrier)) + self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(repr(barrier).endswith('unset, state:0]>')) + + nb_waits = 3 + tasks = [] + for _ in range(nb_waits): + tasks.append(self.loop.create_task(coro())) + test_utils.run_briefly(self.loop) + self.assertTrue("[unset," in repr(barrier)) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith('unset, state:0]>')) + + barrier.reset() + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue("[set," in repr(barrier)) + self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith('unset, state:-1]>')) + test_utils.run_briefly(self.loop) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue("[unset," in repr(barrier)) + self.assertTrue(f"count:0/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith('set, state:0]>')) + test_utils.run_briefly(self.loop) + + barrier.abort() + test_utils.run_briefly(self.loop) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(f"0/{self.N}" in repr(barrier)) + self.assertTrue(repr(barrier).endswith(':-2]>')) + + def test_init(self): + self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) + self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) + + def test_wait(self): + async def coro(result): + r = await barrier.wait() + result.append(r) + + barrier = asyncio.Barrier(self.N) + results = [] + _ = [self.loop.create_task(coro(results)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + self.assertEqual(sum(results), sum(range(self.N))) + self.assertFalse(barrier.broken) + + def test_wait_task_multi(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) + self.assertEqual(barrier.n_waiting, 0) + self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 2) + self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_wait_task(self): + self.N = 1 + barrier = asyncio.Barrier(self.N) + r1 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + r2 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + self.assertEqual(r1, r2) + self.assertFalse(barrier.broken) + + def test_wait_step_by_step(self): + async def coro(result, value): + await barrier.wait() + result.append(value) + + results = [] + barrier = asyncio.Barrier(self.N) + value = 1 + tasks = [] + for n in range(self.N-1): + tasks.append(self.loop.create_task(coro(results, value))) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, n+1) + + tasks.append(self.loop.create_task(coro(results, value))) + test_utils.run_briefly(self.loop) + self.assertEqual(results, [value]* self.N) + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(all(t.done() for t in tasks)) + self.assertTrue(all(t.result() is None for t in tasks)) + self.assertFalse(barrier.broken) + + def test_wait_multi_return(self): + results1 = [] + results2 = [] + results3 = [] + async def coro(): + await barrier.wait() + results1.append(True) + i = await barrier.wait() + results2.append(True) + results3.append(i) + return i + + barrier = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertEqual(len(results1), self.N) + self.assertTrue(all(results1)) + self.assertEqual(len(results2), self.N) + self.assertTrue(all(results2)) + self.assertEqual(sum(res), sum(results3)) + + def test_barrier(self, multipass=1, nn=5): + results = [[] for _ in range(multipass)] + async def coro(result, value): + ret = await barrier.wait() + result.append(value) + return ret + + barrier = asyncio.Barrier(nn) + for i in range(multipass): + _ = [self.loop.create_task(coro(results[i], value)) for value in range(nn)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(results[i]), nn) + self.assertEqual(sum(results[i]), sum(range(nn))) + if i > 0: + self.assertEqual(sum(results[i]), sum(results[i-1])) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) + + def test_barrier_multipass(self): + self.test_barrier(10) + + def test_action_callback(self): + async def coro(result, value): + ret = await barrier.wait() + result.append(value) + return ret + + result = [] + result1 = [] + value = 1 + barrier = asyncio.Barrier(1, action=lambda: result1.append(True)) + _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(result1), self.N) + self.assertTrue(all(result1)) + self.assertEqual(len(result), self.N) + self.assertEqual(sum(result), self.N*value) + self.assertFalse(barrier.broken) + + def test_action_callback_n(self): + async def coro(result, value): + ret = await barrier.wait() + result.append(value) + return ret + + result = [] + result1 = [] + value = 1 + barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) + _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(result1), 1) + self.assertTrue(result1[0]) + self.assertEqual(len(result), self.N) + self.assertEqual(sum(result), value*self.N) + self.assertFalse(barrier.broken) + + def test_action_callback_error(self): + results1 = [] + results2 = [] + results3 = [] + def raise_except(e): + raise e + + async def coro(e): + try: + ret = await barrier.wait() + except e: + results1.append(False) + except: + results2.append(True) + else: + results3.append(None) + + barrier = asyncio.Barrier(self.N, lambda: raise_except(ZeroDivisionError)) + _ = [self.loop.create_task(coro(ZeroDivisionError)) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + self.assertEqual(len(results1), 1) + self.assertFalse(results1[0]) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(all(results2)) + self.assertEqual(len(results3), 0) + self.assertTrue(barrier.broken) + + def test_reset(self): + results1 = [] + results2 = [] + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + results1.append(True) + finally: + await barrier.wait() + results2.append(True) + + async def coro2(): + await barrier.wait() + results2.append(True) + + + barrier = asyncio.Barrier(self.N) + _ = [self.loop.create_task(coro()) for _ in range(self.N-1)] + test_utils.run_briefly(self.loop) + barrier.reset() + test_utils.run_briefly(self.loop) + self.loop.create_task(coro2()) + test_utils.run_briefly(self.loop) + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), self.N-1) + self.assertEqual(len(results2), self.N) + + def test_reset_multi_wait(self): + results1 = [] + results2 = [] + results3 = [] + alls = [] + async def coro(): + i = await barrier1.wait() + alls.append(i) + if len(alls) == self.N: # i == self.N//2: + barrier.reset() + else: + try: + await barrier.wait() + results1.append(True) + except Exception as e: + results2.append(True) + + # Now, pass the barrier again + await barrier1.wait() + results3.append(True) + + barrier = asyncio.Barrier(self.N) + barrier1 = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + def test_abort(self): + results1 = [] + results2 = [] + async def coro(): + try: + i = await barrier.wait() + if i == self.N//2: + raise RuntimeError + await barrier.wait() + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + + barrier = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertTrue(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + + def test_abort_and_reset(self): + results1 = [] + results2 = [] + results3 = [] + async def coro(): + try: + i = await barrier.wait() + if i == self.N//2: + raise RuntimeError + await barrier.wait() + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + barrier.abort() + + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + i = await barrier2.wait() + if i == self.N//2: + barrier.reset() + await barrier2.wait() + await barrier.wait() + results3.append(True) + + barrier = asyncio.Barrier(self.N) + barrier2 = asyncio.Barrier(self.N) + res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertFalse(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + if __name__ == '__main__': unittest.main() From bc3d17a45bc554a2bad15659690368ec0ee4db9e Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 15 Mar 2021 21:04:37 +0100 Subject: [PATCH 24/93] make patchcheck Change after run make patchcheck --- Lib/asyncio/locks.py | 18 ++++++++-------- Lib/test/test_asyncio/test_locks.py | 32 ++++++++++++++--------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index e5fcf9648de5d9..4bc69100ed225d 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -433,7 +433,7 @@ def release(self): # and a 'broken' state in which all tasks get the exception. class Barrier(mixins._LoopBoundMixin): - """Asynchronous equivalent to threading.Barrier + """Asynchronous equivalent to threading.Barrier Implements a Barrier. Useful for synchronizing a fixed number of tasks at known synchronization @@ -445,18 +445,18 @@ def __init__(self, parties, action=None, *, loop=mixins._marker): """Create a barrier, initialised to 'parties' tasks. 'action' is a callable which, when supplied, will be called by one of the tasks after they have all entered the barrier and just prior to - releasing them all. + releasing them all. """ super().__init__(loop=loop) if parties < 1: raise ValueError('parties must be > 0') self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block tasks while wainting tasks are draining or broken + self._blocking = Event() # used block tasks while wainting tasks are draining or broken self._action = action self._parties = parties self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken - self._count = 0 # count waiting tasks + self._count = 0 # count waiting tasks def __repr__(self): res = super().__repr__() @@ -494,7 +494,7 @@ async def _block (self): if self._state in (-1, 1): # It is draining or resetting, wait until done await self._blocking.wait() - + #see if the barrier is in a broken state if self._state < 0: raise BrokenBarrierError @@ -515,7 +515,7 @@ def _release(self): self._state = -2 self._blocking.clear() self._waiting.set() - raise + raise # Wait in the barrier until we are released. Raise an exception # if the barrier is reset or broken. @@ -529,7 +529,7 @@ async def _wait(self): # If we are the last tasks to exit the barrier, signal any tasks # waiting for the barrier to drain. def _exit(self): - if self._count == 0: + if self._count == 0: if self._state == 1: self._state = 0 elif self._state == -1: @@ -543,7 +543,7 @@ def reset(self): Any tasks currently waiting will get the BrokenBarrier exception raised. """ - if self._count > 0: + if self._count > 0: if self._state in (0, 1): #reset the barrier, waking up tasks self._state = -1 @@ -588,4 +588,4 @@ def broken(self): # exception raised by the Barrier class class BrokenBarrierError(RuntimeError): - pass \ No newline at end of file + pass diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 824535a18cb168..42ef44db2d3758 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -70,8 +70,8 @@ def test_lock_doesnt_accept_loop_parameter(self): ): cls(loop=self.loop) - # Barrier object has a positional paramater - # so check alone + # Barrier object has a positional paramater + # so check alone cls = asyncio.Barrier with self.assertRaisesRegex( TypeError, @@ -947,7 +947,7 @@ class BarrierTests(test_utils.TestCase): def setUp(self): super().setUp() self.loop = self.new_test_loop() - self.N = 5 + self.N = 5 async def run_coros(self, n, coro): tasks = [self.loop.create_task(coro()) for _ in range(n)] @@ -969,8 +969,8 @@ async def coro(): self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue(repr(barrier).endswith('unset, state:0]>')) - - nb_waits = 3 + + nb_waits = 3 tasks = [] for _ in range(nb_waits): tasks.append(self.loop.create_task(coro())) @@ -1002,16 +1002,16 @@ def test_init(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - def test_wait(self): + def test_wait_more(self): async def coro(result): r = await barrier.wait() result.append(r) - + barrier = asyncio.Barrier(self.N) results = [] _ = [self.loop.create_task(coro(results)) for _ in range(self.N)] test_utils.run_briefly(self.loop) - self.assertEqual(sum(results), sum(range(self.N))) + self.assertEqual(sum(results), sum(range(self.N))) self.assertFalse(barrier.broken) def test_wait_task_multi(self): @@ -1029,7 +1029,7 @@ def test_wait_task_multi(self): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_wait_task(self): + def test_wait_task_one(self): self.N = 1 barrier = asyncio.Barrier(self.N) r1 = self.loop.run_until_complete(barrier.wait()) @@ -1115,7 +1115,7 @@ async def coro(result, value): value = 1 barrier = asyncio.Barrier(1, action=lambda: result1.append(True)) _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] - test_utils.run_briefly(self.loop) + test_utils.run_briefly(self.loop) self.assertEqual(len(result1), self.N) self.assertTrue(all(result1)) @@ -1134,7 +1134,7 @@ async def coro(result, value): value = 1 barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] - test_utils.run_briefly(self.loop) + test_utils.run_briefly(self.loop) self.assertEqual(len(result1), 1) self.assertTrue(result1[0]) @@ -1173,9 +1173,9 @@ def test_reset(self): results1 = [] results2 = [] async def coro(): - try: + try: await barrier.wait() - except asyncio.BrokenBarrierError: + except asyncio.BrokenBarrierError: results1.append(True) finally: await barrier.wait() @@ -1217,7 +1217,7 @@ async def coro(): # Now, pass the barrier again await barrier1.wait() results3.append(True) - + barrier = asyncio.Barrier(self.N) barrier1 = asyncio.Barrier(self.N) res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) @@ -1235,7 +1235,7 @@ async def coro(): if i == self.N//2: raise RuntimeError await barrier.wait() - results1.append(True) + results1.append(True) except asyncio.BrokenBarrierError: results2.append(True) except RuntimeError: @@ -1272,7 +1272,7 @@ async def coro(): await barrier2.wait() await barrier.wait() results3.append(True) - + barrier = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) From a8b7b1acc1a991d5985c2ca0515ca8215fb2d56f Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 15 Mar 2021 21:06:51 +0100 Subject: [PATCH 25/93] make patchcheck run make patchcheck --- Doc/c-api/exceptions.rst.bak | 1054 +++++++++++++ Doc/c-api/memory.rst.bak | 642 ++++++++ Doc/faq/design.rst.bak | 759 +++++++++ Doc/howto/descriptor.rst.bak | 1575 +++++++++++++++++++ Doc/library/_thread.rst.bak | 215 +++ Doc/library/ast.rst.bak | 1993 ++++++++++++++++++++++++ Doc/library/asyncio-api-index.rst.bak | 221 +++ Doc/library/collections.rst.bak | 1278 +++++++++++++++ Doc/library/dataclasses.rst.bak | 595 +++++++ Doc/library/gc.rst.bak | 322 ++++ Doc/library/importlib.metadata.rst.bak | 262 ++++ Doc/library/logging.rst.bak | 1350 ++++++++++++++++ Doc/library/readline.rst.bak | 361 +++++ Doc/library/sqlite3.rst.bak | 1094 +++++++++++++ Doc/library/statistics.rst.bak | 879 +++++++++++ Doc/library/tempfile.rst.bak | 374 +++++ Doc/library/venv.rst.bak | 496 ++++++ Doc/library/xml.sax.handler.rst.bak | 463 ++++++ Doc/library/zipimport.rst.bak | 209 +++ Doc/whatsnew/3.10.rst.bak | 1315 ++++++++++++++++ 20 files changed, 15457 insertions(+) create mode 100644 Doc/c-api/exceptions.rst.bak create mode 100644 Doc/c-api/memory.rst.bak create mode 100644 Doc/faq/design.rst.bak create mode 100644 Doc/howto/descriptor.rst.bak create mode 100644 Doc/library/_thread.rst.bak create mode 100644 Doc/library/ast.rst.bak create mode 100644 Doc/library/asyncio-api-index.rst.bak create mode 100644 Doc/library/collections.rst.bak create mode 100644 Doc/library/dataclasses.rst.bak create mode 100644 Doc/library/gc.rst.bak create mode 100644 Doc/library/importlib.metadata.rst.bak create mode 100644 Doc/library/logging.rst.bak create mode 100644 Doc/library/readline.rst.bak create mode 100644 Doc/library/sqlite3.rst.bak create mode 100644 Doc/library/statistics.rst.bak create mode 100644 Doc/library/tempfile.rst.bak create mode 100644 Doc/library/venv.rst.bak create mode 100644 Doc/library/xml.sax.handler.rst.bak create mode 100644 Doc/library/zipimport.rst.bak create mode 100644 Doc/whatsnew/3.10.rst.bak diff --git a/Doc/c-api/exceptions.rst.bak b/Doc/c-api/exceptions.rst.bak new file mode 100644 index 00000000000000..4e99a0167a632d --- /dev/null +++ b/Doc/c-api/exceptions.rst.bak @@ -0,0 +1,1054 @@ +.. highlight:: c + + +.. _exceptionhandling: + +****************** +Exception Handling +****************** + +The functions described in this chapter will let you handle and raise Python +exceptions. It is important to understand some of the basics of Python +exception handling. It works somewhat like the POSIX :c:data:`errno` variable: +there is a global indicator (per thread) of the last error that occurred. Most +C API functions don't clear this on success, but will set it to indicate the +cause of the error on failure. Most C API functions also return an error +indicator, usually ``NULL`` if they are supposed to return a pointer, or ``-1`` +if they return an integer (exception: the :c:func:`PyArg_\*` functions +return ``1`` for success and ``0`` for failure). + +Concretely, the error indicator consists of three object pointers: the +exception's type, the exception's value, and the traceback object. Any +of those pointers can be ``NULL`` if non-set (although some combinations are +forbidden, for example you can't have a non-``NULL`` traceback if the exception +type is ``NULL``). + +When a function must fail because some function it called failed, it generally +doesn't set the error indicator; the function it called already set it. It is +responsible for either handling the error and clearing the exception or +returning after cleaning up any resources it holds (such as object references or +memory allocations); it should *not* continue normally if it is not prepared to +handle the error. If returning due to an error, it is important to indicate to +the caller that an error has been set. If the error is not handled or carefully +propagated, additional calls into the Python/C API may not behave as intended +and may fail in mysterious ways. + +.. note:: + The error indicator is **not** the result of :func:`sys.exc_info()`. + The former corresponds to an exception that is not yet caught (and is + therefore still propagating), while the latter returns an exception after + it is caught (and has therefore stopped propagating). + + +Printing and clearing +===================== + + +.. c:function:: void PyErr_Clear() + + Clear the error indicator. If the error indicator is not set, there is no + effect. + + +.. c:function:: void PyErr_PrintEx(int set_sys_last_vars) + + Print a standard traceback to ``sys.stderr`` and clear the error indicator. + **Unless** the error is a ``SystemExit``, in that case no traceback is + printed and the Python process will exit with the error code specified by + the ``SystemExit`` instance. + + Call this function **only** when the error indicator is set. Otherwise it + will cause a fatal error! + + If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, + :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the + type, value and traceback of the printed exception, respectively. + + +.. c:function:: void PyErr_Print() + + Alias for ``PyErr_PrintEx(1)``. + + +.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) + + Call :func:`sys.unraisablehook` using the current exception and *obj* + argument. + + This utility function prints a warning message to ``sys.stderr`` when an + exception has been set but it is impossible for the interpreter to actually + raise the exception. It is used, for example, when an exception occurs in an + :meth:`__del__` method. + + The function is called with a single argument *obj* that identifies the context + in which the unraisable exception occurred. If possible, + the repr of *obj* will be printed in the warning message. + + An exception must be set when calling this function. + + +Raising exceptions +================== + +These functions help you set the current thread's error indicator. +For convenience, some of these functions will always return a +``NULL`` pointer for use in a ``return`` statement. + + +.. c:function:: void PyErr_SetString(PyObject *type, const char *message) + + This is the most common way to set the error indicator. The first argument + specifies the exception type; it is normally one of the standard exceptions, + e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. + The second argument is an error message; it is decoded from ``'utf-8``'. + + +.. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) + + This function is similar to :c:func:`PyErr_SetString` but lets you specify an + arbitrary Python object for the "value" of the exception. + + +.. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) + + This function sets the error indicator and returns ``NULL``. *exception* + should be a Python exception class. The *format* and subsequent + parameters help format the error message; they have the same meaning and + values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded + string. + + +.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs) + + Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather + than a variable number of arguments. + + .. versionadded:: 3.5 + + +.. c:function:: void PyErr_SetNone(PyObject *type) + + This is a shorthand for ``PyErr_SetObject(type, Py_None)``. + + +.. c:function:: int PyErr_BadArgument() + + This is a shorthand for ``PyErr_SetString(PyExc_TypeError, message)``, where + *message* indicates that a built-in operation was invoked with an illegal + argument. It is mostly for internal use. + + +.. c:function:: PyObject* PyErr_NoMemory() + + This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns ``NULL`` + so an object allocation function can write ``return PyErr_NoMemory();`` when it + runs out of memory. + + +.. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) + + .. index:: single: strerror() + + This is a convenience function to raise an exception when a C library function + has returned an error and set the C variable :c:data:`errno`. It constructs a + tuple object whose first item is the integer :c:data:`errno` value and whose + second item is the corresponding error message (gotten from :c:func:`strerror`), + and then calls ``PyErr_SetObject(type, object)``. On Unix, when the + :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, + this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, + leaves it set to that. The function always returns ``NULL``, so a wrapper + function around a system call can write ``return PyErr_SetFromErrno(type);`` + when the system call returns an error. + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) + + Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if + *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as + a third parameter. In the case of :exc:`OSError` exception, + this is used to define the :attr:`filename` attribute of the + exception instance. + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) + + Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second + filename object, for raising errors when a function that takes two filenames + fails. + + .. versionadded:: 3.4 + + +.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) + + Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename + is given as a C string. *filename* is decoded from the :term:`filesystem + encoding and error handler`. + + +.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) + + This is a convenience function to raise :exc:`WindowsError`. If called with + *ierr* of :c:data:`0`, the error code returned by a call to :c:func:`GetLastError` + is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve + the Windows description of error code given by *ierr* or :c:func:`GetLastError`, + then it constructs a tuple object whose first item is the *ierr* value and whose + second item is the corresponding error message (gotten from + :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, + object)``. This function always returns ``NULL``. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr) + + Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter + specifying the exception type to be raised. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the + filename is given as a C string. *filename* is decoded from the filesystem + encoding (:func:`os.fsdecode`). + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an + additional parameter specifying the exception type to be raised. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) + + Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, + but accepts a second filename object. + + .. availability:: Windows. + + .. versionadded:: 3.4 + + +.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) + + Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional + parameter specifying the exception type to be raised. + + .. availability:: Windows. + + +.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path) + + This is a convenience function to raise :exc:`ImportError`. *msg* will be + set as the exception's message string. *name* and *path*, both of which can + be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name`` + and ``path`` attributes. + + .. versionadded:: 3.3 + + +.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) + + Set file, line, and offset information for the current exception. If the + current exception is not a :exc:`SyntaxError`, then it sets additional + attributes, which make the exception printing subsystem think the exception + is a :exc:`SyntaxError`. + + .. versionadded:: 3.4 + + +.. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) + + Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string + decoded from the :term:`filesystem encoding and error handler`. + + .. versionadded:: 3.2 + + +.. c:function:: void PyErr_SyntaxLocation(const char *filename, int lineno) + + Like :c:func:`PyErr_SyntaxLocationEx`, but the col_offset parameter is + omitted. + + +.. c:function:: void PyErr_BadInternalCall() + + This is a shorthand for ``PyErr_SetString(PyExc_SystemError, message)``, + where *message* indicates that an internal operation (e.g. a Python/C API + function) was invoked with an illegal argument. It is mostly for internal + use. + + +Issuing warnings +================ + +Use these functions to issue warnings from C code. They mirror similar +functions exported by the Python :mod:`warnings` module. They normally +print a warning message to *sys.stderr*; however, it is +also possible that the user has specified that warnings are to be turned into +errors, and in that case they will raise an exception. It is also possible that +the functions raise an exception because of a problem with the warning machinery. +The return value is ``0`` if no exception is raised, or ``-1`` if an exception +is raised. (It is not possible to determine whether a warning message is +actually printed, nor what the reason is for the exception; this is +intentional.) If an exception is raised, the caller should do its normal +exception handling (for example, :c:func:`Py_DECREF` owned references and return +an error value). + +.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) + + Issue a warning message. The *category* argument is a warning category (see + below) or ``NULL``; the *message* argument is a UTF-8 encoded string. *stack_level* is a + positive number giving a number of stack frames; the warning will be issued from + the currently executing line of code in that stack frame. A *stack_level* of 1 + is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, + and so forth. + + Warning categories must be subclasses of :c:data:`PyExc_Warning`; + :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; + the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard + Python warning categories are available as global variables whose names are + enumerated at :ref:`standardwarningcategories`. + + For information about warning control, see the documentation for the + :mod:`warnings` module and the :option:`-W` option in the command line + documentation. There is no C API for warning control. + +.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) + + Much like :c:func:`PyErr_SetImportError` but this function allows for + specifying a subclass of :exc:`ImportError` to raise. + + .. versionadded:: 3.6 + + +.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) + + Issue a warning message with explicit control over all warning attributes. This + is a straightforward wrapper around the Python function + :func:`warnings.warn_explicit`, see there for more information. The *module* + and *registry* arguments may be set to ``NULL`` to get the default effect + described there. + + .. versionadded:: 3.4 + + +.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) + + Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and + *module* are UTF-8 encoded strings, and *filename* is decoded from the + :term:`filesystem encoding and error handler`. + + +.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnEx`, but use + :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is + an ASCII-encoded string. + + .. versionadded:: 3.2 + + +.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) + + Function similar to :c:func:`PyErr_WarnFormat`, but *category* is + :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. + + .. versionadded:: 3.6 + + +Querying the error indicator +============================ + +.. c:function:: PyObject* PyErr_Occurred() + + Test whether the error indicator is set. If set, return the exception *type* + (the first argument to the last call to one of the :c:func:`PyErr_Set\*` + functions or to :c:func:`PyErr_Restore`). If not set, return ``NULL``. You do not + own a reference to the return value, so you do not need to :c:func:`Py_DECREF` + it. + + The caller must hold the GIL. + + .. note:: + + Do not compare the return value to a specific exception; use + :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could + easily fail since the exception may be an instance instead of a class, in the + case of a class exception, or it may be a subclass of the expected exception.) + + +.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) + + Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This + should only be called when an exception is actually set; a memory access + violation will occur if no exception has been raised. + + +.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) + + Return true if the *given* exception matches the exception type in *exc*. If + *exc* is a class object, this also returns true when *given* is an instance + of a subclass. If *exc* is a tuple, all exception types in the tuple (and + recursively in subtuples) are searched for a match. + + +.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) + + Retrieve the error indicator into three variables whose addresses are passed. + If the error indicator is not set, set all three variables to ``NULL``. If it is + set, it will be cleared and you own a reference to each object retrieved. The + value and traceback object may be ``NULL`` even when the type object is not. + + .. note:: + + This function is normally only used by code that needs to catch exceptions or + by code that needs to save and restore the error indicator temporarily, e.g.:: + + { + PyObject *type, *value, *traceback; + PyErr_Fetch(&type, &value, &traceback); + + /* ... code that might produce other errors ... */ + + PyErr_Restore(type, value, traceback); + } + + +.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) + + Set the error indicator from the three objects. If the error indicator is + already set, it is cleared first. If the objects are ``NULL``, the error + indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or + traceback. The exception type should be a class. Do not pass an invalid + exception type or value. (Violating these rules will cause subtle problems + later.) This call takes away a reference to each object: you must own a + reference to each object before the call and after the call you no longer own + these references. (If you don't understand this, don't use this function. I + warned you.) + + .. note:: + + This function is normally only used by code that needs to save and restore the + error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current + error indicator. + + +.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) + + Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below + can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is + not an instance of the same class. This function can be used to instantiate + the class in that case. If the values are already normalized, nothing happens. + The delayed normalization is implemented to improve performance. + + .. note:: + + This function *does not* implicitly set the ``__traceback__`` + attribute on the exception value. If setting the traceback + appropriately is desired, the following additional snippet is needed:: + + if (tb != NULL) { + PyException_SetTraceback(val, tb); + } + + +.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) + + Retrieve the exception info, as known from ``sys.exc_info()``. This refers + to an exception that was *already caught*, not to an exception that was + freshly raised. Returns new references for the three objects, any of which + may be ``NULL``. Does not modify the exception info state. + + .. note:: + + This function is not normally used by code that wants to handle exceptions. + Rather, it can be used when code needs to save and restore the exception + state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the + exception state. + + .. versionadded:: 3.3 + + +.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback) + + Set the exception info, as known from ``sys.exc_info()``. This refers + to an exception that was *already caught*, not to an exception that was + freshly raised. This function steals the references of the arguments. + To clear the exception state, pass ``NULL`` for all three arguments. + For general rules about the three arguments, see :c:func:`PyErr_Restore`. + + .. note:: + + This function is not normally used by code that wants to handle exceptions. + Rather, it can be used when code needs to save and restore the exception + state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception + state. + + .. versionadded:: 3.3 + + +Signal Handling +=============== + + +.. c:function:: int PyErr_CheckSignals() + + .. index:: + module: signal + single: SIGINT + single: KeyboardInterrupt (built-in exception) + + This function interacts with Python's signal handling. It checks whether a + signal has been sent to the processes and if so, invokes the corresponding + signal handler. If the :mod:`signal` module is supported, this can invoke a + signal handler written in Python. In all cases, the default effect for + :const:`SIGINT` is to raise the :exc:`KeyboardInterrupt` exception. If an + exception is raised the error indicator is set and the function returns ``-1``; + otherwise the function returns ``0``. The error indicator may or may not be + cleared if it was previously set. + + +.. c:function:: void PyErr_SetInterrupt() + + .. index:: + single: SIGINT + single: KeyboardInterrupt (built-in exception) + + Simulate the effect of a :const:`SIGINT` signal arriving. The next time + :c:func:`PyErr_CheckSignals` is called, the Python signal handler for + :const:`SIGINT` will be called. + + If :const:`SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. + +.. c:function:: int PySignal_SetWakeupFd(int fd) + + This utility function specifies a file descriptor to which the signal number + is written as a single byte whenever a signal is received. *fd* must be + non-blocking. It returns the previous such file descriptor. + + The value ``-1`` disables the feature; this is the initial state. + This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any + error checking. *fd* should be a valid file descriptor. The function should + only be called from the main thread. + + .. versionchanged:: 3.5 + On Windows, the function now also supports socket handles. + + +Exception Classes +================= + +.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict) + + This utility function creates and returns a new exception class. The *name* + argument must be the name of the new exception, a C string of the form + ``module.classname``. The *base* and *dict* arguments are normally ``NULL``. + This creates a class object derived from :exc:`Exception` (accessible in C as + :c:data:`PyExc_Exception`). + + The :attr:`__module__` attribute of the new class is set to the first part (up + to the last dot) of the *name* argument, and the class name is set to the last + part (after the last dot). The *base* argument can be used to specify alternate + base classes; it can either be only one class or a tuple of classes. The *dict* + argument can be used to specify a dictionary of class variables and methods. + + +.. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) + + Same as :c:func:`PyErr_NewException`, except that the new exception class can + easily be given a docstring: If *doc* is non-``NULL``, it will be used as the + docstring for the exception class. + + .. versionadded:: 3.2 + + +Exception Objects +================= + +.. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) + + Return the traceback associated with the exception as a new reference, as + accessible from Python through :attr:`__traceback__`. If there is no + traceback associated, this returns ``NULL``. + + +.. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) + + Set the traceback associated with the exception to *tb*. Use ``Py_None`` to + clear it. + + +.. c:function:: PyObject* PyException_GetContext(PyObject *ex) + + Return the context (another exception instance during whose handling *ex* was + raised) associated with the exception as a new reference, as accessible from + Python through :attr:`__context__`. If there is no context associated, this + returns ``NULL``. + + +.. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) + + Set the context associated with the exception to *ctx*. Use ``NULL`` to clear + it. There is no type check to make sure that *ctx* is an exception instance. + This steals a reference to *ctx*. + + +.. c:function:: PyObject* PyException_GetCause(PyObject *ex) + + Return the cause (either an exception instance, or :const:`None`, + set by ``raise ... from ...``) associated with the exception as a new + reference, as accessible from Python through :attr:`__cause__`. + + +.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) + + Set the cause associated with the exception to *cause*. Use ``NULL`` to clear + it. There is no type check to make sure that *cause* is either an exception + instance or :const:`None`. This steals a reference to *cause*. + + :attr:`__suppress_context__` is implicitly set to ``True`` by this function. + + +.. _unicodeexceptions: + +Unicode Exception Objects +========================= + +The following functions are used to create and modify Unicode exceptions from C. + +.. c:function:: PyObject* PyUnicodeDecodeError_Create(const char *encoding, const char *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeDecodeError` object with the attributes *encoding*, + *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are + UTF-8 encoded strings. + +.. c:function:: PyObject* PyUnicodeEncodeError_Create(const char *encoding, const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeEncodeError` object with the attributes *encoding*, + *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are + UTF-8 encoded strings. + + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", ...)``. + +.. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) + + Create a :class:`UnicodeTranslateError` object with the attributes *object*, + *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. + + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeTranslateError, "Onns", ...)``. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) + + Return the *encoding* attribute of the given exception object. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetObject(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetObject(PyObject *exc) + PyObject* PyUnicodeTranslateError_GetObject(PyObject *exc) + + Return the *object* attribute of the given exception object. + +.. c:function:: int PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) + int PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) + int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) + + Get the *start* attribute of the given exception object and place it into + *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on + failure. + +.. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) + int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) + int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) + + Set the *start* attribute of the given exception object to *start*. Return + ``0`` on success, ``-1`` on failure. + +.. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) + int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) + int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) + + Get the *end* attribute of the given exception object and place it into + *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on + failure. + +.. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) + int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) + int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) + + Set the *end* attribute of the given exception object to *end*. Return ``0`` + on success, ``-1`` on failure. + +.. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) + PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) + PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) + + Return the *reason* attribute of the given exception object. + +.. c:function:: int PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) + int PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) + int PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) + + Set the *reason* attribute of the given exception object to *reason*. Return + ``0`` on success, ``-1`` on failure. + + +.. _recursion: + +Recursion Control +================= + +These two functions provide a way to perform safe recursive calls at the C +level, both in the core and in extension modules. They are needed if the +recursive code does not necessarily invoke Python code (which tracks its +recursion depth automatically). +They are also not needed for *tp_call* implementations +because the :ref:`call protocol ` takes care of recursion handling. + +.. c:function:: int Py_EnterRecursiveCall(const char *where) + + Marks a point where a recursive C-level call is about to be performed. + + If :const:`USE_STACKCHECK` is defined, this function checks if the OS + stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it + sets a :exc:`MemoryError` and returns a nonzero value. + + The function then checks if the recursion limit is reached. If this is the + case, a :exc:`RecursionError` is set and a nonzero value is returned. + Otherwise, zero is returned. + + *where* should be a UTF-8 encoded string such as ``" in instance check"`` to + be concatenated to the :exc:`RecursionError` message caused by the recursion + depth limit. + + .. versionchanged:: 3.9 + This function is now also available in the limited API. + +.. c:function:: void Py_LeaveRecursiveCall(void) + + Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each + *successful* invocation of :c:func:`Py_EnterRecursiveCall`. + + .. versionchanged:: 3.9 + This function is now also available in the limited API. + +Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires +special recursion handling. In addition to protecting the stack, +:c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The +following two functions facilitate this functionality. Effectively, +these are the C equivalent to :func:`reprlib.recursive_repr`. + +.. c:function:: int Py_ReprEnter(PyObject *object) + + Called at the beginning of the :c:member:`~PyTypeObject.tp_repr` implementation to + detect cycles. + + If the object has already been processed, the function returns a + positive integer. In that case the :c:member:`~PyTypeObject.tp_repr` implementation + should return a string object indicating a cycle. As examples, + :class:`dict` objects return ``{...}`` and :class:`list` objects + return ``[...]``. + + The function will return a negative integer if the recursion limit + is reached. In that case the :c:member:`~PyTypeObject.tp_repr` implementation should + typically return ``NULL``. + + Otherwise, the function returns zero and the :c:member:`~PyTypeObject.tp_repr` + implementation can continue normally. + +.. c:function:: void Py_ReprLeave(PyObject *object) + + Ends a :c:func:`Py_ReprEnter`. Must be called once for each + invocation of :c:func:`Py_ReprEnter` that returns zero. + + +.. _standardexceptions: + +Standard Exceptions +=================== + +All standard Python exceptions are available as global variables whose names are +``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_BaseException + single: PyExc_Exception + single: PyExc_ArithmeticError + single: PyExc_AssertionError + single: PyExc_AttributeError + single: PyExc_BlockingIOError + single: PyExc_BrokenPipeError + single: PyExc_BufferError + single: PyExc_ChildProcessError + single: PyExc_ConnectionAbortedError + single: PyExc_ConnectionError + single: PyExc_ConnectionRefusedError + single: PyExc_ConnectionResetError + single: PyExc_EOFError + single: PyExc_FileExistsError + single: PyExc_FileNotFoundError + single: PyExc_FloatingPointError + single: PyExc_GeneratorExit + single: PyExc_ImportError + single: PyExc_IndentationError + single: PyExc_IndexError + single: PyExc_InterruptedError + single: PyExc_IsADirectoryError + single: PyExc_KeyError + single: PyExc_KeyboardInterrupt + single: PyExc_LookupError + single: PyExc_MemoryError + single: PyExc_ModuleNotFoundError + single: PyExc_NameError + single: PyExc_NotADirectoryError + single: PyExc_NotImplementedError + single: PyExc_OSError + single: PyExc_OverflowError + single: PyExc_PermissionError + single: PyExc_ProcessLookupError + single: PyExc_RecursionError + single: PyExc_ReferenceError + single: PyExc_RuntimeError + single: PyExc_StopAsyncIteration + single: PyExc_StopIteration + single: PyExc_SyntaxError + single: PyExc_SystemError + single: PyExc_SystemExit + single: PyExc_TabError + single: PyExc_TimeoutError + single: PyExc_TypeError + single: PyExc_UnboundLocalError + single: PyExc_UnicodeDecodeError + single: PyExc_UnicodeEncodeError + single: PyExc_UnicodeError + single: PyExc_UnicodeTranslateError + single: PyExc_ValueError + single: PyExc_ZeroDivisionError + ++-----------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++=========================================+=================================+==========+ +| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NameError` | :exc:`NameError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TabError` | :exc:`TabError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | ++-----------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | ++-----------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.3 + :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, + :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`, + :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`, + :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`, + :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`, + :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`, + :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` + and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. + +.. versionadded:: 3.5 + :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. + +.. versionadded:: 3.6 + :c:data:`PyExc_ModuleNotFoundError`. + +These are compatibility aliases to :c:data:`PyExc_OSError`: + +.. index:: + single: PyExc_EnvironmentError + single: PyExc_IOError + single: PyExc_WindowsError + ++-------------------------------------+----------+ +| C Name | Notes | ++=====================================+==========+ +| :c:data:`PyExc_EnvironmentError` | | ++-------------------------------------+----------+ +| :c:data:`PyExc_IOError` | | ++-------------------------------------+----------+ +| :c:data:`PyExc_WindowsError` | \(3) | ++-------------------------------------+----------+ + +.. versionchanged:: 3.3 + These aliases used to be separate exception types. + +Notes: + +(1) + This is a base class for other standard exceptions. + +(2) + Only defined on Windows; protect code that uses this by testing that the + preprocessor macro ``MS_WINDOWS`` is defined. + +.. _standardwarningcategories: + +Standard Warning Categories +=========================== + +All standard Python warning categories are available as global variables whose +names are ``PyExc_`` followed by the Python exception name. These have the type +:c:type:`PyObject*`; they are all class objects. For completeness, here are all +the variables: + +.. index:: + single: PyExc_Warning + single: PyExc_BytesWarning + single: PyExc_DeprecationWarning + single: PyExc_FutureWarning + single: PyExc_ImportWarning + single: PyExc_PendingDeprecationWarning + single: PyExc_ResourceWarning + single: PyExc_RuntimeWarning + single: PyExc_SyntaxWarning + single: PyExc_UnicodeWarning + single: PyExc_UserWarning + ++------------------------------------------+---------------------------------+----------+ +| C Name | Python Name | Notes | ++==========================================+=================================+==========+ +| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | ++------------------------------------------+---------------------------------+----------+ +| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | ++------------------------------------------+---------------------------------+----------+ + +.. versionadded:: 3.2 + :c:data:`PyExc_ResourceWarning`. + +Notes: + +(1) + This is a base class for other standard warning categories. diff --git a/Doc/c-api/memory.rst.bak b/Doc/c-api/memory.rst.bak new file mode 100644 index 00000000000000..588668ee853c65 --- /dev/null +++ b/Doc/c-api/memory.rst.bak @@ -0,0 +1,642 @@ +.. highlight:: c + + +.. _memory: + +***************** +Memory Management +***************** + +.. sectionauthor:: Vladimir Marangozov + + + +.. _memoryoverview: + +Overview +======== + +Memory management in Python involves a private heap containing all Python +objects and data structures. The management of this private heap is ensured +internally by the *Python memory manager*. The Python memory manager has +different components which deal with various dynamic storage management aspects, +like sharing, segmentation, preallocation or caching. + +At the lowest level, a raw memory allocator ensures that there is enough room in +the private heap for storing all Python-related data by interacting with the +memory manager of the operating system. On top of the raw memory allocator, +several object-specific allocators operate on the same heap and implement +distinct memory management policies adapted to the peculiarities of every object +type. For example, integer objects are managed differently within the heap than +strings, tuples or dictionaries because integers imply different storage +requirements and speed/space tradeoffs. The Python memory manager thus delegates +some of the work to the object-specific allocators, but ensures that the latter +operate within the bounds of the private heap. + +It is important to understand that the management of the Python heap is +performed by the interpreter itself and that the user has no control over it, +even if they regularly manipulate object pointers to memory blocks inside that +heap. The allocation of heap space for Python objects and other internal +buffers is performed on demand by the Python memory manager through the Python/C +API functions listed in this document. + +.. index:: + single: malloc() + single: calloc() + single: realloc() + single: free() + +To avoid memory corruption, extension writers should never try to operate on +Python objects with the functions exported by the C library: :c:func:`malloc`, +:c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. This will result in mixed +calls between the C allocator and the Python memory manager with fatal +consequences, because they implement different algorithms and operate on +different heaps. However, one may safely allocate and release memory blocks +with the C library allocator for individual purposes, as shown in the following +example:: + + PyObject *res; + char *buf = (char *) malloc(BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + ...Do some I/O operation involving buf... + res = PyBytes_FromString(buf); + free(buf); /* malloc'ed */ + return res; + +In this example, the memory request for the I/O buffer is handled by the C +library allocator. The Python memory manager is involved only in the allocation +of the bytes object returned as a result. + +In most situations, however, it is recommended to allocate memory from the +Python heap specifically because the latter is under control of the Python +memory manager. For example, this is required when the interpreter is extended +with new object types written in C. Another reason for using the Python heap is +the desire to *inform* the Python memory manager about the memory needs of the +extension module. Even when the requested memory is used exclusively for +internal, highly-specific purposes, delegating all memory requests to the Python +memory manager causes the interpreter to have a more accurate image of its +memory footprint as a whole. Consequently, under certain circumstances, the +Python memory manager may or may not trigger appropriate actions, like garbage +collection, memory compaction or other preventive procedures. Note that by using +the C library allocator as shown in the previous example, the allocated memory +for the I/O buffer escapes completely the Python memory manager. + +.. seealso:: + + The :envvar:`PYTHONMALLOC` environment variable can be used to configure + the memory allocators used by Python. + + The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print + statistics of the :ref:`pymalloc memory allocator ` every time a + new pymalloc object arena is created, and on shutdown. + +Allocator Domains +================= + +All allocating functions belong to one of three different "domains" (see also +:c:type:`PyMemAllocatorDomain`). These domains represent different allocation +strategies and are optimized for different purposes. The specific details on +how every domain allocates memory or what internal functions each domain calls +is considered an implementation detail, but for debugging purposes a simplified +table can be found at :ref:`here `. There is no hard +requirement to use the memory returned by the allocation functions belonging to +a given domain for only the purposes hinted by that domain (although this is the +recommended practice). For example, one could use the memory returned by +:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned +by :c:func:`PyObject_Malloc` for allocating memory for buffers. + +The three allocation domains are: + +* Raw domain: intended for allocating memory for general-purpose memory + buffers where the allocation *must* go to the system allocator or where the + allocator can operate without the :term:`GIL`. The memory is requested directly + to the system. + +* "Mem" domain: intended for allocating memory for Python buffers and + general-purpose memory buffers where the allocation must be performed with + the :term:`GIL` held. The memory is taken from the Python private heap. + +* Object domain: intended for allocating memory belonging to Python objects. The + memory is taken from the Python private heap. + +When freeing memory previously allocated by the allocating functions belonging to a +given domain,the matching specific deallocating functions must be used. For example, +:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. + +Raw Memory Interface +==================== + +The following function sets are wrappers to the system allocator. These +functions are thread-safe, the :term:`GIL ` does not +need to be held. + +The :ref:`default raw memory allocator ` uses +the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` +and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting +zero bytes. + +.. versionadded:: 3.4 + +.. c:function:: void* PyMem_RawMalloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. + + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as + if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-``NULL`` pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been + called instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will + be unchanged to the minimum of the old and the new sizes. + + If *p* is ``NULL``, the call is equivalent to ``PyMem_RawMalloc(n)``; else if + *n* is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-``NULL``. + + Unless *p* is ``NULL``, it must have been returned by a previous call to + :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or + :c:func:`PyMem_RawCalloc`. + + If the request fails, :c:func:`PyMem_RawRealloc` returns ``NULL`` and *p* + remains a valid pointer to the previous memory area. + + +.. c:function:: void PyMem_RawFree(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or + :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been + called before, undefined behavior occurs. + + If *p* is ``NULL``, no operation is performed. + + +.. _memoryinterface: + +Memory Interface +================ + +The following function sets, modeled after the ANSI C standard, but specifying +behavior when requesting zero bytes, are available for allocating and releasing +memory from the Python heap. + +The :ref:`default memory allocator ` uses the +:ref:`pymalloc memory allocator `. + +.. warning:: + + The :term:`GIL ` must be held when using these + functions. + +.. versionchanged:: 3.6 + + The default allocator is now pymalloc instead of system :c:func:`malloc`. + +.. c:function:: void* PyMem_Malloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. + + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as + if ``PyMem_Malloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-``NULL`` pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called + instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyMem_Realloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will be + unchanged to the minimum of the old and the new sizes. + + If *p* is ``NULL``, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* + is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-``NULL``. + + Unless *p* is ``NULL``, it must have been returned by a previous call to + :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. + + If the request fails, :c:func:`PyMem_Realloc` returns ``NULL`` and *p* remains + a valid pointer to the previous memory area. + + +.. c:function:: void PyMem_Free(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or + :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called + before, undefined behavior occurs. + + If *p* is ``NULL``, no operation is performed. + +The following type-oriented macros are provided for convenience. Note that +*TYPE* refers to any C type. + + +.. c:function:: TYPE* PyMem_New(TYPE, size_t n) + + Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of + memory. Returns a pointer cast to :c:type:`TYPE*`. The memory will not have + been initialized in any way. + + +.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) + + Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * + sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE*`. On return, + *p* will be a pointer to the new memory area, or ``NULL`` in the event of + failure. + + This is a C preprocessor macro; *p* is always reassigned. Save the original + value of *p* to avoid losing memory when handling errors. + + +.. c:function:: void PyMem_Del(void *p) + + Same as :c:func:`PyMem_Free`. + +In addition, the following macro sets are provided for calling the Python memory +allocator directly, without involving the C API functions listed above. However, +note that their use does not preserve binary compatibility across Python +versions and is therefore deprecated in extension modules. + +* ``PyMem_MALLOC(size)`` +* ``PyMem_NEW(type, size)`` +* ``PyMem_REALLOC(ptr, size)`` +* ``PyMem_RESIZE(ptr, type, size)`` +* ``PyMem_FREE(ptr)`` +* ``PyMem_DEL(ptr)`` + + +Object allocators +================= + +The following function sets, modeled after the ANSI C standard, but specifying +behavior when requesting zero bytes, are available for allocating and releasing +memory from the Python heap. + +.. note:: + There is no guarantee that the memory returned by these allocators can be + succesfully casted to a Python object when intercepting the allocating + functions in this domain by the methods described in + the :ref:`Customize Memory Allocators ` section. + +The :ref:`default object allocator ` uses the +:ref:`pymalloc memory allocator `. + +.. warning:: + + The :term:`GIL ` must be held when using these + functions. + +.. c:function:: void* PyObject_Malloc(size_t n) + + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the + allocated memory, or ``NULL`` if the request fails. + + Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as + if ``PyObject_Malloc(1)`` had been called instead. The memory will not have + been initialized in any way. + + +.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) + + Allocates *nelem* elements each whose size in bytes is *elsize* and returns + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the + request fails. The memory is initialized to zeros. + + Requesting zero elements or elements of size zero bytes returns a distinct + non-``NULL`` pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called + instead. + + .. versionadded:: 3.5 + + +.. c:function:: void* PyObject_Realloc(void *p, size_t n) + + Resizes the memory block pointed to by *p* to *n* bytes. The contents will be + unchanged to the minimum of the old and the new sizes. + + If *p* is ``NULL``, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* + is equal to zero, the memory block is resized but is not freed, and the + returned pointer is non-``NULL``. + + Unless *p* is ``NULL``, it must have been returned by a previous call to + :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. + + If the request fails, :c:func:`PyObject_Realloc` returns ``NULL`` and *p* remains + a valid pointer to the previous memory area. + + +.. c:function:: void PyObject_Free(void *p) + + Frees the memory block pointed to by *p*, which must have been returned by a + previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or + :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called + before, undefined behavior occurs. + + If *p* is ``NULL``, no operation is performed. + + +.. _default-memory-allocators: + +Default Memory Allocators +========================= + +Default memory allocators: + +=============================== ==================== ================== ===================== ==================== +Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc +=============================== ==================== ================== ===================== ==================== +Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` +Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug +Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` +Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug +=============================== ==================== ================== ===================== ==================== + +Legend: + +* Name: value for :envvar:`PYTHONMALLOC` environment variable +* ``malloc``: system allocators from the standard C library, C functions: + :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free` +* ``pymalloc``: :ref:`pymalloc memory allocator ` +* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` + +.. _customize-memory-allocators: + +Customize Memory Allocators +=========================== + +.. versionadded:: 3.4 + +.. c:type:: PyMemAllocatorEx + + Structure used to describe a memory block allocator. The structure has + four fields: + + +----------------------------------------------------------+---------------------------------------+ + | Field | Meaning | + +==========================================================+=======================================+ + | ``void *ctx`` | user context passed as first argument | + +----------------------------------------------------------+---------------------------------------+ + | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | + +----------------------------------------------------------+---------------------------------------+ + | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized | + | | with zeros | + +----------------------------------------------------------+---------------------------------------+ + | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | + +----------------------------------------------------------+---------------------------------------+ + | ``void free(void *ctx, void *ptr)`` | free a memory block | + +----------------------------------------------------------+---------------------------------------+ + + .. versionchanged:: 3.5 + The :c:type:`PyMemAllocator` structure was renamed to + :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. + + +.. c:type:: PyMemAllocatorDomain + + Enum used to identify an allocator domain. Domains: + + .. c:macro:: PYMEM_DOMAIN_RAW + + Functions: + + * :c:func:`PyMem_RawMalloc` + * :c:func:`PyMem_RawRealloc` + * :c:func:`PyMem_RawCalloc` + * :c:func:`PyMem_RawFree` + + .. c:macro:: PYMEM_DOMAIN_MEM + + Functions: + + * :c:func:`PyMem_Malloc`, + * :c:func:`PyMem_Realloc` + * :c:func:`PyMem_Calloc` + * :c:func:`PyMem_Free` + + .. c:macro:: PYMEM_DOMAIN_OBJ + + Functions: + + * :c:func:`PyObject_Malloc` + * :c:func:`PyObject_Realloc` + * :c:func:`PyObject_Calloc` + * :c:func:`PyObject_Free` + +.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) + + Get the memory block allocator of the specified domain. + + +.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) + + Set the memory block allocator of the specified domain. + + The new allocator must return a distinct non-``NULL`` pointer when requesting + zero bytes. + + For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be + thread-safe: the :term:`GIL ` is not held when the + allocator is called. + + If the new allocator is not a hook (does not call the previous allocator), + the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the + debug hooks on top on the new allocator. + + +.. c:function:: void PyMem_SetupDebugHooks(void) + + Setup hooks to detect bugs in the Python memory allocator functions. + + Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``), + freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks + are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``). + + Runtime checks: + + - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer + allocated by :c:func:`PyMem_Malloc` + - Detect write before the start of the buffer (buffer underflow) + - Detect write after the end of the buffer (buffer overflow) + - Check that the :term:`GIL ` is held when + allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: + :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: + :c:func:`PyMem_Malloc`) domains are called + + On error, the debug hooks use the :mod:`tracemalloc` module to get the + traceback where a memory block was allocated. The traceback is only + displayed if :mod:`tracemalloc` is tracing Python memory allocations and the + memory block was traced. + + These hooks are :ref:`installed by default ` if + Python is compiled in debug + mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install + debug hooks on a Python compiled in release mode. + + .. versionchanged:: 3.6 + This function now also works on Python compiled in release mode. + On error, the debug hooks now use :mod:`tracemalloc` to get the traceback + where a memory block was allocated. The debug hooks now also check + if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and + :c:data:`PYMEM_DOMAIN_MEM` domains are called. + + .. versionchanged:: 3.8 + Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and + ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` + and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` + and ``free()``. + + +.. _pymalloc: + +The pymalloc allocator +====================== + +Python has a *pymalloc* allocator optimized for small objects (smaller or equal +to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" +with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and +:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. + +*pymalloc* is the :ref:`default allocator ` of the +:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and +:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. + +The arena allocator uses the following functions: + +* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, +* :c:func:`mmap` and :c:func:`munmap` if available, +* :c:func:`malloc` and :c:func:`free` otherwise. + +Customize pymalloc Arena Allocator +---------------------------------- + +.. versionadded:: 3.4 + +.. c:type:: PyObjectArenaAllocator + + Structure used to describe an arena allocator. The structure has + three fields: + + +--------------------------------------------------+---------------------------------------+ + | Field | Meaning | + +==================================================+=======================================+ + | ``void *ctx`` | user context passed as first argument | + +--------------------------------------------------+---------------------------------------+ + | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | + +--------------------------------------------------+---------------------------------------+ + | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | + +--------------------------------------------------+---------------------------------------+ + +.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) + + Get the arena allocator. + +.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) + + Set the arena allocator. + + +tracemalloc C API +================= + +.. versionadded:: 3.7 + +.. c:function:: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) + + Track an allocated memory block in the :mod:`tracemalloc` module. + + Return ``0`` on success, return ``-1`` on error (failed to allocate memory to + store the trace). Return ``-2`` if tracemalloc is disabled. + + If memory block is already tracked, update the existing trace. + +.. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) + + Untrack an allocated memory block in the :mod:`tracemalloc` module. + Do nothing if the block was not tracked. + + Return ``-2`` if tracemalloc is disabled, otherwise return ``0``. + + +.. _memoryexamples: + +Examples +======== + +Here is the example from section :ref:`memoryoverview`, rewritten so that the +I/O buffer is allocated from the Python heap by using the first function set:: + + PyObject *res; + char *buf = (char *) PyMem_Malloc(BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + /* ...Do some I/O operation involving buf... */ + res = PyBytes_FromString(buf); + PyMem_Free(buf); /* allocated with PyMem_Malloc */ + return res; + +The same code using the type-oriented function set:: + + PyObject *res; + char *buf = PyMem_New(char, BUFSIZ); /* for I/O */ + + if (buf == NULL) + return PyErr_NoMemory(); + /* ...Do some I/O operation involving buf... */ + res = PyBytes_FromString(buf); + PyMem_Del(buf); /* allocated with PyMem_New */ + return res; + +Note that in the two examples above, the buffer is always manipulated via +functions belonging to the same set. Indeed, it is required to use the same +memory API family for a given memory block, so that the risk of mixing different +allocators is reduced to a minimum. The following code sequence contains two +errors, one of which is labeled as *fatal* because it mixes two different +allocators operating on different heaps. :: + + char *buf1 = PyMem_New(char, BUFSIZ); + char *buf2 = (char *) malloc(BUFSIZ); + char *buf3 = (char *) PyMem_Malloc(BUFSIZ); + ... + PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ + free(buf2); /* Right -- allocated via malloc() */ + free(buf1); /* Fatal -- should be PyMem_Del() */ + +In addition to the functions aimed at handling raw memory blocks from the Python +heap, objects in Python are allocated and released with :c:func:`PyObject_New`, +:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`. + +These will be explained in the next chapter on defining and implementing new +object types in C. diff --git a/Doc/faq/design.rst.bak b/Doc/faq/design.rst.bak new file mode 100644 index 00000000000000..7fe1c6d58f58a1 --- /dev/null +++ b/Doc/faq/design.rst.bak @@ -0,0 +1,759 @@ +====================== +Design and History FAQ +====================== + +.. only:: html + + .. contents:: + + +Why does Python use indentation for grouping of statements? +----------------------------------------------------------- + +Guido van Rossum believes that using indentation for grouping is extremely +elegant and contributes a lot to the clarity of the average Python program. +Most people learn to love this feature after a while. + +Since there are no begin/end brackets there cannot be a disagreement between +grouping perceived by the parser and the human reader. Occasionally C +programmers will encounter a fragment of code like this:: + + if (x <= y) + x++; + y--; + z++; + +Only the ``x++`` statement is executed if the condition is true, but the +indentation leads many to believe otherwise. Even experienced C programmers will +sometimes stare at it a long time wondering as to why ``y`` is being decremented even +for ``x > y``. + +Because there are no begin/end brackets, Python is much less prone to +coding-style conflicts. In C there are many different ways to place the braces. +After becoming used to reading and writing code using a particular style, +it is normal to feel somewhat uneasy when reading (or being required to write) +in a different one. + + +Many coding styles place begin/end brackets on a line by themselves. This makes +programs considerably longer and wastes valuable screen space, making it harder +to get a good overview of a program. Ideally, a function should fit on one +screen (say, 20--30 lines). 20 lines of Python can do a lot more work than 20 +lines of C. This is not solely due to the lack of begin/end brackets -- the +lack of declarations and the high-level data types are also responsible -- but +the indentation-based syntax certainly helps. + + +Why am I getting strange results with simple arithmetic operations? +------------------------------------------------------------------- + +See the next question. + + +Why are floating-point calculations so inaccurate? +-------------------------------------------------- + +Users are often surprised by results like this:: + + >>> 1.2 - 1.0 + 0.19999999999999996 + +and think it is a bug in Python. It's not. This has little to do with Python, +and much more to do with how the underlying platform handles floating-point +numbers. + +The :class:`float` type in CPython uses a C ``double`` for storage. A +:class:`float` object's value is stored in binary floating-point with a fixed +precision (typically 53 bits) and Python uses C operations, which in turn rely +on the hardware implementation in the processor, to perform floating-point +operations. This means that as far as floating-point operations are concerned, +Python behaves like many popular languages including C and Java. + +Many numbers that can be written easily in decimal notation cannot be expressed +exactly in binary floating-point. For example, after:: + + >>> x = 1.2 + +the value stored for ``x`` is a (very good) approximation to the decimal value +``1.2``, but is not exactly equal to it. On a typical machine, the actual +stored value is:: + + 1.0011001100110011001100110011001100110011001100110011 (binary) + +which is exactly:: + + 1.1999999999999999555910790149937383830547332763671875 (decimal) + +The typical precision of 53 bits provides Python floats with 15--16 +decimal digits of accuracy. + +For a fuller explanation, please see the :ref:`floating point arithmetic +` chapter in the Python tutorial. + + +Why are Python strings immutable? +--------------------------------- + +There are several advantages. + +One is performance: knowing that a string is immutable means we can allocate +space for it at creation time, and the storage requirements are fixed and +unchanging. This is also one of the reasons for the distinction between tuples +and lists. + +Another advantage is that strings in Python are considered as "elemental" as +numbers. No amount of activity will change the value 8 to anything else, and in +Python, no amount of activity will change the string "eight" to anything else. + + +.. _why-self: + +Why must 'self' be used explicitly in method definitions and calls? +------------------------------------------------------------------- + +The idea was borrowed from Modula-3. It turns out to be very useful, for a +variety of reasons. + +First, it's more obvious that you are using a method or instance attribute +instead of a local variable. Reading ``self.x`` or ``self.meth()`` makes it +absolutely clear that an instance variable or method is used even if you don't +know the class definition by heart. In C++, you can sort of tell by the lack of +a local variable declaration (assuming globals are rare or easily recognizable) +-- but in Python, there are no local variable declarations, so you'd have to +look up the class definition to be sure. Some C++ and Java coding standards +call for instance attributes to have an ``m_`` prefix, so this explicitness is +still useful in those languages, too. + +Second, it means that no special syntax is necessary if you want to explicitly +reference or call the method from a particular class. In C++, if you want to +use a method from a base class which is overridden in a derived class, you have +to use the ``::`` operator -- in Python you can write +``baseclass.methodname(self, )``. This is particularly useful +for :meth:`__init__` methods, and in general in cases where a derived class +method wants to extend the base class method of the same name and thus has to +call the base class method somehow. + +Finally, for instance variables it solves a syntactic problem with assignment: +since local variables in Python are (by definition!) those variables to which a +value is assigned in a function body (and that aren't explicitly declared +global), there has to be some way to tell the interpreter that an assignment was +meant to assign to an instance variable instead of to a local variable, and it +should preferably be syntactic (for efficiency reasons). C++ does this through +declarations, but Python doesn't have declarations and it would be a pity having +to introduce them just for this purpose. Using the explicit ``self.var`` solves +this nicely. Similarly, for using instance variables, having to write +``self.var`` means that references to unqualified names inside a method don't +have to search the instance's directories. To put it another way, local +variables and instance variables live in two different namespaces, and you need +to tell Python which namespace to use. + + +.. _why-can-t-i-use-an-assignment-in-an-expression: + +Why can't I use an assignment in an expression? +----------------------------------------------- + +Starting in Python 3.8, you can! + +Assignment expressions using the walrus operator `:=` assign a variable in an +expression:: + + while chunk := fp.read(200): + print(chunk) + +See :pep:`572` for more information. + + + +Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))? +---------------------------------------------------------------------------------------------------------------- + +As Guido said: + + (a) For some operations, prefix notation just reads better than + postfix -- prefix (and infix!) operations have a long tradition in + mathematics which likes notations where the visuals help the + mathematician thinking about a problem. Compare the easy with which we + rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of + doing the same thing using a raw OO notation. + + (b) When I read code that says len(x) I *know* that it is asking for + the length of something. This tells me two things: the result is an + integer, and the argument is some kind of container. To the contrary, + when I read x.len(), I have to already know that x is some kind of + container implementing an interface or inheriting from a class that + has a standard len(). Witness the confusion we occasionally have when + a class that is not implementing a mapping has a get() or keys() + method, or something that isn't a file has a write() method. + + -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html + + +Why is join() a string method instead of a list or tuple method? +---------------------------------------------------------------- + +Strings became much more like other standard types starting in Python 1.6, when +methods were added which give the same functionality that has always been +available using the functions of the string module. Most of these new methods +have been widely accepted, but the one which appears to make some programmers +feel uncomfortable is:: + + ", ".join(['1', '2', '4', '8', '16']) + +which gives the result:: + + "1, 2, 4, 8, 16" + +There are two common arguments against this usage. + +The first runs along the lines of: "It looks really ugly using a method of a +string literal (string constant)", to which the answer is that it might, but a +string literal is just a fixed value. If the methods are to be allowed on names +bound to strings there is no logical reason to make them unavailable on +literals. + +The second objection is typically cast as: "I am really telling a sequence to +join its members together with a string constant". Sadly, you aren't. For some +reason there seems to be much less difficulty with having :meth:`~str.split` as +a string method, since in that case it is easy to see that :: + + "1, 2, 4, 8, 16".split(", ") + +is an instruction to a string literal to return the substrings delimited by the +given separator (or, by default, arbitrary runs of white space). + +:meth:`~str.join` is a string method because in using it you are telling the +separator string to iterate over a sequence of strings and insert itself between +adjacent elements. This method can be used with any argument which obeys the +rules for sequence objects, including any new classes you might define yourself. +Similar methods exist for bytes and bytearray objects. + + +How fast are exceptions? +------------------------ + +A try/except block is extremely efficient if no exceptions are raised. Actually +catching an exception is expensive. In versions of Python prior to 2.0 it was +common to use this idiom:: + + try: + value = mydict[key] + except KeyError: + mydict[key] = getvalue(key) + value = mydict[key] + +This only made sense when you expected the dict to have the key almost all the +time. If that wasn't the case, you coded it like this:: + + if key in mydict: + value = mydict[key] + else: + value = mydict[key] = getvalue(key) + +For this specific case, you could also use ``value = dict.setdefault(key, +getvalue(key))``, but only if the ``getvalue()`` call is cheap enough because it +is evaluated in all cases. + + +Why isn't there a switch or case statement in Python? +----------------------------------------------------- + +You can do this easily enough with a sequence of ``if... elif... elif... else``. +For literal values, or constants within a namespace, you can also use a +``match ... case`` statement. + +For cases where you need to choose from a very large number of possibilities, +you can create a dictionary mapping case values to functions to call. For +example:: + + def function_1(...): + ... + + functions = {'a': function_1, + 'b': function_2, + 'c': self.method_1, ...} + + func = functions[value] + func() + +For calling methods on objects, you can simplify yet further by using the +:func:`getattr` built-in to retrieve methods with a particular name:: + + def visit_a(self, ...): + ... + ... + + def dispatch(self, value): + method_name = 'visit_' + str(value) + method = getattr(self, method_name) + method() + +It's suggested that you use a prefix for the method names, such as ``visit_`` in +this example. Without such a prefix, if values are coming from an untrusted +source, an attacker would be able to call any method on your object. + + +Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? +-------------------------------------------------------------------------------------------------------- + +Answer 1: Unfortunately, the interpreter pushes at least one C stack frame for +each Python stack frame. Also, extensions can call back into Python at almost +random moments. Therefore, a complete threads implementation requires thread +support for C. + +Answer 2: Fortunately, there is `Stackless Python `_, +which has a completely redesigned interpreter loop that avoids the C stack. + + +Why can't lambda expressions contain statements? +------------------------------------------------ + +Python lambda expressions cannot contain statements because Python's syntactic +framework can't handle statements nested inside expressions. However, in +Python, this is not a serious problem. Unlike lambda forms in other languages, +where they add functionality, Python lambdas are only a shorthand notation if +you're too lazy to define a function. + +Functions are already first class objects in Python, and can be declared in a +local scope. Therefore the only advantage of using a lambda instead of a +locally-defined function is that you don't need to invent a name for the +function -- but that's just a local variable to which the function object (which +is exactly the same type of object that a lambda expression yields) is assigned! + + +Can Python be compiled to machine code, C or some other language? +----------------------------------------------------------------- + +`Cython `_ compiles a modified version of Python with +optional annotations into C extensions. `Nuitka `_ is +an up-and-coming compiler of Python into C++ code, aiming to support the full +Python language. For compiling to Java you can consider +`VOC `_. + + +How does Python manage memory? +------------------------------ + +The details of Python memory management depend on the implementation. The +standard implementation of Python, :term:`CPython`, uses reference counting to +detect inaccessible objects, and another mechanism to collect reference cycles, +periodically executing a cycle detection algorithm which looks for inaccessible +cycles and deletes the objects involved. The :mod:`gc` module provides functions +to perform a garbage collection, obtain debugging statistics, and tune the +collector's parameters. + +Other implementations (such as `Jython `_ or +`PyPy `_), however, can rely on a different mechanism +such as a full-blown garbage collector. This difference can cause some +subtle porting problems if your Python code depends on the behavior of the +reference counting implementation. + +In some Python implementations, the following code (which is fine in CPython) +will probably run out of file descriptors:: + + for file in very_long_list_of_files: + f = open(file) + c = f.read(1) + +Indeed, using CPython's reference counting and destructor scheme, each new +assignment to *f* closes the previous file. With a traditional GC, however, +those file objects will only get collected (and closed) at varying and possibly +long intervals. + +If you want to write code that will work with any Python implementation, +you should explicitly close the file or use the :keyword:`with` statement; +this will work regardless of memory management scheme:: + + for file in very_long_list_of_files: + with open(file) as f: + c = f.read(1) + + +Why doesn't CPython use a more traditional garbage collection scheme? +--------------------------------------------------------------------- + +For one thing, this is not a C standard feature and hence it's not portable. +(Yes, we know about the Boehm GC library. It has bits of assembler code for +*most* common platforms, not for all of them, and although it is mostly +transparent, it isn't completely transparent; patches are required to get +Python to work with it.) + +Traditional GC also becomes a problem when Python is embedded into other +applications. While in a standalone Python it's fine to replace the standard +malloc() and free() with versions provided by the GC library, an application +embedding Python may want to have its *own* substitute for malloc() and free(), +and may not want Python's. Right now, CPython works with anything that +implements malloc() and free() properly. + + +Why isn't all memory freed when CPython exits? +---------------------------------------------- + +Objects referenced from the global namespaces of Python modules are not always +deallocated when Python exits. This may happen if there are circular +references. There are also certain bits of memory that are allocated by the C +library that are impossible to free (e.g. a tool like Purify will complain about +these). Python is, however, aggressive about cleaning up memory on exit and +does try to destroy every single object. + +If you want to force Python to delete certain things on deallocation use the +:mod:`atexit` module to run a function that will force those deletions. + + +Why are there separate tuple and list data types? +------------------------------------------------- + +Lists and tuples, while similar in many respects, are generally used in +fundamentally different ways. Tuples can be thought of as being similar to +Pascal records or C structs; they're small collections of related data which may +be of different types which are operated on as a group. For example, a +Cartesian coordinate is appropriately represented as a tuple of two or three +numbers. + +Lists, on the other hand, are more like arrays in other languages. They tend to +hold a varying number of objects all of which have the same type and which are +operated on one-by-one. For example, ``os.listdir('.')`` returns a list of +strings representing the files in the current directory. Functions which +operate on this output would generally not break if you added another file or +two to the directory. + +Tuples are immutable, meaning that once a tuple has been created, you can't +replace any of its elements with a new value. Lists are mutable, meaning that +you can always change a list's elements. Only immutable elements can be used as +dictionary keys, and hence only tuples and not lists can be used as keys. + + +How are lists implemented in CPython? +------------------------------------- + +CPython's lists are really variable-length arrays, not Lisp-style linked lists. +The implementation uses a contiguous array of references to other objects, and +keeps a pointer to this array and the array's length in a list head structure. + +This makes indexing a list ``a[i]`` an operation whose cost is independent of +the size of the list or the value of the index. + +When items are appended or inserted, the array of references is resized. Some +cleverness is applied to improve the performance of appending items repeatedly; +when the array must be grown, some extra space is allocated so the next few +times don't require an actual resize. + + +How are dictionaries implemented in CPython? +-------------------------------------------- + +CPython's dictionaries are implemented as resizable hash tables. Compared to +B-trees, this gives better performance for lookup (the most common operation by +far) under most circumstances, and the implementation is simpler. + +Dictionaries work by computing a hash code for each key stored in the dictionary +using the :func:`hash` built-in function. The hash code varies widely depending +on the key and a per-process seed; for example, "Python" could hash to +-539294296 while "python", a string that differs by a single bit, could hash +to 1142331976. The hash code is then used to calculate a location in an +internal array where the value will be stored. Assuming that you're storing +keys that all have different hash values, this means that dictionaries take +constant time -- O(1), in Big-O notation -- to retrieve a key. + + +Why must dictionary keys be immutable? +-------------------------------------- + +The hash table implementation of dictionaries uses a hash value calculated from +the key value to find the key. If the key were a mutable object, its value +could change, and thus its hash could also change. But since whoever changes +the key object can't tell that it was being used as a dictionary key, it can't +move the entry around in the dictionary. Then, when you try to look up the same +object in the dictionary it won't be found because its hash value is different. +If you tried to look up the old value it wouldn't be found either, because the +value of the object found in that hash bin would be different. + +If you want a dictionary indexed with a list, simply convert the list to a tuple +first; the function ``tuple(L)`` creates a tuple with the same entries as the +list ``L``. Tuples are immutable and can therefore be used as dictionary keys. + +Some unacceptable solutions that have been proposed: + +- Hash lists by their address (object ID). This doesn't work because if you + construct a new list with the same value it won't be found; e.g.:: + + mydict = {[1, 2]: '12'} + print(mydict[[1, 2]]) + + would raise a :exc:`KeyError` exception because the id of the ``[1, 2]`` used in the + second line differs from that in the first line. In other words, dictionary + keys should be compared using ``==``, not using :keyword:`is`. + +- Make a copy when using a list as a key. This doesn't work because the list, + being a mutable object, could contain a reference to itself, and then the + copying code would run into an infinite loop. + +- Allow lists as keys but tell the user not to modify them. This would allow a + class of hard-to-track bugs in programs when you forgot or modified a list by + accident. It also invalidates an important invariant of dictionaries: every + value in ``d.keys()`` is usable as a key of the dictionary. + +- Mark lists as read-only once they are used as a dictionary key. The problem + is that it's not just the top-level object that could change its value; you + could use a tuple containing a list as a key. Entering anything as a key into + a dictionary would require marking all objects reachable from there as + read-only -- and again, self-referential objects could cause an infinite loop. + +There is a trick to get around this if you need to, but use it at your own risk: +You can wrap a mutable structure inside a class instance which has both a +:meth:`__eq__` and a :meth:`__hash__` method. You must then make sure that the +hash value for all such wrapper objects that reside in a dictionary (or other +hash based structure), remain fixed while the object is in the dictionary (or +other structure). :: + + class ListWrapper: + def __init__(self, the_list): + self.the_list = the_list + + def __eq__(self, other): + return self.the_list == other.the_list + + def __hash__(self): + l = self.the_list + result = 98767 - len(l)*555 + for i, el in enumerate(l): + try: + result = result + (hash(el) % 9999999) * 1001 + i + except Exception: + result = (result % 7777777) + i * 333 + return result + +Note that the hash computation is complicated by the possibility that some +members of the list may be unhashable and also by the possibility of arithmetic +overflow. + +Furthermore it must always be the case that if ``o1 == o2`` (ie ``o1.__eq__(o2) +is True``) then ``hash(o1) == hash(o2)`` (ie, ``o1.__hash__() == o2.__hash__()``), +regardless of whether the object is in a dictionary or not. If you fail to meet +these restrictions dictionaries and other hash based structures will misbehave. + +In the case of ListWrapper, whenever the wrapper object is in a dictionary the +wrapped list must not change to avoid anomalies. Don't do this unless you are +prepared to think hard about the requirements and the consequences of not +meeting them correctly. Consider yourself warned. + + +Why doesn't list.sort() return the sorted list? +----------------------------------------------- + +In situations where performance matters, making a copy of the list just to sort +it would be wasteful. Therefore, :meth:`list.sort` sorts the list in place. In +order to remind you of that fact, it does not return the sorted list. This way, +you won't be fooled into accidentally overwriting a list when you need a sorted +copy but also need to keep the unsorted version around. + +If you want to return a new list, use the built-in :func:`sorted` function +instead. This function creates a new list from a provided iterable, sorts +it and returns it. For example, here's how to iterate over the keys of a +dictionary in sorted order:: + + for key in sorted(mydict): + ... # do whatever with mydict[key]... + + +How do you specify and enforce an interface spec in Python? +----------------------------------------------------------- + +An interface specification for a module as provided by languages such as C++ and +Java describes the prototypes for the methods and functions of the module. Many +feel that compile-time enforcement of interface specifications helps in the +construction of large programs. + +Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes +(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check +whether an instance or a class implements a particular ABC. The +:mod:`collections.abc` module defines a set of useful ABCs such as +:class:`~collections.abc.Iterable`, :class:`~collections.abc.Container`, and +:class:`~collections.abc.MutableMapping`. + +For Python, many of the advantages of interface specifications can be obtained +by an appropriate test discipline for components. + +A good test suite for a module can both provide a regression test and serve as a +module interface specification and a set of examples. Many Python modules can +be run as a script to provide a simple "self test." Even modules which use +complex external interfaces can often be tested in isolation using trivial +"stub" emulations of the external interface. The :mod:`doctest` and +:mod:`unittest` modules or third-party test frameworks can be used to construct +exhaustive test suites that exercise every line of code in a module. + +An appropriate testing discipline can help build large complex applications in +Python as well as having interface specifications would. In fact, it can be +better because an interface specification cannot test certain properties of a +program. For example, the :meth:`append` method is expected to add new elements +to the end of some internal list; an interface specification cannot test that +your :meth:`append` implementation will actually do this correctly, but it's +trivial to check this property in a test suite. + +Writing test suites is very helpful, and you might want to design your code to +make it easily tested. One increasingly popular technique, test-driven +development, calls for writing parts of the test suite first, before you write +any of the actual code. Of course Python allows you to be sloppy and not write +test cases at all. + + +Why is there no goto? +--------------------- + +You can use exceptions to provide a "structured goto" that even works across +function calls. Many feel that exceptions can conveniently emulate all +reasonable uses of the "go" or "goto" constructs of C, Fortran, and other +languages. For example:: + + class label(Exception): pass # declare a label + + try: + ... + if condition: raise label() # goto label + ... + except label: # where to goto + pass + ... + +This doesn't allow you to jump into the middle of a loop, but that's usually +considered an abuse of goto anyway. Use sparingly. + + +Why can't raw strings (r-strings) end with a backslash? +------------------------------------------------------- + +More precisely, they can't end with an odd number of backslashes: the unpaired +backslash at the end escapes the closing quote character, leaving an +unterminated string. + +Raw strings were designed to ease creating input for processors (chiefly regular +expression engines) that want to do their own backslash escape processing. Such +processors consider an unmatched trailing backslash to be an error anyway, so +raw strings disallow that. In return, they allow you to pass on the string +quote character by escaping it with a backslash. These rules work well when +r-strings are used for their intended purpose. + +If you're trying to build Windows pathnames, note that all Windows system calls +accept forward slashes too:: + + f = open("/mydir/file.txt") # works fine! + +If you're trying to build a pathname for a DOS command, try e.g. one of :: + + dir = r"\this\is\my\dos\dir" "\\" + dir = r"\this\is\my\dos\dir\ "[:-1] + dir = "\\this\\is\\my\\dos\\dir\\" + + +Why doesn't Python have a "with" statement for attribute assignments? +--------------------------------------------------------------------- + +Python has a 'with' statement that wraps the execution of a block, calling code +on the entrance and exit from the block. Some languages have a construct that +looks like this:: + + with obj: + a = 1 # equivalent to obj.a = 1 + total = total + 1 # obj.total = obj.total + 1 + +In Python, such a construct would be ambiguous. + +Other languages, such as Object Pascal, Delphi, and C++, use static types, so +it's possible to know, in an unambiguous way, what member is being assigned +to. This is the main point of static typing -- the compiler *always* knows the +scope of every variable at compile time. + +Python uses dynamic types. It is impossible to know in advance which attribute +will be referenced at runtime. Member attributes may be added or removed from +objects on the fly. This makes it impossible to know, from a simple reading, +what attribute is being referenced: a local one, a global one, or a member +attribute? + +For instance, take the following incomplete snippet:: + + def foo(a): + with a: + print(x) + +The snippet assumes that "a" must have a member attribute called "x". However, +there is nothing in Python that tells the interpreter this. What should happen +if "a" is, let us say, an integer? If there is a global variable named "x", +will it be used inside the with block? As you see, the dynamic nature of Python +makes such choices much harder. + +The primary benefit of "with" and similar language features (reduction of code +volume) can, however, easily be achieved in Python by assignment. Instead of:: + + function(args).mydict[index][index].a = 21 + function(args).mydict[index][index].b = 42 + function(args).mydict[index][index].c = 63 + +write this:: + + ref = function(args).mydict[index][index] + ref.a = 21 + ref.b = 42 + ref.c = 63 + +This also has the side-effect of increasing execution speed because name +bindings are resolved at run-time in Python, and the second version only needs +to perform the resolution once. + + +Why are colons required for the if/while/def/class statements? +-------------------------------------------------------------- + +The colon is required primarily to enhance readability (one of the results of +the experimental ABC language). Consider this:: + + if a == b + print(a) + +versus :: + + if a == b: + print(a) + +Notice how the second one is slightly easier to read. Notice further how a +colon sets off the example in this FAQ answer; it's a standard usage in English. + +Another minor reason is that the colon makes it easier for editors with syntax +highlighting; they can look for colons to decide when indentation needs to be +increased instead of having to do a more elaborate parsing of the program text. + + +Why does Python allow commas at the end of lists and tuples? +------------------------------------------------------------ + +Python lets you add a trailing comma at the end of lists, tuples, and +dictionaries:: + + [1, 2, 3,] + ('a', 'b', 'c',) + d = { + "A": [1, 5], + "B": [6, 7], # last trailing comma is optional but good style + } + + +There are several reasons to allow this. + +When you have a literal value for a list, tuple, or dictionary spread across +multiple lines, it's easier to add more elements because you don't have to +remember to add a comma to the previous line. The lines can also be reordered +without creating a syntax error. + +Accidentally omitting the comma can lead to errors that are hard to diagnose. +For example:: + + x = [ + "fee", + "fie" + "foo", + "fum" + ] + +This list looks like it has four elements, but it actually contains three: +"fee", "fiefoo" and "fum". Always adding the comma avoids this source of error. + +Allowing the trailing comma may also make programmatic code generation easier. diff --git a/Doc/howto/descriptor.rst.bak b/Doc/howto/descriptor.rst.bak new file mode 100644 index 00000000000000..94a8b4e6b40b96 --- /dev/null +++ b/Doc/howto/descriptor.rst.bak @@ -0,0 +1,1575 @@ +.. _descriptorhowto: + +====================== +Descriptor HowTo Guide +====================== + +:Author: Raymond Hettinger +:Contact: + +.. Contents:: + + +:term:`Descriptors ` let objects customize attribute lookup, +storage, and deletion. + +This guide has four major sections: + +1) The "primer" gives a basic overview, moving gently from simple examples, + adding one feature at a time. Start here if you're new to descriptors. + +2) The second section shows a complete, practical descriptor example. If you + already know the basics, start there. + +3) The third section provides a more technical tutorial that goes into the + detailed mechanics of how descriptors work. Most people don't need this + level of detail. + +4) The last section has pure Python equivalents for built-in descriptors that + are written in C. Read this if you're curious about how functions turn + into bound methods or about the implementation of common tools like + :func:`classmethod`, :func:`staticmethod`, :func:`property`, and + :term:`__slots__`. + + +Primer +^^^^^^ + +In this primer, we start with the most basic possible example and then we'll +add new capabilities one by one. + + +Simple example: A descriptor that returns a constant +---------------------------------------------------- + +The :class:`Ten` class is a descriptor whose :meth:`__get__` method always +returns the constant ``10``: + +.. testcode:: + + class Ten: + def __get__(self, obj, objtype=None): + return 10 + +To use the descriptor, it must be stored as a class variable in another class: + +.. testcode:: + + class A: + x = 5 # Regular class attribute + y = Ten() # Descriptor instance + +An interactive session shows the difference between normal attribute lookup +and descriptor lookup: + +.. doctest:: + + >>> a = A() # Make an instance of class A + >>> a.x # Normal attribute lookup + 5 + >>> a.y # Descriptor lookup + 10 + +In the ``a.x`` attribute lookup, the dot operator finds ``'x': 5`` +in the class dictionary. In the ``a.y`` lookup, the dot operator +finds a descriptor instance, recognized by its ``__get__`` method. +Calling that method returns ``10``. + +Note that the value ``10`` is not stored in either the class dictionary or the +instance dictionary. Instead, the value ``10`` is computed on demand. + +This example shows how a simple descriptor works, but it isn't very useful. +For retrieving constants, normal attribute lookup would be better. + +In the next section, we'll create something more useful, a dynamic lookup. + + +Dynamic lookups +--------------- + +Interesting descriptors typically run computations instead of returning +constants: + +.. testcode:: + + import os + + class DirectorySize: + + def __get__(self, obj, objtype=None): + return len(os.listdir(obj.dirname)) + + class Directory: + + size = DirectorySize() # Descriptor instance + + def __init__(self, dirname): + self.dirname = dirname # Regular instance attribute + +An interactive session shows that the lookup is dynamic — it computes +different, updated answers each time:: + + >>> s = Directory('songs') + >>> g = Directory('games') + >>> s.size # The songs directory has twenty files + 20 + >>> g.size # The games directory has three files + 3 + >>> open('games/newfile').close() # Add a fourth file to the directory + >>> g.size # File count is automatically updated + 4 + +Besides showing how descriptors can run computations, this example also +reveals the purpose of the parameters to :meth:`__get__`. The *self* +parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is +either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that +lets the :meth:`__get__` method learn the target directory. The *objtype* +parameter is the class *Directory*. + + +Managed attributes +------------------ + +A popular use for descriptors is managing access to instance data. The +descriptor is assigned to a public attribute in the class dictionary while the +actual data is stored as a private attribute in the instance dictionary. The +descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when +the public attribute is accessed. + +In the following example, *age* is the public attribute and *_age* is the +private attribute. When the public attribute is accessed, the descriptor logs +the lookup or update: + +.. testcode:: + + import logging + + logging.basicConfig(level=logging.INFO) + + class LoggedAgeAccess: + + def __get__(self, obj, objtype=None): + value = obj._age + logging.info('Accessing %r giving %r', 'age', value) + return value + + def __set__(self, obj, value): + logging.info('Updating %r to %r', 'age', value) + obj._age = value + + class Person: + + age = LoggedAgeAccess() # Descriptor instance + + def __init__(self, name, age): + self.name = name # Regular instance attribute + self.age = age # Calls __set__() + + def birthday(self): + self.age += 1 # Calls both __get__() and __set__() + + +An interactive session shows that all access to the managed attribute *age* is +logged, but that the regular attribute *name* is not logged: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: + + >>> mary = Person('Mary M', 30) # The initial age update is logged + INFO:root:Updating 'age' to 30 + >>> dave = Person('David D', 40) + INFO:root:Updating 'age' to 40 + + >>> vars(mary) # The actual data is in a private attribute + {'name': 'Mary M', '_age': 30} + >>> vars(dave) + {'name': 'David D', '_age': 40} + + >>> mary.age # Access the data and log the lookup + INFO:root:Accessing 'age' giving 30 + 30 + >>> mary.birthday() # Updates are logged as well + INFO:root:Accessing 'age' giving 30 + INFO:root:Updating 'age' to 31 + + >>> dave.name # Regular attribute lookup isn't logged + 'David D' + >>> dave.age # Only the managed attribute is logged + INFO:root:Accessing 'age' giving 40 + 40 + +One major issue with this example is that the private name *_age* is hardwired in +the *LoggedAgeAccess* class. That means that each instance can only have one +logged attribute and that its name is unchangeable. In the next example, +we'll fix that problem. + + +Customized names +---------------- + +When a class uses descriptors, it can inform each descriptor about which +variable name was used. + +In this example, the :class:`Person` class has two descriptor instances, +*name* and *age*. When the :class:`Person` class is defined, it makes a +callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can +be recorded, giving each descriptor its own *public_name* and *private_name*: + +.. testcode:: + + import logging + + logging.basicConfig(level=logging.INFO) + + class LoggedAccess: + + def __set_name__(self, owner, name): + self.public_name = name + self.private_name = '_' + name + + def __get__(self, obj, objtype=None): + value = getattr(obj, self.private_name) + logging.info('Accessing %r giving %r', self.public_name, value) + return value + + def __set__(self, obj, value): + logging.info('Updating %r to %r', self.public_name, value) + setattr(obj, self.private_name, value) + + class Person: + + name = LoggedAccess() # First descriptor instance + age = LoggedAccess() # Second descriptor instance + + def __init__(self, name, age): + self.name = name # Calls the first descriptor + self.age = age # Calls the second descriptor + + def birthday(self): + self.age += 1 + +An interactive session shows that the :class:`Person` class has called +:meth:`__set_name__` so that the field names would be recorded. Here +we call :func:`vars` to look up the descriptor without triggering it: + +.. doctest:: + + >>> vars(vars(Person)['name']) + {'public_name': 'name', 'private_name': '_name'} + >>> vars(vars(Person)['age']) + {'public_name': 'age', 'private_name': '_age'} + +The new class now logs access to both *name* and *age*: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: + + >>> pete = Person('Peter P', 10) + INFO:root:Updating 'name' to 'Peter P' + INFO:root:Updating 'age' to 10 + >>> kate = Person('Catherine C', 20) + INFO:root:Updating 'name' to 'Catherine C' + INFO:root:Updating 'age' to 20 + +The two *Person* instances contain only the private names:: + + >>> vars(pete) + {'_name': 'Peter P', '_age': 10} + >>> vars(kate) + {'_name': 'Catherine C', '_age': 20} + + +Closing thoughts +---------------- + +A :term:`descriptor` is what we call any object that defines :meth:`__get__`, +:meth:`__set__`, or :meth:`__delete__`. + +Optionally, descriptors can have a :meth:`__set_name__` method. This is only +used in cases where a descriptor needs to know either the class where it was +created or the name of class variable it was assigned to. (This method, if +present, is called even if the class is not a descriptor.) + +Descriptors get invoked by the dot operator during attribute lookup. If a +descriptor is accessed indirectly with ``vars(some_class)[descriptor_name]``, +the descriptor instance is returned without invoking it. + +Descriptors only work when used as class variables. When put in instances, +they have no effect. + +The main motivation for descriptors is to provide a hook allowing objects +stored in class variables to control what happens during attribute lookup. + +Traditionally, the calling class controls what happens during lookup. +Descriptors invert that relationship and allow the data being looked-up to +have a say in the matter. + +Descriptors are used throughout the language. It is how functions turn into +bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, +:func:`property`, and :func:`functools.cached_property` are all implemented as +descriptors. + + +Complete Practical Example +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In this example, we create a practical and powerful tool for locating +notoriously hard to find data corruption bugs. + + +Validator class +--------------- + +A validator is a descriptor for managed attribute access. Prior to storing +any data, it verifies that the new value meets various type and range +restrictions. If those restrictions aren't met, it raises an exception to +prevent data corruption at its source. + +This :class:`Validator` class is both an :term:`abstract base class` and a +managed attribute descriptor: + +.. testcode:: + + from abc import ABC, abstractmethod + + class Validator(ABC): + + def __set_name__(self, owner, name): + self.private_name = '_' + name + + def __get__(self, obj, objtype=None): + return getattr(obj, self.private_name) + + def __set__(self, obj, value): + self.validate(value) + setattr(obj, self.private_name, value) + + @abstractmethod + def validate(self, value): + pass + +Custom validators need to inherit from :class:`Validator` and must supply a +:meth:`validate` method to test various restrictions as needed. + + +Custom validators +----------------- + +Here are three practical data validation utilities: + +1) :class:`OneOf` verifies that a value is one of a restricted set of options. + +2) :class:`Number` verifies that a value is either an :class:`int` or + :class:`float`. Optionally, it verifies that a value is between a given + minimum or maximum. + +3) :class:`String` verifies that a value is a :class:`str`. Optionally, it + validates a given minimum or maximum length. It can validate a + user-defined `predicate + `_ as well. + +.. testcode:: + + class OneOf(Validator): + + def __init__(self, *options): + self.options = set(options) + + def validate(self, value): + if value not in self.options: + raise ValueError(f'Expected {value!r} to be one of {self.options!r}') + + class Number(Validator): + + def __init__(self, minvalue=None, maxvalue=None): + self.minvalue = minvalue + self.maxvalue = maxvalue + + def validate(self, value): + if not isinstance(value, (int, float)): + raise TypeError(f'Expected {value!r} to be an int or float') + if self.minvalue is not None and value < self.minvalue: + raise ValueError( + f'Expected {value!r} to be at least {self.minvalue!r}' + ) + if self.maxvalue is not None and value > self.maxvalue: + raise ValueError( + f'Expected {value!r} to be no more than {self.maxvalue!r}' + ) + + class String(Validator): + + def __init__(self, minsize=None, maxsize=None, predicate=None): + self.minsize = minsize + self.maxsize = maxsize + self.predicate = predicate + + def validate(self, value): + if not isinstance(value, str): + raise TypeError(f'Expected {value!r} to be an str') + if self.minsize is not None and len(value) < self.minsize: + raise ValueError( + f'Expected {value!r} to be no smaller than {self.minsize!r}' + ) + if self.maxsize is not None and len(value) > self.maxsize: + raise ValueError( + f'Expected {value!r} to be no bigger than {self.maxsize!r}' + ) + if self.predicate is not None and not self.predicate(value): + raise ValueError( + f'Expected {self.predicate} to be true for {value!r}' + ) + + +Practical application +--------------------- + +Here's how the data validators can be used in a real class: + +.. testcode:: + + class Component: + + name = String(minsize=3, maxsize=10, predicate=str.isupper) + kind = OneOf('wood', 'metal', 'plastic') + quantity = Number(minvalue=0) + + def __init__(self, name, kind, quantity): + self.name = name + self.kind = kind + self.quantity = quantity + +The descriptors prevent invalid instances from being created: + +.. doctest:: + + >>> Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase + Traceback (most recent call last): + ... + ValueError: Expected to be true for 'Widget' + + >>> Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled + Traceback (most recent call last): + ... + ValueError: Expected 'metle' to be one of {'metal', 'plastic', 'wood'} + + >>> Component('WIDGET', 'metal', -5) # Blocked: -5 is negative + Traceback (most recent call last): + ... + ValueError: Expected -5 to be at least 0 + >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number + Traceback (most recent call last): + ... + TypeError: Expected 'V' to be an int or float + + >>> c = Component('WIDGET', 'metal', 5) # Allowed: The inputs are valid + + +Technical Tutorial +^^^^^^^^^^^^^^^^^^ + +What follows is a more technical tutorial for the mechanics and details of how +descriptors work. + + +Abstract +-------- + +Defines descriptors, summarizes the protocol, and shows how descriptors are +called. Provides an example showing how object relational mappings work. + +Learning about descriptors not only provides access to a larger toolset, it +creates a deeper understanding of how Python works. + + +Definition and introduction +--------------------------- + +In general, a descriptor is an attribute value that has one of the methods in +the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, +and :meth:`__delete__`. If any of those methods are defined for an +attribute, it is said to be a :term:`descriptor`. + +The default behavior for attribute access is to get, set, or delete the +attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain +starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and +continuing through the method resolution order of ``type(a)``. If the +looked-up value is an object defining one of the descriptor methods, then Python +may override the default behavior and invoke the descriptor method instead. +Where this occurs in the precedence chain depends on which descriptor methods +were defined. + +Descriptors are a powerful, general purpose protocol. They are the mechanism +behind properties, methods, static methods, class methods, and +:func:`super()`. They are used throughout Python itself. Descriptors +simplify the underlying C code and offer a flexible set of new tools for +everyday Python programs. + + +Descriptor protocol +------------------- + +``descr.__get__(self, obj, type=None) -> value`` + +``descr.__set__(self, obj, value) -> None`` + +``descr.__delete__(self, obj) -> None`` + +That is all there is to it. Define any of these methods and an object is +considered a descriptor and can override default behavior upon being looked up +as an attribute. + +If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered +a data descriptor. Descriptors that only define :meth:`__get__` are called +non-data descriptors (they are often used for methods but other uses are +possible). + +Data and non-data descriptors differ in how overrides are calculated with +respect to entries in an instance's dictionary. If an instance's dictionary +has an entry with the same name as a data descriptor, the data descriptor +takes precedence. If an instance's dictionary has an entry with the same +name as a non-data descriptor, the dictionary entry takes precedence. + +To make a read-only data descriptor, define both :meth:`__get__` and +:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when +called. Defining the :meth:`__set__` method with an exception raising +placeholder is enough to make it a data descriptor. + + +Overview of descriptor invocation +--------------------------------- + +A descriptor can be called directly with ``desc.__get__(obj)`` or +``desc.__get__(None, cls)``. + +But it is more common for a descriptor to be invoked automatically from +attribute access. + +The expression ``obj.x`` looks up the attribute ``x`` in the chain of +namespaces for ``obj``. If the search finds a descriptor outside of the +instance ``__dict__``, its :meth:`__get__` method is invoked according to the +precedence rules listed below. + +The details of invocation depend on whether ``obj`` is an object, class, or +instance of super. + + +Invocation from an instance +--------------------------- + +Instance lookup scans through a chain of namespaces giving data descriptors +the highest priority, followed by instance variables, then non-data +descriptors, then class variables, and lastly :meth:`__getattr__` if it is +provided. + +If a descriptor is found for ``a.x``, then it is invoked with: +``desc.__get__(a, type(a))``. + +The logic for a dotted lookup is in :meth:`object.__getattribute__`. Here is +a pure Python equivalent: + +.. testcode:: + + def object_getattribute(obj, name): + "Emulate PyObject_GenericGetAttr() in Objects/object.c" + null = object() + objtype = type(obj) + cls_var = getattr(objtype, name, null) + descr_get = getattr(type(cls_var), '__get__', null) + if descr_get is not null: + if (hasattr(type(cls_var), '__set__') + or hasattr(type(cls_var), '__delete__')): + return descr_get(cls_var, obj, objtype) # data descriptor + if hasattr(obj, '__dict__') and name in vars(obj): + return vars(obj)[name] # instance variable + if descr_get is not null: + return descr_get(cls_var, obj, objtype) # non-data descriptor + if cls_var is not null: + return cls_var # class variable + raise AttributeError(name) + + +.. testcode:: + :hide: + + # Test the fidelity of object_getattribute() by comparing it with the + # normal object.__getattribute__(). The former will be accessed by + # square brackets and the latter by the dot operator. + + class Object: + + def __getitem__(obj, name): + try: + return object_getattribute(obj, name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + + class DualOperator(Object): + + x = 10 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + @property + def p3(self): + return 3 * self.x + + def m5(self, y): + return 5 * y + + def m7(self, y): + return 7 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + class DualOperatorWithSlots: + + __getitem__ = Object.__getitem__ + + __slots__ = ['z'] + + x = 15 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + def m5(self, y): + return 5 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + +.. doctest:: + :hide: + + >>> a = DualOperator(11) + >>> vars(a).update(p3 = '_p3', m7 = '_m7') + >>> a.x == a['x'] == 10 + True + >>> a.z == a['z'] == 11 + True + >>> a.p2 == a['p2'] == 20 + True + >>> a.p3 == a['p3'] == 30 + True + >>> a.m5(100) == a.m5(100) == 500 + True + >>> a.m7 == a['m7'] == '_m7' + True + >>> a.g == a['g'] == ('getattr_hook', a, 'g') + True + + >>> b = DualOperatorWithSlots(22) + >>> b.x == b['x'] == 15 + True + >>> b.z == b['z'] == 22 + True + >>> b.p2 == b['p2'] == 30 + True + >>> b.m5(200) == b['m5'](200) == 1000 + True + >>> b.g == b['g'] == ('getattr_hook', b, 'g') + True + + +Interestingly, attribute lookup doesn't call :meth:`object.__getattribute__` +directly. Instead, both the dot operator and the :func:`getattr` function +perform attribute lookup by way of a helper function: + +.. testcode:: + + def getattr_hook(obj, name): + "Emulate slot_tp_getattr_hook() in Objects/typeobject.c" + try: + return obj.__getattribute__(name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + +So if :meth:`__getattr__` exists, it is called whenever :meth:`__getattribute__` +raises :exc:`AttributeError` (either directly or in one of the descriptor calls). + +Also, if a user calls :meth:`object.__getattribute__` directly, the +:meth:`__getattr__` hook is bypassed entirely. + + +Invocation from a class +----------------------- + +The logic for a dotted lookup such as ``A.x`` is in +:meth:`type.__getattribute__`. The steps are similar to those for +:meth:`object.__getattribute__` but the instance dictionary lookup is replaced +by a search through the class's :term:`method resolution order`. + +If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. + +The full C implementation can be found in :c:func:`type_getattro()` and +:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`. + + +Invocation from super +--------------------- + +The logic for super's dotted lookup is in the :meth:`__getattribute__` method for +object returned by :class:`super()`. + +A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` +for the base class ``B`` immediately following ``A`` and then returns +``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned +unchanged. + +The full C implementation can be found in :c:func:`super_getattro()` in +:source:`Objects/typeobject.c`. A pure Python equivalent can be found in +`Guido's Tutorial +`_. + + +Summary of invocation logic +--------------------------- + +The mechanism for descriptors is embedded in the :meth:`__getattribute__()` +methods for :class:`object`, :class:`type`, and :func:`super`. + +The important points to remember are: + +* Descriptors are invoked by the :meth:`__getattribute__` method. + +* Classes inherit this machinery from :class:`object`, :class:`type`, or + :func:`super`. + +* Overriding :meth:`__getattribute__` prevents automatic descriptor calls + because all the descriptor logic is in that method. + +* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make + different calls to :meth:`__get__`. The first includes the instance and may + include the class. The second puts in ``None`` for the instance and always + includes the class. + +* Data descriptors always override instance dictionaries. + +* Non-data descriptors may be overridden by instance dictionaries. + + +Automatic name notification +--------------------------- + +Sometimes it is desirable for a descriptor to know what class variable name it +was assigned to. When a new class is created, the :class:`type` metaclass +scans the dictionary of the new class. If any of the entries are descriptors +and if they define :meth:`__set_name__`, that method is called with two +arguments. The *owner* is the class where the descriptor is used, and the +*name* is the class variable the descriptor was assigned to. + +The implementation details are in :c:func:`type_new()` and +:c:func:`set_names()` in :source:`Objects/typeobject.c`. + +Since the update logic is in :meth:`type.__new__`, notifications only take +place at the time of class creation. If descriptors are added to the class +afterwards, :meth:`__set_name__` will need to be called manually. + + +ORM example +----------- + +The following code is simplified skeleton showing how data descriptors could +be used to implement an `object relational mapping +`_. + +The essential idea is that the data is stored in an external database. The +Python instances only hold keys to the database's tables. Descriptors take +care of lookups or updates: + +.. testcode:: + + class Field: + + def __set_name__(self, owner, name): + self.fetch = f'SELECT {name} FROM {owner.table} WHERE {owner.key}=?;' + self.store = f'UPDATE {owner.table} SET {name}=? WHERE {owner.key}=?;' + + def __get__(self, obj, objtype=None): + return conn.execute(self.fetch, [obj.key]).fetchone()[0] + + def __set__(self, obj, value): + conn.execute(self.store, [value, obj.key]) + conn.commit() + +We can use the :class:`Field` class to define `models +`_ that describe the schema for +each table in a database: + +.. testcode:: + + class Movie: + table = 'Movies' # Table name + key = 'title' # Primary key + director = Field() + year = Field() + + def __init__(self, key): + self.key = key + + class Song: + table = 'Music' + key = 'title' + artist = Field() + year = Field() + genre = Field() + + def __init__(self, key): + self.key = key + +To use the models, first connect to the database:: + + >>> import sqlite3 + >>> conn = sqlite3.connect('entertainment.db') + +An interactive session shows how data is retrieved from the database and how +it can be updated: + +.. testsetup:: + + song_data = [ + ('Country Roads', 'John Denver', 1972), + ('Me and Bobby McGee', 'Janice Joplin', 1971), + ('Coal Miners Daughter', 'Loretta Lynn', 1970), + ] + + movie_data = [ + ('Star Wars', 'George Lucas', 1977), + ('Jaws', 'Steven Spielberg', 1975), + ('Aliens', 'James Cameron', 1986), + ] + + import sqlite3 + + conn = sqlite3.connect(':memory:') + conn.execute('CREATE TABLE Music (title text, artist text, year integer);') + conn.execute('CREATE INDEX MusicNdx ON Music (title);') + conn.executemany('INSERT INTO Music VALUES (?, ?, ?);', song_data) + conn.execute('CREATE TABLE Movies (title text, director text, year integer);') + conn.execute('CREATE INDEX MovieNdx ON Music (title);') + conn.executemany('INSERT INTO Movies VALUES (?, ?, ?);', movie_data) + conn.commit() + +.. doctest:: + + >>> Movie('Star Wars').director + 'George Lucas' + >>> jaws = Movie('Jaws') + >>> f'Released in {jaws.year} by {jaws.director}' + 'Released in 1975 by Steven Spielberg' + + >>> Song('Country Roads').artist + 'John Denver' + + >>> Movie('Star Wars').director = 'J.J. Abrams' + >>> Movie('Star Wars').director + 'J.J. Abrams' + + +Pure Python Equivalents +^^^^^^^^^^^^^^^^^^^^^^^ + +The descriptor protocol is simple and offers exciting possibilities. Several +use cases are so common that they have been prepackaged into built-in tools. +Properties, bound methods, static methods, class methods, and \_\_slots\_\_ are +all based on the descriptor protocol. + + +Properties +---------- + +Calling :func:`property` is a succinct way of building a data descriptor that +triggers a function call upon access to an attribute. Its signature is:: + + property(fget=None, fset=None, fdel=None, doc=None) -> property + +The documentation shows a typical use to define a managed attribute ``x``: + +.. testcode:: + + class C: + def getx(self): return self.__x + def setx(self, value): self.__x = value + def delx(self): del self.__x + x = property(getx, setx, delx, "I'm the 'x' property.") + +To see how :func:`property` is implemented in terms of the descriptor protocol, +here is a pure Python equivalent: + +.. testcode:: + + class Property: + "Emulate PyProperty_Type() in Objects/descrobject.c" + + def __init__(self, fget=None, fset=None, fdel=None, doc=None): + self.fget = fget + self.fset = fset + self.fdel = fdel + if doc is None and fget is not None: + doc = fget.__doc__ + self.__doc__ = doc + self._name = '' + + def __set_name__(self, owner, name): + self._name = name + + def __get__(self, obj, objtype=None): + if obj is None: + return self + if self.fget is None: + raise AttributeError(f'unreadable attribute {self._name}') + return self.fget(obj) + + def __set__(self, obj, value): + if self.fset is None: + raise AttributeError(f"can't set attribute {self._name}") + self.fset(obj, value) + + def __delete__(self, obj): + if self.fdel is None: + raise AttributeError(f"can't delete attribute {self._name}") + self.fdel(obj) + + def getter(self, fget): + prop = type(self)(fget, self.fset, self.fdel, self.__doc__) + prop._name = self._name + return prop + + def setter(self, fset): + prop = type(self)(self.fget, fset, self.fdel, self.__doc__) + prop._name = self._name + return prop + + def deleter(self, fdel): + prop = type(self)(self.fget, self.fset, fdel, self.__doc__) + prop._name = self._name + return prop + +.. testcode:: + :hide: + + # Verify the Property() emulation + + class CC: + def getx(self): + return self.__x + def setx(self, value): + self.__x = value + def delx(self): + del self.__x + x = Property(getx, setx, delx, "I'm the 'x' property.") + + # Now do it again but use the decorator style + + class CCC: + @Property + def x(self): + return self.__x + @x.setter + def x(self, value): + self.__x = value + @x.deleter + def x(self): + del self.__x + + +.. doctest:: + :hide: + + >>> cc = CC() + >>> hasattr(cc, 'x') + False + >>> cc.x = 33 + >>> cc.x + 33 + >>> del cc.x + >>> hasattr(cc, 'x') + False + + >>> ccc = CCC() + >>> hasattr(ccc, 'x') + False + >>> ccc.x = 333 + >>> ccc.x == 333 + True + >>> del ccc.x + >>> hasattr(ccc, 'x') + False + +The :func:`property` builtin helps whenever a user interface has granted +attribute access and then subsequent changes require the intervention of a +method. + +For instance, a spreadsheet class may grant access to a cell value through +``Cell('b10').value``. Subsequent improvements to the program require the cell +to be recalculated on every access; however, the programmer does not want to +affect existing client code accessing the attribute directly. The solution is +to wrap access to the value attribute in a property data descriptor: + +.. testcode:: + + class Cell: + ... + + @property + def value(self): + "Recalculate the cell before returning value" + self.recalc() + return self._value + +Either the built-in :func:`property` or our :func:`Property` equivalent would +work in this example. + + +Functions and methods +--------------------- + +Python's object oriented features are built upon a function based environment. +Using non-data descriptors, the two are merged seamlessly. + +Functions stored in class dictionaries get turned into methods when invoked. +Methods only differ from regular functions in that the object instance is +prepended to the other arguments. By convention, the instance is called +*self* but could be called *this* or any other variable name. + +Methods can be created manually with :class:`types.MethodType` which is +roughly equivalent to: + +.. testcode:: + + class MethodType: + "Emulate Py_MethodType in Objects/classobject.c" + + def __init__(self, func, obj): + self.__func__ = func + self.__self__ = obj + + def __call__(self, *args, **kwargs): + func = self.__func__ + obj = self.__self__ + return func(obj, *args, **kwargs) + +To support automatic creation of methods, functions include the +:meth:`__get__` method for binding methods during attribute access. This +means that functions are non-data descriptors that return bound methods +during dotted lookup from an instance. Here's how it works: + +.. testcode:: + + class Function: + ... + + def __get__(self, obj, objtype=None): + "Simulate func_descr_get() in Objects/funcobject.c" + if obj is None: + return self + return MethodType(self, obj) + +Running the following class in the interpreter shows how the function +descriptor works in practice: + +.. testcode:: + + class D: + def f(self, x): + return x + +The function has a :term:`qualified name` attribute to support introspection: + +.. doctest:: + + >>> D.f.__qualname__ + 'D.f' + +Accessing the function through the class dictionary does not invoke +:meth:`__get__`. Instead, it just returns the underlying function object:: + + >>> D.__dict__['f'] + + +Dotted access from a class calls :meth:`__get__` which just returns the +underlying function unchanged:: + + >>> D.f + + +The interesting behavior occurs during dotted access from an instance. The +dotted lookup calls :meth:`__get__` which returns a bound method object:: + + >>> d = D() + >>> d.f + > + +Internally, the bound method stores the underlying function and the bound +instance:: + + >>> d.f.__func__ + + + >>> d.f.__self__ + <__main__.D object at 0x1012e1f98> + +If you have ever wondered where *self* comes from in regular methods or where +*cls* comes from in class methods, this is it! + + +Static methods +-------------- + +Non-data descriptors provide a simple mechanism for variations on the usual +patterns of binding functions into methods. + +To recap, functions have a :meth:`__get__` method so that they can be converted +to a method when accessed as attributes. The non-data descriptor transforms an +``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` +becomes ``f(*args)``. + +This chart summarizes the binding and its two most useful variants: + + +-----------------+----------------------+------------------+ + | Transformation | Called from an | Called from a | + | | object | class | + +=================+======================+==================+ + | function | f(obj, \*args) | f(\*args) | + +-----------------+----------------------+------------------+ + | staticmethod | f(\*args) | f(\*args) | + +-----------------+----------------------+------------------+ + | classmethod | f(type(obj), \*args) | f(cls, \*args) | + +-----------------+----------------------+------------------+ + +Static methods return the underlying function without changes. Calling either +``c.f`` or ``C.f`` is the equivalent of a direct lookup into +``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a +result, the function becomes identically accessible from either an object or a +class. + +Good candidates for static methods are methods that do not reference the +``self`` variable. + +For instance, a statistics package may include a container class for +experimental data. The class provides normal methods for computing the average, +mean, median, and other descriptive statistics that depend on the data. However, +there may be useful functions which are conceptually related but do not depend +on the data. For instance, ``erf(x)`` is handy conversion routine that comes up +in statistical work but does not directly depend on a particular dataset. +It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or +``Sample.erf(1.5) --> .9332``. + +Since static methods return the underlying function with no changes, the +example calls are unexciting: + +.. testcode:: + + class E: + @staticmethod + def f(x): + print(x) + +.. doctest:: + + >>> E.f(3) + 3 + >>> E().f(3) + 3 + +Using the non-data descriptor protocol, a pure Python version of +:func:`staticmethod` would look like this: + +.. doctest:: + + class StaticMethod: + "Emulate PyStaticMethod_Type() in Objects/funcobject.c" + + def __init__(self, f): + self.f = f + + def __get__(self, obj, objtype=None): + return self.f + + +Class methods +------------- + +Unlike static methods, class methods prepend the class reference to the +argument list before calling the function. This format is the same +for whether the caller is an object or a class: + +.. testcode:: + + class F: + @classmethod + def f(cls, x): + return cls.__name__, x + +.. doctest:: + + >>> F.f(3) + ('F', 3) + >>> F().f(3) + ('F', 3) + +This behavior is useful whenever the method only needs to have a class +reference and does not rely on data stored in a specific instance. One use for +class methods is to create alternate class constructors. For example, the +classmethod :func:`dict.fromkeys` creates a new dictionary from a list of +keys. The pure Python equivalent is: + +.. testcode:: + + class Dict(dict): + @classmethod + def fromkeys(cls, iterable, value=None): + "Emulate dict_fromkeys() in Objects/dictobject.c" + d = cls() + for key in iterable: + d[key] = value + return d + +Now a new dictionary of unique keys can be constructed like this: + +.. doctest:: + + >>> d = Dict.fromkeys('abracadabra') + >>> type(d) is Dict + True + >>> d + {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} + +Using the non-data descriptor protocol, a pure Python version of +:func:`classmethod` would look like this: + +.. testcode:: + + class ClassMethod: + "Emulate PyClassMethod_Type() in Objects/funcobject.c" + + def __init__(self, f): + self.f = f + + def __get__(self, obj, cls=None): + if cls is None: + cls = type(obj) + if hasattr(obj, '__get__'): + return self.f.__get__(cls) + return MethodType(self.f, cls) + +.. testcode:: + :hide: + + # Verify the emulation works + class T: + @ClassMethod + def cm(cls, x, y): + return (cls, x, y) + +.. doctest:: + :hide: + + >>> T.cm(11, 22) + (, 11, 22) + + # Also call it from an instance + >>> t = T() + >>> t.cm(11, 22) + (, 11, 22) + +The code path for ``hasattr(obj, '__get__')`` was added in Python 3.9 and +makes it possible for :func:`classmethod` to support chained decorators. +For example, a classmethod and property could be chained together: + +.. testcode:: + + class G: + @classmethod + @property + def __doc__(cls): + return f'A doc for {cls.__name__!r}' + +.. doctest:: + + >>> G.__doc__ + "A doc for 'G'" + + +Member objects and __slots__ +---------------------------- + +When a class defines ``__slots__``, it replaces instance dictionaries with a +fixed-length array of slot values. From a user point of view that has +several effects: + +1. Provides immediate detection of bugs due to misspelled attribute +assignments. Only attribute names specified in ``__slots__`` are allowed: + +.. testcode:: + + class Vehicle: + __slots__ = ('id_number', 'make', 'model') + +.. doctest:: + + >>> auto = Vehicle() + >>> auto.id_nubmer = 'VYE483814LQEX' + Traceback (most recent call last): + ... + AttributeError: 'Vehicle' object has no attribute 'id_nubmer' + +2. Helps create immutable objects where descriptors manage access to private +attributes stored in ``__slots__``: + +.. testcode:: + + class Immutable: + + __slots__ = ('_dept', '_name') # Replace the instance dictionary + + def __init__(self, dept, name): + self._dept = dept # Store to private attribute + self._name = name # Store to private attribute + + @property # Read-only descriptor + def dept(self): + return self._dept + + @property + def name(self): # Read-only descriptor + return self._name + +.. doctest:: + + >>> mark = Immutable('Botany', 'Mark Watney') + >>> mark.dept + 'Botany' + >>> mark.dept = 'Space Pirate' + Traceback (most recent call last): + ... + AttributeError: can't set attribute + >>> mark.location = 'Mars' + Traceback (most recent call last): + ... + AttributeError: 'Immutable' object has no attribute 'location' + +3. Saves memory. On a 64-bit Linux build, an instance with two attributes +takes 48 bytes with ``__slots__`` and 152 bytes without. This `flyweight +design pattern `_ likely only +matters when a large number of instances are going to be created. + +4. Improves speed. Reading instance variables is 35% faster with +``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). + +5. Blocks tools like :func:`functools.cached_property` which require an +instance dictionary to function correctly: + +.. testcode:: + + from functools import cached_property + + class CP: + __slots__ = () # Eliminates the instance dict + + @cached_property # Requires an instance dict + def pi(self): + return 4 * sum((-1.0)**n / (2.0*n + 1.0) + for n in reversed(range(100_000))) + +.. doctest:: + + >>> CP().pi + Traceback (most recent call last): + ... + TypeError: No '__dict__' attribute on 'CP' instance to cache 'pi' property. + +It is not possible to create an exact drop-in pure Python version of +``__slots__`` because it requires direct access to C structures and control +over object memory allocation. However, we can build a mostly faithful +simulation where the actual C structure for slots is emulated by a private +``_slotvalues`` list. Reads and writes to that private structure are managed +by member descriptors: + +.. testcode:: + + null = object() + + class Member: + + def __init__(self, name, clsname, offset): + 'Emulate PyMemberDef in Include/structmember.h' + # Also see descr_new() in Objects/descrobject.c + self.name = name + self.clsname = clsname + self.offset = offset + + def __get__(self, obj, objtype=None): + 'Emulate member_get() in Objects/descrobject.c' + # Also see PyMember_GetOne() in Python/structmember.c + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + return value + + def __set__(self, obj, value): + 'Emulate member_set() in Objects/descrobject.c' + obj._slotvalues[self.offset] = value + + def __delete__(self, obj): + 'Emulate member_delete() in Objects/descrobject.c' + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + obj._slotvalues[self.offset] = null + + def __repr__(self): + 'Emulate member_repr() in Objects/descrobject.c' + return f'' + +The :meth:`type.__new__` method takes care of adding member objects to class +variables: + +.. testcode:: + + class Type(type): + 'Simulate how the type metaclass adds member objects for slots' + + def __new__(mcls, clsname, bases, mapping): + 'Emuluate type_new() in Objects/typeobject.c' + # type_new() calls PyTypeReady() which calls add_methods() + slot_names = mapping.get('slot_names', []) + for offset, name in enumerate(slot_names): + mapping[name] = Member(name, clsname, offset) + return type.__new__(mcls, clsname, bases, mapping) + +The :meth:`object.__new__` method takes care of creating instances that have +slots instead of an instance dictionary. Here is a rough simulation in pure +Python: + +.. testcode:: + + class Object: + 'Simulate how object.__new__() allocates memory for __slots__' + + def __new__(cls, *args): + 'Emulate object_new() in Objects/typeobject.c' + inst = super().__new__(cls) + if hasattr(cls, 'slot_names'): + empty_slots = [null] * len(cls.slot_names) + object.__setattr__(inst, '_slotvalues', empty_slots) + return inst + + def __setattr__(self, name, value): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__setattr__(name, value) + + def __delattr__(self, name): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__delattr__(name) + +To use the simulation in a real class, just inherit from :class:`Object` and +set the :term:`metaclass` to :class:`Type`: + +.. testcode:: + + class H(Object, metaclass=Type): + 'Instance variables stored in slots' + + slot_names = ['x', 'y'] + + def __init__(self, x, y): + self.x = x + self.y = y + +At this point, the metaclass has loaded member objects for *x* and *y*:: + + >>> from pprint import pp + >>> pp(dict(vars(H))) + {'__module__': '__main__', + '__doc__': 'Instance variables stored in slots', + 'slot_names': ['x', 'y'], + '__init__': , + 'x': , + 'y': } + +.. doctest:: + :hide: + + # We test this separately because the preceding section is not + # doctestable due to the hex memory address for the __init__ function + >>> isinstance(vars(H)['x'], Member) + True + >>> isinstance(vars(H)['y'], Member) + True + +When instances are created, they have a ``slot_values`` list where the +attributes are stored: + +.. doctest:: + + >>> h = H(10, 20) + >>> vars(h) + {'_slotvalues': [10, 20]} + >>> h.x = 55 + >>> vars(h) + {'_slotvalues': [55, 20]} + +Misspelled or unassigned attributes will raise an exception: + +.. doctest:: + + >>> h.xz + Traceback (most recent call last): + ... + AttributeError: 'H' object has no attribute 'xz' + +.. doctest:: + :hide: + + # Examples for deleted attributes are not shown because this section + # is already a bit lengthy. We still test that code here. + >>> del h.x + >>> hasattr(h, 'x') + False + + # Also test the code for uninitialized slots + >>> class HU(Object, metaclass=Type): + ... slot_names = ['x', 'y'] + ... + >>> hu = HU() + >>> hasattr(hu, 'x') + False + >>> hasattr(hu, 'y') + False diff --git a/Doc/library/_thread.rst.bak b/Doc/library/_thread.rst.bak new file mode 100644 index 00000000000000..bd653ab32bb9c4 --- /dev/null +++ b/Doc/library/_thread.rst.bak @@ -0,0 +1,215 @@ +:mod:`_thread` --- Low-level threading API +========================================== + +.. module:: _thread + :synopsis: Low-level threading API. + +.. index:: + single: light-weight processes + single: processes, light-weight + single: binary semaphores + single: semaphores, binary + +-------------- + +This module provides low-level primitives for working with multiple threads +(also called :dfn:`light-weight processes` or :dfn:`tasks`) --- multiple threads of +control sharing their global data space. For synchronization, simple locks +(also called :dfn:`mutexes` or :dfn:`binary semaphores`) are provided. +The :mod:`threading` module provides an easier to use and higher-level +threading API built on top of this module. + +.. index:: + single: pthreads + pair: threads; POSIX + +.. versionchanged:: 3.7 + This module used to be optional, it is now always available. + +This module defines the following constants and functions: + +.. exception:: error + + Raised on thread-specific errors. + + .. versionchanged:: 3.3 + This is now a synonym of the built-in :exc:`RuntimeError`. + + +.. data:: LockType + + This is the type of lock objects. + + +.. function:: start_new_thread(function, args[, kwargs]) + + Start a new thread and return its identifier. The thread executes the + function *function* with the argument list *args* (which must be a tuple). + The optional *kwargs* argument specifies a dictionary of keyword arguments. + + When the function returns, the thread silently exits. + + When the function terminates with an unhandled exception, + :func:`sys.unraisablehook` is called to handle the exception. The *object* + attribute of the hook argument is *function*. By default, a stack trace is + printed and then the thread exits (but other threads continue to run). + + When the function raises a :exc:`SystemExit` exception, it is silently + ignored. + + .. versionchanged:: 3.8 + :func:`sys.unraisablehook` is now used to handle unhandled exceptions. + + +.. function:: interrupt_main() + + Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main + thread. A thread can use this function to interrupt the main thread. + + If :data:`signal.SIGINT` isn't handled by Python (it was set to + :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does + nothing. + + +.. function:: exit() + + Raise the :exc:`SystemExit` exception. When not caught, this will cause the + thread to exit silently. + +.. + function:: exit_prog(status) + + Exit all threads and report the value of the integer argument + *status* as the exit status of the entire program. + **Caveat:** code in pending :keyword:`finally` clauses, in this thread + or in other threads, is not executed. + + +.. function:: allocate_lock() + + Return a new lock object. Methods of locks are described below. The lock is + initially unlocked. + + +.. function:: get_ident() + + Return the 'thread identifier' of the current thread. This is a nonzero + integer. Its value has no direct meaning; it is intended as a magic cookie to + be used e.g. to index a dictionary of thread-specific data. Thread identifiers + may be recycled when a thread exits and another thread is created. + + +.. function:: get_native_id() + + Return the native integral Thread ID of the current thread assigned by the kernel. + This is a non-negative integer. + Its value may be used to uniquely identify this particular thread system-wide + (until the thread terminates, after which the value may be recycled by the OS). + + .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. + + .. versionadded:: 3.8 + + +.. function:: stack_size([size]) + + Return the thread stack size used when creating new threads. The optional + *size* argument specifies the stack size to be used for subsequently created + threads, and must be 0 (use platform or configured default) or a positive + integer value of at least 32,768 (32 KiB). If *size* is not specified, + 0 is used. If changing the thread stack size is + unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is + invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB + is currently the minimum supported stack size value to guarantee sufficient + stack space for the interpreter itself. Note that some platforms may have + particular restrictions on values for the stack size, such as requiring a + minimum stack size > 32 KiB or requiring allocation in multiples of the system + memory page size - platform documentation should be referred to for more + information (4 KiB pages are common; using multiples of 4096 for the stack size is + the suggested approach in the absence of more specific information). + + .. availability:: Windows, systems with POSIX threads. + + +.. data:: TIMEOUT_MAX + + The maximum value allowed for the *timeout* parameter of + :meth:`Lock.acquire`. Specifying a timeout greater than this value will + raise an :exc:`OverflowError`. + + .. versionadded:: 3.2 + + +Lock objects have the following methods: + + +.. method:: lock.acquire(waitflag=1, timeout=-1) + + Without any optional argument, this method acquires the lock unconditionally, if + necessary waiting until it is released by another thread (only one thread at a + time can acquire a lock --- that's their reason for existence). + + If the integer *waitflag* argument is present, the action depends on its + value: if it is zero, the lock is only acquired if it can be acquired + immediately without waiting, while if it is nonzero, the lock is acquired + unconditionally as above. + + If the floating-point *timeout* argument is present and positive, it + specifies the maximum wait time in seconds before returning. A negative + *timeout* argument specifies an unbounded wait. You cannot specify + a *timeout* if *waitflag* is zero. + + The return value is ``True`` if the lock is acquired successfully, + ``False`` if not. + + .. versionchanged:: 3.2 + The *timeout* parameter is new. + + .. versionchanged:: 3.2 + Lock acquires can now be interrupted by signals on POSIX. + + +.. method:: lock.release() + + Releases the lock. The lock must have been acquired earlier, but not + necessarily by the same thread. + + +.. method:: lock.locked() + + Return the status of the lock: ``True`` if it has been acquired by some thread, + ``False`` if not. + +In addition to these methods, lock objects can also be used via the +:keyword:`with` statement, e.g.:: + + import _thread + + a_lock = _thread.allocate_lock() + + with a_lock: + print("a_lock is locked while this executes") + +**Caveats:** + + .. index:: module: signal + +* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` + exception will be received by an arbitrary thread. (When the :mod:`signal` + module is available, interrupts always go to the main thread.) + +* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is + equivalent to calling :func:`_thread.exit`. + +* It is not possible to interrupt the :meth:`acquire` method on a lock --- the + :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. + +* When the main thread exits, it is system defined whether the other threads + survive. On most systems, they are killed without executing + :keyword:`try` ... :keyword:`finally` clauses or executing object + destructors. + +* When the main thread exits, it does not do any of its usual cleanup (except + that :keyword:`try` ... :keyword:`finally` clauses are honored), and the + standard I/O files are not flushed. + diff --git a/Doc/library/ast.rst.bak b/Doc/library/ast.rst.bak new file mode 100644 index 00000000000000..b5e1be82691157 --- /dev/null +++ b/Doc/library/ast.rst.bak @@ -0,0 +1,1993 @@ +:mod:`ast` --- Abstract Syntax Trees +==================================== + +.. module:: ast + :synopsis: Abstract Syntax Tree classes and manipulation. + +.. sectionauthor:: Martin v. Löwis +.. sectionauthor:: Georg Brandl + +.. testsetup:: + + import ast + +**Source code:** :source:`Lib/ast.py` + +-------------- + +The :mod:`ast` module helps Python applications to process trees of the Python +abstract syntax grammar. The abstract syntax itself might change with each +Python release; this module helps to find out programmatically what the current +grammar looks like. + +An abstract syntax tree can be generated by passing :data:`ast.PyCF_ONLY_AST` as +a flag to the :func:`compile` built-in function, or using the :func:`parse` +helper provided in this module. The result will be a tree of objects whose +classes all inherit from :class:`ast.AST`. An abstract syntax tree can be +compiled into a Python code object using the built-in :func:`compile` function. + + +.. _abstract-grammar: + +Abstract Grammar +---------------- + +The abstract grammar is currently defined as follows: + +.. literalinclude:: ../../Parser/Python.asdl + :language: asdl + + +Node classes +------------ + +.. class:: AST + + This is the base of all AST node classes. The actual node classes are + derived from the :file:`Parser/Python.asdl` file, which is reproduced + :ref:`below `. They are defined in the :mod:`_ast` C + module and re-exported in :mod:`ast`. + + There is one class defined for each left-hand side symbol in the abstract + grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, + there is one class defined for each constructor on the right-hand side; these + classes inherit from the classes for the left-hand side trees. For example, + :class:`ast.BinOp` inherits from :class:`ast.expr`. For production rules + with alternatives (aka "sums"), the left-hand side class is abstract: only + instances of specific constructor nodes are ever created. + + .. index:: single: ? (question mark); in AST grammar + .. index:: single: * (asterisk); in AST grammar + + .. attribute:: _fields + + Each concrete class has an attribute :attr:`_fields` which gives the names + of all child nodes. + + Each instance of a concrete class has one attribute for each child node, + of the type as defined in the grammar. For example, :class:`ast.BinOp` + instances have an attribute :attr:`left` of type :class:`ast.expr`. + + If these attributes are marked as optional in the grammar (using a + question mark), the value might be ``None``. If the attributes can have + zero-or-more values (marked with an asterisk), the values are represented + as Python lists. All possible attributes must be present and have valid + values when compiling an AST with :func:`compile`. + + .. attribute:: lineno + col_offset + end_lineno + end_col_offset + + Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have + :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and + :attr:`end_col_offset` attributes. The :attr:`lineno` and :attr:`end_lineno` + are the first and last line numbers of source text span (1-indexed so the + first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset` + are the corresponding UTF-8 byte offsets of the first and last tokens that + generated the node. The UTF-8 offset is recorded because the parser uses + UTF-8 internally. + + Note that the end positions are not required by the compiler and are + therefore optional. The end offset is *after* the last symbol, for example + one can get the source segment of a one-line expression node using + ``source_line[node.col_offset : node.end_col_offset]``. + + The constructor of a class :class:`ast.T` parses its arguments as follows: + + * If there are positional arguments, there must be as many as there are items + in :attr:`T._fields`; they will be assigned as attributes of these names. + * If there are keyword arguments, they will set the attributes of the same + names to the given values. + + For example, to create and populate an :class:`ast.UnaryOp` node, you could + use :: + + node = ast.UnaryOp() + node.op = ast.USub() + node.operand = ast.Constant() + node.operand.value = 5 + node.operand.lineno = 0 + node.operand.col_offset = 0 + node.lineno = 0 + node.col_offset = 0 + + or the more compact :: + + node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), + lineno=0, col_offset=0) + +.. versionchanged:: 3.8 + + Class :class:`ast.Constant` is now used for all constants. + +.. versionchanged:: 3.9 + + Simple indices are represented by their value, extended slices are + represented as tuples. + +.. deprecated:: 3.8 + + Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, + :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, + but they will be removed in future Python releases. In the meantime, + instantiating them will return an instance of a different class. + +.. deprecated:: 3.9 + + Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still + available, but they will be removed in future Python releases. + In the meantime, instantiating them will return an instance of + a different class. + +.. note:: + The descriptions of the specific node classes displayed here + were initially adapted from the fantastic `Green Tree + Snakes `__ project and + all its contributors. + +Literals +^^^^^^^^ + +.. class:: Constant(value) + + A constant value. The ``value`` attribute of the ``Constant`` literal contains the + Python object it represents. The values represented can be simple types + such as a number, string or ``None``, but also immutable container types + (tuples and frozensets) if all of their elements are constant. + + .. doctest:: + + >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) + Expression( + body=Constant(value=123)) + + +.. class:: FormattedValue(value, conversion, format_spec) + + Node representing a single formatting field in an f-string. If the string + contains a single formatting field and nothing else the node can be + isolated otherwise it appears in :class:`JoinedStr`. + + * ``value`` is any expression node (such as a literal, a variable, or a + function call). + * ``conversion`` is an integer: + + * -1: no formatting + * 115: ``!s`` string formatting + * 114: ``!r`` repr formatting + * 97: ``!a`` ascii formatting + + * ``format_spec`` is a :class:`JoinedStr` node representing the formatting + of the value, or ``None`` if no format was specified. Both + ``conversion`` and ``format_spec`` can be set at the same time. + + +.. class:: JoinedStr(values) + + An f-string, comprising a series of :class:`FormattedValue` and :class:`Constant` + nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('f"sin({a}) is {sin(a):.3}"', mode='eval'), indent=4)) + Expression( + body=JoinedStr( + values=[ + Constant(value='sin('), + FormattedValue( + value=Name(id='a', ctx=Load()), + conversion=-1), + Constant(value=') is '), + FormattedValue( + value=Call( + func=Name(id='sin', ctx=Load()), + args=[ + Name(id='a', ctx=Load())], + keywords=[]), + conversion=-1, + format_spec=JoinedStr( + values=[ + Constant(value='.3')]))])) + + +.. class:: List(elts, ctx) + Tuple(elts, ctx) + + A list or tuple. ``elts`` holds a list of nodes representing the elements. + ``ctx`` is :class:`Store` if the container is an assignment target (i.e. + ``(x,y)=something``), and :class:`Load` otherwise. + + .. doctest:: + + >>> print(ast.dump(ast.parse('[1, 2, 3]', mode='eval'), indent=4)) + Expression( + body=List( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + ctx=Load())) + >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) + Expression( + body=Tuple( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)], + ctx=Load())) + + +.. class:: Set(elts) + + A set. ``elts`` holds a list of nodes representing the set's elements. + + .. doctest:: + + >>> print(ast.dump(ast.parse('{1, 2, 3}', mode='eval'), indent=4)) + Expression( + body=Set( + elts=[ + Constant(value=1), + Constant(value=2), + Constant(value=3)])) + + +.. class:: Dict(keys, values) + + A dictionary. ``keys`` and ``values`` hold lists of nodes representing the + keys and the values respectively, in matching order (what would be returned + when calling :code:`dictionary.keys()` and :code:`dictionary.values()`). + + When doing dictionary unpacking using dictionary literals the expression to be + expanded goes in the ``values`` list, with a ``None`` at the corresponding + position in ``keys``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('{"a":1, **d}', mode='eval'), indent=4)) + Expression( + body=Dict( + keys=[ + Constant(value='a'), + None], + values=[ + Constant(value=1), + Name(id='d', ctx=Load())])) + + +Variables +^^^^^^^^^ + +.. class:: Name(id, ctx) + + A variable name. ``id`` holds the name as a string, and ``ctx`` is one of + the following types. + + +.. class:: Load() + Store() + Del() + + Variable references can be used to load the value of a variable, to assign + a new value to it, or to delete it. Variable references are given a context + to distinguish these cases. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a'), indent=4)) + Module( + body=[ + Expr( + value=Name(id='a', ctx=Load()))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a = 1'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Name(id='a', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('del a'), indent=4)) + Module( + body=[ + Delete( + targets=[ + Name(id='a', ctx=Del())])], + type_ignores=[]) + + +.. class:: Starred(value, ctx) + + A ``*var`` variable reference. ``value`` holds the variable, typically a + :class:`Name` node. This type must be used when building a :class:`Call` + node with ``*args``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a, *b = it'), indent=4)) + Module( + body=[ + Assign( + targets=[ + Tuple( + elts=[ + Name(id='a', ctx=Store()), + Starred( + value=Name(id='b', ctx=Store()), + ctx=Store())], + ctx=Store())], + value=Name(id='it', ctx=Load()))], + type_ignores=[]) + + +Expressions +^^^^^^^^^^^ + +.. class:: Expr(value) + + When an expression, such as a function call, appears as a statement by itself + with its return value not used or stored, it is wrapped in this container. + ``value`` holds one of the other nodes in this section, a :class:`Constant`, a + :class:`Name`, a :class:`Lambda`, a :class:`Yield` or :class:`YieldFrom` node. + + .. doctest:: + + >>> print(ast.dump(ast.parse('-a'), indent=4)) + Module( + body=[ + Expr( + value=UnaryOp( + op=USub(), + operand=Name(id='a', ctx=Load())))], + type_ignores=[]) + + +.. class:: UnaryOp(op, operand) + + A unary operation. ``op`` is the operator, and ``operand`` any expression + node. + + +.. class:: UAdd + USub + Not + Invert + + Unary operator tokens. :class:`Not` is the ``not`` keyword, :class:`Invert` + is the ``~`` operator. + + .. doctest:: + + >>> print(ast.dump(ast.parse('not x', mode='eval'), indent=4)) + Expression( + body=UnaryOp( + op=Not(), + operand=Name(id='x', ctx=Load()))) + + +.. class:: BinOp(left, op, right) + + A binary operation (like addition or division). ``op`` is the operator, and + ``left`` and ``right`` are any expression nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) + Expression( + body=BinOp( + left=Name(id='x', ctx=Load()), + op=Add(), + right=Name(id='y', ctx=Load()))) + + +.. class:: Add + Sub + Mult + Div + FloorDiv + Mod + Pow + LShift + RShift + BitOr + BitXor + BitAnd + MatMult + + Binary operator tokens. + + +.. class:: BoolOp(op, values) + + A boolean operation, 'or' or 'and'. ``op`` is :class:`Or` or :class:`And`. + ``values`` are the values involved. Consecutive operations with the same + operator, such as ``a or b or c``, are collapsed into one node with several + values. + + This doesn't include ``not``, which is a :class:`UnaryOp`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x or y', mode='eval'), indent=4)) + Expression( + body=BoolOp( + op=Or(), + values=[ + Name(id='x', ctx=Load()), + Name(id='y', ctx=Load())])) + + +.. class:: And + Or + + Boolean operator tokens. + + +.. class:: Compare(left, ops, comparators) + + A comparison of two or more values. ``left`` is the first value in the + comparison, ``ops`` the list of operators, and ``comparators`` the list + of values after the first element in the comparison. + + .. doctest:: + + >>> print(ast.dump(ast.parse('1 <= a < 10', mode='eval'), indent=4)) + Expression( + body=Compare( + left=Constant(value=1), + ops=[ + LtE(), + Lt()], + comparators=[ + Name(id='a', ctx=Load()), + Constant(value=10)])) + + +.. class:: Eq + NotEq + Lt + LtE + Gt + GtE + Is + IsNot + In + NotIn + + Comparison operator tokens. + + +.. class:: Call(func, args, keywords, starargs, kwargs) + + A function call. ``func`` is the function, which will often be a + :class:`Name` or :class:`Attribute` object. Of the arguments: + + * ``args`` holds a list of the arguments passed by position. + * ``keywords`` holds a list of :class:`keyword` objects representing + arguments passed by keyword. + + When creating a ``Call`` node, ``args`` and ``keywords`` are required, but + they can be empty lists. ``starargs`` and ``kwargs`` are optional. + + .. doctest:: + + >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) + Expression( + body=Call( + func=Name(id='func', ctx=Load()), + args=[ + Name(id='a', ctx=Load()), + Starred( + value=Name(id='d', ctx=Load()), + ctx=Load())], + keywords=[ + keyword( + arg='b', + value=Name(id='c', ctx=Load())), + keyword( + value=Name(id='e', ctx=Load()))])) + + +.. class:: keyword(arg, value) + + A keyword argument to a function call or class definition. ``arg`` is a raw + string of the parameter name, ``value`` is a node to pass in. + + +.. class:: IfExp(test, body, orelse) + + An expression such as ``a if b else c``. Each field holds a single node, so + in the following example, all three are :class:`Name` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) + Expression( + body=IfExp( + test=Name(id='b', ctx=Load()), + body=Name(id='a', ctx=Load()), + orelse=Name(id='c', ctx=Load()))) + + +.. class:: Attribute(value, attr, ctx) + + Attribute access, e.g. ``d.keys``. ``value`` is a node, typically a + :class:`Name`. ``attr`` is a bare string giving the name of the attribute, + and ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` according to how + the attribute is acted on. + + .. doctest:: + + >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) + Expression( + body=Attribute( + value=Name(id='snake', ctx=Load()), + attr='colour', + ctx=Load())) + + +.. class:: NamedExpr(target, value) + + A named expression. This AST node is produced by the assignment expressions + operator (also known as the walrus operator). As opposed to the :class:`Assign` + node in which the first argument can be multiple nodes, in this case both + ``target`` and ``value`` must be single nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('(x := 4)', mode='eval'), indent=4)) + Expression( + body=NamedExpr( + target=Name(id='x', ctx=Store()), + value=Constant(value=4))) + + +Subscripting +~~~~~~~~~~~~ + +.. class:: Subscript(value, slice, ctx) + + A subscript, such as ``l[1]``. ``value`` is the subscripted object + (usually sequence or mapping). ``slice`` is an index, slice or key. + It can be a :class:`Tuple` and contain a :class:`Slice`. + ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` + according to the action performed with the subscript. + + .. doctest:: + + >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) + Expression( + body=Subscript( + value=Name(id='l', ctx=Load()), + slice=Tuple( + elts=[ + Slice( + lower=Constant(value=1), + upper=Constant(value=2)), + Constant(value=3)], + ctx=Load()), + ctx=Load())) + + +.. class:: Slice(lower, upper, step) + + Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``). + Can occur only inside the *slice* field of :class:`Subscript`, either + directly or as an element of :class:`Tuple`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) + Expression( + body=Subscript( + value=Name(id='l', ctx=Load()), + slice=Slice( + lower=Constant(value=1), + upper=Constant(value=2)), + ctx=Load())) + + +Comprehensions +~~~~~~~~~~~~~~ + +.. class:: ListComp(elt, generators) + SetComp(elt, generators) + GeneratorExp(elt, generators) + DictComp(key, value, generators) + + List and set comprehensions, generator expressions, and dictionary + comprehensions. ``elt`` (or ``key`` and ``value``) is a single node + representing the part that will be evaluated for each item. + + ``generators`` is a list of :class:`comprehension` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) + Expression( + body=ListComp( + elt=Name(id='x', ctx=Load()), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) + Expression( + body=DictComp( + key=Name(id='x', ctx=Load()), + value=BinOp( + left=Name(id='x', ctx=Load()), + op=Pow(), + right=Constant(value=2)), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) + Expression( + body=SetComp( + elt=Name(id='x', ctx=Load()), + generators=[ + comprehension( + target=Name(id='x', ctx=Store()), + iter=Name(id='numbers', ctx=Load()), + ifs=[], + is_async=0)])) + + +.. class:: comprehension(target, iter, ifs, is_async) + + One ``for`` clause in a comprehension. ``target`` is the reference to use for + each element - typically a :class:`Name` or :class:`Tuple` node. ``iter`` + is the object to iterate over. ``ifs`` is a list of test expressions: each + ``for`` clause can have multiple ``ifs``. + + ``is_async`` indicates a comprehension is asynchronous (using an + ``async for`` instead of ``for``). The value is an integer (0 or 1). + + .. doctest:: + + >>> print(ast.dump(ast.parse('[ord(c) for line in file for c in line]', mode='eval'), + ... indent=4)) # Multiple comprehensions in one. + Expression( + body=ListComp( + elt=Call( + func=Name(id='ord', ctx=Load()), + args=[ + Name(id='c', ctx=Load())], + keywords=[]), + generators=[ + comprehension( + target=Name(id='line', ctx=Store()), + iter=Name(id='file', ctx=Load()), + ifs=[], + is_async=0), + comprehension( + target=Name(id='c', ctx=Store()), + iter=Name(id='line', ctx=Load()), + ifs=[], + is_async=0)])) + + >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), + ... indent=4)) # generator comprehension + Expression( + body=GeneratorExp( + elt=BinOp( + left=Name(id='n', ctx=Load()), + op=Pow(), + right=Constant(value=2)), + generators=[ + comprehension( + target=Name(id='n', ctx=Store()), + iter=Name(id='it', ctx=Load()), + ifs=[ + Compare( + left=Name(id='n', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=5)]), + Compare( + left=Name(id='n', ctx=Load()), + ops=[ + Lt()], + comparators=[ + Constant(value=10)])], + is_async=0)])) + + >>> print(ast.dump(ast.parse('[i async for i in soc]', mode='eval'), + ... indent=4)) # Async comprehension + Expression( + body=ListComp( + elt=Name(id='i', ctx=Load()), + generators=[ + comprehension( + target=Name(id='i', ctx=Store()), + iter=Name(id='soc', ctx=Load()), + ifs=[], + is_async=1)])) + +Statements +^^^^^^^^^^ + +.. class:: Assign(targets, value, type_comment) + + An assignment. ``targets`` is a list of nodes, and ``value`` is a single node. + + Multiple nodes in ``targets`` represents assigning the same value to each. + Unpacking is represented by putting a :class:`Tuple` or :class:`List` + within ``targets``. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + .. doctest:: + + >>> print(ast.dump(ast.parse('a = b = 1'), indent=4)) # Multiple assignment + Module( + body=[ + Assign( + targets=[ + Name(id='a', ctx=Store()), + Name(id='b', ctx=Store())], + value=Constant(value=1))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking + Module( + body=[ + Assign( + targets=[ + Tuple( + elts=[ + Name(id='a', ctx=Store()), + Name(id='b', ctx=Store())], + ctx=Store())], + value=Name(id='c', ctx=Load()))], + type_ignores=[]) + + +.. class:: AnnAssign(target, annotation, value, simple) + + An assignment with a type annotation. ``target`` is a single node and can + be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. + ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` + node. ``value`` is a single optional node. ``simple`` is a boolean integer + set to True for a :class:`Name` node in ``target`` that do not appear in + between parenthesis and are hence pure names and not expressions. + + .. doctest:: + + >>> print(ast.dump(ast.parse('c: int'), indent=4)) + Module( + body=[ + AnnAssign( + target=Name(id='c', ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=1)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis + Module( + body=[ + AnnAssign( + target=Name(id='a', ctx=Store()), + annotation=Name(id='int', ctx=Load()), + value=Constant(value=1), + simple=0)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation + Module( + body=[ + AnnAssign( + target=Attribute( + value=Name(id='a', ctx=Load()), + attr='b', + ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=0)], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation + Module( + body=[ + AnnAssign( + target=Subscript( + value=Name(id='a', ctx=Load()), + slice=Constant(value=1), + ctx=Store()), + annotation=Name(id='int', ctx=Load()), + simple=0)], + type_ignores=[]) + + +.. class:: AugAssign(target, op, value) + + Augmented assignment, such as ``a += 1``. In the following example, + ``target`` is a :class:`Name` node for ``x`` (with the :class:`Store` + context), ``op`` is :class:`Add`, and ``value`` is a :class:`Constant` with + value for 1. + + The ``target`` attribute connot be of class :class:`Tuple` or :class:`List`, + unlike the targets of :class:`Assign`. + + .. doctest:: + + >>> print(ast.dump(ast.parse('x += 2'), indent=4)) + Module( + body=[ + AugAssign( + target=Name(id='x', ctx=Store()), + op=Add(), + value=Constant(value=2))], + type_ignores=[]) + + +.. class:: Raise(exc, cause) + + A ``raise`` statement. ``exc`` is the exception object to be raised, normally a + :class:`Call` or :class:`Name`, or ``None`` for a standalone ``raise``. + ``cause`` is the optional part for ``y`` in ``raise x from y``. + + .. doctest:: + + >>> print(ast.dump(ast.parse('raise x from y'), indent=4)) + Module( + body=[ + Raise( + exc=Name(id='x', ctx=Load()), + cause=Name(id='y', ctx=Load()))], + type_ignores=[]) + + +.. class:: Assert(test, msg) + + An assertion. ``test`` holds the condition, such as a :class:`Compare` node. + ``msg`` holds the failure message. + + .. doctest:: + + >>> print(ast.dump(ast.parse('assert x,y'), indent=4)) + Module( + body=[ + Assert( + test=Name(id='x', ctx=Load()), + msg=Name(id='y', ctx=Load()))], + type_ignores=[]) + + +.. class:: Delete(targets) + + Represents a ``del`` statement. ``targets`` is a list of nodes, such as + :class:`Name`, :class:`Attribute` or :class:`Subscript` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('del x,y,z'), indent=4)) + Module( + body=[ + Delete( + targets=[ + Name(id='x', ctx=Del()), + Name(id='y', ctx=Del()), + Name(id='z', ctx=Del())])], + type_ignores=[]) + + +.. class:: Pass() + + A ``pass`` statement. + + .. doctest:: + + >>> print(ast.dump(ast.parse('pass'), indent=4)) + Module( + body=[ + Pass()], + type_ignores=[]) + + +Other statements which are only applicable inside functions or loops are +described in other sections. + +Imports +~~~~~~~ + +.. class:: Import(names) + + An import statement. ``names`` is a list of :class:`alias` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse('import x,y,z'), indent=4)) + Module( + body=[ + Import( + names=[ + alias(name='x'), + alias(name='y'), + alias(name='z')])], + type_ignores=[]) + + +.. class:: ImportFrom(module, names, level) + + Represents ``from x import y``. ``module`` is a raw string of the 'from' name, + without any leading dots, or ``None`` for statements such as ``from . import foo``. + ``level`` is an integer holding the level of the relative import (0 means + absolute import). + + .. doctest:: + + >>> print(ast.dump(ast.parse('from y import x,y,z'), indent=4)) + Module( + body=[ + ImportFrom( + module='y', + names=[ + alias(name='x'), + alias(name='y'), + alias(name='z')], + level=0)], + type_ignores=[]) + + +.. class:: alias(name, asname) + + Both parameters are raw strings of the names. ``asname`` can be ``None`` if + the regular name is to be used. + + .. doctest:: + + >>> print(ast.dump(ast.parse('from ..foo.bar import a as b, c'), indent=4)) + Module( + body=[ + ImportFrom( + module='foo.bar', + names=[ + alias(name='a', asname='b'), + alias(name='c')], + level=2)], + type_ignores=[]) + +Control flow +^^^^^^^^^^^^ + +.. note:: + Optional clauses such as ``else`` are stored as an empty list if they're + not present. + +.. class:: If(test, body, orelse) + + An ``if`` statement. ``test`` holds a single node, such as a :class:`Compare` + node. ``body`` and ``orelse`` each hold a list of nodes. + + ``elif`` clauses don't have a special representation in the AST, but rather + appear as extra :class:`If` nodes within the ``orelse`` section of the + previous one. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... if x: + ... ... + ... elif y: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + If( + test=Name(id='x', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + If( + test=Name(id='y', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +.. class:: For(target, iter, body, orelse, type_comment) + + A ``for`` loop. ``target`` holds the variable(s) the loop assigns to, as a + single :class:`Name`, :class:`Tuple` or :class:`List` node. ``iter`` holds + the item to be looped over, again as a single node. ``body`` and ``orelse`` + contain lists of nodes to execute. Those in ``orelse`` are executed if the + loop finishes normally, rather than via a ``break`` statement. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... for x in y: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + For( + target=Name(id='x', ctx=Store()), + iter=Name(id='y', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: While(test, body, orelse) + + A ``while`` loop. ``test`` holds the condition, such as a :class:`Compare` + node. + + .. doctest:: + + >> print(ast.dump(ast.parse(""" + ... while x: + ... ... + ... else: + ... ... + ... """), indent=4)) + Module( + body=[ + While( + test=Name(id='x', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))], + orelse=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: Break + Continue + + The ``break`` and ``continue`` statements. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... for a in b: + ... if a > 5: + ... break + ... else: + ... continue + ... + ... """), indent=4)) + Module( + body=[ + For( + target=Name(id='a', ctx=Store()), + iter=Name(id='b', ctx=Load()), + body=[ + If( + test=Compare( + left=Name(id='a', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=5)]), + body=[ + Break()], + orelse=[ + Continue()])], + orelse=[])], + type_ignores=[]) + + +.. class:: Try(body, handlers, orelse, finalbody) + + ``try`` blocks. All attributes are list of nodes to execute, except for + ``handlers``, which is a list of :class:`ExceptHandler` nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... try: + ... ... + ... except Exception: + ... ... + ... except OtherException as e: + ... ... + ... else: + ... ... + ... finally: + ... ... + ... """), indent=4)) + Module( + body=[ + Try( + body=[ + Expr( + value=Constant(value=Ellipsis))], + handlers=[ + ExceptHandler( + type=Name(id='Exception', ctx=Load()), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + ExceptHandler( + type=Name(id='OtherException', ctx=Load()), + name='e', + body=[ + Expr( + value=Constant(value=Ellipsis))])], + orelse=[ + Expr( + value=Constant(value=Ellipsis))], + finalbody=[ + Expr( + value=Constant(value=Ellipsis))])], + type_ignores=[]) + + +.. class:: ExceptHandler(type, name, body) + + A single ``except`` clause. ``type`` is the exception type it will match, + typically a :class:`Name` node (or ``None`` for a catch-all ``except:`` clause). + ``name`` is a raw string for the name to hold the exception, or ``None`` if + the clause doesn't have ``as foo``. ``body`` is a list of nodes. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... try: + ... a + 1 + ... except TypeError: + ... pass + ... """), indent=4)) + Module( + body=[ + Try( + body=[ + Expr( + value=BinOp( + left=Name(id='a', ctx=Load()), + op=Add(), + right=Constant(value=1)))], + handlers=[ + ExceptHandler( + type=Name(id='TypeError', ctx=Load()), + body=[ + Pass()])], + orelse=[], + finalbody=[])], + type_ignores=[]) + + +.. class:: With(items, body, type_comment) + + A ``with`` block. ``items`` is a list of :class:`withitem` nodes representing + the context managers, and ``body`` is the indented block inside the context. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + +.. class:: withitem(context_expr, optional_vars) + + A single context manager in a ``with`` block. ``context_expr`` is the context + manager, often a :class:`Call` node. ``optional_vars`` is a :class:`Name`, + :class:`Tuple` or :class:`List` for the ``as foo`` part, or ``None`` if that + isn't used. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... with a as b, c as d: + ... something(b, d) + ... """), indent=4)) + Module( + body=[ + With( + items=[ + withitem( + context_expr=Name(id='a', ctx=Load()), + optional_vars=Name(id='b', ctx=Store())), + withitem( + context_expr=Name(id='c', ctx=Load()), + optional_vars=Name(id='d', ctx=Store()))], + body=[ + Expr( + value=Call( + func=Name(id='something', ctx=Load()), + args=[ + Name(id='b', ctx=Load()), + Name(id='d', ctx=Load())], + keywords=[]))])], + type_ignores=[]) + + +.. class:: Match(subject, cases) + + A ``match`` statement. ``subject`` holds the subject of the match (the object + that is being matched against the cases) and ``cases`` contains an iterable of + :class:`match_case` nodes with the different cases. + + +.. class:: match_case(context_expr, optional_vars) + + A single case pattern in a ``match`` statement. ``pattern`` contains the + match pattern that will be used to match the subject against. Notice that + the meaning of the :class:`AST` nodes in this attribute have a different + meaning than in other places, as they represent patterns to match against. + The ``guard`` attribute contains an expression that will be evaluated if + the pattern matches the subject. If the pattern matches and the ``guard`` condition + is truthy, the body of the case shall be executed. ``body`` contains a list + of nodes to execute if the guard is truthy. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] if x>0: + ... ... + ... case tuple(): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=List( + elts=[ + Name(id='x', ctx=Store())], + ctx=Load()), + guard=Compare( + left=Name(id='x', ctx=Load()), + ops=[ + Gt()], + comparators=[ + Constant(value=0)]), + body=[ + Expr( + value=Constant(value=Ellipsis))]), + match_case( + pattern=Call( + func=Name(id='tuple', ctx=Load()), + args=[], + keywords=[]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + +.. class:: MatchAs(pattern, name) + + A match "as-pattern". The as-pattern matches whatever pattern is on its + left-hand side, but also binds the value to a name. ``pattern`` contains + the match pattern that will be used to match the subject agsinst. The ``name`` + attribute contains the name that will be binded if the pattern is successful. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] as y: + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchAs( + pattern=List( + elts=[ + Name(id='x', ctx=Store())], + ctx=Load()), + name='y'), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +.. class:: MatchOr(patterns) + + A match "or-pattern". An or-pattern matches each of its subpatterns in turn + to the subject, until one succeeds. The or-pattern is then deemed to + succeed. If none of the subpatterns succeed the or-pattern fails. The + ``patterns`` attribute contains a list of match patterns nodes that will be + matched against the subject. + + .. doctest:: + + >>> print(ast.dump(ast.parse(""" + ... match x: + ... case [x] | (y): + ... ... + ... """), indent=4)) + Module( + body=[ + Match( + subject=Name(id='x', ctx=Load()), + cases=[ + match_case( + pattern=MatchOr( + patterns=[ + List( + elts=[ + Name(id='x', ctx=Store())], + ctx=Load()), + Name(id='y', ctx=Store())]), + body=[ + Expr( + value=Constant(value=Ellipsis))])])], + type_ignores=[]) + + +Function and class definitions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment) + + A function definition. + + * ``name`` is a raw string of the function name. + * ``args`` is a :class:`arguments` node. + * ``body`` is the list of nodes inside the function. + * ``decorator_list`` is the list of decorators to be applied, stored outermost + first (i.e. the first in the list will be applied last). + * ``returns`` is the return annotation. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment. + + +.. class:: Lambda(args, body) + + ``lambda`` is a minimal function definition that can be used inside an + expression. Unlike :class:`FunctionDef`, ``body`` holds a single node. + + .. doctest:: + + >>> print(ast.dump(ast.parse('lambda x,y: ...'), indent=4)) + Module( + body=[ + Expr( + value=Lambda( + args=arguments( + posonlyargs=[], + args=[ + arg(arg='x'), + arg(arg='y')], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=Constant(value=Ellipsis)))], + type_ignores=[]) + + +.. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) + + The arguments for a function. + + * ``posonlyargs``, ``args`` and ``kwonlyargs`` are lists of :class:`arg` nodes. + * ``vararg`` and ``kwarg`` are single :class:`arg` nodes, referring to the + ``*args, **kwargs`` parameters. + * ``kw_defaults`` is a list of default values for keyword-only arguments. If + one is ``None``, the corresponding argument is required. + * ``defaults`` is a list of default values for arguments that can be passed + positionally. If there are fewer defaults, they correspond to the last n + arguments. + + +.. class:: arg(arg, annotation, type_comment) + + A single argument in a list. ``arg`` is a raw string of the argument + name, ``annotation`` is its annotation, such as a :class:`Str` or + :class:`Name` node. + + .. attribute:: type_comment + + ``type_comment`` is an optional string with the type annotation as a comment + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... @decorator1 + ... @decorator2 + ... def f(a: 'annotation', b=1, c=2, *d, e, f=3, **g) -> 'return annotation': + ... pass + ... """), indent=4)) + Module( + body=[ + FunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[ + arg( + arg='a', + annotation=Constant(value='annotation')), + arg(arg='b'), + arg(arg='c')], + vararg=arg(arg='d'), + kwonlyargs=[ + arg(arg='e'), + arg(arg='f')], + kw_defaults=[ + None, + Constant(value=3)], + kwarg=arg(arg='g'), + defaults=[ + Constant(value=1), + Constant(value=2)]), + body=[ + Pass()], + decorator_list=[ + Name(id='decorator1', ctx=Load()), + Name(id='decorator2', ctx=Load())], + returns=Constant(value='return annotation'))], + type_ignores=[]) + + +.. class:: Return(value) + + A ``return`` statement. + + .. doctest:: + + >>> print(ast.dump(ast.parse('return 4'), indent=4)) + Module( + body=[ + Return( + value=Constant(value=4))], + type_ignores=[]) + + +.. class:: Yield(value) + YieldFrom(value) + + A ``yield`` or ``yield from`` expression. Because these are expressions, they + must be wrapped in a :class:`Expr` node if the value sent back is not used. + + .. doctest:: + + >>> print(ast.dump(ast.parse('yield x'), indent=4)) + Module( + body=[ + Expr( + value=Yield( + value=Name(id='x', ctx=Load())))], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('yield from x'), indent=4)) + Module( + body=[ + Expr( + value=YieldFrom( + value=Name(id='x', ctx=Load())))], + type_ignores=[]) + + +.. class:: Global(names) + Nonlocal(names) + + ``global`` and ``nonlocal`` statements. ``names`` is a list of raw strings. + + .. doctest:: + + >>> print(ast.dump(ast.parse('global x,y,z'), indent=4)) + Module( + body=[ + Global( + names=[ + 'x', + 'y', + 'z'])], + type_ignores=[]) + + >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) + Module( + body=[ + Nonlocal( + names=[ + 'x', + 'y', + 'z'])], + type_ignores=[]) + + +.. class:: ClassDef(name, bases, keywords, starargs, kwargs, body, decorator_list) + + A class definition. + + * ``name`` is a raw string for the class name + * ``bases`` is a list of nodes for explicitly specified base classes. + * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. + Other keywords will be passed to the metaclass, as per `PEP-3115 + `_. + * ``starargs`` and ``kwargs`` are each a single node, as in a function call. + starargs will be expanded to join the list of base classes, and kwargs will + be passed to the metaclass. + * ``body`` is a list of nodes representing the code within the class + definition. + * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. + + .. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... @decorator1 + ... @decorator2 + ... class Foo(base1, base2, metaclass=meta): + ... pass + ... """), indent=4)) + Module( + body=[ + ClassDef( + name='Foo', + bases=[ + Name(id='base1', ctx=Load()), + Name(id='base2', ctx=Load())], + keywords=[ + keyword( + arg='metaclass', + value=Name(id='meta', ctx=Load()))], + body=[ + Pass()], + decorator_list=[ + Name(id='decorator1', ctx=Load()), + Name(id='decorator2', ctx=Load())])], + type_ignores=[]) + +Async and await +^^^^^^^^^^^^^^^ + +.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment) + + An ``async def`` function definition. Has the same fields as + :class:`FunctionDef`. + + +.. class:: Await(value) + + An ``await`` expression. ``value`` is what it waits for. + Only valid in the body of an :class:`AsyncFunctionDef`. + +.. doctest:: + + >>> print(ast.dump(ast.parse("""\ + ... async def f(): + ... await other_func() + ... """), indent=4)) + Module( + body=[ + AsyncFunctionDef( + name='f', + args=arguments( + posonlyargs=[], + args=[], + kwonlyargs=[], + kw_defaults=[], + defaults=[]), + body=[ + Expr( + value=Await( + value=Call( + func=Name(id='other_func', ctx=Load()), + args=[], + keywords=[])))], + decorator_list=[])], + type_ignores=[]) + + +.. class:: AsyncFor(target, iter, body, orelse, type_comment) + AsyncWith(items, body, type_comment) + + ``async for`` loops and ``async with`` context managers. They have the same + fields as :class:`For` and :class:`With`, respectively. Only valid in the + body of an :class:`AsyncFunctionDef`. + +.. note:: + When a string is parsed by :func:`ast.parse`, operator nodes (subclasses + of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, + :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree + will be singletons. Changes to one will be reflected in all other + occurrences of the same value (e.g. :class:`ast.Add`). + + +:mod:`ast` Helpers +------------------ + +Apart from the node classes, the :mod:`ast` module defines these utility functions +and classes for traversing abstract syntax trees: + +.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) + + Parse the source into an AST node. Equivalent to ``compile(source, + filename, mode, ast.PyCF_ONLY_AST)``. + + If ``type_comments=True`` is given, the parser is modified to check + and return type comments as specified by :pep:`484` and :pep:`526`. + This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the + flags passed to :func:`compile()`. This will report syntax errors + for misplaced type comments. Without this flag, type comments will + be ignored, and the ``type_comment`` field on selected AST nodes + will always be ``None``. In addition, the locations of ``# type: + ignore`` comments will be returned as the ``type_ignores`` + attribute of :class:`Module` (otherwise it is always an empty list). + + In addition, if ``mode`` is ``'func_type'``, the input syntax is + modified to correspond to :pep:`484` "signature type comments", + e.g. ``(str, int) -> List[str]``. + + Also, setting ``feature_version`` to a tuple ``(major, minor)`` + will attempt to parse using that Python version's grammar. + Currently ``major`` must equal to ``3``. For example, setting + ``feature_version=(3, 4)`` will allow the use of ``async`` and + ``await`` as variable names. The lowest supported version is + ``(3, 4)``; the highest is ``sys.version_info[0:2]``. + + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + + .. versionchanged:: 3.8 + Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. + + +.. function:: unparse(ast_obj) + + Unparse an :class:`ast.AST` object and generate a string with code + that would produce an equivalent :class:`ast.AST` object if parsed + back with :func:`ast.parse`. + + .. warning:: + The produced code string will not necessarily be equal to the original + code that generated the :class:`ast.AST` object (without any compiler + optimizations, such as constant tuples/frozensets). + + .. warning:: + Trying to unparse a highly complex expression would result with + :exc:`RecursionError`. + + .. versionadded:: 3.9 + + +.. function:: literal_eval(node_or_string) + + Safely evaluate an expression node or a string containing a Python literal or + container display. The string or node provided may only consist of the + following Python literal structures: strings, bytes, numbers, tuples, lists, + dicts, sets, booleans, ``None`` and ``Ellipsis``. + + This can be used for safely evaluating strings containing Python values from + untrusted sources without the need to parse the values oneself. It is not + capable of evaluating arbitrarily complex expressions, for example involving + operators or indexing. + + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + + It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, + :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed + input. + + .. versionchanged:: 3.2 + Now allows bytes and set literals. + + .. versionchanged:: 3.9 + Now supports creating empty sets with ``'set()'``. + + .. versionchanged:: 3.10 + For string inputs, leading spaces and tabs are now stripped. + + +.. function:: get_docstring(node, clean=True) + + Return the docstring of the given *node* (which must be a + :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`, + or :class:`Module` node), or ``None`` if it has no docstring. + If *clean* is true, clean up the docstring's indentation with + :func:`inspect.cleandoc`. + + .. versionchanged:: 3.5 + :class:`AsyncFunctionDef` is now supported. + + +.. function:: get_source_segment(source, node, *, padded=False) + + Get source code segment of the *source* that generated *node*. + If some location information (:attr:`lineno`, :attr:`end_lineno`, + :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. + + If *padded* is ``True``, the first line of a multi-line statement will + be padded with spaces to match its original position. + + .. versionadded:: 3.8 + + +.. function:: fix_missing_locations(node) + + When you compile a node tree with :func:`compile`, the compiler expects + :attr:`lineno` and :attr:`col_offset` attributes for every node that supports + them. This is rather tedious to fill in for generated nodes, so this helper + adds these attributes recursively where not already set, by setting them to + the values of the parent node. It works recursively starting at *node*. + + +.. function:: increment_lineno(node, n=1) + + Increment the line number and end line number of each node in the tree + starting at *node* by *n*. This is useful to "move code" to a different + location in a file. + + +.. function:: copy_location(new_node, old_node) + + Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, + and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, + and return *new_node*. + + +.. function:: iter_fields(node) + + Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` + that is present on *node*. + + +.. function:: iter_child_nodes(node) + + Yield all direct child nodes of *node*, that is, all fields that are nodes + and all items of fields that are lists of nodes. + + +.. function:: walk(node) + + Recursively yield all descendant nodes in the tree starting at *node* + (including *node* itself), in no specified order. This is useful if you only + want to modify nodes in place and don't care about the context. + + +.. class:: NodeVisitor() + + A node visitor base class that walks the abstract syntax tree and calls a + visitor function for every node found. This function may return a value + which is forwarded by the :meth:`visit` method. + + This class is meant to be subclassed, with the subclass adding visitor + methods. + + .. method:: visit(node) + + Visit a node. The default implementation calls the method called + :samp:`self.visit_{classname}` where *classname* is the name of the node + class, or :meth:`generic_visit` if that method doesn't exist. + + .. method:: generic_visit(node) + + This visitor calls :meth:`visit` on all children of the node. + + Note that child nodes of nodes that have a custom visitor method won't be + visited unless the visitor calls :meth:`generic_visit` or visits them + itself. + + Don't use the :class:`NodeVisitor` if you want to apply changes to nodes + during traversal. For this a special visitor exists + (:class:`NodeTransformer`) that allows modifications. + + .. deprecated:: 3.8 + + Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, + :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated + now and will not be called in future Python versions. Add the + :meth:`visit_Constant` method to handle all constant nodes. + + +.. class:: NodeTransformer() + + A :class:`NodeVisitor` subclass that walks the abstract syntax tree and + allows modification of nodes. + + The :class:`NodeTransformer` will walk the AST and use the return value of + the visitor methods to replace or remove the old node. If the return value + of the visitor method is ``None``, the node will be removed from its + location, otherwise it is replaced with the return value. The return value + may be the original node in which case no replacement takes place. + + Here is an example transformer that rewrites all occurrences of name lookups + (``foo``) to ``data['foo']``:: + + class RewriteName(NodeTransformer): + + def visit_Name(self, node): + return Subscript( + value=Name(id='data', ctx=Load()), + slice=Constant(value=node.id), + ctx=node.ctx + ) + + Keep in mind that if the node you're operating on has child nodes you must + either transform the child nodes yourself or call the :meth:`generic_visit` + method for the node first. + + For nodes that were part of a collection of statements (that applies to all + statement nodes), the visitor may also return a list of nodes rather than + just a single node. + + If :class:`NodeTransformer` introduces new nodes (that weren't part of + original tree) without giving them location information (such as + :attr:`lineno`), :func:`fix_missing_locations` should be called with + the new sub-tree to recalculate the location information:: + + tree = ast.parse('foo', mode='eval') + new_tree = fix_missing_locations(RewriteName().visit(tree)) + + Usually you use the transformer like this:: + + node = YourTransformer().visit(node) + + +.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) + + Return a formatted dump of the tree in *node*. This is mainly useful for + debugging purposes. If *annotate_fields* is true (by default), + the returned string will show the names and the values for fields. + If *annotate_fields* is false, the result string will be more compact by + omitting unambiguous field names. Attributes such as line + numbers and column offsets are not dumped by default. If this is wanted, + *include_attributes* can be set to true. + + If *indent* is a non-negative integer or string, then the tree will be + pretty-printed with that indent level. An indent level + of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) + selects the single line representation. Using a positive integer indent + indents that many spaces per level. If *indent* is a string (such as ``"\t"``), + that string is used to indent each level. + + .. versionchanged:: 3.9 + Added the *indent* option. + + +.. _ast-compiler-flags: + +Compiler Flags +-------------- + +The following flags may be passed to :func:`compile` in order to change +effects on the compilation of a program: + +.. data:: PyCF_ALLOW_TOP_LEVEL_AWAIT + + Enables support for top-level ``await``, ``async for``, ``async with`` + and async comprehensions. + + .. versionadded:: 3.8 + +.. data:: PyCF_ONLY_AST + + Generates and returns an abstract syntax tree instead of returning a + compiled code object. + +.. data:: PyCF_TYPE_COMMENTS + + Enables support for :pep:`484` and :pep:`526` style type comments + (``# type: ``, ``# type: ignore ``). + + .. versionadded:: 3.8 + + +.. _ast-cli: + +Command-Line Usage +------------------ + +.. versionadded:: 3.9 + +The :mod:`ast` module can be executed as a script from the command line. +It is as simple as: + +.. code-block:: sh + + python -m ast [-m ] [-a] [infile] + +The following options are accepted: + +.. program:: ast + +.. cmdoption:: -h, --help + + Show the help message and exit. + +.. cmdoption:: -m + --mode + + Specify what kind of code must be compiled, like the *mode* argument + in :func:`parse`. + +.. cmdoption:: --no-type-comments + + Don't parse type comments. + +.. cmdoption:: -a, --include-attributes + + Include attributes such as line numbers and column offsets. + +.. cmdoption:: -i + --indent + + Indentation of nodes in AST (number of spaces). + +If :file:`infile` is specified its contents are parsed to AST and dumped +to stdout. Otherwise, the content is read from stdin. + + +.. seealso:: + + `Green Tree Snakes `_, an external + documentation resource, has good details on working with Python ASTs. + + `ASTTokens `_ + annotates Python ASTs with the positions of tokens and text in the source + code that generated them. This is helpful for tools that make source code + transformations. + + `leoAst.py `_ unifies the + token-based and parse-tree-based views of python programs by inserting + two-way links between tokens and ast nodes. + + `LibCST `_ parses code as a Concrete Syntax + Tree that looks like an ast tree and keeps all formatting details. It's + useful for building automated refactoring (codemod) applications and + linters. + + `Parso `_ is a Python parser that supports + error recovery and round-trip parsing for different Python versions (in + multiple Python versions). Parso is also able to list multiple syntax errors + in your python file. diff --git a/Doc/library/asyncio-api-index.rst.bak b/Doc/library/asyncio-api-index.rst.bak new file mode 100644 index 00000000000000..047e5bbc58ccad --- /dev/null +++ b/Doc/library/asyncio-api-index.rst.bak @@ -0,0 +1,221 @@ +.. currentmodule:: asyncio + + +==================== +High-level API Index +==================== + +This page lists all high-level async/await enabled asyncio APIs. + + +Tasks +===== + +Utilities to run asyncio programs, create Tasks, and +await on multiple things with timeouts. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :func:`run` + - Create event loop, run a coroutine, close the loop. + + * - :func:`create_task` + - Start an asyncio Task. + + * - ``await`` :func:`sleep` + - Sleep for a number of seconds. + + * - ``await`` :func:`gather` + - Schedule and wait for things concurrently. + + * - ``await`` :func:`wait_for` + - Run with a timeout. + + * - ``await`` :func:`shield` + - Shield from cancellation. + + * - ``await`` :func:`wait` + - Monitor for completion. + + * - :func:`current_task` + - Return the current Task. + + * - :func:`all_tasks` + - Return all tasks for an event loop. + + * - :class:`Task` + - Task object. + + * - :func:`to_thread` + - Asychronously run a function in a separate OS thread. + + * - :func:`run_coroutine_threadsafe` + - Schedule a coroutine from another OS thread. + + * - ``for in`` :func:`as_completed` + - Monitor for completion with a ``for`` loop. + + +.. rubric:: Examples + +* :ref:`Using asyncio.gather() to run things in parallel + `. + +* :ref:`Using asyncio.wait_for() to enforce a timeout + `. + +* :ref:`Cancellation `. + +* :ref:`Using asyncio.sleep() `. + +* See also the main :ref:`Tasks documentation page `. + + +Queues +====== + +Queues should be used to distribute work amongst multiple asyncio Tasks, +implement connection pools, and pub/sub patterns. + + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :class:`Queue` + - A FIFO queue. + + * - :class:`PriorityQueue` + - A priority queue. + + * - :class:`LifoQueue` + - A LIFO queue. + + +.. rubric:: Examples + +* :ref:`Using asyncio.Queue to distribute workload between several + Tasks `. + +* See also the :ref:`Queues documentation page `. + + +Subprocesses +============ + +Utilities to spawn subprocesses and run shell commands. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :func:`create_subprocess_exec` + - Create a subprocess. + + * - ``await`` :func:`create_subprocess_shell` + - Run a shell command. + + +.. rubric:: Examples + +* :ref:`Executing a shell command `. + +* See also the :ref:`subprocess APIs ` + documentation. + + +Streams +======= + +High-level APIs to work with network IO. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - ``await`` :func:`open_connection` + - Establish a TCP connection. + + * - ``await`` :func:`open_unix_connection` + - Establish a Unix socket connection. + + * - ``await`` :func:`start_server` + - Start a TCP server. + + * - ``await`` :func:`start_unix_server` + - Start a Unix socket server. + + * - :class:`StreamReader` + - High-level async/await object to receive network data. + + * - :class:`StreamWriter` + - High-level async/await object to send network data. + + +.. rubric:: Examples + +* :ref:`Example TCP client `. + +* See also the :ref:`streams APIs ` + documentation. + + +Synchronization +=============== + +Threading-like synchronization primitives that can be used in Tasks. + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + * - :class:`Lock` + - A mutex lock. + + * - :class:`Event` + - An event object. + + * - :class:`Condition` + - A condition object. + + * - :class:`Semaphore` + - A semaphore. + + * - :class:`BoundedSemaphore` + - A bounded semaphore. + + +.. rubric:: Examples + +* :ref:`Using asyncio.Event `. + +* See also the documentation of asyncio + :ref:`synchronization primitives `. + + +Exceptions +========== + +.. list-table:: + :widths: 50 50 + :class: full-width-table + + + * - :exc:`asyncio.TimeoutError` + - Raised on timeout by functions like :func:`wait_for`. + Keep in mind that ``asyncio.TimeoutError`` is **unrelated** + to the built-in :exc:`TimeoutError` exception. + + * - :exc:`asyncio.CancelledError` + - Raised when a Task is cancelled. See also :meth:`Task.cancel`. + + +.. rubric:: Examples + +* :ref:`Handling CancelledError to run code on cancellation request + `. + +* See also the full list of + :ref:`asyncio-specific exceptions `. diff --git a/Doc/library/collections.rst.bak b/Doc/library/collections.rst.bak new file mode 100644 index 00000000000000..aa0acfaae45a5f --- /dev/null +++ b/Doc/library/collections.rst.bak @@ -0,0 +1,1278 @@ +:mod:`collections` --- Container datatypes +========================================== + +.. module:: collections + :synopsis: Container datatypes + +.. moduleauthor:: Raymond Hettinger +.. sectionauthor:: Raymond Hettinger + +**Source code:** :source:`Lib/collections/__init__.py` + +.. testsetup:: * + + from collections import * + import itertools + __name__ = '' + +-------------- + +This module implements specialized container datatypes providing alternatives to +Python's general purpose built-in containers, :class:`dict`, :class:`list`, +:class:`set`, and :class:`tuple`. + +===================== ==================================================================== +:func:`namedtuple` factory function for creating tuple subclasses with named fields +:class:`deque` list-like container with fast appends and pops on either end +:class:`ChainMap` dict-like class for creating a single view of multiple mappings +:class:`Counter` dict subclass for counting hashable objects +:class:`OrderedDict` dict subclass that remembers the order entries were added +:class:`defaultdict` dict subclass that calls a factory function to supply missing values +:class:`UserDict` wrapper around dictionary objects for easier dict subclassing +:class:`UserList` wrapper around list objects for easier list subclassing +:class:`UserString` wrapper around string objects for easier string subclassing +===================== ==================================================================== + + +:class:`ChainMap` objects +------------------------- + +.. versionadded:: 3.3 + +A :class:`ChainMap` class is provided for quickly linking a number of mappings +so they can be treated as a single unit. It is often much faster than creating +a new dictionary and running multiple :meth:`~dict.update` calls. + +The class can be used to simulate nested scopes and is useful in templating. + +.. class:: ChainMap(*maps) + + A :class:`ChainMap` groups multiple dicts or other mappings together to + create a single, updateable view. If no *maps* are specified, a single empty + dictionary is provided so that a new chain always has at least one mapping. + + The underlying mappings are stored in a list. That list is public and can + be accessed or updated using the *maps* attribute. There is no other state. + + Lookups search the underlying mappings successively until a key is found. In + contrast, writes, updates, and deletions only operate on the first mapping. + + A :class:`ChainMap` incorporates the underlying mappings by reference. So, if + one of the underlying mappings gets updated, those changes will be reflected + in :class:`ChainMap`. + + All of the usual dictionary methods are supported. In addition, there is a + *maps* attribute, a method for creating new subcontexts, and a property for + accessing all but the first mapping: + + .. attribute:: maps + + A user updateable list of mappings. The list is ordered from + first-searched to last-searched. It is the only stored state and can + be modified to change which mappings are searched. The list should + always contain at least one mapping. + + .. method:: new_child(m=None) + + Returns a new :class:`ChainMap` containing a new map followed by + all of the maps in the current instance. If ``m`` is specified, + it becomes the new map at the front of the list of mappings; if not + specified, an empty dict is used, so that a call to ``d.new_child()`` + is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for + creating subcontexts that can be updated without altering values in any + of the parent mappings. + + .. versionchanged:: 3.4 + The optional ``m`` parameter was added. + + .. attribute:: parents + + Property returning a new :class:`ChainMap` containing all of the maps in + the current instance except the first one. This is useful for skipping + the first map in the search. Use cases are similar to those for the + :keyword:`nonlocal` keyword used in :term:`nested scopes `. The use cases also parallel those for the built-in + :func:`super` function. A reference to ``d.parents`` is equivalent to: + ``ChainMap(*d.maps[1:])``. + + Note, the iteration order of a :class:`ChainMap()` is determined by + scanning the mappings last to first:: + + >>> baseline = {'music': 'bach', 'art': 'rembrandt'} + >>> adjustments = {'art': 'van gogh', 'opera': 'carmen'} + >>> list(ChainMap(adjustments, baseline)) + ['music', 'art', 'opera'] + + This gives the same ordering as a series of :meth:`dict.update` calls + starting with the last mapping:: + + >>> combined = baseline.copy() + >>> combined.update(adjustments) + >>> list(combined) + ['music', 'art', 'opera'] + + .. versionchanged:: 3.9 + Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. + +.. seealso:: + + * The `MultiContext class + `_ + in the Enthought `CodeTools package + `_ has options to support + writing to any mapping in the chain. + + * Django's `Context class + `_ + for templating is a read-only chain of mappings. It also features + pushing and popping of contexts similar to the + :meth:`~collections.ChainMap.new_child` method and the + :attr:`~collections.ChainMap.parents` property. + + * The `Nested Contexts recipe + `_ has options to control + whether writes and other mutations apply only to the first mapping or to + any mapping in the chain. + + * A `greatly simplified read-only version of Chainmap + `_. + + +:class:`ChainMap` Examples and Recipes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This section shows various approaches to working with chained maps. + + +Example of simulating Python's internal lookup chain:: + + import builtins + pylookup = ChainMap(locals(), globals(), vars(builtins)) + +Example of letting user specified command-line arguments take precedence over +environment variables which in turn take precedence over default values:: + + import os, argparse + + defaults = {'color': 'red', 'user': 'guest'} + + parser = argparse.ArgumentParser() + parser.add_argument('-u', '--user') + parser.add_argument('-c', '--color') + namespace = parser.parse_args() + command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} + + combined = ChainMap(command_line_args, os.environ, defaults) + print(combined['color']) + print(combined['user']) + +Example patterns for using the :class:`ChainMap` class to simulate nested +contexts:: + + c = ChainMap() # Create root context + d = c.new_child() # Create nested child context + e = c.new_child() # Child of c, independent from d + e.maps[0] # Current context dictionary -- like Python's locals() + e.maps[-1] # Root context -- like Python's globals() + e.parents # Enclosing context chain -- like Python's nonlocals + + d['x'] = 1 # Set value in current context + d['x'] # Get first key in the chain of contexts + del d['x'] # Delete from current context + list(d) # All nested values + k in d # Check all nested values + len(d) # Number of nested values + d.items() # All nested items + dict(d) # Flatten into a regular dictionary + +The :class:`ChainMap` class only makes updates (writes and deletions) to the +first mapping in the chain while lookups will search the full chain. However, +if deep writes and deletions are desired, it is easy to make a subclass that +updates keys found deeper in the chain:: + + class DeepChainMap(ChainMap): + 'Variant of ChainMap that allows direct updates to inner scopes' + + def __setitem__(self, key, value): + for mapping in self.maps: + if key in mapping: + mapping[key] = value + return + self.maps[0][key] = value + + def __delitem__(self, key): + for mapping in self.maps: + if key in mapping: + del mapping[key] + return + raise KeyError(key) + + >>> d = DeepChainMap({'zebra': 'black'}, {'elephant': 'blue'}, {'lion': 'yellow'}) + >>> d['lion'] = 'orange' # update an existing key two levels down + >>> d['snake'] = 'red' # new keys get added to the topmost dict + >>> del d['elephant'] # remove an existing key one level down + >>> d # display result + DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) + + +:class:`Counter` objects +------------------------ + +A counter tool is provided to support convenient and rapid tallies. +For example:: + + >>> # Tally occurrences of words in a list + >>> cnt = Counter() + >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: + ... cnt[word] += 1 + >>> cnt + Counter({'blue': 3, 'red': 2, 'green': 1}) + + >>> # Find the ten most common words in Hamlet + >>> import re + >>> words = re.findall(r'\w+', open('hamlet.txt').read().lower()) + >>> Counter(words).most_common(10) + [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), + ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] + +.. class:: Counter([iterable-or-mapping]) + + A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. + It is a collection where elements are stored as dictionary keys + and their counts are stored as dictionary values. Counts are allowed to be + any integer value including zero or negative counts. The :class:`Counter` + class is similar to bags or multisets in other languages. + + Elements are counted from an *iterable* or initialized from another + *mapping* (or counter): + + >>> c = Counter() # a new, empty counter + >>> c = Counter('gallahad') # a new counter from an iterable + >>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping + >>> c = Counter(cats=4, dogs=8) # a new counter from keyword args + + Counter objects have a dictionary interface except that they return a zero + count for missing items instead of raising a :exc:`KeyError`: + + >>> c = Counter(['eggs', 'ham']) + >>> c['bacon'] # count of a missing element is zero + 0 + + Setting a count to zero does not remove an element from a counter. + Use ``del`` to remove it entirely: + + >>> c['sausage'] = 0 # counter entry with a zero count + >>> del c['sausage'] # del actually removes the entry + + .. versionadded:: 3.1 + + .. versionchanged:: 3.7 As a :class:`dict` subclass, :class:`Counter` + Inherited the capability to remember insertion order. Math operations + on *Counter* objects also preserve order. Results are ordered + according to when an element is first encountered in the left operand + and then by the order encountered in the right operand. + + Counter objects support three methods beyond those available for all + dictionaries: + + .. method:: elements() + + Return an iterator over elements repeating each as many times as its + count. Elements are returned in the order first encountered. If an + element's count is less than one, :meth:`elements` will ignore it. + + >>> c = Counter(a=4, b=2, c=0, d=-2) + >>> sorted(c.elements()) + ['a', 'a', 'a', 'a', 'b', 'b'] + + .. method:: most_common([n]) + + Return a list of the *n* most common elements and their counts from the + most common to the least. If *n* is omitted or ``None``, + :meth:`most_common` returns *all* elements in the counter. + Elements with equal counts are ordered in the order first encountered: + + >>> Counter('abracadabra').most_common(3) + [('a', 5), ('b', 2), ('r', 2)] + + .. method:: subtract([iterable-or-mapping]) + + Elements are subtracted from an *iterable* or from another *mapping* + (or counter). Like :meth:`dict.update` but subtracts counts instead + of replacing them. Both inputs and outputs may be zero or negative. + + >>> c = Counter(a=4, b=2, c=0, d=-2) + >>> d = Counter(a=1, b=2, c=3, d=4) + >>> c.subtract(d) + >>> c + Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6}) + + .. versionadded:: 3.2 + + The usual dictionary methods are available for :class:`Counter` objects + except for two which work differently for counters. + + .. method:: fromkeys(iterable) + + This class method is not implemented for :class:`Counter` objects. + + .. method:: update([iterable-or-mapping]) + + Elements are counted from an *iterable* or added-in from another + *mapping* (or counter). Like :meth:`dict.update` but adds counts + instead of replacing them. Also, the *iterable* is expected to be a + sequence of elements, not a sequence of ``(key, value)`` pairs. + +Counters support rich comparison operators for equality, subset, and +superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. +All of those tests treat missing elements as having zero counts so that +``Counter(a=1) == Counter(a=1, b=0)`` returns true. + +.. versionadded:: 3.10 + Rich comparison operations we were added + +.. versionchanged:: 3.10 + In equality tests, missing elements are treated as having zero counts. + Formerly, ``Counter(a=3)`` and ``Counter(a=3, b=0)`` were considered + distinct. + +Common patterns for working with :class:`Counter` objects:: + + sum(c.values()) # total of all counts + c.clear() # reset all counts + list(c) # list unique elements + set(c) # convert to a set + dict(c) # convert to a regular dictionary + c.items() # convert to a list of (elem, cnt) pairs + Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs + c.most_common()[:-n-1:-1] # n least common elements + +c # remove zero and negative counts + +Several mathematical operations are provided for combining :class:`Counter` +objects to produce multisets (counters that have counts greater than zero). +Addition and subtraction combine counters by adding or subtracting the counts +of corresponding elements. Intersection and union return the minimum and +maximum of corresponding counts. Each operation can accept inputs with signed +counts, but the output will exclude results with counts of zero or less. + + >>> c = Counter(a=3, b=1) + >>> d = Counter(a=1, b=2) + >>> c + d # add two counters together: c[x] + d[x] + Counter({'a': 4, 'b': 3}) + >>> c - d # subtract (keeping only positive counts) + Counter({'a': 2}) + >>> c & d # intersection: min(c[x], d[x]) # doctest: +SKIP + Counter({'a': 1, 'b': 1}) + >>> c | d # union: max(c[x], d[x]) + Counter({'a': 3, 'b': 2}) + +Unary addition and subtraction are shortcuts for adding an empty counter +or subtracting from an empty counter. + + >>> c = Counter(a=2, b=-4) + >>> +c + Counter({'a': 2}) + >>> -c + Counter({'b': 4}) + +.. versionadded:: 3.3 + Added support for unary plus, unary minus, and in-place multiset operations. + +.. note:: + + Counters were primarily designed to work with positive integers to represent + running counts; however, care was taken to not unnecessarily preclude use + cases needing other types or negative values. To help with those use cases, + this section documents the minimum range and type restrictions. + + * The :class:`Counter` class itself is a dictionary subclass with no + restrictions on its keys and values. The values are intended to be numbers + representing counts, but you *could* store anything in the value field. + + * The :meth:`~Counter.most_common` method requires only that the values be orderable. + + * For in-place operations such as ``c[key] += 1``, the value type need only + support addition and subtraction. So fractions, floats, and decimals would + work and negative values are supported. The same is also true for + :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values + for both inputs and outputs. + + * The multiset methods are designed only for use cases with positive values. + The inputs may be negative or zero, but only outputs with positive values + are created. There are no type restrictions, but the value type needs to + support addition, subtraction, and comparison. + + * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and + negative counts. + +.. seealso:: + + * `Bag class `_ + in Smalltalk. + + * Wikipedia entry for `Multisets `_. + + * `C++ multisets `_ + tutorial with examples. + + * For mathematical operations on multisets and their use cases, see + *Knuth, Donald. The Art of Computer Programming Volume II, + Section 4.6.3, Exercise 19*. + + * To enumerate all distinct multisets of a given size over a given set of + elements, see :func:`itertools.combinations_with_replacement`:: + + map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC + + +:class:`deque` objects +---------------------- + +.. class:: deque([iterable, [maxlen]]) + + Returns a new deque object initialized left-to-right (using :meth:`append`) with + data from *iterable*. If *iterable* is not specified, the new deque is empty. + + Deques are a generalization of stacks and queues (the name is pronounced "deck" + and is short for "double-ended queue"). Deques support thread-safe, memory + efficient appends and pops from either side of the deque with approximately the + same O(1) performance in either direction. + + Though :class:`list` objects support similar operations, they are optimized for + fast fixed-length operations and incur O(n) memory movement costs for + ``pop(0)`` and ``insert(0, v)`` operations which change both the size and + position of the underlying data representation. + + + If *maxlen* is not specified or is ``None``, deques may grow to an + arbitrary length. Otherwise, the deque is bounded to the specified maximum + length. Once a bounded length deque is full, when new items are added, a + corresponding number of items are discarded from the opposite end. Bounded + length deques provide functionality similar to the ``tail`` filter in + Unix. They are also useful for tracking transactions and other pools of data + where only the most recent activity is of interest. + + + Deque objects support the following methods: + + .. method:: append(x) + + Add *x* to the right side of the deque. + + + .. method:: appendleft(x) + + Add *x* to the left side of the deque. + + + .. method:: clear() + + Remove all elements from the deque leaving it with length 0. + + + .. method:: copy() + + Create a shallow copy of the deque. + + .. versionadded:: 3.5 + + + .. method:: count(x) + + Count the number of deque elements equal to *x*. + + .. versionadded:: 3.2 + + + .. method:: extend(iterable) + + Extend the right side of the deque by appending elements from the iterable + argument. + + + .. method:: extendleft(iterable) + + Extend the left side of the deque by appending elements from *iterable*. + Note, the series of left appends results in reversing the order of + elements in the iterable argument. + + + .. method:: index(x[, start[, stop]]) + + Return the position of *x* in the deque (at or after index *start* + and before index *stop*). Returns the first match or raises + :exc:`ValueError` if not found. + + .. versionadded:: 3.5 + + + .. method:: insert(i, x) + + Insert *x* into the deque at position *i*. + + If the insertion would cause a bounded deque to grow beyond *maxlen*, + an :exc:`IndexError` is raised. + + .. versionadded:: 3.5 + + + .. method:: pop() + + Remove and return an element from the right side of the deque. If no + elements are present, raises an :exc:`IndexError`. + + + .. method:: popleft() + + Remove and return an element from the left side of the deque. If no + elements are present, raises an :exc:`IndexError`. + + + .. method:: remove(value) + + Remove the first occurrence of *value*. If not found, raises a + :exc:`ValueError`. + + + .. method:: reverse() + + Reverse the elements of the deque in-place and then return ``None``. + + .. versionadded:: 3.2 + + + .. method:: rotate(n=1) + + Rotate the deque *n* steps to the right. If *n* is negative, rotate + to the left. + + When the deque is not empty, rotating one step to the right is equivalent + to ``d.appendleft(d.pop())``, and rotating one step to the left is + equivalent to ``d.append(d.popleft())``. + + + Deque objects also provide one read-only attribute: + + .. attribute:: maxlen + + Maximum size of a deque or ``None`` if unbounded. + + .. versionadded:: 3.1 + + +In addition to the above, deques support iteration, pickling, ``len(d)``, +``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with +the :keyword:`in` operator, and subscript references such as ``d[0]`` to access +the first element. Indexed access is O(1) at both ends but slows to O(n) in +the middle. For fast random access, use lists instead. + +Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, +and ``__imul__()``. + +Example: + +.. doctest:: + + >>> from collections import deque + >>> d = deque('ghi') # make a new deque with three items + >>> for elem in d: # iterate over the deque's elements + ... print(elem.upper()) + G + H + I + + >>> d.append('j') # add a new entry to the right side + >>> d.appendleft('f') # add a new entry to the left side + >>> d # show the representation of the deque + deque(['f', 'g', 'h', 'i', 'j']) + + >>> d.pop() # return and remove the rightmost item + 'j' + >>> d.popleft() # return and remove the leftmost item + 'f' + >>> list(d) # list the contents of the deque + ['g', 'h', 'i'] + >>> d[0] # peek at leftmost item + 'g' + >>> d[-1] # peek at rightmost item + 'i' + + >>> list(reversed(d)) # list the contents of a deque in reverse + ['i', 'h', 'g'] + >>> 'h' in d # search the deque + True + >>> d.extend('jkl') # add multiple elements at once + >>> d + deque(['g', 'h', 'i', 'j', 'k', 'l']) + >>> d.rotate(1) # right rotation + >>> d + deque(['l', 'g', 'h', 'i', 'j', 'k']) + >>> d.rotate(-1) # left rotation + >>> d + deque(['g', 'h', 'i', 'j', 'k', 'l']) + + >>> deque(reversed(d)) # make a new deque in reverse order + deque(['l', 'k', 'j', 'i', 'h', 'g']) + >>> d.clear() # empty the deque + >>> d.pop() # cannot pop from an empty deque + Traceback (most recent call last): + File "", line 1, in -toplevel- + d.pop() + IndexError: pop from an empty deque + + >>> d.extendleft('abc') # extendleft() reverses the input order + >>> d + deque(['c', 'b', 'a']) + + +:class:`deque` Recipes +^^^^^^^^^^^^^^^^^^^^^^ + +This section shows various approaches to working with deques. + +Bounded length deques provide functionality similar to the ``tail`` filter +in Unix:: + + def tail(filename, n=10): + 'Return the last n lines of a file' + with open(filename) as f: + return deque(f, n) + +Another approach to using deques is to maintain a sequence of recently +added elements by appending to the right and popping to the left:: + + def moving_average(iterable, n=3): + # moving_average([40, 30, 50, 46, 39, 44]) --> 40.0 42.0 45.0 43.0 + # http://en.wikipedia.org/wiki/Moving_average + it = iter(iterable) + d = deque(itertools.islice(it, n-1)) + d.appendleft(0) + s = sum(d) + for elem in it: + s += elem - d.popleft() + d.append(elem) + yield s / n + +A `round-robin scheduler +`_ can be implemented with +input iterators stored in a :class:`deque`. Values are yielded from the active +iterator in position zero. If that iterator is exhausted, it can be removed +with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with +the :meth:`~deque.rotate` method:: + + def roundrobin(*iterables): + "roundrobin('ABC', 'D', 'EF') --> A D E B F C" + iterators = deque(map(iter, iterables)) + while iterators: + try: + while True: + yield next(iterators[0]) + iterators.rotate(-1) + except StopIteration: + # Remove an exhausted iterator. + iterators.popleft() + +The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and +deletion. For example, a pure Python implementation of ``del d[n]`` relies on +the ``rotate()`` method to position elements to be popped:: + + def delete_nth(d, n): + d.rotate(-n) + d.popleft() + d.rotate(n) + +To implement :class:`deque` slicing, use a similar approach applying +:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove +old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then +reverse the rotation. +With minor variations on that approach, it is easy to implement Forth style +stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, +``rot``, and ``roll``. + + +:class:`defaultdict` objects +---------------------------- + +.. class:: defaultdict([default_factory[, ...]]) + + Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the + built-in :class:`dict` class. It overrides one method and adds one writable + instance variable. The remaining functionality is the same as for the + :class:`dict` class and is not documented here. + + The first argument provides the initial value for the :attr:`default_factory` + attribute; it defaults to ``None``. All remaining arguments are treated the same + as if they were passed to the :class:`dict` constructor, including keyword + arguments. + + + :class:`defaultdict` objects support the following method in addition to the + standard :class:`dict` operations: + + .. method:: __missing__(key) + + If the :attr:`default_factory` attribute is ``None``, this raises a + :exc:`KeyError` exception with the *key* as argument. + + If :attr:`default_factory` is not ``None``, it is called without arguments + to provide a default value for the given *key*, this value is inserted in + the dictionary for the *key*, and returned. + + If calling :attr:`default_factory` raises an exception this exception is + propagated unchanged. + + This method is called by the :meth:`__getitem__` method of the + :class:`dict` class when the requested key is not found; whatever it + returns or raises is then returned or raised by :meth:`__getitem__`. + + Note that :meth:`__missing__` is *not* called for any operations besides + :meth:`__getitem__`. This means that :meth:`get` will, like normal + dictionaries, return ``None`` as a default rather than using + :attr:`default_factory`. + + + :class:`defaultdict` objects support the following instance variable: + + + .. attribute:: default_factory + + This attribute is used by the :meth:`__missing__` method; it is + initialized from the first argument to the constructor, if present, or to + ``None``, if absent. + + .. versionchanged:: 3.9 + Added merge (``|``) and update (``|=``) operators, specified in + :pep:`584`. + + +:class:`defaultdict` Examples +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a +sequence of key-value pairs into a dictionary of lists: + + >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] + >>> d = defaultdict(list) + >>> for k, v in s: + ... d[k].append(v) + ... + >>> sorted(d.items()) + [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] + +When each key is encountered for the first time, it is not already in the +mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` +function which returns an empty :class:`list`. The :meth:`list.append` +operation then attaches the value to the new list. When keys are encountered +again, the look-up proceeds normally (returning the list for that key) and the +:meth:`list.append` operation adds another value to the list. This technique is +simpler and faster than an equivalent technique using :meth:`dict.setdefault`: + + >>> d = {} + >>> for k, v in s: + ... d.setdefault(k, []).append(v) + ... + >>> sorted(d.items()) + [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] + +Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the +:class:`defaultdict` useful for counting (like a bag or multiset in other +languages): + + >>> s = 'mississippi' + >>> d = defaultdict(int) + >>> for k in s: + ... d[k] += 1 + ... + >>> sorted(d.items()) + [('i', 4), ('m', 1), ('p', 2), ('s', 4)] + +When a letter is first encountered, it is missing from the mapping, so the +:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of +zero. The increment operation then builds up the count for each letter. + +The function :func:`int` which always returns zero is just a special case of +constant functions. A faster and more flexible way to create constant functions +is to use a lambda function which can supply any constant value (not just +zero): + + >>> def constant_factory(value): + ... return lambda: value + >>> d = defaultdict(constant_factory('')) + >>> d.update(name='John', action='ran') + >>> '%(name)s %(action)s to %(object)s' % d + 'John ran to ' + +Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the +:class:`defaultdict` useful for building a dictionary of sets: + + >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] + >>> d = defaultdict(set) + >>> for k, v in s: + ... d[k].add(v) + ... + >>> sorted(d.items()) + [('blue', {2, 4}), ('red', {1, 3})] + + +:func:`namedtuple` Factory Function for Tuples with Named Fields +---------------------------------------------------------------- + +Named tuples assign meaning to each position in a tuple and allow for more readable, +self-documenting code. They can be used wherever regular tuples are used, and +they add the ability to access fields by name instead of position index. + +.. function:: namedtuple(typename, field_names, *, rename=False, defaults=None, module=None) + + Returns a new tuple subclass named *typename*. The new subclass is used to + create tuple-like objects that have fields accessible by attribute lookup as + well as being indexable and iterable. Instances of the subclass also have a + helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` + method which lists the tuple contents in a ``name=value`` format. + + The *field_names* are a sequence of strings such as ``['x', 'y']``. + Alternatively, *field_names* can be a single string with each fieldname + separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. + + Any valid Python identifier may be used for a fieldname except for names + starting with an underscore. Valid identifiers consist of letters, digits, + and underscores but do not start with a digit or underscore and cannot be + a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, + or *raise*. + + If *rename* is true, invalid fieldnames are automatically replaced + with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is + converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword + ``def`` and the duplicate fieldname ``abc``. + + *defaults* can be ``None`` or an :term:`iterable` of default values. + Since fields with a default value must come after any fields without a + default, the *defaults* are applied to the rightmost parameters. For + example, if the fieldnames are ``['x', 'y', 'z']`` and the defaults are + ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to + ``1``, and ``z`` will default to ``2``. + + If *module* is defined, the ``__module__`` attribute of the named tuple is + set to that value. + + Named tuple instances do not have per-instance dictionaries, so they are + lightweight and require no more memory than regular tuples. + + To support pickling, the named tuple class should be assigned to a variable + that matches *typename*. + + .. versionchanged:: 3.1 + Added support for *rename*. + + .. versionchanged:: 3.6 + The *verbose* and *rename* parameters became + :ref:`keyword-only arguments `. + + .. versionchanged:: 3.6 + Added the *module* parameter. + + .. versionchanged:: 3.7 + Removed the *verbose* parameter and the :attr:`_source` attribute. + + .. versionchanged:: 3.7 + Added the *defaults* parameter and the :attr:`_field_defaults` + attribute. + +.. doctest:: + :options: +NORMALIZE_WHITESPACE + + >>> # Basic example + >>> Point = namedtuple('Point', ['x', 'y']) + >>> p = Point(11, y=22) # instantiate with positional or keyword arguments + >>> p[0] + p[1] # indexable like the plain tuple (11, 22) + 33 + >>> x, y = p # unpack like a regular tuple + >>> x, y + (11, 22) + >>> p.x + p.y # fields also accessible by name + 33 + >>> p # readable __repr__ with a name=value style + Point(x=11, y=22) + +Named tuples are especially useful for assigning field names to result tuples returned +by the :mod:`csv` or :mod:`sqlite3` modules:: + + EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') + + import csv + for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): + print(emp.name, emp.title) + + import sqlite3 + conn = sqlite3.connect('/companydata') + cursor = conn.cursor() + cursor.execute('SELECT name, age, title, department, paygrade FROM employees') + for emp in map(EmployeeRecord._make, cursor.fetchall()): + print(emp.name, emp.title) + +In addition to the methods inherited from tuples, named tuples support +three additional methods and two attributes. To prevent conflicts with +field names, the method and attribute names start with an underscore. + +.. classmethod:: somenamedtuple._make(iterable) + + Class method that makes a new instance from an existing sequence or iterable. + + .. doctest:: + + >>> t = [11, 22] + >>> Point._make(t) + Point(x=11, y=22) + +.. method:: somenamedtuple._asdict() + + Return a new :class:`dict` which maps field names to their corresponding + values: + + .. doctest:: + + >>> p = Point(x=11, y=22) + >>> p._asdict() + {'x': 11, 'y': 22} + + .. versionchanged:: 3.1 + Returns an :class:`OrderedDict` instead of a regular :class:`dict`. + + .. versionchanged:: 3.8 + Returns a regular :class:`dict` instead of an :class:`OrderedDict`. + As of Python 3.7, regular dicts are guaranteed to be ordered. If the + extra features of :class:`OrderedDict` are required, the suggested + remediation is to cast the result to the desired type: + ``OrderedDict(nt._asdict())``. + +.. method:: somenamedtuple._replace(**kwargs) + + Return a new instance of the named tuple replacing specified fields with new + values:: + + >>> p = Point(x=11, y=22) + >>> p._replace(x=33) + Point(x=33, y=22) + + >>> for partnum, record in inventory.items(): + ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) + +.. attribute:: somenamedtuple._fields + + Tuple of strings listing the field names. Useful for introspection + and for creating new named tuple types from existing named tuples. + + .. doctest:: + + >>> p._fields # view the field names + ('x', 'y') + + >>> Color = namedtuple('Color', 'red green blue') + >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) + >>> Pixel(11, 22, 128, 255, 0) + Pixel(x=11, y=22, red=128, green=255, blue=0) + +.. attribute:: somenamedtuple._field_defaults + + Dictionary mapping field names to default values. + + .. doctest:: + + >>> Account = namedtuple('Account', ['type', 'balance'], defaults=[0]) + >>> Account._field_defaults + {'balance': 0} + >>> Account('premium') + Account(type='premium', balance=0) + +To retrieve a field whose name is stored in a string, use the :func:`getattr` +function: + + >>> getattr(p, 'x') + 11 + +To convert a dictionary to a named tuple, use the double-star-operator +(as described in :ref:`tut-unpacking-arguments`): + + >>> d = {'x': 11, 'y': 22} + >>> Point(**d) + Point(x=11, y=22) + +Since a named tuple is a regular Python class, it is easy to add or change +functionality with a subclass. Here is how to add a calculated field and +a fixed-width print format: + +.. doctest:: + + >>> class Point(namedtuple('Point', ['x', 'y'])): + ... __slots__ = () + ... @property + ... def hypot(self): + ... return (self.x ** 2 + self.y ** 2) ** 0.5 + ... def __str__(self): + ... return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) + + >>> for p in Point(3, 4), Point(14, 5/7): + ... print(p) + Point: x= 3.000 y= 4.000 hypot= 5.000 + Point: x=14.000 y= 0.714 hypot=14.018 + +The subclass shown above sets ``__slots__`` to an empty tuple. This helps +keep memory requirements low by preventing the creation of instance dictionaries. + +Subclassing is not useful for adding new, stored fields. Instead, simply +create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: + + >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) + +Docstrings can be customized by making direct assignments to the ``__doc__`` +fields: + + >>> Book = namedtuple('Book', ['id', 'title', 'authors']) + >>> Book.__doc__ += ': Hardcover book in active collection' + >>> Book.id.__doc__ = '13-digit ISBN' + >>> Book.title.__doc__ = 'Title of first printing' + >>> Book.authors.__doc__ = 'List of authors sorted by last name' + +.. versionchanged:: 3.5 + Property docstrings became writeable. + +.. seealso:: + + * See :class:`typing.NamedTuple` for a way to add type hints for named + tuples. It also provides an elegant notation using the :keyword:`class` + keyword:: + + class Component(NamedTuple): + part_number: int + weight: float + description: Optional[str] = None + + * See :meth:`types.SimpleNamespace` for a mutable namespace based on an + underlying dictionary instead of a tuple. + + * The :mod:`dataclasses` module provides a decorator and functions for + automatically adding generated special methods to user-defined classes. + + +:class:`OrderedDict` objects +---------------------------- + +Ordered dictionaries are just like regular dictionaries but have some extra +capabilities relating to ordering operations. They have become less +important now that the built-in :class:`dict` class gained the ability +to remember insertion order (this new behavior became guaranteed in +Python 3.7). + +Some differences from :class:`dict` still remain: + +* The regular :class:`dict` was designed to be very good at mapping + operations. Tracking insertion order was secondary. + +* The :class:`OrderedDict` was designed to be good at reordering operations. + Space efficiency, iteration speed, and the performance of update + operations were secondary. + +* Algorithmically, :class:`OrderedDict` can handle frequent reordering + operations better than :class:`dict`. This makes it suitable for tracking + recent accesses (for example in an `LRU cache + `_). + +* The equality operation for :class:`OrderedDict` checks for matching order. + +* The :meth:`popitem` method of :class:`OrderedDict` has a different + signature. It accepts an optional argument to specify which item is popped. + +* :class:`OrderedDict` has a :meth:`move_to_end` method to + efficiently reposition an element to an endpoint. + +* Until Python 3.8, :class:`dict` lacked a :meth:`__reversed__` method. + + +.. class:: OrderedDict([items]) + + Return an instance of a :class:`dict` subclass that has methods + specialized for rearranging dictionary order. + + .. versionadded:: 3.1 + + .. method:: popitem(last=True) + + The :meth:`popitem` method for ordered dictionaries returns and removes a + (key, value) pair. The pairs are returned in + :abbr:`LIFO (last-in, first-out)` order if *last* is true + or :abbr:`FIFO (first-in, first-out)` order if false. + + .. method:: move_to_end(key, last=True) + + Move an existing *key* to either end of an ordered dictionary. The item + is moved to the right end if *last* is true (the default) or to the + beginning if *last* is false. Raises :exc:`KeyError` if the *key* does + not exist:: + + >>> d = OrderedDict.fromkeys('abcde') + >>> d.move_to_end('b') + >>> ''.join(d.keys()) + 'acdeb' + >>> d.move_to_end('b', last=False) + >>> ''.join(d.keys()) + 'bacde' + + .. versionadded:: 3.2 + +In addition to the usual mapping methods, ordered dictionaries also support +reverse iteration using :func:`reversed`. + +Equality tests between :class:`OrderedDict` objects are order-sensitive +and are implemented as ``list(od1.items())==list(od2.items())``. +Equality tests between :class:`OrderedDict` objects and other +:class:`~collections.abc.Mapping` objects are order-insensitive like regular +dictionaries. This allows :class:`OrderedDict` objects to be substituted +anywhere a regular dictionary is used. + +.. versionchanged:: 3.5 + The items, keys, and values :term:`views ` + of :class:`OrderedDict` now support reverse iteration using :func:`reversed`. + +.. versionchanged:: 3.6 + With the acceptance of :pep:`468`, order is retained for keyword arguments + passed to the :class:`OrderedDict` constructor and its :meth:`update` + method. + +.. versionchanged:: 3.9 + Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. + + +:class:`OrderedDict` Examples and Recipes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +It is straightforward to create an ordered dictionary variant +that remembers the order the keys were *last* inserted. +If a new entry overwrites an existing entry, the +original insertion position is changed and moved to the end:: + + class LastUpdatedOrderedDict(OrderedDict): + 'Store items in the order the keys were last added' + + def __setitem__(self, key, value): + super().__setitem__(key, value) + self.move_to_end(key) + +An :class:`OrderedDict` would also be useful for implementing +variants of :func:`functools.lru_cache`:: + + class LRU(OrderedDict): + 'Limit size, evicting the least recently looked-up key when full' + + def __init__(self, maxsize=128, /, *args, **kwds): + self.maxsize = maxsize + super().__init__(*args, **kwds) + + def __getitem__(self, key): + value = super().__getitem__(key) + self.move_to_end(key) + return value + + def __setitem__(self, key, value): + if key in self: + self.move_to_end(key) + super().__setitem__(key, value) + if len(self) > self.maxsize: + oldest = next(iter(self)) + del self[oldest] + + +:class:`UserDict` objects +------------------------- + +The class, :class:`UserDict` acts as a wrapper around dictionary objects. +The need for this class has been partially supplanted by the ability to +subclass directly from :class:`dict`; however, this class can be easier +to work with because the underlying dictionary is accessible as an +attribute. + +.. class:: UserDict([initialdata]) + + Class that simulates a dictionary. The instance's contents are kept in a + regular dictionary, which is accessible via the :attr:`data` attribute of + :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is + initialized with its contents; note that a reference to *initialdata* will not + be kept, allowing it be used for other purposes. + + In addition to supporting the methods and operations of mappings, + :class:`UserDict` instances provide the following attribute: + + .. attribute:: data + + A real dictionary used to store the contents of the :class:`UserDict` + class. + + + +:class:`UserList` objects +------------------------- + +This class acts as a wrapper around list objects. It is a useful base class +for your own list-like classes which can inherit from them and override +existing methods or add new ones. In this way, one can add new behaviors to +lists. + +The need for this class has been partially supplanted by the ability to +subclass directly from :class:`list`; however, this class can be easier +to work with because the underlying list is accessible as an attribute. + +.. class:: UserList([list]) + + Class that simulates a list. The instance's contents are kept in a regular + list, which is accessible via the :attr:`data` attribute of :class:`UserList` + instances. The instance's contents are initially set to a copy of *list*, + defaulting to the empty list ``[]``. *list* can be any iterable, for + example a real Python list or a :class:`UserList` object. + + In addition to supporting the methods and operations of mutable sequences, + :class:`UserList` instances provide the following attribute: + + .. attribute:: data + + A real :class:`list` object used to store the contents of the + :class:`UserList` class. + +**Subclassing requirements:** Subclasses of :class:`UserList` are expected to +offer a constructor which can be called with either no arguments or one +argument. List operations which return a new sequence attempt to create an +instance of the actual implementation class. To do so, it assumes that the +constructor can be called with a single parameter, which is a sequence object +used as a data source. + +If a derived class does not wish to comply with this requirement, all of the +special methods supported by this class will need to be overridden; please +consult the sources for information about the methods which need to be provided +in that case. + +:class:`UserString` objects +--------------------------- + +The class, :class:`UserString` acts as a wrapper around string objects. +The need for this class has been partially supplanted by the ability to +subclass directly from :class:`str`; however, this class can be easier +to work with because the underlying string is accessible as an +attribute. + +.. class:: UserString(seq) + + Class that simulates a string object. The instance's + content is kept in a regular string object, which is accessible via the + :attr:`data` attribute of :class:`UserString` instances. The instance's + contents are initially set to a copy of *seq*. The *seq* argument can + be any object which can be converted into a string using the built-in + :func:`str` function. + + In addition to supporting the methods and operations of strings, + :class:`UserString` instances provide the following attribute: + + .. attribute:: data + + A real :class:`str` object used to store the contents of the + :class:`UserString` class. + + .. versionchanged:: 3.5 + New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, + ``format_map``, ``isprintable``, and ``maketrans``. diff --git a/Doc/library/dataclasses.rst.bak b/Doc/library/dataclasses.rst.bak new file mode 100644 index 00000000000000..e706f7fcc566d8 --- /dev/null +++ b/Doc/library/dataclasses.rst.bak @@ -0,0 +1,595 @@ +:mod:`dataclasses` --- Data Classes +=================================== + +.. module:: dataclasses + :synopsis: Generate special methods on user-defined classes. + +.. moduleauthor:: Eric V. Smith +.. sectionauthor:: Eric V. Smith + +**Source code:** :source:`Lib/dataclasses.py` + +-------------- + +This module provides a decorator and functions for automatically +adding generated :term:`special method`\s such as :meth:`__init__` and +:meth:`__repr__` to user-defined classes. It was originally described +in :pep:`557`. + +The member variables to use in these generated methods are defined +using :pep:`526` type annotations. For example this code:: + + from dataclasses import dataclass + + @dataclass + class InventoryItem: + """Class for keeping track of an item in inventory.""" + name: str + unit_price: float + quantity_on_hand: int = 0 + + def total_cost(self) -> float: + return self.unit_price * self.quantity_on_hand + +Will add, among other things, a :meth:`__init__` that looks like:: + + def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0): + self.name = name + self.unit_price = unit_price + self.quantity_on_hand = quantity_on_hand + +Note that this method is automatically added to the class: it is not +directly specified in the ``InventoryItem`` definition shown above. + +.. versionadded:: 3.7 + +Module-level decorators, classes, and functions +----------------------------------------------- + +.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + + This function is a :term:`decorator` that is used to add generated + :term:`special method`\s to classes, as described below. + + The :func:`dataclass` decorator examines the class to find + ``field``\s. A ``field`` is defined as class variable that has a + :term:`type annotation `. With two + exceptions described below, nothing in :func:`dataclass` + examines the type specified in the variable annotation. + + The order of the fields in all of the generated methods is the + order in which they appear in the class definition. + + The :func:`dataclass` decorator will add various "dunder" methods to + the class, described below. If any of the added methods already + exist on the class, the behavior depends on the parameter, as documented + below. The decorator returns the same class that is called on; no new + class is created. + + If :func:`dataclass` is used just as a simple decorator with no parameters, + it acts as if it has the default values documented in this + signature. That is, these three uses of :func:`dataclass` are + equivalent:: + + @dataclass + class C: + ... + + @dataclass() + class C: + ... + + @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + class C: + ... + + The parameters to :func:`dataclass` are: + + - ``init``: If true (the default), a :meth:`__init__` method will be + generated. + + If the class already defines :meth:`__init__`, this parameter is + ignored. + + - ``repr``: If true (the default), a :meth:`__repr__` method will be + generated. The generated repr string will have the class name and + the name and repr of each field, in the order they are defined in + the class. Fields that are marked as being excluded from the repr + are not included. For example: + ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. + + If the class already defines :meth:`__repr__`, this parameter is + ignored. + + - ``eq``: If true (the default), an :meth:`__eq__` method will be + generated. This method compares the class as if it were a tuple + of its fields, in order. Both instances in the comparison must + be of the identical type. + + If the class already defines :meth:`__eq__`, this parameter is + ignored. + + - ``order``: If true (the default is ``False``), :meth:`__lt__`, + :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` methods will be + generated. These compare the class as if it were a tuple of its + fields, in order. Both instances in the comparison must be of the + identical type. If ``order`` is true and ``eq`` is false, a + :exc:`ValueError` is raised. + + If the class already defines any of :meth:`__lt__`, + :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then + :exc:`TypeError` is raised. + + - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method + is generated according to how ``eq`` and ``frozen`` are set. + + :meth:`__hash__` is used by built-in :meth:`hash()`, and when objects are + added to hashed collections such as dictionaries and sets. Having a + :meth:`__hash__` implies that instances of the class are immutable. + Mutability is a complicated property that depends on the programmer's + intent, the existence and behavior of :meth:`__eq__`, and the values of + the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. + + By default, :func:`dataclass` will not implicitly add a :meth:`__hash__` + method unless it is safe to do so. Neither will it add or change an + existing explicitly defined :meth:`__hash__` method. Setting the class + attribute ``__hash__ = None`` has a specific meaning to Python, as + described in the :meth:`__hash__` documentation. + + If :meth:`__hash__` is not explicit defined, or if it is set to ``None``, + then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. + Although not recommended, you can force :func:`dataclass` to create a + :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case + if your class is logically immutable but can nonetheless be mutated. + This is a specialized use case and should be considered carefully. + + Here are the rules governing implicit creation of a :meth:`__hash__` + method. Note that you cannot both have an explicit :meth:`__hash__` + method in your dataclass and set ``unsafe_hash=True``; this will result + in a :exc:`TypeError`. + + If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will + generate a :meth:`__hash__` method for you. If ``eq`` is true and + ``frozen`` is false, :meth:`__hash__` will be set to ``None``, marking it + unhashable (which it is, since it is mutable). If ``eq`` is false, + :meth:`__hash__` will be left untouched meaning the :meth:`__hash__` + method of the superclass will be used (if the superclass is + :class:`object`, this means it will fall back to id-based hashing). + + - ``frozen``: If true (the default is ``False``), assigning to fields will + generate an exception. This emulates read-only frozen instances. If + :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then + :exc:`TypeError` is raised. See the discussion below. + + ``field``\s may optionally specify a default value, using normal + Python syntax:: + + @dataclass + class C: + a: int # 'a' has no default value + b: int = 0 # assign a default value for 'b' + + In this example, both ``a`` and ``b`` will be included in the added + :meth:`__init__` method, which will be defined as:: + + def __init__(self, a: int, b: int = 0): + + :exc:`TypeError` will be raised if a field without a default value + follows a field with a default value. This is true either when this + occurs in a single class, or as a result of class inheritance. + +.. function:: field(*, default=MISSING, default_factory=MISSING, repr=True, hash=None, init=True, compare=True, metadata=None) + + For common and simple use cases, no other functionality is + required. There are, however, some dataclass features that + require additional per-field information. To satisfy this need for + additional information, you can replace the default field value + with a call to the provided :func:`field` function. For example:: + + @dataclass + class C: + mylist: list[int] = field(default_factory=list) + + c = C() + c.mylist += [1, 2, 3] + + As shown above, the ``MISSING`` value is a sentinel object used to + detect if the ``default`` and ``default_factory`` parameters are + provided. This sentinel is used because ``None`` is a valid value + for ``default``. No code should directly use the ``MISSING`` + value. + + The parameters to :func:`field` are: + + - ``default``: If provided, this will be the default value for this + field. This is needed because the :meth:`field` call itself + replaces the normal position of the default value. + + - ``default_factory``: If provided, it must be a zero-argument + callable that will be called when a default value is needed for + this field. Among other purposes, this can be used to specify + fields with mutable default values, as discussed below. It is an + error to specify both ``default`` and ``default_factory``. + + - ``init``: If true (the default), this field is included as a + parameter to the generated :meth:`__init__` method. + + - ``repr``: If true (the default), this field is included in the + string returned by the generated :meth:`__repr__` method. + + - ``compare``: If true (the default), this field is included in the + generated equality and comparison methods (:meth:`__eq__`, + :meth:`__gt__`, et al.). + + - ``hash``: This can be a bool or ``None``. If true, this field is + included in the generated :meth:`__hash__` method. If ``None`` (the + default), use the value of ``compare``: this would normally be + the expected behavior. A field should be considered in the hash + if it's used for comparisons. Setting this value to anything + other than ``None`` is discouraged. + + One possible reason to set ``hash=False`` but ``compare=True`` + would be if a field is expensive to compute a hash value for, + that field is needed for equality testing, and there are other + fields that contribute to the type's hash value. Even if a field + is excluded from the hash, it will still be used for comparisons. + + - ``metadata``: This can be a mapping or None. None is treated as + an empty dict. This value is wrapped in + :func:`~types.MappingProxyType` to make it read-only, and exposed + on the :class:`Field` object. It is not used at all by Data + Classes, and is provided as a third-party extension mechanism. + Multiple third-parties can each have their own key, to use as a + namespace in the metadata. + + If the default value of a field is specified by a call to + :func:`field()`, then the class attribute for this field will be + replaced by the specified ``default`` value. If no ``default`` is + provided, then the class attribute will be deleted. The intent is + that after the :func:`dataclass` decorator runs, the class + attributes will all contain the default values for the fields, just + as if the default value itself were specified. For example, + after:: + + @dataclass + class C: + x: int + y: int = field(repr=False) + z: int = field(repr=False, default=10) + t: int = 20 + + The class attribute ``C.z`` will be ``10``, the class attribute + ``C.t`` will be ``20``, and the class attributes ``C.x`` and + ``C.y`` will not be set. + +.. class:: Field + + :class:`Field` objects describe each defined field. These objects + are created internally, and are returned by the :func:`fields` + module-level method (see below). Users should never instantiate a + :class:`Field` object directly. Its documented attributes are: + + - ``name``: The name of the field. + + - ``type``: The type of the field. + + - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, + ``compare``, and ``metadata`` have the identical meaning and + values as they do in the :func:`field` declaration. + + Other attributes may exist, but they are private and must not be + inspected or relied on. + +.. function:: fields(class_or_instance) + + Returns a tuple of :class:`Field` objects that define the fields for this + dataclass. Accepts either a dataclass, or an instance of a dataclass. + Raises :exc:`TypeError` if not passed a dataclass or instance of one. + Does not return pseudo-fields which are ``ClassVar`` or ``InitVar``. + +.. function:: asdict(instance, *, dict_factory=dict) + + Converts the dataclass ``instance`` to a dict (by using the + factory function ``dict_factory``). Each dataclass is converted + to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, + lists, and tuples are recursed into. For example:: + + @dataclass + class Point: + x: int + y: int + + @dataclass + class C: + mylist: list[Point] + + p = Point(10, 20) + assert asdict(p) == {'x': 10, 'y': 20} + + c = C([Point(0, 0), Point(10, 4)]) + assert asdict(c) == {'mylist': [{'x': 0, 'y': 0}, {'x': 10, 'y': 4}]} + + Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. + +.. function:: astuple(instance, *, tuple_factory=tuple) + + Converts the dataclass ``instance`` to a tuple (by using the + factory function ``tuple_factory``). Each dataclass is converted + to a tuple of its field values. dataclasses, dicts, lists, and + tuples are recursed into. + + Continuing from the previous example:: + + assert astuple(p) == (10, 20) + assert astuple(c) == ([(0, 0), (10, 4)],) + + Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. + +.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) + + Creates a new dataclass with name ``cls_name``, fields as defined + in ``fields``, base classes as given in ``bases``, and initialized + with a namespace as given in ``namespace``. ``fields`` is an + iterable whose elements are each either ``name``, ``(name, type)``, + or ``(name, type, Field)``. If just ``name`` is supplied, + ``typing.Any`` is used for ``type``. The values of ``init``, + ``repr``, ``eq``, ``order``, ``unsafe_hash``, and ``frozen`` have + the same meaning as they do in :func:`dataclass`. + + This function is not strictly required, because any Python + mechanism for creating a new class with ``__annotations__`` can + then apply the :func:`dataclass` function to convert that class to + a dataclass. This function is provided as a convenience. For + example:: + + C = make_dataclass('C', + [('x', int), + 'y', + ('z', int, field(default=5))], + namespace={'add_one': lambda self: self.x + 1}) + + Is equivalent to:: + + @dataclass + class C: + x: int + y: 'typing.Any' + z: int = 5 + + def add_one(self): + return self.x + 1 + +.. function:: replace(instance, /, **changes) + + Creates a new object of the same type of ``instance``, replacing + fields with values from ``changes``. If ``instance`` is not a Data + Class, raises :exc:`TypeError`. If values in ``changes`` do not + specify fields, raises :exc:`TypeError`. + + The newly returned object is created by calling the :meth:`__init__` + method of the dataclass. This ensures that + :meth:`__post_init__`, if present, is also called. + + Init-only variables without default values, if any exist, must be + specified on the call to :func:`replace` so that they can be passed to + :meth:`__init__` and :meth:`__post_init__`. + + It is an error for ``changes`` to contain any fields that are + defined as having ``init=False``. A :exc:`ValueError` will be raised + in this case. + + Be forewarned about how ``init=False`` fields work during a call to + :func:`replace`. They are not copied from the source object, but + rather are initialized in :meth:`__post_init__`, if they're + initialized at all. It is expected that ``init=False`` fields will + be rarely and judiciously used. If they are used, it might be wise + to have alternate class constructors, or perhaps a custom + ``replace()`` (or similarly named) method which handles instance + copying. + +.. function:: is_dataclass(class_or_instance) + + Return ``True`` if its parameter is a dataclass or an instance of one, + otherwise return ``False``. + + If you need to know if a class is an instance of a dataclass (and + not a dataclass itself), then add a further check for ``not + isinstance(obj, type)``:: + + def is_dataclass_instance(obj): + return is_dataclass(obj) and not isinstance(obj, type) + +Post-init processing +-------------------- + +The generated :meth:`__init__` code will call a method named +:meth:`__post_init__`, if :meth:`__post_init__` is defined on the +class. It will normally be called as ``self.__post_init__()``. +However, if any ``InitVar`` fields are defined, they will also be +passed to :meth:`__post_init__` in the order they were defined in the +class. If no :meth:`__init__` method is generated, then +:meth:`__post_init__` will not automatically be called. + +Among other uses, this allows for initializing field values that +depend on one or more other fields. For example:: + + @dataclass + class C: + a: float + b: float + c: float = field(init=False) + + def __post_init__(self): + self.c = self.a + self.b + +See the section below on init-only variables for ways to pass +parameters to :meth:`__post_init__`. Also see the warning about how +:func:`replace` handles ``init=False`` fields. + +Class variables +--------------- + +One of two places where :func:`dataclass` actually inspects the type +of a field is to determine if a field is a class variable as defined +in :pep:`526`. It does this by checking if the type of the field is +``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded +from consideration as a field and is ignored by the dataclass +mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the +module-level :func:`fields` function. + +Init-only variables +------------------- + +The other place where :func:`dataclass` inspects a type annotation is to +determine if a field is an init-only variable. It does this by seeing +if the type of a field is of type ``dataclasses.InitVar``. If a field +is an ``InitVar``, it is considered a pseudo-field called an init-only +field. As it is not a true field, it is not returned by the +module-level :func:`fields` function. Init-only fields are added as +parameters to the generated :meth:`__init__` method, and are passed to +the optional :meth:`__post_init__` method. They are not otherwise used +by dataclasses. + +For example, suppose a field will be initialized from a database, if a +value is not provided when creating the class:: + + @dataclass + class C: + i: int + j: int = None + database: InitVar[DatabaseType] = None + + def __post_init__(self, database): + if self.j is None and database is not None: + self.j = database.lookup('j') + + c = C(10, database=my_database) + +In this case, :func:`fields` will return :class:`Field` objects for ``i`` and +``j``, but not for ``database``. + +Frozen instances +---------------- + +It is not possible to create truly immutable Python objects. However, +by passing ``frozen=True`` to the :meth:`dataclass` decorator you can +emulate immutability. In that case, dataclasses will add +:meth:`__setattr__` and :meth:`__delattr__` methods to the class. These +methods will raise a :exc:`FrozenInstanceError` when invoked. + +There is a tiny performance penalty when using ``frozen=True``: +:meth:`__init__` cannot use simple assignment to initialize fields, and +must use :meth:`object.__setattr__`. + +Inheritance +----------- + +When the dataclass is being created by the :meth:`dataclass` decorator, +it looks through all of the class's base classes in reverse MRO (that +is, starting at :class:`object`) and, for each dataclass that it finds, +adds the fields from that base class to an ordered mapping of fields. +After all of the base class fields are added, it adds its own fields +to the ordered mapping. All of the generated methods will use this +combined, calculated ordered mapping of fields. Because the fields +are in insertion order, derived classes override base classes. An +example:: + + @dataclass + class Base: + x: Any = 15.0 + y: int = 0 + + @dataclass + class C(Base): + z: int = 10 + x: int = 15 + +The final list of fields is, in order, ``x``, ``y``, ``z``. The final +type of ``x`` is ``int``, as specified in class ``C``. + +The generated :meth:`__init__` method for ``C`` will look like:: + + def __init__(self, x: int = 15, y: int = 0, z: int = 10): + +Default factory functions +------------------------- + + If a :func:`field` specifies a ``default_factory``, it is called with + zero arguments when a default value for the field is needed. For + example, to create a new instance of a list, use:: + + mylist: list = field(default_factory=list) + + If a field is excluded from :meth:`__init__` (using ``init=False``) + and the field also specifies ``default_factory``, then the default + factory function will always be called from the generated + :meth:`__init__` function. This happens because there is no other + way to give the field an initial value. + +Mutable default values +---------------------- + + Python stores default member variable values in class attributes. + Consider this example, not using dataclasses:: + + class C: + x = [] + def add(self, element): + self.x.append(element) + + o1 = C() + o2 = C() + o1.add(1) + o2.add(2) + assert o1.x == [1, 2] + assert o1.x is o2.x + + Note that the two instances of class ``C`` share the same class + variable ``x``, as expected. + + Using dataclasses, *if* this code was valid:: + + @dataclass + class D: + x: List = [] + def add(self, element): + self.x += element + + it would generate code similar to:: + + class D: + x = [] + def __init__(self, x=x): + self.x = x + def add(self, element): + self.x += element + + assert D().x is D().x + + This has the same issue as the original example using class ``C``. + That is, two instances of class ``D`` that do not specify a value for + ``x`` when creating a class instance will share the same copy of + ``x``. Because dataclasses just use normal Python class creation + they also share this behavior. There is no general way for Data + Classes to detect this condition. Instead, dataclasses will raise a + :exc:`TypeError` if it detects a default parameter of type ``list``, + ``dict``, or ``set``. This is a partial solution, but it does protect + against many common errors. + + Using default factory functions is a way to create new instances of + mutable types as default values for fields:: + + @dataclass + class D: + x: list = field(default_factory=list) + + assert D().x is not D().x + +Exceptions +---------- + +.. exception:: FrozenInstanceError + + Raised when an implicitly defined :meth:`__setattr__` or + :meth:`__delattr__` is called on a dataclass which was defined with + ``frozen=True``. diff --git a/Doc/library/gc.rst.bak b/Doc/library/gc.rst.bak new file mode 100644 index 00000000000000..a3d201d5055c8e --- /dev/null +++ b/Doc/library/gc.rst.bak @@ -0,0 +1,322 @@ +:mod:`gc` --- Garbage Collector interface +========================================= + +.. module:: gc + :synopsis: Interface to the cycle-detecting garbage collector. + +.. moduleauthor:: Neil Schemenauer +.. sectionauthor:: Neil Schemenauer + +-------------- + +This module provides an interface to the optional garbage collector. It +provides the ability to disable the collector, tune the collection frequency, +and set debugging options. It also provides access to unreachable objects that +the collector found but cannot free. Since the collector supplements the +reference counting already used in Python, you can disable the collector if you +are sure your program does not create reference cycles. Automatic collection +can be disabled by calling ``gc.disable()``. To debug a leaking program call +``gc.set_debug(gc.DEBUG_LEAK)``. Notice that this includes +``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in +gc.garbage for inspection. + +The :mod:`gc` module provides the following functions: + + +.. function:: enable() + + Enable automatic garbage collection. + + +.. function:: disable() + + Disable automatic garbage collection. + + +.. function:: isenabled() + + Return ``True`` if automatic collection is enabled. + + +.. function:: collect(generation=2) + + With no arguments, run a full collection. The optional argument *generation* + may be an integer specifying which generation to collect (from 0 to 2). A + :exc:`ValueError` is raised if the generation number is invalid. The number of + unreachable objects found is returned. + + The free lists maintained for a number of built-in types are cleared + whenever a full collection or collection of the highest generation (2) + is run. Not all items in some free lists may be freed due to the + particular implementation, in particular :class:`float`. + + +.. function:: set_debug(flags) + + Set the garbage collection debugging flags. Debugging information will be + written to ``sys.stderr``. See below for a list of debugging flags which can be + combined using bit operations to control debugging. + + +.. function:: get_debug() + + Return the debugging flags currently set. + + +.. function:: get_objects(generation=None) + + Returns a list of all objects tracked by the collector, excluding the list + returned. If *generation* is not None, return only the objects tracked by + the collector that are in that generation. + + .. versionchanged:: 3.8 + New *generation* parameter. + +.. function:: get_stats() + + Return a list of three per-generation dictionaries containing collection + statistics since interpreter start. The number of keys may change + in the future, but currently each dictionary will contain the following + items: + + * ``collections`` is the number of times this generation was collected; + + * ``collected`` is the total number of objects collected inside this + generation; + + * ``uncollectable`` is the total number of objects which were found + to be uncollectable (and were therefore moved to the :data:`garbage` + list) inside this generation. + + .. versionadded:: 3.4 + + +.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) + + Set the garbage collection thresholds (the collection frequency). Setting + *threshold0* to zero disables collection. + + The GC classifies objects into three generations depending on how many + collection sweeps they have survived. New objects are placed in the youngest + generation (generation ``0``). If an object survives a collection it is moved + into the next older generation. Since generation ``2`` is the oldest + generation, objects in that generation remain there after a collection. In + order to decide when to run, the collector keeps track of the number object + allocations and deallocations since the last collection. When the number of + allocations minus the number of deallocations exceeds *threshold0*, collection + starts. Initially only generation ``0`` is examined. If generation ``0`` has + been examined more than *threshold1* times since generation ``1`` has been + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. + + +.. function:: get_count() + + Return the current collection counts as a tuple of ``(count0, count1, + count2)``. + + +.. function:: get_threshold() + + Return the current collection thresholds as a tuple of ``(threshold0, + threshold1, threshold2)``. + + +.. function:: get_referrers(*objs) + + Return the list of objects that directly refer to any of objs. This function + will only locate those containers which support garbage collection; extension + types which do refer to other objects but do not support garbage collection will + not be found. + + Note that objects which have already been dereferenced, but which live in cycles + and have not yet been collected by the garbage collector can be listed among the + resulting referrers. To get only currently live objects, call :func:`collect` + before calling :func:`get_referrers`. + + .. warning:: + Care must be taken when using objects returned by :func:`get_referrers` because + some of them could still be under construction and hence in a temporarily + invalid state. Avoid using :func:`get_referrers` for any purpose other than + debugging. + + +.. function:: get_referents(*objs) + + Return a list of objects directly referred to by any of the arguments. The + referents returned are those objects visited by the arguments' C-level + :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually + directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects + that support garbage collection, and are only required to visit objects that may + be involved in a cycle. So, for example, if an integer is directly reachable + from an argument, that integer object may or may not appear in the result list. + + +.. function:: is_tracked(obj) + + Returns ``True`` if the object is currently tracked by the garbage collector, + ``False`` otherwise. As a general rule, instances of atomic types aren't + tracked and instances of non-atomic types (containers, user-defined + objects...) are. However, some type-specific optimizations can be present + in order to suppress the garbage collector footprint of simple instances + (e.g. dicts containing only atomic keys and values):: + + >>> gc.is_tracked(0) + False + >>> gc.is_tracked("a") + False + >>> gc.is_tracked([]) + True + >>> gc.is_tracked({}) + False + >>> gc.is_tracked({"a": 1}) + False + >>> gc.is_tracked({"a": []}) + True + + .. versionadded:: 3.1 + + +.. function:: is_finalized(obj) + + Returns ``True`` if the given object has been finalized by the + garbage collector, ``False`` otherwise. :: + + >>> x = None + >>> class Lazarus: + ... def __del__(self): + ... global x + ... x = self + ... + >>> lazarus = Lazarus() + >>> gc.is_finalized(lazarus) + False + >>> del lazarus + >>> gc.is_finalized(x) + True + + .. versionadded:: 3.9 + + +.. function:: freeze() + + Freeze all the objects tracked by gc - move them to a permanent generation + and ignore all the future collections. This can be used before a POSIX + fork() call to make the gc copy-on-write friendly or to speed up collection. + Also collection before a POSIX fork() call may free pages for future + allocation which can cause copy-on-write too so it's advised to disable gc + in parent process and freeze before fork and enable gc in child process. + + .. versionadded:: 3.7 + + +.. function:: unfreeze() + + Unfreeze the objects in the permanent generation, put them back into the + oldest generation. + + .. versionadded:: 3.7 + + +.. function:: get_freeze_count() + + Return the number of objects in the permanent generation. + + .. versionadded:: 3.7 + + +The following variables are provided for read-only access (you can mutate the +values but should not rebind them): + +.. data:: garbage + + A list of objects which the collector found to be unreachable but could + not be freed (uncollectable objects). Starting with Python 3.4, this + list should be empty most of the time, except when using instances of + C extension types with a non-``NULL`` ``tp_del`` slot. + + If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be + added to this list rather than freed. + + .. versionchanged:: 3.2 + If this list is non-empty at :term:`interpreter shutdown`, a + :exc:`ResourceWarning` is emitted, which is silent by default. If + :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects + are printed. + + .. versionchanged:: 3.4 + Following :pep:`442`, objects with a :meth:`__del__` method don't end + up in :attr:`gc.garbage` anymore. + +.. data:: callbacks + + A list of callbacks that will be invoked by the garbage collector before and + after collection. The callbacks will be called with two arguments, + *phase* and *info*. + + *phase* can be one of two values: + + "start": The garbage collection is about to start. + + "stop": The garbage collection has finished. + + *info* is a dict providing more information for the callback. The following + keys are currently defined: + + "generation": The oldest generation being collected. + + "collected": When *phase* is "stop", the number of objects + successfully collected. + + "uncollectable": When *phase* is "stop", the number of objects + that could not be collected and were put in :data:`garbage`. + + Applications can add their own callbacks to this list. The primary + use cases are: + + Gathering statistics about garbage collection, such as how often + various generations are collected, and how long the collection + takes. + + Allowing applications to identify and clear their own uncollectable + types when they appear in :data:`garbage`. + + .. versionadded:: 3.3 + + +The following constants are provided for use with :func:`set_debug`: + + +.. data:: DEBUG_STATS + + Print statistics during collection. This information can be useful when tuning + the collection frequency. + + +.. data:: DEBUG_COLLECTABLE + + Print information on collectable objects found. + + +.. data:: DEBUG_UNCOLLECTABLE + + Print information of uncollectable objects found (objects which are not + reachable but cannot be freed by the collector). These objects will be added + to the ``garbage`` list. + + .. versionchanged:: 3.2 + Also print the contents of the :data:`garbage` list at + :term:`interpreter shutdown`, if it isn't empty. + +.. data:: DEBUG_SAVEALL + + When set, all unreachable objects found will be appended to *garbage* rather + than being freed. This can be useful for debugging a leaking program. + + +.. data:: DEBUG_LEAK + + The debugging flags necessary for the collector to print information about a + leaking program (equal to ``DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | + DEBUG_SAVEALL``). diff --git a/Doc/library/importlib.metadata.rst.bak b/Doc/library/importlib.metadata.rst.bak new file mode 100644 index 00000000000000..7f154ea02cc4f2 --- /dev/null +++ b/Doc/library/importlib.metadata.rst.bak @@ -0,0 +1,262 @@ +.. _using: + +================================= + Using :mod:`!importlib.metadata` +================================= + +.. note:: + This functionality is provisional and may deviate from the usual + version semantics of the standard library. + +``importlib.metadata`` is a library that provides for access to installed +package metadata. Built in part on Python's import system, this library +intends to replace similar functionality in the `entry point +API`_ and `metadata API`_ of ``pkg_resources``. Along with +:mod:`importlib.resources` in Python 3.7 +and newer (backported as `importlib_resources`_ for older versions of +Python), this can eliminate the need to use the older and less efficient +``pkg_resources`` package. + +By "installed package" we generally mean a third-party package installed into +Python's ``site-packages`` directory via tools such as `pip +`_. Specifically, +it means a package with either a discoverable ``dist-info`` or ``egg-info`` +directory, and metadata defined by :pep:`566` or its older specifications. +By default, package metadata can live on the file system or in zip archives on +:data:`sys.path`. Through an extension mechanism, the metadata can live almost +anywhere. + + +Overview +======== + +Let's say you wanted to get the version string for a package you've installed +using ``pip``. We start by creating a virtual environment and installing +something into it: + +.. code-block:: shell-session + + $ python3 -m venv example + $ source example/bin/activate + (example) $ pip install wheel + +You can get the version string for ``wheel`` by running the following: + +.. code-block:: pycon + + (example) $ python + >>> from importlib.metadata import version # doctest: +SKIP + >>> version('wheel') # doctest: +SKIP + '0.32.3' + +You can also get the set of entry points keyed by group, such as +``console_scripts``, ``distutils.commands`` and others. Each group contains a +sequence of :ref:`EntryPoint ` objects. + +You can get the :ref:`metadata for a distribution `:: + + >>> list(metadata('wheel')) # doctest: +SKIP + ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] + +You can also get a :ref:`distribution's version number `, list its +:ref:`constituent files `, and get a list of the distribution's +:ref:`requirements`. + + +Functional API +============== + +This package provides the following functionality via its public API. + + +.. _entry-points: + +Entry points +------------ + +The ``entry_points()`` function returns a dictionary of all entry points, +keyed by group. Entry points are represented by ``EntryPoint`` instances; +each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and +a ``.load()`` method to resolve the value. There are also ``.module``, +``.attr``, and ``.extras`` attributes for getting the components of the +``.value`` attribute:: + + >>> eps = entry_points() # doctest: +SKIP + >>> list(eps) # doctest: +SKIP + ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] + >>> scripts = eps['console_scripts'] # doctest: +SKIP + >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP + >>> wheel # doctest: +SKIP + EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> wheel.module # doctest: +SKIP + 'wheel.cli' + >>> wheel.attr # doctest: +SKIP + 'main' + >>> wheel.extras # doctest: +SKIP + [] + >>> main = wheel.load() # doctest: +SKIP + >>> main # doctest: +SKIP + + +The ``group`` and ``name`` are arbitrary values defined by the package author +and usually a client will wish to resolve all entry points for a particular +group. Read `the setuptools docs +`_ +for more information on entry points, their definition, and usage. + + +.. _metadata: + +Distribution metadata +--------------------- + +Every distribution includes some metadata, which you can extract using the +``metadata()`` function:: + + >>> wheel_metadata = metadata('wheel') # doctest: +SKIP + +The keys of the returned data structure, a ``PackageMetadata``, +name the metadata keywords, and +the values are returned unparsed from the distribution metadata:: + + >>> wheel_metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + + +.. _version: + +Distribution versions +--------------------- + +The ``version()`` function is the quickest way to get a distribution's version +number, as a string:: + + >>> version('wheel') # doctest: +SKIP + '0.32.3' + + +.. _files: + +Distribution files +------------------ + +You can also get the full set of files contained within a distribution. The +``files()`` function takes a distribution package name and returns all of the +files installed by this distribution. Each file object returned is a +``PackagePath``, a :class:`pathlib.Path` derived object with additional ``dist``, +``size``, and ``hash`` properties as indicated by the metadata. For example:: + + >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP + >>> util # doctest: +SKIP + PackagePath('wheel/util.py') + >>> util.size # doctest: +SKIP + 859 + >>> util.dist # doctest: +SKIP + + >>> util.hash # doctest: +SKIP + + +Once you have the file, you can also read its contents:: + + >>> print(util.read_text()) # doctest: +SKIP + import base64 + import sys + ... + def as_bytes(s): + if isinstance(s, text_type): + return s.encode('utf-8') + return s + +In the case where the metadata file listing files +(RECORD or SOURCES.txt) is missing, ``files()`` will +return ``None``. The caller may wish to wrap calls to +``files()`` in `always_iterable +`_ +or otherwise guard against this condition if the target +distribution is not known to have the metadata present. + +.. _requirements: + +Distribution requirements +------------------------- + +To get the full set of requirements for a distribution, use the ``requires()`` +function:: + + >>> requires('wheel') # doctest: +SKIP + ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] + + +Distributions +============= + +While the above API is the most common and convenient usage, you can get all +of that information from the ``Distribution`` class. A ``Distribution`` is an +abstract object that represents the metadata for a Python package. You can +get the ``Distribution`` instance:: + + >>> from importlib.metadata import distribution # doctest: +SKIP + >>> dist = distribution('wheel') # doctest: +SKIP + +Thus, an alternative way to get the version number is through the +``Distribution`` instance:: + + >>> dist.version # doctest: +SKIP + '0.32.3' + +There are all kinds of additional metadata available on the ``Distribution`` +instance:: + + >>> dist.metadata['Requires-Python'] # doctest: +SKIP + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> dist.metadata['License'] # doctest: +SKIP + 'MIT' + +The full set of available metadata is not described here. See :pep:`566` +for additional details. + + +Extending the search algorithm +============================== + +Because package metadata is not available through :data:`sys.path` searches, or +package loaders directly, the metadata for a package is found through import +system :ref:`finders `. To find a distribution package's metadata, +``importlib.metadata`` queries the list of :term:`meta path finders ` on +:data:`sys.meta_path`. + +The default ``PathFinder`` for Python includes a hook that calls into +``importlib.metadata.MetadataPathFinder`` for finding distributions +loaded from typical file-system-based paths. + +The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the +interface expected of finders by Python's import system. +``importlib.metadata`` extends this protocol by looking for an optional +``find_distributions`` callable on the finders from +:data:`sys.meta_path` and presents this extended interface as the +``DistributionFinder`` abstract base class, which defines this abstract +method:: + + @abc.abstractmethod + def find_distributions(context=DistributionFinder.Context()): + """Return an iterable of all Distribution instances capable of + loading the metadata for packages for the indicated ``context``. + """ + +The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` +properties indicating the path to search and name to match and may +supply other relevant context. + +What this means in practice is that to support finding distribution package +metadata in locations other than the file system, subclass +``Distribution`` and implement the abstract methods. Then from +a custom finder, return instances of this derived ``Distribution`` in the +``find_distributions()`` method. + + +.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points +.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html + + +.. rubric:: Footnotes diff --git a/Doc/library/logging.rst.bak b/Doc/library/logging.rst.bak new file mode 100644 index 00000000000000..431a5849fa9bff --- /dev/null +++ b/Doc/library/logging.rst.bak @@ -0,0 +1,1350 @@ +:mod:`logging` --- Logging facility for Python +============================================== + +.. module:: logging + :synopsis: Flexible event logging system for applications. + +.. moduleauthor:: Vinay Sajip +.. sectionauthor:: Vinay Sajip + +**Source code:** :source:`Lib/logging/__init__.py` + +.. index:: pair: Errors; logging + +.. sidebar:: Important + + This page contains the API reference information. For tutorial + information and discussion of more advanced topics, see + + * :ref:`Basic Tutorial ` + * :ref:`Advanced Tutorial ` + * :ref:`Logging Cookbook ` + +-------------- + +This module defines functions and classes which implement a flexible event +logging system for applications and libraries. + +The key benefit of having the logging API provided by a standard library module +is that all Python modules can participate in logging, so your application log +can include your own messages integrated with messages from third-party +modules. + +The module provides a lot of functionality and flexibility. If you are +unfamiliar with logging, the best way to get to grips with it is to see the +tutorials (see the links on the right). + +The basic classes defined by the module, together with their functions, are +listed below. + +* Loggers expose the interface that application code directly uses. +* Handlers send the log records (created by loggers) to the appropriate + destination. +* Filters provide a finer grained facility for determining which log records + to output. +* Formatters specify the layout of log records in the final output. + + +.. _logger: + +Logger Objects +-------------- + +Loggers have the following attributes and methods. Note that Loggers should +*NEVER* be instantiated directly, but always through the module-level function +``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same +name will always return a reference to the same Logger object. + +The ``name`` is potentially a period-separated hierarchical value, like +``foo.bar.baz`` (though it could also be just plain ``foo``, for example). +Loggers that are further down in the hierarchical list are children of loggers +higher up in the list. For example, given a logger with a name of ``foo``, +loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all +descendants of ``foo``. The logger name hierarchy is analogous to the Python +package hierarchy, and identical to it if you organise your loggers on a +per-module basis using the recommended construction +``logging.getLogger(__name__)``. That's because in a module, ``__name__`` +is the module's name in the Python package namespace. + + +.. class:: Logger + + .. attribute:: Logger.propagate + + If this attribute evaluates to true, events logged to this logger will be + passed to the handlers of higher level (ancestor) loggers, in addition to + any handlers attached to this logger. Messages are passed directly to the + ancestor loggers' handlers - neither the level nor filters of the ancestor + loggers in question are considered. + + If this evaluates to false, logging messages are not passed to the handlers + of ancestor loggers. + + The constructor sets this attribute to ``True``. + + .. note:: If you attach a handler to a logger *and* one or more of its + ancestors, it may emit the same record multiple times. In general, you + should not need to attach a handler to more than one logger - if you just + attach it to the appropriate logger which is highest in the logger + hierarchy, then it will see all events logged by all descendant loggers, + provided that their propagate setting is left set to ``True``. A common + scenario is to attach handlers only to the root logger, and to let + propagation take care of the rest. + + .. method:: Logger.setLevel(level) + + Sets the threshold for this logger to *level*. Logging messages which are less + severe than *level* will be ignored; logging messages which have severity *level* + or higher will be emitted by whichever handler or handlers service this logger, + unless a handler's level has been set to a higher severity level than *level*. + + When a logger is created, the level is set to :const:`NOTSET` (which causes + all messages to be processed when the logger is the root logger, or delegation + to the parent when the logger is a non-root logger). Note that the root logger + is created with level :const:`WARNING`. + + The term 'delegation to the parent' means that if a logger has a level of + NOTSET, its chain of ancestor loggers is traversed until either an ancestor with + a level other than NOTSET is found, or the root is reached. + + If an ancestor is found with a level other than NOTSET, then that ancestor's + level is treated as the effective level of the logger where the ancestor search + began, and is used to determine how a logging event is handled. + + If the root is reached, and it has a level of NOTSET, then all messages will be + processed. Otherwise, the root's level will be used as the effective level. + + See :ref:`levels` for a list of levels. + + .. versionchanged:: 3.2 + The *level* parameter now accepts a string representation of the + level such as 'INFO' as an alternative to the integer constants + such as :const:`INFO`. Note, however, that levels are internally stored + as integers, and methods such as e.g. :meth:`getEffectiveLevel` and + :meth:`isEnabledFor` will return/expect to be passed integers. + + + .. method:: Logger.isEnabledFor(level) + + Indicates if a message of severity *level* would be processed by this logger. + This method checks first the module-level level set by + ``logging.disable(level)`` and then the logger's effective level as determined + by :meth:`getEffectiveLevel`. + + + .. method:: Logger.getEffectiveLevel() + + Indicates the effective level for this logger. If a value other than + :const:`NOTSET` has been set using :meth:`setLevel`, it is returned. Otherwise, + the hierarchy is traversed towards the root until a value other than + :const:`NOTSET` is found, and that value is returned. The value returned is + an integer, typically one of :const:`logging.DEBUG`, :const:`logging.INFO` + etc. + + + .. method:: Logger.getChild(suffix) + + Returns a logger which is a descendant to this logger, as determined by the suffix. + Thus, ``logging.getLogger('abc').getChild('def.ghi')`` would return the same + logger as would be returned by ``logging.getLogger('abc.def.ghi')``. This is a + convenience method, useful when the parent logger is named using e.g. ``__name__`` + rather than a literal string. + + .. versionadded:: 3.2 + + + .. method:: Logger.debug(msg, *args, **kwargs) + + Logs a message with level :const:`DEBUG` on this logger. The *msg* is the + message format string, and the *args* are the arguments which are merged into + *msg* using the string formatting operator. (Note that this means that you can + use keywords in the format string, together with a single dictionary argument.) + No % formatting operation is performed on *msg* when no *args* are supplied. + + There are four keyword arguments in *kwargs* which are inspected: + *exc_info*, *stack_info*, *stacklevel* and *extra*. + + If *exc_info* does not evaluate as false, it causes exception information to be + added to the logging message. If an exception tuple (in the format returned by + :func:`sys.exc_info`) or an exception instance is provided, it is used; + otherwise, :func:`sys.exc_info` is called to get the exception information. + + The second optional keyword argument is *stack_info*, which defaults to + ``False``. If true, stack information is added to the logging + message, including the actual logging call. Note that this is not the same + stack information as that displayed through specifying *exc_info*: The + former is stack frames from the bottom of the stack up to the logging call + in the current thread, whereas the latter is information about stack frames + which have been unwound, following an exception, while searching for + exception handlers. + + You can specify *stack_info* independently of *exc_info*, e.g. to just show + how you got to a certain point in your code, even when no exceptions were + raised. The stack frames are printed following a header line which says: + + .. code-block:: none + + Stack (most recent call last): + + This mimics the ``Traceback (most recent call last):`` which is used when + displaying exception frames. + + The third optional keyword argument is *stacklevel*, which defaults to ``1``. + If greater than 1, the corresponding number of stack frames are skipped + when computing the line number and function name set in the :class:`LogRecord` + created for the logging event. This can be used in logging helpers so that + the function name, filename and line number recorded are not the information + for the helper function/method, but rather its caller. The name of this + parameter mirrors the equivalent one in the :mod:`warnings` module. + + The fourth keyword argument is *extra* which can be used to pass a + dictionary which is used to populate the __dict__ of the :class:`LogRecord` + created for the logging event with user-defined attributes. These custom + attributes can then be used as you like. For example, they could be + incorporated into logged messages. For example:: + + FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' + logging.basicConfig(format=FORMAT) + d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} + logger = logging.getLogger('tcpserver') + logger.warning('Protocol problem: %s', 'connection reset', extra=d) + + would print something like + + .. code-block:: none + + 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset + + The keys in the dictionary passed in *extra* should not clash with the keys used + by the logging system. (See the :class:`Formatter` documentation for more + information on which keys are used by the logging system.) + + If you choose to use these attributes in logged messages, you need to exercise + some care. In the above example, for instance, the :class:`Formatter` has been + set up with a format string which expects 'clientip' and 'user' in the attribute + dictionary of the :class:`LogRecord`. If these are missing, the message will + not be logged because a string formatting exception will occur. So in this case, + you always need to pass the *extra* dictionary with these keys. + + While this might be annoying, this feature is intended for use in specialized + circumstances, such as multi-threaded servers where the same code executes in + many contexts, and interesting conditions which arise are dependent on this + context (such as remote client IP address and authenticated user name, in the + above example). In such circumstances, it is likely that specialized + :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. + + .. versionchanged:: 3.2 + The *stack_info* parameter was added. + + .. versionchanged:: 3.5 + The *exc_info* parameter can now accept exception instances. + + .. versionchanged:: 3.8 + The *stacklevel* parameter was added. + + + .. method:: Logger.info(msg, *args, **kwargs) + + Logs a message with level :const:`INFO` on this logger. The arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.warning(msg, *args, **kwargs) + + Logs a message with level :const:`WARNING` on this logger. The arguments are + interpreted as for :meth:`debug`. + + .. note:: There is an obsolete method ``warn`` which is functionally + identical to ``warning``. As ``warn`` is deprecated, please do not use + it - use ``warning`` instead. + + .. method:: Logger.error(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on this logger. The arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.critical(msg, *args, **kwargs) + + Logs a message with level :const:`CRITICAL` on this logger. The arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.log(level, msg, *args, **kwargs) + + Logs a message with integer level *level* on this logger. The other arguments are + interpreted as for :meth:`debug`. + + + .. method:: Logger.exception(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on this logger. The arguments are + interpreted as for :meth:`debug`. Exception info is added to the logging + message. This method should only be called from an exception handler. + + + .. method:: Logger.addFilter(filter) + + Adds the specified filter *filter* to this logger. + + + .. method:: Logger.removeFilter(filter) + + Removes the specified filter *filter* from this logger. + + + .. method:: Logger.filter(record) + + Apply this logger's filters to the record and return ``True`` if the + record is to be processed. The filters are consulted in turn, until one of + them returns a false value. If none of them return a false value, the record + will be processed (passed to handlers). If one returns a false value, no + further processing of the record occurs. + + + .. method:: Logger.addHandler(hdlr) + + Adds the specified handler *hdlr* to this logger. + + + .. method:: Logger.removeHandler(hdlr) + + Removes the specified handler *hdlr* from this logger. + + + .. method:: Logger.findCaller(stack_info=False, stacklevel=1) + + Finds the caller's source filename and line number. Returns the filename, line + number, function name and stack information as a 4-element tuple. The stack + information is returned as ``None`` unless *stack_info* is ``True``. + + The *stacklevel* parameter is passed from code calling the :meth:`debug` + and other APIs. If greater than 1, the excess is used to skip stack frames + before determining the values to be returned. This will generally be useful + when calling logging APIs from helper/wrapper code, so that the information + in the event log refers not to the helper/wrapper code, but to the code that + calls it. + + + .. method:: Logger.handle(record) + + Handles a record by passing it to all handlers associated with this logger and + its ancestors (until a false value of *propagate* is found). This method is used + for unpickled records received from a socket, as well as those created locally. + Logger-level filtering is applied using :meth:`~Logger.filter`. + + + .. method:: Logger.makeRecord(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) + + This is a factory method which can be overridden in subclasses to create + specialized :class:`LogRecord` instances. + + .. method:: Logger.hasHandlers() + + Checks to see if this logger has any handlers configured. This is done by + looking for handlers in this logger and its parents in the logger hierarchy. + Returns ``True`` if a handler was found, else ``False``. The method stops searching + up the hierarchy whenever a logger with the 'propagate' attribute set to + false is found - that will be the last logger which is checked for the + existence of handlers. + + .. versionadded:: 3.2 + + .. versionchanged:: 3.7 + Loggers can now be pickled and unpickled. + +.. _levels: + +Logging Levels +-------------- + +The numeric values of logging levels are given in the following table. These are +primarily of interest if you want to define your own levels, and need them to +have specific values relative to the predefined levels. If you define a level +with the same numeric value, it overwrites the predefined value; the predefined +name is lost. + ++--------------+---------------+ +| Level | Numeric value | ++==============+===============+ +| ``CRITICAL`` | 50 | ++--------------+---------------+ +| ``ERROR`` | 40 | ++--------------+---------------+ +| ``WARNING`` | 30 | ++--------------+---------------+ +| ``INFO`` | 20 | ++--------------+---------------+ +| ``DEBUG`` | 10 | ++--------------+---------------+ +| ``NOTSET`` | 0 | ++--------------+---------------+ + + +.. _handler: + +Handler Objects +--------------- + +Handlers have the following attributes and methods. Note that :class:`Handler` +is never instantiated directly; this class acts as a base for more useful +subclasses. However, the :meth:`__init__` method in subclasses needs to call +:meth:`Handler.__init__`. + +.. class:: Handler + + .. method:: Handler.__init__(level=NOTSET) + + Initializes the :class:`Handler` instance by setting its level, setting the list + of filters to the empty list and creating a lock (using :meth:`createLock`) for + serializing access to an I/O mechanism. + + + .. method:: Handler.createLock() + + Initializes a thread lock which can be used to serialize access to underlying + I/O functionality which may not be threadsafe. + + + .. method:: Handler.acquire() + + Acquires the thread lock created with :meth:`createLock`. + + + .. method:: Handler.release() + + Releases the thread lock acquired with :meth:`acquire`. + + + .. method:: Handler.setLevel(level) + + Sets the threshold for this handler to *level*. Logging messages which are + less severe than *level* will be ignored. When a handler is created, the + level is set to :const:`NOTSET` (which causes all messages to be + processed). + + See :ref:`levels` for a list of levels. + + .. versionchanged:: 3.2 + The *level* parameter now accepts a string representation of the + level such as 'INFO' as an alternative to the integer constants + such as :const:`INFO`. + + + .. method:: Handler.setFormatter(fmt) + + Sets the :class:`Formatter` for this handler to *fmt*. + + + .. method:: Handler.addFilter(filter) + + Adds the specified filter *filter* to this handler. + + + .. method:: Handler.removeFilter(filter) + + Removes the specified filter *filter* from this handler. + + + .. method:: Handler.filter(record) + + Apply this handler's filters to the record and return ``True`` if the + record is to be processed. The filters are consulted in turn, until one of + them returns a false value. If none of them return a false value, the record + will be emitted. If one returns a false value, the handler will not emit the + record. + + + .. method:: Handler.flush() + + Ensure all logging output has been flushed. This version does nothing and is + intended to be implemented by subclasses. + + + .. method:: Handler.close() + + Tidy up any resources used by the handler. This version does no output but + removes the handler from an internal list of handlers which is closed when + :func:`shutdown` is called. Subclasses should ensure that this gets called + from overridden :meth:`close` methods. + + + .. method:: Handler.handle(record) + + Conditionally emits the specified logging record, depending on filters which may + have been added to the handler. Wraps the actual emission of the record with + acquisition/release of the I/O thread lock. + + + .. method:: Handler.handleError(record) + + This method should be called from handlers when an exception is encountered + during an :meth:`emit` call. If the module-level attribute + ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is + what is mostly wanted for a logging system - most users will not care about + errors in the logging system, they are more interested in application + errors. You could, however, replace this with a custom handler if you wish. + The specified record is the one which was being processed when the exception + occurred. (The default value of ``raiseExceptions`` is ``True``, as that is + more useful during development). + + + .. method:: Handler.format(record) + + Do formatting for a record - if a formatter is set, use it. Otherwise, use the + default formatter for the module. + + + .. method:: Handler.emit(record) + + Do whatever it takes to actually log the specified logging record. This version + is intended to be implemented by subclasses and so raises a + :exc:`NotImplementedError`. + +For a list of handlers included as standard, see :mod:`logging.handlers`. + +.. _formatter-objects: + +Formatter Objects +----------------- + +.. currentmodule:: logging + +:class:`Formatter` objects have the following attributes and methods. They are +responsible for converting a :class:`LogRecord` to (usually) a string which can +be interpreted by either a human or an external system. The base +:class:`Formatter` allows a formatting string to be specified. If none is +supplied, the default value of ``'%(message)s'`` is used, which just includes +the message in the logging call. To have additional items of information in the +formatted output (such as a timestamp), keep reading. + +A Formatter can be initialized with a format string which makes use of knowledge +of the :class:`LogRecord` attributes - such as the default value mentioned above +making use of the fact that the user's message and arguments are pre-formatted +into a :class:`LogRecord`'s *message* attribute. This format string contains +standard Python %-style mapping keys. See section :ref:`old-string-formatting` +for more information on string formatting. + +The useful mapping keys in a :class:`LogRecord` are given in the section on +:ref:`logrecord-attributes`. + + +.. class:: Formatter(fmt=None, datefmt=None, style='%', validate=True, *, defaults=None) + + Returns a new instance of the :class:`Formatter` class. The instance is + initialized with a format string for the message as a whole, as well as a + format string for the date/time portion of a message. If no *fmt* is + specified, ``'%(message)s'`` is used. If no *datefmt* is specified, a format + is used which is described in the :meth:`formatTime` documentation. + + The *style* parameter can be one of '%', '{' or '$' and determines how + the format string will be merged with its data: using one of %-formatting, + :meth:`str.format` or :class:`string.Template`. This only applies to the + format string *fmt* (e.g. ``'%(message)s'`` or ``{message}``), not to the + actual log messages passed to ``Logger.debug`` etc; see + :ref:`formatting-styles` for more information on using {- and $-formatting + for log messages. + + The *defaults* parameter can be a dictionary with default values to use in + custom fields. For example: + ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` + + .. versionchanged:: 3.2 + The *style* parameter was added. + + .. versionchanged:: 3.8 + The *validate* parameter was added. Incorrect or mismatched style and fmt + will raise a ``ValueError``. + For example: ``logging.Formatter('%(asctime)s - %(message)s', style='{')``. + + .. versionchanged:: 3.10 + The *defaults* parameter was added. + + .. method:: format(record) + + The record's attribute dictionary is used as the operand to a string + formatting operation. Returns the resulting string. Before formatting the + dictionary, a couple of preparatory steps are carried out. The *message* + attribute of the record is computed using *msg* % *args*. If the + formatting string contains ``'(asctime)'``, :meth:`formatTime` is called + to format the event time. If there is exception information, it is + formatted using :meth:`formatException` and appended to the message. Note + that the formatted exception information is cached in attribute + *exc_text*. This is useful because the exception information can be + pickled and sent across the wire, but you should be careful if you have + more than one :class:`Formatter` subclass which customizes the formatting + of exception information. In this case, you will have to clear the cached + value (by setting the *exc_text* attribute to ``None``) after a formatter + has done its formatting, so that the next formatter to handle the event + doesn't use the cached value, but recalculates it afresh. + + If stack information is available, it's appended after the exception + information, using :meth:`formatStack` to transform it if necessary. + + + .. method:: formatTime(record, datefmt=None) + + This method should be called from :meth:`format` by a formatter which + wants to make use of a formatted time. This method can be overridden in + formatters to provide for any specific requirement, but the basic behavior + is as follows: if *datefmt* (a string) is specified, it is used with + :func:`time.strftime` to format the creation time of the + record. Otherwise, the format '%Y-%m-%d %H:%M:%S,uuu' is used, where the + uuu part is a millisecond value and the other letters are as per the + :func:`time.strftime` documentation. An example time in this format is + ``2003-01-23 00:29:50,411``. The resulting string is returned. + + This function uses a user-configurable function to convert the creation + time to a tuple. By default, :func:`time.localtime` is used; to change + this for a particular formatter instance, set the ``converter`` attribute + to a function with the same signature as :func:`time.localtime` or + :func:`time.gmtime`. To change it for all formatters, for example if you + want all logging times to be shown in GMT, set the ``converter`` + attribute in the ``Formatter`` class. + + .. versionchanged:: 3.3 + Previously, the default format was hard-coded as in this example: + ``2010-09-06 22:38:15,292`` where the part before the comma is + handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the + part after the comma is a millisecond value. Because strptime does not + have a format placeholder for milliseconds, the millisecond value is + appended using another format string, ``'%s,%03d'`` --- and both of these + format strings have been hardcoded into this method. With the change, + these strings are defined as class-level attributes which can be + overridden at the instance level when desired. The names of the + attributes are ``default_time_format`` (for the strptime format string) + and ``default_msec_format`` (for appending the millisecond value). + + .. versionchanged:: 3.9 + The ``default_msec_format`` can be ``None``. + + .. method:: formatException(exc_info) + + Formats the specified exception information (a standard exception tuple as + returned by :func:`sys.exc_info`) as a string. This default implementation + just uses :func:`traceback.print_exception`. The resulting string is + returned. + + .. method:: formatStack(stack_info) + + Formats the specified stack information (a string as returned by + :func:`traceback.print_stack`, but with the last newline removed) as a + string. This default implementation just returns the input value. + +.. _filter: + +Filter Objects +-------------- + +``Filters`` can be used by ``Handlers`` and ``Loggers`` for more sophisticated +filtering than is provided by levels. The base filter class only allows events +which are below a certain point in the logger hierarchy. For example, a filter +initialized with 'A.B' will allow events logged by loggers 'A.B', 'A.B.C', +'A.B.C.D', 'A.B.D' etc. but not 'A.BB', 'B.A.B' etc. If initialized with the +empty string, all events are passed. + + +.. class:: Filter(name='') + + Returns an instance of the :class:`Filter` class. If *name* is specified, it + names a logger which, together with its children, will have its events allowed + through the filter. If *name* is the empty string, allows every event. + + + .. method:: filter(record) + + Is the specified record to be logged? Returns zero for no, nonzero for + yes. If deemed appropriate, the record may be modified in-place by this + method. + +Note that filters attached to handlers are consulted before an event is +emitted by the handler, whereas filters attached to loggers are consulted +whenever an event is logged (using :meth:`debug`, :meth:`info`, +etc.), before sending an event to handlers. This means that events which have +been generated by descendant loggers will not be filtered by a logger's filter +setting, unless the filter has also been applied to those descendant loggers. + +You don't actually need to subclass ``Filter``: you can pass any instance +which has a ``filter`` method with the same semantics. + +.. versionchanged:: 3.2 + You don't need to create specialized ``Filter`` classes, or use other + classes with a ``filter`` method: you can use a function (or other + callable) as a filter. The filtering logic will check to see if the filter + object has a ``filter`` attribute: if it does, it's assumed to be a + ``Filter`` and its :meth:`~Filter.filter` method is called. Otherwise, it's + assumed to be a callable and called with the record as the single + parameter. The returned value should conform to that returned by + :meth:`~Filter.filter`. + +Although filters are used primarily to filter records based on more +sophisticated criteria than levels, they get to see every record which is +processed by the handler or logger they're attached to: this can be useful if +you want to do things like counting how many records were processed by a +particular logger or handler, or adding, changing or removing attributes in +the :class:`LogRecord` being processed. Obviously changing the LogRecord needs +to be done with some care, but it does allow the injection of contextual +information into logs (see :ref:`filters-contextual`). + +.. _log-record: + +LogRecord Objects +----------------- + +:class:`LogRecord` instances are created automatically by the :class:`Logger` +every time something is logged, and can be created manually via +:func:`makeLogRecord` (for example, from a pickled event received over the +wire). + + +.. class:: LogRecord(name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None) + + Contains all the information pertinent to the event being logged. + + The primary information is passed in :attr:`msg` and :attr:`args`, which + are combined using ``msg % args`` to create the :attr:`message` field of the + record. + + :param name: The name of the logger used to log the event represented by + this LogRecord. Note that this name will always have this + value, even though it may be emitted by a handler attached to + a different (ancestor) logger. + :param level: The numeric level of the logging event (one of DEBUG, INFO etc.) + Note that this is converted to *two* attributes of the LogRecord: + ``levelno`` for the numeric value and ``levelname`` for the + corresponding level name. + :param pathname: The full pathname of the source file where the logging call + was made. + :param lineno: The line number in the source file where the logging call was + made. + :param msg: The event description message, possibly a format string with + placeholders for variable data. + :param args: Variable data to merge into the *msg* argument to obtain the + event description. + :param exc_info: An exception tuple with the current exception information, + or ``None`` if no exception information is available. + :param func: The name of the function or method from which the logging call + was invoked. + :param sinfo: A text string representing stack information from the base of + the stack in the current thread, up to the logging call. + + .. method:: getMessage() + + Returns the message for this :class:`LogRecord` instance after merging any + user-supplied arguments with the message. If the user-supplied message + argument to the logging call is not a string, :func:`str` is called on it to + convert it to a string. This allows use of user-defined classes as + messages, whose ``__str__`` method can return the actual format string to + be used. + + .. versionchanged:: 3.2 + The creation of a :class:`LogRecord` has been made more configurable by + providing a factory which is used to create the record. The factory can be + set using :func:`getLogRecordFactory` and :func:`setLogRecordFactory` + (see this for the factory's signature). + + This functionality can be used to inject your own values into a + :class:`LogRecord` at creation time. You can use the following pattern:: + + old_factory = logging.getLogRecordFactory() + + def record_factory(*args, **kwargs): + record = old_factory(*args, **kwargs) + record.custom_attribute = 0xdecafbad + return record + + logging.setLogRecordFactory(record_factory) + + With this pattern, multiple factories could be chained, and as long + as they don't overwrite each other's attributes or unintentionally + overwrite the standard attributes listed above, there should be no + surprises. + + +.. _logrecord-attributes: + +LogRecord attributes +-------------------- + +The LogRecord has a number of attributes, most of which are derived from the +parameters to the constructor. (Note that the names do not always correspond +exactly between the LogRecord constructor parameters and the LogRecord +attributes.) These attributes can be used to merge data from the record into +the format string. The following table lists (in alphabetical order) the +attribute names, their meanings and the corresponding placeholder in a %-style +format string. + +If you are using {}-formatting (:func:`str.format`), you can use +``{attrname}`` as the placeholder in the format string. If you are using +$-formatting (:class:`string.Template`), use the form ``${attrname}``. In +both cases, of course, replace ``attrname`` with the actual attribute name +you want to use. + +In the case of {}-formatting, you can specify formatting flags by placing them +after the attribute name, separated from it with a colon. For example: a +placeholder of ``{msecs:03d}`` would format a millisecond value of ``4`` as +``004``. Refer to the :meth:`str.format` documentation for full details on +the options available to you. + ++----------------+-------------------------+-----------------------------------------------+ +| Attribute name | Format | Description | ++================+=========================+===============================================+ +| args | You shouldn't need to | The tuple of arguments merged into ``msg`` to | +| | format this yourself. | produce ``message``, or a dict whose values | +| | | are used for the merge (when there is only one| +| | | argument, and it is a dictionary). | ++----------------+-------------------------+-----------------------------------------------+ +| asctime | ``%(asctime)s`` | Human-readable time when the | +| | | :class:`LogRecord` was created. By default | +| | | this is of the form '2003-07-08 16:49:45,896' | +| | | (the numbers after the comma are millisecond | +| | | portion of the time). | ++----------------+-------------------------+-----------------------------------------------+ +| created | ``%(created)f`` | Time when the :class:`LogRecord` was created | +| | | (as returned by :func:`time.time`). | ++----------------+-------------------------+-----------------------------------------------+ +| exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | +| | format this yourself. | if no exception has occurred, ``None``. | ++----------------+-------------------------+-----------------------------------------------+ +| filename | ``%(filename)s`` | Filename portion of ``pathname``. | ++----------------+-------------------------+-----------------------------------------------+ +| funcName | ``%(funcName)s`` | Name of function containing the logging call. | ++----------------+-------------------------+-----------------------------------------------+ +| levelname | ``%(levelname)s`` | Text logging level for the message | +| | | (``'DEBUG'``, ``'INFO'``, ``'WARNING'``, | +| | | ``'ERROR'``, ``'CRITICAL'``). | ++----------------+-------------------------+-----------------------------------------------+ +| levelno | ``%(levelno)s`` | Numeric logging level for the message | +| | | (:const:`DEBUG`, :const:`INFO`, | +| | | :const:`WARNING`, :const:`ERROR`, | +| | | :const:`CRITICAL`). | ++----------------+-------------------------+-----------------------------------------------+ +| lineno | ``%(lineno)d`` | Source line number where the logging call was | +| | | issued (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| message | ``%(message)s`` | The logged message, computed as ``msg % | +| | | args``. This is set when | +| | | :meth:`Formatter.format` is invoked. | ++----------------+-------------------------+-----------------------------------------------+ +| module | ``%(module)s`` | Module (name portion of ``filename``). | ++----------------+-------------------------+-----------------------------------------------+ +| msecs | ``%(msecs)d`` | Millisecond portion of the time when the | +| | | :class:`LogRecord` was created. | ++----------------+-------------------------+-----------------------------------------------+ +| msg | You shouldn't need to | The format string passed in the original | +| | format this yourself. | logging call. Merged with ``args`` to | +| | | produce ``message``, or an arbitrary object | +| | | (see :ref:`arbitrary-object-messages`). | ++----------------+-------------------------+-----------------------------------------------+ +| name | ``%(name)s`` | Name of the logger used to log the call. | ++----------------+-------------------------+-----------------------------------------------+ +| pathname | ``%(pathname)s`` | Full pathname of the source file where the | +| | | logging call was issued (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| process | ``%(process)d`` | Process ID (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| processName | ``%(processName)s`` | Process name (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| relativeCreated| ``%(relativeCreated)d`` | Time in milliseconds when the LogRecord was | +| | | created, relative to the time the logging | +| | | module was loaded. | ++----------------+-------------------------+-----------------------------------------------+ +| stack_info | You shouldn't need to | Stack frame information (where available) | +| | format this yourself. | from the bottom of the stack in the current | +| | | thread, up to and including the stack frame | +| | | of the logging call which resulted in the | +| | | creation of this record. | ++----------------+-------------------------+-----------------------------------------------+ +| thread | ``%(thread)d`` | Thread ID (if available). | ++----------------+-------------------------+-----------------------------------------------+ +| threadName | ``%(threadName)s`` | Thread name (if available). | ++----------------+-------------------------+-----------------------------------------------+ + +.. versionchanged:: 3.1 + *processName* was added. + + +.. _logger-adapter: + +LoggerAdapter Objects +--------------------- + +:class:`LoggerAdapter` instances are used to conveniently pass contextual +information into logging calls. For a usage example, see the section on +:ref:`adding contextual information to your logging output `. + +.. class:: LoggerAdapter(logger, extra) + + Returns an instance of :class:`LoggerAdapter` initialized with an + underlying :class:`Logger` instance and a dict-like object. + + .. method:: process(msg, kwargs) + + Modifies the message and/or keyword arguments passed to a logging call in + order to insert contextual information. This implementation takes the object + passed as *extra* to the constructor and adds it to *kwargs* using key + 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the + (possibly modified) versions of the arguments passed in. + +In addition to the above, :class:`LoggerAdapter` supports the following +methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, +:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, +:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, +:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and +:meth:`~Logger.hasHandlers`. These methods have the same signatures as their +counterparts in :class:`Logger`, so you can use the two types of instances +interchangeably. + +.. versionchanged:: 3.2 + The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, + :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added + to :class:`LoggerAdapter`. These methods delegate to the underlying logger. + + +Thread Safety +------------- + +The logging module is intended to be thread-safe without any special work +needing to be done by its clients. It achieves this though using threading +locks; there is one lock to serialize access to the module's shared data, and +each handler also creates a lock to serialize access to its underlying I/O. + +If you are implementing asynchronous signal handlers using the :mod:`signal` +module, you may not be able to use logging from within such handlers. This is +because lock implementations in the :mod:`threading` module are not always +re-entrant, and so cannot be invoked from such signal handlers. + + +Module-Level Functions +---------------------- + +In addition to the classes described above, there are a number of module-level +functions. + + +.. function:: getLogger(name=None) + + Return a logger with the specified name or, if name is ``None``, return a + logger which is the root logger of the hierarchy. If specified, the name is + typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. + Choice of these names is entirely up to the developer who is using logging. + + All calls to this function with a given name return the same logger instance. + This means that logger instances never need to be passed between different parts + of an application. + + +.. function:: getLoggerClass() + + Return either the standard :class:`Logger` class, or the last class passed to + :func:`setLoggerClass`. This function may be called from within a new class + definition, to ensure that installing a customized :class:`Logger` class will + not undo customizations already applied by other code. For example:: + + class MyLogger(logging.getLoggerClass()): + # ... override behaviour here + + +.. function:: getLogRecordFactory() + + Return a callable which is used to create a :class:`LogRecord`. + + .. versionadded:: 3.2 + This function has been provided, along with :func:`setLogRecordFactory`, + to allow developers more control over how the :class:`LogRecord` + representing a logging event is constructed. + + See :func:`setLogRecordFactory` for more information about the how the + factory is called. + +.. function:: debug(msg, *args, **kwargs) + + Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the + message format string, and the *args* are the arguments which are merged into + *msg* using the string formatting operator. (Note that this means that you can + use keywords in the format string, together with a single dictionary argument.) + + There are three keyword arguments in *kwargs* which are inspected: *exc_info* + which, if it does not evaluate as false, causes exception information to be + added to the logging message. If an exception tuple (in the format returned by + :func:`sys.exc_info`) or an exception instance is provided, it is used; + otherwise, :func:`sys.exc_info` is called to get the exception information. + + The second optional keyword argument is *stack_info*, which defaults to + ``False``. If true, stack information is added to the logging + message, including the actual logging call. Note that this is not the same + stack information as that displayed through specifying *exc_info*: The + former is stack frames from the bottom of the stack up to the logging call + in the current thread, whereas the latter is information about stack frames + which have been unwound, following an exception, while searching for + exception handlers. + + You can specify *stack_info* independently of *exc_info*, e.g. to just show + how you got to a certain point in your code, even when no exceptions were + raised. The stack frames are printed following a header line which says: + + .. code-block:: none + + Stack (most recent call last): + + This mimics the ``Traceback (most recent call last):`` which is used when + displaying exception frames. + + The third optional keyword argument is *extra* which can be used to pass a + dictionary which is used to populate the __dict__ of the LogRecord created for + the logging event with user-defined attributes. These custom attributes can then + be used as you like. For example, they could be incorporated into logged + messages. For example:: + + FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' + logging.basicConfig(format=FORMAT) + d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} + logging.warning('Protocol problem: %s', 'connection reset', extra=d) + + would print something like: + + .. code-block:: none + + 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset + + The keys in the dictionary passed in *extra* should not clash with the keys used + by the logging system. (See the :class:`Formatter` documentation for more + information on which keys are used by the logging system.) + + If you choose to use these attributes in logged messages, you need to exercise + some care. In the above example, for instance, the :class:`Formatter` has been + set up with a format string which expects 'clientip' and 'user' in the attribute + dictionary of the LogRecord. If these are missing, the message will not be + logged because a string formatting exception will occur. So in this case, you + always need to pass the *extra* dictionary with these keys. + + While this might be annoying, this feature is intended for use in specialized + circumstances, such as multi-threaded servers where the same code executes in + many contexts, and interesting conditions which arise are dependent on this + context (such as remote client IP address and authenticated user name, in the + above example). In such circumstances, it is likely that specialized + :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. + + .. versionchanged:: 3.2 + The *stack_info* parameter was added. + +.. function:: info(msg, *args, **kwargs) + + Logs a message with level :const:`INFO` on the root logger. The arguments are + interpreted as for :func:`debug`. + + +.. function:: warning(msg, *args, **kwargs) + + Logs a message with level :const:`WARNING` on the root logger. The arguments + are interpreted as for :func:`debug`. + + .. note:: There is an obsolete function ``warn`` which is functionally + identical to ``warning``. As ``warn`` is deprecated, please do not use + it - use ``warning`` instead. + + +.. function:: error(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on the root logger. The arguments are + interpreted as for :func:`debug`. + + +.. function:: critical(msg, *args, **kwargs) + + Logs a message with level :const:`CRITICAL` on the root logger. The arguments + are interpreted as for :func:`debug`. + + +.. function:: exception(msg, *args, **kwargs) + + Logs a message with level :const:`ERROR` on the root logger. The arguments are + interpreted as for :func:`debug`. Exception info is added to the logging + message. This function should only be called from an exception handler. + +.. function:: log(level, msg, *args, **kwargs) + + Logs a message with level *level* on the root logger. The other arguments are + interpreted as for :func:`debug`. + + .. note:: The above module-level convenience functions, which delegate to the + root logger, call :func:`basicConfig` to ensure that at least one handler + is available. Because of this, they should *not* be used in threads, + in versions of Python earlier than 2.7.1 and 3.2, unless at least one + handler has been added to the root logger *before* the threads are + started. In earlier versions of Python, due to a thread safety shortcoming + in :func:`basicConfig`, this can (under rare circumstances) lead to + handlers being added multiple times to the root logger, which can in turn + lead to multiple messages for the same event. + +.. function:: disable(level=CRITICAL) + + Provides an overriding level *level* for all loggers which takes precedence over + the logger's own level. When the need arises to temporarily throttle logging + output down across the whole application, this function can be useful. Its + effect is to disable all logging calls of severity *level* and below, so that + if you call it with a value of INFO, then all INFO and DEBUG events would be + discarded, whereas those of severity WARNING and above would be processed + according to the logger's effective level. If + ``logging.disable(logging.NOTSET)`` is called, it effectively removes this + overriding level, so that logging output again depends on the effective + levels of individual loggers. + + Note that if you have defined any custom logging level higher than + ``CRITICAL`` (this is not recommended), you won't be able to rely on the + default value for the *level* parameter, but will have to explicitly supply a + suitable value. + + .. versionchanged:: 3.7 + The *level* parameter was defaulted to level ``CRITICAL``. See + :issue:`28524` for more information about this change. + +.. function:: addLevelName(level, levelName) + + Associates level *level* with text *levelName* in an internal dictionary, which is + used to map numeric levels to a textual representation, for example when a + :class:`Formatter` formats a message. This function can also be used to define + your own levels. The only constraints are that all levels used must be + registered using this function, levels should be positive integers and they + should increase in increasing order of severity. + + .. note:: If you are thinking of defining your own levels, please see the + section on :ref:`custom-levels`. + +.. function:: getLevelName(level) + + Returns the textual representation of logging level *level*. If the level is one + of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, + :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you + have associated levels with names using :func:`addLevelName` then the name you + have associated with *level* is returned. If a numeric value corresponding to one + of the defined levels is passed in, the corresponding string representation is + returned. Otherwise, the string 'Level %s' % level is returned. + + .. note:: Levels are internally integers (as they need to be compared in the + logging logic). This function is used to convert between an integer level + and the level name displayed in the formatted log output by means of the + ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). + + .. versionchanged:: 3.4 + In Python versions earlier than 3.4, this function could also be passed a + text level, and would return the corresponding numeric value of the level. + This undocumented behaviour was considered a mistake, and was removed in + Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. + +.. function:: makeLogRecord(attrdict) + + Creates and returns a new :class:`LogRecord` instance whose attributes are + defined by *attrdict*. This function is useful for taking a pickled + :class:`LogRecord` attribute dictionary, sent over a socket, and reconstituting + it as a :class:`LogRecord` instance at the receiving end. + + +.. function:: basicConfig(**kwargs) + + Does basic configuration for the logging system by creating a + :class:`StreamHandler` with a default :class:`Formatter` and adding it to the + root logger. The functions :func:`debug`, :func:`info`, :func:`warning`, + :func:`error` and :func:`critical` will call :func:`basicConfig` automatically + if no handlers are defined for the root logger. + + This function does nothing if the root logger already has handlers + configured, unless the keyword argument *force* is set to ``True``. + + .. note:: This function should be called from the main thread + before other threads are started. In versions of Python prior to + 2.7.1 and 3.2, if this function is called from multiple threads, + it is possible (in rare circumstances) that a handler will be added + to the root logger more than once, leading to unexpected results + such as messages being duplicated in the log. + + The following keyword arguments are supported. + + .. tabularcolumns:: |l|L| + + +--------------+---------------------------------------------+ + | Format | Description | + +==============+=============================================+ + | *filename* | Specifies that a :class:`FileHandler` be | + | | created, using the specified filename, | + | | rather than a :class:`StreamHandler`. | + +--------------+---------------------------------------------+ + | *filemode* | If *filename* is specified, open the file | + | | in this :ref:`mode `. Defaults | + | | to ``'a'``. | + +--------------+---------------------------------------------+ + | *format* | Use the specified format string for the | + | | handler. Defaults to attributes | + | | ``levelname``, ``name`` and ``message`` | + | | separated by colons. | + +--------------+---------------------------------------------+ + | *datefmt* | Use the specified date/time format, as | + | | accepted by :func:`time.strftime`. | + +--------------+---------------------------------------------+ + | *style* | If *format* is specified, use this style | + | | for the format string. One of ``'%'``, | + | | ``'{'`` or ``'$'`` for :ref:`printf-style | + | | `, | + | | :meth:`str.format` or | + | | :class:`string.Template` respectively. | + | | Defaults to ``'%'``. | + +--------------+---------------------------------------------+ + | *level* | Set the root logger level to the specified | + | | :ref:`level `. | + +--------------+---------------------------------------------+ + | *stream* | Use the specified stream to initialize the | + | | :class:`StreamHandler`. Note that this | + | | argument is incompatible with *filename* - | + | | if both are present, a ``ValueError`` is | + | | raised. | + +--------------+---------------------------------------------+ + | *handlers* | If specified, this should be an iterable of | + | | already created handlers to add to the root | + | | logger. Any handlers which don't already | + | | have a formatter set will be assigned the | + | | default formatter created in this function. | + | | Note that this argument is incompatible | + | | with *filename* or *stream* - if both | + | | are present, a ``ValueError`` is raised. | + +--------------+---------------------------------------------+ + | *force* | If this keyword argument is specified as | + | | true, any existing handlers attached to the | + | | root logger are removed and closed, before | + | | carrying out the configuration as specified | + | | by the other arguments. | + +--------------+---------------------------------------------+ + | *encoding* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | :class:`FileHandler` is created, and thus | + | | used when opening the output file. | + +--------------+---------------------------------------------+ + | *errors* | If this keyword argument is specified along | + | | with *filename*, its value is used when the | + | | :class:`FileHandler` is created, and thus | + | | used when opening the output file. If not | + | | specified, the value 'backslashreplace' is | + | | used. Note that if ``None`` is specified, | + | | it will be passed as such to :func:`open`, | + | | which means that it will be treated the | + | | same as passing 'errors'. | + +--------------+---------------------------------------------+ + + .. versionchanged:: 3.2 + The *style* argument was added. + + .. versionchanged:: 3.3 + The *handlers* argument was added. Additional checks were added to + catch situations where incompatible arguments are specified (e.g. + *handlers* together with *stream* or *filename*, or *stream* + together with *filename*). + + .. versionchanged:: 3.8 + The *force* argument was added. + + .. versionchanged:: 3.9 + The *encoding* and *errors* arguments were added. + +.. function:: shutdown() + + Informs the logging system to perform an orderly shutdown by flushing and + closing all handlers. This should be called at application exit and no + further use of the logging system should be made after this call. + + When the logging module is imported, it registers this function as an exit + handler (see :mod:`atexit`), so normally there's no need to do that + manually. + + +.. function:: setLoggerClass(klass) + + Tells the logging system to use the class *klass* when instantiating a logger. + The class should define :meth:`__init__` such that only a name argument is + required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This + function is typically called before any loggers are instantiated by applications + which need to use custom logger behavior. After this call, as at any other + time, do not instantiate loggers directly using the subclass: continue to use + the :func:`logging.getLogger` API to get your loggers. + + +.. function:: setLogRecordFactory(factory) + + Set a callable which is used to create a :class:`LogRecord`. + + :param factory: The factory callable to be used to instantiate a log record. + + .. versionadded:: 3.2 + This function has been provided, along with :func:`getLogRecordFactory`, to + allow developers more control over how the :class:`LogRecord` representing + a logging event is constructed. + + The factory has the following signature: + + ``factory(name, level, fn, lno, msg, args, exc_info, func=None, sinfo=None, **kwargs)`` + + :name: The logger name. + :level: The logging level (numeric). + :fn: The full pathname of the file where the logging call was made. + :lno: The line number in the file where the logging call was made. + :msg: The logging message. + :args: The arguments for the logging message. + :exc_info: An exception tuple, or ``None``. + :func: The name of the function or method which invoked the logging + call. + :sinfo: A stack traceback such as is provided by + :func:`traceback.print_stack`, showing the call hierarchy. + :kwargs: Additional keyword arguments. + + +Module-Level Attributes +----------------------- + +.. attribute:: lastResort + + A "handler of last resort" is available through this attribute. This + is a :class:`StreamHandler` writing to ``sys.stderr`` with a level of + ``WARNING``, and is used to handle logging events in the absence of any + logging configuration. The end result is to just print the message to + ``sys.stderr``. This replaces the earlier error message saying that + "no handlers could be found for logger XYZ". If you need the earlier + behaviour for some reason, ``lastResort`` can be set to ``None``. + + .. versionadded:: 3.2 + +Integration with the warnings module +------------------------------------ + +The :func:`captureWarnings` function can be used to integrate :mod:`logging` +with the :mod:`warnings` module. + +.. function:: captureWarnings(capture) + + This function is used to turn the capture of warnings by logging on and + off. + + If *capture* is ``True``, warnings issued by the :mod:`warnings` module will + be redirected to the logging system. Specifically, a warning will be + formatted using :func:`warnings.formatwarning` and the resulting string + logged to a logger named ``'py.warnings'`` with a severity of :const:`WARNING`. + + If *capture* is ``False``, the redirection of warnings to the logging system + will stop, and warnings will be redirected to their original destinations + (i.e. those in effect before ``captureWarnings(True)`` was called). + + +.. seealso:: + + Module :mod:`logging.config` + Configuration API for the logging module. + + Module :mod:`logging.handlers` + Useful handlers included with the logging module. + + :pep:`282` - A Logging System + The proposal which described this feature for inclusion in the Python standard + library. + + `Original Python logging package `_ + This is the original source for the :mod:`logging` package. The version of the + package available from this site is suitable for use with Python 1.5.2, 2.1.x + and 2.2.x, which do not include the :mod:`logging` package in the standard + library. diff --git a/Doc/library/readline.rst.bak b/Doc/library/readline.rst.bak new file mode 100644 index 00000000000000..3ff64885f7fced --- /dev/null +++ b/Doc/library/readline.rst.bak @@ -0,0 +1,361 @@ +:mod:`readline` --- GNU readline interface +========================================== + +.. module:: readline + :platform: Unix + :synopsis: GNU readline support for Python. + +.. sectionauthor:: Skip Montanaro + +-------------- + +The :mod:`readline` module defines a number of functions to facilitate +completion and reading/writing of history files from the Python interpreter. +This module can be used directly, or via the :mod:`rlcompleter` module, which +supports completion of Python identifiers at the interactive prompt. Settings +made using this module affect the behaviour of both the interpreter's +interactive prompt and the prompts offered by the built-in :func:`input` +function. + +Readline keybindings may be configured via an initialization file, typically +``.inputrc`` in your home directory. See `Readline Init File +`_ +in the GNU Readline manual for information about the format and +allowable constructs of that file, and the capabilities of the +Readline library in general. + +.. note:: + + The underlying Readline library API may be implemented by + the ``libedit`` library instead of GNU readline. + On macOS the :mod:`readline` module detects which library is being used + at run time. + + The configuration file for ``libedit`` is different from that + of GNU readline. If you programmatically load configuration strings + you can check for the text "libedit" in :const:`readline.__doc__` + to differentiate between GNU readline and libedit. + + If you use *editline*/``libedit`` readline emulation on macOS, the + initialization file located in your home directory is named + ``.editrc``. For example, the following content in ``~/.editrc`` will + turn ON *vi* keybindings and TAB completion:: + + python:bind -v + python:bind ^I rl_complete + + +Init file +--------- + +The following functions relate to the init file and user configuration: + + +.. function:: parse_and_bind(string) + + Execute the init line provided in the *string* argument. This calls + :c:func:`rl_parse_and_bind` in the underlying library. + + +.. function:: read_init_file([filename]) + + Execute a readline initialization file. The default filename is the last filename + used. This calls :c:func:`rl_read_init_file` in the underlying library. + + +Line buffer +----------- + +The following functions operate on the line buffer: + + +.. function:: get_line_buffer() + + Return the current contents of the line buffer (:c:data:`rl_line_buffer` + in the underlying library). + + +.. function:: insert_text(string) + + Insert text into the line buffer at the cursor position. This calls + :c:func:`rl_insert_text` in the underlying library, but ignores + the return value. + + +.. function:: redisplay() + + Change what's displayed on the screen to reflect the current contents of the + line buffer. This calls :c:func:`rl_redisplay` in the underlying library. + + +History file +------------ + +The following functions operate on a history file: + + +.. function:: read_history_file([filename]) + + Load a readline history file, and append it to the history list. + The default filename is :file:`~/.history`. This calls + :c:func:`read_history` in the underlying library. + + +.. function:: write_history_file([filename]) + + Save the history list to a readline history file, overwriting any + existing file. The default filename is :file:`~/.history`. This calls + :c:func:`write_history` in the underlying library. + + +.. function:: append_history_file(nelements[, filename]) + + Append the last *nelements* items of history to a file. The default filename is + :file:`~/.history`. The file must already exist. This calls + :c:func:`append_history` in the underlying library. This function + only exists if Python was compiled for a version of the library + that supports it. + + .. versionadded:: 3.5 + + +.. function:: get_history_length() + set_history_length(length) + + Set or return the desired number of lines to save in the history file. + The :func:`write_history_file` function uses this value to truncate + the history file, by calling :c:func:`history_truncate_file` in + the underlying library. Negative values imply + unlimited history file size. + + +History list +------------ + +The following functions operate on a global history list: + + +.. function:: clear_history() + + Clear the current history. This calls :c:func:`clear_history` in the + underlying library. The Python function only exists if Python was + compiled for a version of the library that supports it. + + +.. function:: get_current_history_length() + + Return the number of items currently in the history. (This is different from + :func:`get_history_length`, which returns the maximum number of lines that will + be written to a history file.) + + +.. function:: get_history_item(index) + + Return the current contents of history item at *index*. The item index + is one-based. This calls :c:func:`history_get` in the underlying library. + + +.. function:: remove_history_item(pos) + + Remove history item specified by its position from the history. + The position is zero-based. This calls :c:func:`remove_history` in + the underlying library. + + +.. function:: replace_history_item(pos, line) + + Replace history item specified by its position with *line*. + The position is zero-based. This calls :c:func:`replace_history_entry` + in the underlying library. + + +.. function:: add_history(line) + + Append *line* to the history buffer, as if it was the last line typed. + This calls :c:func:`add_history` in the underlying library. + + +.. function:: set_auto_history(enabled) + + Enable or disable automatic calls to :c:func:`add_history` when reading + input via readline. The *enabled* argument should be a Boolean value + that when true, enables auto history, and that when false, disables + auto history. + + .. versionadded:: 3.6 + + .. impl-detail:: + Auto history is enabled by default, and changes to this do not persist + across multiple sessions. + + +Startup hooks +------------- + + +.. function:: set_startup_hook([function]) + + Set or remove the function invoked by the :c:data:`rl_startup_hook` + callback of the underlying library. If *function* is specified, it will + be used as the new hook function; if omitted or ``None``, any function + already installed is removed. The hook is called with no + arguments just before readline prints the first prompt. + + +.. function:: set_pre_input_hook([function]) + + Set or remove the function invoked by the :c:data:`rl_pre_input_hook` + callback of the underlying library. If *function* is specified, it will + be used as the new hook function; if omitted or ``None``, any + function already installed is removed. The hook is called + with no arguments after the first prompt has been printed and just before + readline starts reading input characters. This function only exists + if Python was compiled for a version of the library that supports it. + + +Completion +---------- + +The following functions relate to implementing a custom word completion +function. This is typically operated by the Tab key, and can suggest and +automatically complete a word being typed. By default, Readline is set up +to be used by :mod:`rlcompleter` to complete Python identifiers for +the interactive interpreter. If the :mod:`readline` module is to be used +with a custom completer, a different set of word delimiters should be set. + + +.. function:: set_completer([function]) + + Set or remove the completer function. If *function* is specified, it will be + used as the new completer function; if omitted or ``None``, any completer + function already installed is removed. The completer function is called as + ``function(text, state)``, for *state* in ``0``, ``1``, ``2``, ..., until it + returns a non-string value. It should return the next possible completion + starting with *text*. + + The installed completer function is invoked by the *entry_func* callback + passed to :c:func:`rl_completion_matches` in the underlying library. + The *text* string comes from the first parameter to the + :c:data:`rl_attempted_completion_function` callback of the + underlying library. + + +.. function:: get_completer() + + Get the completer function, or ``None`` if no completer function has been set. + + +.. function:: get_completion_type() + + Get the type of completion being attempted. This returns the + :c:data:`rl_completion_type` variable in the underlying library as + an integer. + + +.. function:: get_begidx() + get_endidx() + + Get the beginning or ending index of the completion scope. + These indexes are the *start* and *end* arguments passed to the + :c:data:`rl_attempted_completion_function` callback of the + underlying library. The values may be different in the same + input editing scenario based on the underlying C readline implemtation. + Ex: libedit is known to behave differently than libreadline. + + +.. function:: set_completer_delims(string) + get_completer_delims() + + Set or get the word delimiters for completion. These determine the + start of the word to be considered for completion (the completion scope). + These functions access the :c:data:`rl_completer_word_break_characters` + variable in the underlying library. + + +.. function:: set_completion_display_matches_hook([function]) + + Set or remove the completion display function. If *function* is + specified, it will be used as the new completion display function; + if omitted or ``None``, any completion display function already + installed is removed. This sets or clears the + :c:data:`rl_completion_display_matches_hook` callback in the + underlying library. The completion display function is called as + ``function(substitution, [matches], longest_match_length)`` once + each time matches need to be displayed. + + +.. _readline-example: + +Example +------- + +The following example demonstrates how to use the :mod:`readline` module's +history reading and writing functions to automatically load and save a history +file named :file:`.python_history` from the user's home directory. The code +below would normally be executed automatically during interactive sessions +from the user's :envvar:`PYTHONSTARTUP` file. :: + + import atexit + import os + import readline + + histfile = os.path.join(os.path.expanduser("~"), ".python_history") + try: + readline.read_history_file(histfile) + # default history len is -1 (infinite), which may grow unruly + readline.set_history_length(1000) + except FileNotFoundError: + pass + + atexit.register(readline.write_history_file, histfile) + +This code is actually automatically run when Python is run in +:ref:`interactive mode ` (see :ref:`rlcompleter-config`). + +The following example achieves the same goal but supports concurrent interactive +sessions, by only appending the new history. :: + + import atexit + import os + import readline + histfile = os.path.join(os.path.expanduser("~"), ".python_history") + + try: + readline.read_history_file(histfile) + h_len = readline.get_current_history_length() + except FileNotFoundError: + open(histfile, 'wb').close() + h_len = 0 + + def save(prev_h_len, histfile): + new_h_len = readline.get_current_history_length() + readline.set_history_length(1000) + readline.append_history_file(new_h_len - prev_h_len, histfile) + atexit.register(save, h_len, histfile) + +The following example extends the :class:`code.InteractiveConsole` class to +support history save/restore. :: + + import atexit + import code + import os + import readline + + class HistoryConsole(code.InteractiveConsole): + def __init__(self, locals=None, filename="", + histfile=os.path.expanduser("~/.console-history")): + code.InteractiveConsole.__init__(self, locals, filename) + self.init_history(histfile) + + def init_history(self, histfile): + readline.parse_and_bind("tab: complete") + if hasattr(readline, "read_history_file"): + try: + readline.read_history_file(histfile) + except FileNotFoundError: + pass + atexit.register(self.save_history, histfile) + + def save_history(self, histfile): + readline.set_history_length(1000) + readline.write_history_file(histfile) diff --git a/Doc/library/sqlite3.rst.bak b/Doc/library/sqlite3.rst.bak new file mode 100644 index 00000000000000..172ce6c6bb03ba --- /dev/null +++ b/Doc/library/sqlite3.rst.bak @@ -0,0 +1,1094 @@ +:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases +============================================================ + +.. module:: sqlite3 + :synopsis: A DB-API 2.0 implementation using SQLite 3.x. + +.. sectionauthor:: Gerhard Häring + +**Source code:** :source:`Lib/sqlite3/` + +-------------- + +SQLite is a C library that provides a lightweight disk-based database that +doesn't require a separate server process and allows accessing the database +using a nonstandard variant of the SQL query language. Some applications can use +SQLite for internal data storage. It's also possible to prototype an +application using SQLite and then port the code to a larger database such as +PostgreSQL or Oracle. + +The sqlite3 module was written by Gerhard Häring. It provides a SQL interface +compliant with the DB-API 2.0 specification described by :pep:`249`, and +requires SQLite 3.7.15 or newer. + +To use the module, you must first create a :class:`Connection` object that +represents the database. Here the data will be stored in the +:file:`example.db` file:: + + import sqlite3 + con = sqlite3.connect('example.db') + +You can also supply the special name ``:memory:`` to create a database in RAM. + +Once you have a :class:`Connection`, you can create a :class:`Cursor` object +and call its :meth:`~Cursor.execute` method to perform SQL commands:: + + cur = con.cursor() + + # Create table + cur.execute('''CREATE TABLE stocks + (date text, trans text, symbol text, qty real, price real)''') + + # Insert a row of data + cur.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") + + # Save (commit) the changes + con.commit() + + # We can also close the connection if we are done with it. + # Just be sure any changes have been committed or they will be lost. + con.close() + +The data you've saved is persistent and is available in subsequent sessions:: + + import sqlite3 + con = sqlite3.connect('example.db') + cur = con.cursor() + +Usually your SQL operations will need to use values from Python variables. You +shouldn't assemble your query using Python's string operations because doing so +is insecure; it makes your program vulnerable to an SQL injection attack +(see https://xkcd.com/327/ for humorous example of what can go wrong). + +Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder +wherever you want to use a value, and then provide a tuple of values as the +second argument to the cursor's :meth:`~Cursor.execute` method. (Other database +modules may use a different placeholder, such as ``%s`` or ``:1``.) For +example:: + + # Never do this -- insecure! + symbol = 'RHAT' + cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) + + # Do this instead + t = ('RHAT',) + cur.execute('SELECT * FROM stocks WHERE symbol=?', t) + print(cur.fetchone()) + + # Larger example that inserts many records at a time + purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), + ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), + ('2006-04-06', 'SELL', 'IBM', 500, 53.00), + ] + cur.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) + +To retrieve data after executing a SELECT statement, you can either treat the +cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to +retrieve a single matching row, or call :meth:`~Cursor.fetchall` to get a list of the +matching rows. + +This example uses the iterator form:: + + >>> for row in cur.execute('SELECT * FROM stocks ORDER BY price'): + print(row) + + ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) + ('2006-03-28', 'BUY', 'IBM', 1000, 45.0) + ('2006-04-06', 'SELL', 'IBM', 500, 53.0) + ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) + + +.. seealso:: + + https://www.sqlite.org + The SQLite web page; the documentation describes the syntax and the + available data types for the supported SQL dialect. + + https://www.w3schools.com/sql/ + Tutorial, reference and examples for learning SQL syntax. + + :pep:`249` - Database API Specification 2.0 + PEP written by Marc-André Lemburg. + + +.. _sqlite3-module-contents: + +Module functions and constants +------------------------------ + + +.. data:: version + + The version number of this module, as a string. This is not the version of + the SQLite library. + + +.. data:: version_info + + The version number of this module, as a tuple of integers. This is not the + version of the SQLite library. + + +.. data:: sqlite_version + + The version number of the run-time SQLite library, as a string. + + +.. data:: sqlite_version_info + + The version number of the run-time SQLite library, as a tuple of integers. + + +.. data:: PARSE_DECLTYPES + + This constant is meant to be used with the *detect_types* parameter of the + :func:`connect` function. + + Setting it makes the :mod:`sqlite3` module parse the declared type for each + column it returns. It will parse out the first word of the declared type, + i. e. for "integer primary key", it will parse out "integer", or for + "number(10)" it will parse out "number". Then for that column, it will look + into the converters dictionary and use the converter function registered for + that type there. + + +.. data:: PARSE_COLNAMES + + This constant is meant to be used with the *detect_types* parameter of the + :func:`connect` function. + + Setting this makes the SQLite interface parse the column name for each column it + returns. It will look for a string formed [mytype] in there, and then decide + that 'mytype' is the type of the column. It will try to find an entry of + 'mytype' in the converters dictionary and then use the converter function found + there to return the value. The column name found in :attr:`Cursor.description` + does not include the type, i. e. if you use something like + ``'as "Expiration date [datetime]"'`` in your SQL, then we will parse out + everything until the first ``'['`` for the column name and strip + the preceeding space: the column name would simply be "Expiration date". + + +.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) + + Opens a connection to the SQLite database file *database*. By default returns a + :class:`Connection` object, unless a custom *factory* is given. + + *database* is a :term:`path-like object` giving the pathname (absolute or + relative to the current working directory) of the database file to be opened. + You can use ``":memory:"`` to open a database connection to a database that + resides in RAM instead of on disk. + + When a database is accessed by multiple connections, and one of the processes + modifies the database, the SQLite database is locked until that transaction is + committed. The *timeout* parameter specifies how long the connection should wait + for the lock to go away until raising an exception. The default for the timeout + parameter is 5.0 (five seconds). + + For the *isolation_level* parameter, please see the + :attr:`~Connection.isolation_level` property of :class:`Connection` objects. + + SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If + you want to use other types you must add support for them yourself. The + *detect_types* parameter and the using custom **converters** registered with the + module-level :func:`register_converter` function allow you to easily do that. + + *detect_types* defaults to 0 (i. e. off, no type detection), you can set it to + any combination of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to turn + type detection on. Due to SQLite behaviour, types can't be detected for generated + fields (for example ``max(data)``), even when *detect_types* parameter is set. In + such case, the returned type is :class:`str`. + + By default, *check_same_thread* is :const:`True` and only the creating thread may + use the connection. If set :const:`False`, the returned connection may be shared + across multiple threads. When using multiple threads with the same connection + writing operations should be serialized by the user to avoid data corruption. + + By default, the :mod:`sqlite3` module uses its :class:`Connection` class for the + connect call. You can, however, subclass the :class:`Connection` class and make + :func:`connect` use your class instead by providing your class for the *factory* + parameter. + + Consult the section :ref:`sqlite3-types` of this manual for details. + + The :mod:`sqlite3` module internally uses a statement cache to avoid SQL parsing + overhead. If you want to explicitly set the number of statements that are cached + for the connection, you can set the *cached_statements* parameter. The currently + implemented default is to cache 100 statements. + + If *uri* is true, *database* is interpreted as a URI. This allows you + to specify options. For example, to open a database in read-only mode + you can use:: + + db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) + + More information about this feature, including a list of recognized options, can + be found in the `SQLite URI documentation `_. + + .. audit-event:: sqlite3.connect database sqlite3.connect + + .. versionchanged:: 3.4 + Added the *uri* parameter. + + .. versionchanged:: 3.7 + *database* can now also be a :term:`path-like object`, not only a string. + + +.. function:: register_converter(typename, callable) + + Registers a callable to convert a bytestring from the database into a custom + Python type. The callable will be invoked for all database values that are of + the type *typename*. Confer the parameter *detect_types* of the :func:`connect` + function for how the type detection works. Note that *typename* and the name of + the type in your query are matched in case-insensitive manner. + + +.. function:: register_adapter(type, callable) + + Registers a callable to convert the custom Python type *type* into one of + SQLite's supported types. The callable *callable* accepts as single parameter + the Python value, and must return a value of the following types: int, + float, str or bytes. + + +.. function:: complete_statement(sql) + + Returns :const:`True` if the string *sql* contains one or more complete SQL + statements terminated by semicolons. It does not verify that the SQL is + syntactically correct, only that there are no unclosed string literals and the + statement is terminated by a semicolon. + + This can be used to build a shell for SQLite, as in the following example: + + + .. literalinclude:: ../includes/sqlite3/complete_statement.py + + +.. function:: enable_callback_tracebacks(flag) + + By default you will not get any tracebacks in user-defined functions, + aggregates, converters, authorizer callbacks etc. If you want to debug them, + you can call this function with *flag* set to ``True``. Afterwards, you will + get tracebacks from callbacks on ``sys.stderr``. Use :const:`False` to + disable the feature again. + + +.. _sqlite3-connection-objects: + +Connection Objects +------------------ + +.. class:: Connection + + A SQLite database connection has the following attributes and methods: + + .. attribute:: isolation_level + + Get or set the current default isolation level. :const:`None` for autocommit mode or + one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section + :ref:`sqlite3-controlling-transactions` for a more detailed explanation. + + .. attribute:: in_transaction + + :const:`True` if a transaction is active (there are uncommitted changes), + :const:`False` otherwise. Read-only attribute. + + .. versionadded:: 3.2 + + .. method:: cursor(factory=Cursor) + + The cursor method accepts a single optional parameter *factory*. If + supplied, this must be a callable returning an instance of :class:`Cursor` + or its subclasses. + + .. method:: commit() + + This method commits the current transaction. If you don't call this method, + anything you did since the last call to ``commit()`` is not visible from + other database connections. If you wonder why you don't see the data you've + written to the database, please check you didn't forget to call this method. + + .. method:: rollback() + + This method rolls back any changes to the database since the last call to + :meth:`commit`. + + .. method:: close() + + This closes the database connection. Note that this does not automatically + call :meth:`commit`. If you just close your database connection without + calling :meth:`commit` first, your changes will be lost! + + .. method:: execute(sql[, parameters]) + + This is a nonstandard shortcut that creates a cursor object by calling + the :meth:`~Connection.cursor` method, calls the cursor's + :meth:`~Cursor.execute` method with the *parameters* given, and returns + the cursor. + + .. method:: executemany(sql[, parameters]) + + This is a nonstandard shortcut that creates a cursor object by + calling the :meth:`~Connection.cursor` method, calls the cursor's + :meth:`~Cursor.executemany` method with the *parameters* given, and + returns the cursor. + + .. method:: executescript(sql_script) + + This is a nonstandard shortcut that creates a cursor object by + calling the :meth:`~Connection.cursor` method, calls the cursor's + :meth:`~Cursor.executescript` method with the given *sql_script*, and + returns the cursor. + + .. method:: create_function(name, num_params, func, *, deterministic=False) + + Creates a user-defined function that you can later use from within SQL + statements under the function name *name*. *num_params* is the number of + parameters the function accepts (if *num_params* is -1, the function may + take any number of arguments), and *func* is a Python callable that is + called as the SQL function. If *deterministic* is true, the created function + is marked as `deterministic `_, which + allows SQLite to perform additional optimizations. This flag is supported by + SQLite 3.8.3 or higher, :exc:`NotSupportedError` will be raised if used + with older versions. + + The function can return any of the types supported by SQLite: bytes, str, int, + float and ``None``. + + .. versionchanged:: 3.8 + The *deterministic* parameter was added. + + Example: + + .. literalinclude:: ../includes/sqlite3/md5func.py + + + .. method:: create_aggregate(name, num_params, aggregate_class) + + Creates a user-defined aggregate function. + + The aggregate class must implement a ``step`` method, which accepts the number + of parameters *num_params* (if *num_params* is -1, the function may take + any number of arguments), and a ``finalize`` method which will return the + final result of the aggregate. + + The ``finalize`` method can return any of the types supported by SQLite: + bytes, str, int, float and ``None``. + + Example: + + .. literalinclude:: ../includes/sqlite3/mysumaggr.py + + + .. method:: create_collation(name, callable) + + Creates a collation with the specified *name* and *callable*. The callable will + be passed two string arguments. It should return -1 if the first is ordered + lower than the second, 0 if they are ordered equal and 1 if the first is ordered + higher than the second. Note that this controls sorting (ORDER BY in SQL) so + your comparisons don't affect other SQL operations. + + Note that the callable will get its parameters as Python bytestrings, which will + normally be encoded in UTF-8. + + The following example shows a custom collation that sorts "the wrong way": + + .. literalinclude:: ../includes/sqlite3/collation_reverse.py + + To remove a collation, call ``create_collation`` with ``None`` as callable:: + + con.create_collation("reverse", None) + + + .. method:: interrupt() + + You can call this method from a different thread to abort any queries that might + be executing on the connection. The query will then abort and the caller will + get an exception. + + + .. method:: set_authorizer(authorizer_callback) + + This routine registers a callback. The callback is invoked for each attempt to + access a column of a table in the database. The callback should return + :const:`SQLITE_OK` if access is allowed, :const:`SQLITE_DENY` if the entire SQL + statement should be aborted with an error and :const:`SQLITE_IGNORE` if the + column should be treated as a NULL value. These constants are available in the + :mod:`sqlite3` module. + + The first argument to the callback signifies what kind of operation is to be + authorized. The second and third argument will be arguments or :const:`None` + depending on the first argument. The 4th argument is the name of the database + ("main", "temp", etc.) if applicable. The 5th argument is the name of the + inner-most trigger or view that is responsible for the access attempt or + :const:`None` if this access attempt is directly from input SQL code. + + Please consult the SQLite documentation about the possible values for the first + argument and the meaning of the second and third argument depending on the first + one. All necessary constants are available in the :mod:`sqlite3` module. + + + .. method:: set_progress_handler(handler, n) + + This routine registers a callback. The callback is invoked for every *n* + instructions of the SQLite virtual machine. This is useful if you want to + get called from SQLite during long-running operations, for example to update + a GUI. + + If you want to clear any previously installed progress handler, call the + method with :const:`None` for *handler*. + + Returning a non-zero value from the handler function will terminate the + currently executing query and cause it to raise an :exc:`OperationalError` + exception. + + + .. method:: set_trace_callback(trace_callback) + + Registers *trace_callback* to be called for each SQL statement that is + actually executed by the SQLite backend. + + The only argument passed to the callback is the statement (as string) that + is being executed. The return value of the callback is ignored. Note that + the backend does not only run statements passed to the :meth:`Cursor.execute` + methods. Other sources include the transaction management of the Python + module and the execution of triggers defined in the current database. + + Passing :const:`None` as *trace_callback* will disable the trace callback. + + .. versionadded:: 3.3 + + + .. method:: enable_load_extension(enabled) + + This routine allows/disallows the SQLite engine to load SQLite extensions + from shared libraries. SQLite extensions can define new functions, + aggregates or whole new virtual table implementations. One well-known + extension is the fulltext-search extension distributed with SQLite. + + Loadable extensions are disabled by default. See [#f1]_. + + .. versionadded:: 3.2 + + .. literalinclude:: ../includes/sqlite3/load_extension.py + + .. method:: load_extension(path) + + This routine loads a SQLite extension from a shared library. You have to + enable extension loading with :meth:`enable_load_extension` before you can + use this routine. + + Loadable extensions are disabled by default. See [#f1]_. + + .. versionadded:: 3.2 + + .. attribute:: row_factory + + You can change this attribute to a callable that accepts the cursor and the + original row as a tuple and will return the real result row. This way, you can + implement more advanced ways of returning results, such as returning an object + that can also access columns by name. + + Example: + + .. literalinclude:: ../includes/sqlite3/row_factory.py + + If returning a tuple doesn't suffice and you want name-based access to + columns, you should consider setting :attr:`row_factory` to the + highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both + index-based and case-insensitive name-based access to columns with almost no + memory overhead. It will probably be better than your own custom + dictionary-based approach or even a db_row based solution. + + .. XXX what's a db_row-based solution? + + + .. attribute:: text_factory + + Using this attribute you can control what objects are returned for the ``TEXT`` + data type. By default, this attribute is set to :class:`str` and the + :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to + return bytestrings instead, you can set it to :class:`bytes`. + + You can also set it to any other callable that accepts a single bytestring + parameter and returns the resulting object. + + See the following example code for illustration: + + .. literalinclude:: ../includes/sqlite3/text_factory.py + + + .. attribute:: total_changes + + Returns the total number of database rows that have been modified, inserted, or + deleted since the database connection was opened. + + + .. method:: iterdump + + Returns an iterator to dump the database in an SQL text format. Useful when + saving an in-memory database for later restoration. This function provides + the same capabilities as the :kbd:`.dump` command in the :program:`sqlite3` + shell. + + Example:: + + # Convert file existing_db.db to SQL dump file dump.sql + import sqlite3 + + con = sqlite3.connect('existing_db.db') + with open('dump.sql', 'w') as f: + for line in con.iterdump(): + f.write('%s\n' % line) + con.close() + + + .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) + + This method makes a backup of a SQLite database even while it's being accessed + by other clients, or concurrently by the same connection. The copy will be + written into the mandatory argument *target*, that must be another + :class:`Connection` instance. + + By default, or when *pages* is either ``0`` or a negative integer, the entire + database is copied in a single step; otherwise the method performs a loop + copying up to *pages* pages at a time. + + If *progress* is specified, it must either be ``None`` or a callable object that + will be executed at each iteration with three integer arguments, respectively + the *status* of the last iteration, the *remaining* number of pages still to be + copied and the *total* number of pages. + + The *name* argument specifies the database name that will be copied: it must be + a string containing either ``"main"``, the default, to indicate the main + database, ``"temp"`` to indicate the temporary database or the name specified + after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached + database. + + The *sleep* argument specifies the number of seconds to sleep by between + successive attempts to backup remaining pages, can be specified either as an + integer or a floating point value. + + Example 1, copy an existing database into another:: + + import sqlite3 + + def progress(status, remaining, total): + print(f'Copied {total-remaining} of {total} pages...') + + con = sqlite3.connect('existing_db.db') + bck = sqlite3.connect('backup.db') + with bck: + con.backup(bck, pages=1, progress=progress) + bck.close() + con.close() + + Example 2, copy an existing database into a transient copy:: + + import sqlite3 + + source = sqlite3.connect('existing_db.db') + dest = sqlite3.connect(':memory:') + source.backup(dest) + + .. versionadded:: 3.7 + + +.. _sqlite3-cursor-objects: + +Cursor Objects +-------------- + +.. class:: Cursor + + A :class:`Cursor` instance has the following attributes and methods. + + .. index:: single: ? (question mark); in SQL statements + .. index:: single: : (colon); in SQL statements + + .. method:: execute(sql[, parameters]) + + Executes an SQL statement. The SQL statement may be parameterized (i. e. + placeholders instead of SQL literals). The :mod:`sqlite3` module supports two + kinds of placeholders: question marks (qmark style) and named placeholders + (named style). + + Here's an example of both styles: + + .. literalinclude:: ../includes/sqlite3/execute_1.py + + :meth:`execute` will only execute a single SQL statement. If you try to execute + more than one statement with it, it will raise a :exc:`.Warning`. Use + :meth:`executescript` if you want to execute multiple SQL statements with one + call. + + + .. method:: executemany(sql, seq_of_parameters) + + Executes an SQL command against all parameter sequences or mappings found in + the sequence *seq_of_parameters*. The :mod:`sqlite3` module also allows + using an :term:`iterator` yielding parameters instead of a sequence. + + .. literalinclude:: ../includes/sqlite3/executemany_1.py + + Here's a shorter example using a :term:`generator`: + + .. literalinclude:: ../includes/sqlite3/executemany_2.py + + + .. method:: executescript(sql_script) + + This is a nonstandard convenience method for executing multiple SQL statements + at once. It issues a ``COMMIT`` statement first, then executes the SQL script it + gets as a parameter. + + *sql_script* can be an instance of :class:`str`. + + Example: + + .. literalinclude:: ../includes/sqlite3/executescript.py + + + .. method:: fetchone() + + Fetches the next row of a query result set, returning a single sequence, + or :const:`None` when no more data is available. + + + .. method:: fetchmany(size=cursor.arraysize) + + Fetches the next set of rows of a query result, returning a list. An empty + list is returned when no more rows are available. + + The number of rows to fetch per call is specified by the *size* parameter. + If it is not given, the cursor's arraysize determines the number of rows + to be fetched. The method should try to fetch as many rows as indicated by + the size parameter. If this is not possible due to the specified number of + rows not being available, fewer rows may be returned. + + Note there are performance considerations involved with the *size* parameter. + For optimal performance, it is usually best to use the arraysize attribute. + If the *size* parameter is used, then it is best for it to retain the same + value from one :meth:`fetchmany` call to the next. + + .. method:: fetchall() + + Fetches all (remaining) rows of a query result, returning a list. Note that + the cursor's arraysize attribute can affect the performance of this operation. + An empty list is returned when no rows are available. + + .. method:: close() + + Close the cursor now (rather than whenever ``__del__`` is called). + + The cursor will be unusable from this point forward; a :exc:`ProgrammingError` + exception will be raised if any operation is attempted with the cursor. + + .. attribute:: rowcount + + Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this + attribute, the database engine's own support for the determination of "rows + affected"/"rows selected" is quirky. + + For :meth:`executemany` statements, the number of modifications are summed up + into :attr:`rowcount`. + + As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in + case no ``executeXX()`` has been performed on the cursor or the rowcount of the + last operation is not determinable by the interface". This includes ``SELECT`` + statements because we cannot determine the number of rows a query produced + until all rows were fetched. + + .. attribute:: lastrowid + + This read-only attribute provides the rowid of the last modified row. It is + only set if you issued an ``INSERT`` or a ``REPLACE`` statement using the + :meth:`execute` method. For operations other than ``INSERT`` or + ``REPLACE`` or when :meth:`executemany` is called, :attr:`lastrowid` is + set to :const:`None`. + + If the ``INSERT`` or ``REPLACE`` statement failed to insert the previous + successful rowid is returned. + + .. versionchanged:: 3.6 + Added support for the ``REPLACE`` statement. + + .. attribute:: arraysize + + Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. + The default value is 1 which means a single row would be fetched per call. + + .. attribute:: description + + This read-only attribute provides the column names of the last query. To + remain compatible with the Python DB API, it returns a 7-tuple for each + column where the last six items of each tuple are :const:`None`. + + It is set for ``SELECT`` statements without any matching rows as well. + + .. attribute:: connection + + This read-only attribute provides the SQLite database :class:`Connection` + used by the :class:`Cursor` object. A :class:`Cursor` object created by + calling :meth:`con.cursor() ` will have a + :attr:`connection` attribute that refers to *con*:: + + >>> con = sqlite3.connect(":memory:") + >>> cur = con.cursor() + >>> cur.connection == con + True + +.. _sqlite3-row-objects: + +Row Objects +----------- + +.. class:: Row + + A :class:`Row` instance serves as a highly optimized + :attr:`~Connection.row_factory` for :class:`Connection` objects. + It tries to mimic a tuple in most of its features. + + It supports mapping access by column name and index, iteration, + representation, equality testing and :func:`len`. + + If two :class:`Row` objects have exactly the same columns and their + members are equal, they compare equal. + + .. method:: keys + + This method returns a list of column names. Immediately after a query, + it is the first member of each tuple in :attr:`Cursor.description`. + + .. versionchanged:: 3.5 + Added support of slicing. + +Let's assume we initialize a table as in the example given above:: + + con = sqlite3.connect(":memory:") + cur = con.cursor() + cur.execute('''create table stocks + (date text, trans text, symbol text, + qty real, price real)''') + cur.execute("""insert into stocks + values ('2006-01-05','BUY','RHAT',100,35.14)""") + con.commit() + cur.close() + +Now we plug :class:`Row` in:: + + >>> con.row_factory = sqlite3.Row + >>> cur = con.cursor() + >>> cur.execute('select * from stocks') + + >>> r = cur.fetchone() + >>> type(r) + + >>> tuple(r) + ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14) + >>> len(r) + 5 + >>> r[2] + 'RHAT' + >>> r.keys() + ['date', 'trans', 'symbol', 'qty', 'price'] + >>> r['qty'] + 100.0 + >>> for member in r: + ... print(member) + ... + 2006-01-05 + BUY + RHAT + 100.0 + 35.14 + + +.. _sqlite3-exceptions: + +Exceptions +---------- + +.. exception:: Warning + + A subclass of :exc:`Exception`. + +.. exception:: Error + + The base class of the other exceptions in this module. It is a subclass + of :exc:`Exception`. + +.. exception:: DatabaseError + + Exception raised for errors that are related to the database. + +.. exception:: IntegrityError + + Exception raised when the relational integrity of the database is affected, + e.g. a foreign key check fails. It is a subclass of :exc:`DatabaseError`. + +.. exception:: ProgrammingError + + Exception raised for programming errors, e.g. table not found or already + exists, syntax error in the SQL statement, wrong number of parameters + specified, etc. It is a subclass of :exc:`DatabaseError`. + +.. exception:: OperationalError + + Exception raised for errors that are related to the database's operation + and not necessarily under the control of the programmer, e.g. an unexpected + disconnect occurs, the data source name is not found, a transaction could + not be processed, etc. It is a subclass of :exc:`DatabaseError`. + +.. exception:: NotSupportedError + + Exception raised in case a method or database API was used which is not + supported by the database, e.g. calling the :meth:`~Connection.rollback` + method on a connection that does not support transaction or has + transactions turned off. It is a subclass of :exc:`DatabaseError`. + + +.. _sqlite3-types: + +SQLite and Python types +----------------------- + + +Introduction +^^^^^^^^^^^^ + +SQLite natively supports the following types: ``NULL``, ``INTEGER``, +``REAL``, ``TEXT``, ``BLOB``. + +The following Python types can thus be sent to SQLite without any problem: + ++-------------------------------+-------------+ +| Python type | SQLite type | ++===============================+=============+ +| :const:`None` | ``NULL`` | ++-------------------------------+-------------+ +| :class:`int` | ``INTEGER`` | ++-------------------------------+-------------+ +| :class:`float` | ``REAL`` | ++-------------------------------+-------------+ +| :class:`str` | ``TEXT`` | ++-------------------------------+-------------+ +| :class:`bytes` | ``BLOB`` | ++-------------------------------+-------------+ + + +This is how SQLite types are converted to Python types by default: + ++-------------+----------------------------------------------+ +| SQLite type | Python type | ++=============+==============================================+ +| ``NULL`` | :const:`None` | ++-------------+----------------------------------------------+ +| ``INTEGER`` | :class:`int` | ++-------------+----------------------------------------------+ +| ``REAL`` | :class:`float` | ++-------------+----------------------------------------------+ +| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | +| | :class:`str` by default | ++-------------+----------------------------------------------+ +| ``BLOB`` | :class:`bytes` | ++-------------+----------------------------------------------+ + +The type system of the :mod:`sqlite3` module is extensible in two ways: you can +store additional Python types in a SQLite database via object adaptation, and +you can let the :mod:`sqlite3` module convert SQLite types to different Python +types via converters. + + +Using adapters to store additional Python types in SQLite databases +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +As described before, SQLite supports only a limited set of types natively. To +use other Python types with SQLite, you must **adapt** them to one of the +sqlite3 module's supported types for SQLite: one of NoneType, int, float, +str, bytes. + +There are two ways to enable the :mod:`sqlite3` module to adapt a custom Python +type to one of the supported ones. + + +Letting your object adapt itself +"""""""""""""""""""""""""""""""" + +This is a good approach if you write the class yourself. Let's suppose you have +a class like this:: + + class Point: + def __init__(self, x, y): + self.x, self.y = x, y + +Now you want to store the point in a single SQLite column. First you'll have to +choose one of the supported types to be used for representing the point. +Let's just use str and separate the coordinates using a semicolon. Then you need +to give your class a method ``__conform__(self, protocol)`` which must return +the converted value. The parameter *protocol* will be :class:`PrepareProtocol`. + +.. literalinclude:: ../includes/sqlite3/adapter_point_1.py + + +Registering an adapter callable +""""""""""""""""""""""""""""""" + +The other possibility is to create a function that converts the type to the +string representation and register the function with :meth:`register_adapter`. + +.. literalinclude:: ../includes/sqlite3/adapter_point_2.py + +The :mod:`sqlite3` module has two default adapters for Python's built-in +:class:`datetime.date` and :class:`datetime.datetime` types. Now let's suppose +we want to store :class:`datetime.datetime` objects not in ISO representation, +but as a Unix timestamp. + +.. literalinclude:: ../includes/sqlite3/adapter_datetime.py + + +Converting SQLite values to custom Python types +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Writing an adapter lets you send custom Python types to SQLite. But to make it +really useful we need to make the Python to SQLite to Python roundtrip work. + +Enter converters. + +Let's go back to the :class:`Point` class. We stored the x and y coordinates +separated via semicolons as strings in SQLite. + +First, we'll define a converter function that accepts the string as a parameter +and constructs a :class:`Point` object from it. + +.. note:: + + Converter functions **always** get called with a :class:`bytes` object, no + matter under which data type you sent the value to SQLite. + +:: + + def convert_point(s): + x, y = map(float, s.split(b";")) + return Point(x, y) + +Now you need to make the :mod:`sqlite3` module know that what you select from +the database is actually a point. There are two ways of doing this: + +* Implicitly via the declared type + +* Explicitly via the column name + +Both ways are described in section :ref:`sqlite3-module-contents`, in the entries +for the constants :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES`. + +The following example illustrates both approaches. + +.. literalinclude:: ../includes/sqlite3/converter_point.py + + +Default adapters and converters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +There are default adapters for the date and datetime types in the datetime +module. They will be sent as ISO dates/ISO timestamps to SQLite. + +The default converters are registered under the name "date" for +:class:`datetime.date` and under the name "timestamp" for +:class:`datetime.datetime`. + +This way, you can use date/timestamps from Python without any additional +fiddling in most cases. The format of the adapters is also compatible with the +experimental SQLite date/time functions. + +The following example demonstrates this. + +.. literalinclude:: ../includes/sqlite3/pysqlite_datetime.py + +If a timestamp stored in SQLite has a fractional part longer than 6 +numbers, its value will be truncated to microsecond precision by the +timestamp converter. + + +.. _sqlite3-controlling-transactions: + +Controlling Transactions +------------------------ + +The underlying ``sqlite3`` library operates in ``autocommit`` mode by default, +but the Python :mod:`sqlite3` module by default does not. + +``autocommit`` mode means that statements that modify the database take effect +immediately. A ``BEGIN`` or ``SAVEPOINT`` statement disables ``autocommit`` +mode, and a ``COMMIT``, a ``ROLLBACK``, or a ``RELEASE`` that ends the +outermost transaction, turns ``autocommit`` mode back on. + +The Python :mod:`sqlite3` module by default issues a ``BEGIN`` statement +implicitly before a Data Modification Language (DML) statement (i.e. +``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). + +You can control which kind of ``BEGIN`` statements :mod:`sqlite3` implicitly +executes via the *isolation_level* parameter to the :func:`connect` +call, or via the :attr:`isolation_level` property of connections. +If you specify no *isolation_level*, a plain ``BEGIN`` is used, which is +equivalent to specifying ``DEFERRED``. Other possible values are ``IMMEDIATE`` +and ``EXCLUSIVE``. + +You can disable the :mod:`sqlite3` module's implicit transaction management by +setting :attr:`isolation_level` to ``None``. This will leave the underlying +``sqlite3`` library operating in ``autocommit`` mode. You can then completely +control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, +``SAVEPOINT``, and ``RELEASE`` statements in your code. + +.. versionchanged:: 3.6 + :mod:`sqlite3` used to implicitly commit an open transaction before DDL + statements. This is no longer the case. + + +Using :mod:`sqlite3` efficiently +-------------------------------- + + +Using shortcut methods +^^^^^^^^^^^^^^^^^^^^^^ + +Using the nonstandard :meth:`execute`, :meth:`executemany` and +:meth:`executescript` methods of the :class:`Connection` object, your code can +be written more concisely because you don't have to create the (often +superfluous) :class:`Cursor` objects explicitly. Instead, the :class:`Cursor` +objects are created implicitly and these shortcut methods return the cursor +objects. This way, you can execute a ``SELECT`` statement and iterate over it +directly using only a single call on the :class:`Connection` object. + +.. literalinclude:: ../includes/sqlite3/shortcut_methods.py + + +Accessing columns by name instead of by index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +One useful feature of the :mod:`sqlite3` module is the built-in +:class:`sqlite3.Row` class designed to be used as a row factory. + +Rows wrapped with this class can be accessed both by index (like tuples) and +case-insensitively by name: + +.. literalinclude:: ../includes/sqlite3/rowclass.py + + +Using the connection as a context manager +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Connection objects can be used as context managers +that automatically commit or rollback transactions. In the event of an +exception, the transaction is rolled back; otherwise, the transaction is +committed: + +.. literalinclude:: ../includes/sqlite3/ctx_manager.py + + +.. rubric:: Footnotes + +.. [#f1] The sqlite3 module is not built with loadable extension support by + default, because some platforms (notably Mac OS X) have SQLite + libraries which are compiled without this feature. To get loadable + extension support, you must pass --enable-loadable-sqlite-extensions to + configure. diff --git a/Doc/library/statistics.rst.bak b/Doc/library/statistics.rst.bak new file mode 100644 index 00000000000000..6b6d3154a28810 --- /dev/null +++ b/Doc/library/statistics.rst.bak @@ -0,0 +1,879 @@ +:mod:`statistics` --- Mathematical statistics functions +======================================================= + +.. module:: statistics + :synopsis: Mathematical statistics functions + +.. moduleauthor:: Steven D'Aprano +.. sectionauthor:: Steven D'Aprano + +.. versionadded:: 3.4 + +**Source code:** :source:`Lib/statistics.py` + +.. testsetup:: * + + from statistics import * + __name__ = '' + +-------------- + +This module provides functions for calculating mathematical statistics of +numeric (:class:`~numbers.Real`-valued) data. + +The module is not intended to be a competitor to third-party libraries such +as `NumPy `_, `SciPy `_, or +proprietary full-featured statistics packages aimed at professional +statisticians such as Minitab, SAS and Matlab. It is aimed at the level of +graphing and scientific calculators. + +Unless explicitly noted, these functions support :class:`int`, +:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. +Behaviour with other types (whether in the numeric tower or not) is +currently unsupported. Collections with a mix of types are also undefined +and implementation-dependent. If your input data consists of mixed types, +you may be able to use :func:`map` to ensure a consistent result, for +example: ``map(float, input_data)``. + +Averages and measures of central location +----------------------------------------- + +These functions calculate an average or typical value from a population +or sample. + +======================= =============================================================== +:func:`mean` Arithmetic mean ("average") of data. +:func:`fmean` Fast, floating point arithmetic mean. +:func:`geometric_mean` Geometric mean of data. +:func:`harmonic_mean` Harmonic mean of data. +:func:`median` Median (middle value) of data. +:func:`median_low` Low median of data. +:func:`median_high` High median of data. +:func:`median_grouped` Median, or 50th percentile, of grouped data. +:func:`mode` Single mode (most common value) of discrete or nominal data. +:func:`multimode` List of modes (most common values) of discrete or nomimal data. +:func:`quantiles` Divide data into intervals with equal probability. +======================= =============================================================== + +Measures of spread +------------------ + +These functions calculate a measure of how much the population or sample +tends to deviate from the typical or average values. + +======================= ============================================= +:func:`pstdev` Population standard deviation of data. +:func:`pvariance` Population variance of data. +:func:`stdev` Sample standard deviation of data. +:func:`variance` Sample variance of data. +======================= ============================================= + + +Function details +---------------- + +Note: The functions do not require the data given to them to be sorted. +However, for reading convenience, most of the examples show sorted sequences. + +.. function:: mean(data) + + Return the sample arithmetic mean of *data* which can be a sequence or iterable. + + The arithmetic mean is the sum of the data divided by the number of data + points. It is commonly called "the average", although it is only one of many + different mathematical averages. It is a measure of the central location of + the data. + + If *data* is empty, :exc:`StatisticsError` will be raised. + + Some examples of use: + + .. doctest:: + + >>> mean([1, 2, 3, 4, 4]) + 2.8 + >>> mean([-1.0, 2.5, 3.25, 5.75]) + 2.625 + + >>> from fractions import Fraction as F + >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) + Fraction(13, 21) + + >>> from decimal import Decimal as D + >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) + Decimal('0.5625') + + .. note:: + + The mean is strongly affected by outliers and is not a robust estimator + for central location: the mean is not necessarily a typical example of + the data points. For more robust measures of central location, see + :func:`median` and :func:`mode`. + + The sample mean gives an unbiased estimate of the true population mean, + so that when taken on average over all the possible samples, + ``mean(sample)`` converges on the true mean of the entire population. If + *data* represents the entire population rather than a sample, then + ``mean(data)`` is equivalent to calculating the true population mean μ. + + +.. function:: fmean(data) + + Convert *data* to floats and compute the arithmetic mean. + + This runs faster than the :func:`mean` function and it always returns a + :class:`float`. The *data* may be a sequence or iterable. If the input + dataset is empty, raises a :exc:`StatisticsError`. + + .. doctest:: + + >>> fmean([3.5, 4.0, 5.25]) + 4.25 + + .. versionadded:: 3.8 + + +.. function:: geometric_mean(data) + + Convert *data* to floats and compute the geometric mean. + + The geometric mean indicates the central tendency or typical value of the + *data* using the product of the values (as opposed to the arithmetic mean + which uses their sum). + + Raises a :exc:`StatisticsError` if the input dataset is empty, + if it contains a zero, or if it contains a negative value. + The *data* may be a sequence or iterable. + + No special efforts are made to achieve exact results. + (However, this may change in the future.) + + .. doctest:: + + >>> round(geometric_mean([54, 24, 36]), 1) + 36.0 + + .. versionadded:: 3.8 + + +.. function:: harmonic_mean(data, weights=None) + + Return the harmonic mean of *data*, a sequence or iterable of + real-valued numbers. If *weights* is omitted or *None*, then + equal weighting is assumed. + + The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the + reciprocals of the data. For example, the harmonic mean of three values *a*, + *b* and *c* will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the + values is zero, the result will be zero. + + The harmonic mean is a type of average, a measure of the central + location of the data. It is often appropriate when averaging + ratios or rates, for example speeds. + + Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. + What is the average speed? + + .. doctest:: + + >>> harmonic_mean([40, 60]) + 48.0 + + Suppose a car travels 40 km/hr for 5 km, and when traffic clears, + speeds-up to 60 km/hr for the remaining 30 km of the journey. What + is the average speed? + + .. doctest:: + + >>> harmonic_mean([40, 60], weights=[5, 30]) + 56.0 + + :exc:`StatisticsError` is raised if *data* is empty, any element + is less than zero, or if the weighted sum isn't positive. + + The current algorithm has an early-out when it encounters a zero + in the input. This means that the subsequent inputs are not tested + for validity. (This behavior may change in the future.) + + .. versionadded:: 3.6 + + .. versionchanged:: 3.10 + Added support for *weights*. + +.. function:: median(data) + + Return the median (middle value) of numeric data, using the common "mean of + middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. + *data* can be a sequence or iterable. + + The median is a robust measure of central location and is less affected by + the presence of outliers. When the number of data points is odd, the + middle data point is returned: + + .. doctest:: + + >>> median([1, 3, 5]) + 3 + + When the number of data points is even, the median is interpolated by taking + the average of the two middle values: + + .. doctest:: + + >>> median([1, 3, 5, 7]) + 4.0 + + This is suited for when your data is discrete, and you don't mind that the + median may not be an actual data point. + + If the data is ordinal (supports order operations) but not numeric (doesn't + support addition), consider using :func:`median_low` or :func:`median_high` + instead. + +.. function:: median_low(data) + + Return the low median of numeric data. If *data* is empty, + :exc:`StatisticsError` is raised. *data* can be a sequence or iterable. + + The low median is always a member of the data set. When the number of data + points is odd, the middle value is returned. When it is even, the smaller of + the two middle values is returned. + + .. doctest:: + + >>> median_low([1, 3, 5]) + 3 + >>> median_low([1, 3, 5, 7]) + 3 + + Use the low median when your data are discrete and you prefer the median to + be an actual data point rather than interpolated. + + +.. function:: median_high(data) + + Return the high median of data. If *data* is empty, :exc:`StatisticsError` + is raised. *data* can be a sequence or iterable. + + The high median is always a member of the data set. When the number of data + points is odd, the middle value is returned. When it is even, the larger of + the two middle values is returned. + + .. doctest:: + + >>> median_high([1, 3, 5]) + 3 + >>> median_high([1, 3, 5, 7]) + 5 + + Use the high median when your data are discrete and you prefer the median to + be an actual data point rather than interpolated. + + +.. function:: median_grouped(data, interval=1) + + Return the median of grouped continuous data, calculated as the 50th + percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` + is raised. *data* can be a sequence or iterable. + + .. doctest:: + + >>> median_grouped([52, 52, 53, 54]) + 52.5 + + In the following example, the data are rounded, so that each value represents + the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 + is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data + given, the middle value falls somewhere in the class 3.5--4.5, and + interpolation is used to estimate it: + + .. doctest:: + + >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) + 3.7 + + Optional argument *interval* represents the class interval, and defaults + to 1. Changing the class interval naturally will change the interpolation: + + .. doctest:: + + >>> median_grouped([1, 3, 3, 5, 7], interval=1) + 3.25 + >>> median_grouped([1, 3, 3, 5, 7], interval=2) + 3.5 + + This function does not check whether the data points are at least + *interval* apart. + + .. impl-detail:: + + Under some circumstances, :func:`median_grouped` may coerce data points to + floats. This behaviour is likely to change in the future. + + .. seealso:: + + * "Statistics for the Behavioral Sciences", Frederick J Gravetter and + Larry B Wallnau (8th Edition). + + * The `SSMEDIAN + `_ + function in the Gnome Gnumeric spreadsheet, including `this discussion + `_. + + +.. function:: mode(data) + + Return the single most common data point from discrete or nominal *data*. + The mode (when it exists) is the most typical value and serves as a + measure of central location. + + If there are multiple modes with the same frequency, returns the first one + encountered in the *data*. If the smallest or largest of those is + desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``. + If the input *data* is empty, :exc:`StatisticsError` is raised. + + ``mode`` assumes discrete data and returns a single value. This is the + standard treatment of the mode as commonly taught in schools: + + .. doctest:: + + >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) + 3 + + The mode is unique in that it is the only statistic in this package that + also applies to nominal (non-numeric) data: + + .. doctest:: + + >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) + 'red' + + .. versionchanged:: 3.8 + Now handles multimodal datasets by returning the first mode encountered. + Formerly, it raised :exc:`StatisticsError` when more than one mode was + found. + + +.. function:: multimode(data) + + Return a list of the most frequently occurring values in the order they + were first encountered in the *data*. Will return more than one result if + there are multiple modes or an empty list if the *data* is empty: + + .. doctest:: + + >>> multimode('aabbbbccddddeeffffgg') + ['b', 'd', 'f'] + >>> multimode('') + [] + + .. versionadded:: 3.8 + + +.. function:: pstdev(data, mu=None) + + Return the population standard deviation (the square root of the population + variance). See :func:`pvariance` for arguments and other details. + + .. doctest:: + + >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) + 0.986893273527251 + + +.. function:: pvariance(data, mu=None) + + Return the population variance of *data*, a non-empty sequence or iterable + of real-valued numbers. Variance, or second moment about the mean, is a + measure of the variability (spread or dispersion) of data. A large + variance indicates that the data is spread out; a small variance indicates + it is clustered closely around the mean. + + If the optional second argument *mu* is given, it is typically the mean of + the *data*. It can also be used to compute the second moment around a + point that is not the mean. If it is missing or ``None`` (the default), + the arithmetic mean is automatically calculated. + + Use this function to calculate the variance from the entire population. To + estimate the variance from a sample, the :func:`variance` function is usually + a better choice. + + Raises :exc:`StatisticsError` if *data* is empty. + + Examples: + + .. doctest:: + + >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] + >>> pvariance(data) + 1.25 + + If you have already calculated the mean of your data, you can pass it as the + optional second argument *mu* to avoid recalculation: + + .. doctest:: + + >>> mu = mean(data) + >>> pvariance(data, mu) + 1.25 + + Decimals and Fractions are supported: + + .. doctest:: + + >>> from decimal import Decimal as D + >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('24.815') + + >>> from fractions import Fraction as F + >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) + Fraction(13, 72) + + .. note:: + + When called with the entire population, this gives the population variance + σ². When called on a sample instead, this is the biased sample variance + s², also known as variance with N degrees of freedom. + + If you somehow know the true population mean μ, you may use this + function to calculate the variance of a sample, giving the known + population mean as the second argument. Provided the data points are a + random sample of the population, the result will be an unbiased estimate + of the population variance. + + +.. function:: stdev(data, xbar=None) + + Return the sample standard deviation (the square root of the sample + variance). See :func:`variance` for arguments and other details. + + .. doctest:: + + >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) + 1.0810874155219827 + + +.. function:: variance(data, xbar=None) + + Return the sample variance of *data*, an iterable of at least two real-valued + numbers. Variance, or second moment about the mean, is a measure of the + variability (spread or dispersion) of data. A large variance indicates that + the data is spread out; a small variance indicates it is clustered closely + around the mean. + + If the optional second argument *xbar* is given, it should be the mean of + *data*. If it is missing or ``None`` (the default), the mean is + automatically calculated. + + Use this function when your data is a sample from a population. To calculate + the variance from the entire population, see :func:`pvariance`. + + Raises :exc:`StatisticsError` if *data* has fewer than two values. + + Examples: + + .. doctest:: + + >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] + >>> variance(data) + 1.3720238095238095 + + If you have already calculated the mean of your data, you can pass it as the + optional second argument *xbar* to avoid recalculation: + + .. doctest:: + + >>> m = mean(data) + >>> variance(data, m) + 1.3720238095238095 + + This function does not attempt to verify that you have passed the actual mean + as *xbar*. Using arbitrary values for *xbar* can lead to invalid or + impossible results. + + Decimal and Fraction values are supported: + + .. doctest:: + + >>> from decimal import Decimal as D + >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) + Decimal('31.01875') + + >>> from fractions import Fraction as F + >>> variance([F(1, 6), F(1, 2), F(5, 3)]) + Fraction(67, 108) + + .. note:: + + This is the sample variance s² with Bessel's correction, also known as + variance with N-1 degrees of freedom. Provided that the data points are + representative (e.g. independent and identically distributed), the result + should be an unbiased estimate of the true population variance. + + If you somehow know the actual population mean μ you should pass it to the + :func:`pvariance` function as the *mu* parameter to get the variance of a + sample. + +.. function:: quantiles(data, *, n=4, method='exclusive') + + Divide *data* into *n* continuous intervals with equal probability. + Returns a list of ``n - 1`` cut points separating the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. Set + *n* to 100 for percentiles which gives the 99 cuts points that separate + *data* into 100 equal sized groups. Raises :exc:`StatisticsError` if *n* + is not least 1. + + The *data* can be any iterable containing sample data. For meaningful + results, the number of data points in *data* should be larger than *n*. + Raises :exc:`StatisticsError` if there are not at least two data points. + + The cut points are linearly interpolated from the + two nearest data points. For example, if a cut point falls one-third + of the distance between two sample values, ``100`` and ``112``, the + cut-point will evaluate to ``104``. + + The *method* for computing quantiles can be varied depending on + whether the *data* includes or excludes the lowest and + highest possible values from the population. + + The default *method* is "exclusive" and is used for data sampled from + a population that can have more extreme values than found in the + samples. The portion of the population falling below the *i-th* of + *m* sorted data points is computed as ``i / (m + 1)``. Given nine + sample values, the method sorts them and assigns the following + percentiles: 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%. + + Setting the *method* to "inclusive" is used for describing population + data or for samples that are known to include the most extreme values + from the population. The minimum value in *data* is treated as the 0th + percentile and the maximum value is treated as the 100th percentile. + The portion of the population falling below the *i-th* of *m* sorted + data points is computed as ``(i - 1) / (m - 1)``. Given 11 sample + values, the method sorts them and assigns the following percentiles: + 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%. + + .. doctest:: + + # Decile cut points for empirically sampled data + >>> data = [105, 129, 87, 86, 111, 111, 89, 81, 108, 92, 110, + ... 100, 75, 105, 103, 109, 76, 119, 99, 91, 103, 129, + ... 106, 101, 84, 111, 74, 87, 86, 103, 103, 106, 86, + ... 111, 75, 87, 102, 121, 111, 88, 89, 101, 106, 95, + ... 103, 107, 101, 81, 109, 104] + >>> [round(q, 1) for q in quantiles(data, n=10)] + [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] + + .. versionadded:: 3.8 + + +Exceptions +---------- + +A single exception is defined: + +.. exception:: StatisticsError + + Subclass of :exc:`ValueError` for statistics-related exceptions. + + +:class:`NormalDist` objects +--------------------------- + +:class:`NormalDist` is a tool for creating and manipulating normal +distributions of a `random variable +`_. It is a +class that treats the mean and standard deviation of data +measurements as a single entity. + +Normal distributions arise from the `Central Limit Theorem +`_ and have a wide range +of applications in statistics. + +.. class:: NormalDist(mu=0.0, sigma=1.0) + + Returns a new *NormalDist* object where *mu* represents the `arithmetic + mean `_ and *sigma* + represents the `standard deviation + `_. + + If *sigma* is negative, raises :exc:`StatisticsError`. + + .. attribute:: mean + + A read-only property for the `arithmetic mean + `_ of a normal + distribution. + + .. attribute:: median + + A read-only property for the `median + `_ of a normal + distribution. + + .. attribute:: mode + + A read-only property for the `mode + `_ of a normal + distribution. + + .. attribute:: stdev + + A read-only property for the `standard deviation + `_ of a normal + distribution. + + .. attribute:: variance + + A read-only property for the `variance + `_ of a normal + distribution. Equal to the square of the standard deviation. + + .. classmethod:: NormalDist.from_samples(data) + + Makes a normal distribution instance with *mu* and *sigma* parameters + estimated from the *data* using :func:`fmean` and :func:`stdev`. + + The *data* can be any :term:`iterable` and should consist of values + that can be converted to type :class:`float`. If *data* does not + contain at least two elements, raises :exc:`StatisticsError` because it + takes at least one point to estimate a central value and at least two + points to estimate dispersion. + + .. method:: NormalDist.samples(n, *, seed=None) + + Generates *n* random samples for a given mean and standard deviation. + Returns a :class:`list` of :class:`float` values. + + If *seed* is given, creates a new instance of the underlying random + number generator. This is useful for creating reproducible results, + even in a multi-threading context. + + .. method:: NormalDist.pdf(x) + + Using a `probability density function (pdf) + `_, compute + the relative likelihood that a random variable *X* will be near the + given value *x*. Mathematically, it is the limit of the ratio ``P(x <= + X < x+dx) / dx`` as *dx* approaches zero. + + The relative likelihood is computed as the probability of a sample + occurring in a narrow range divided by the width of the range (hence + the word "density"). Since the likelihood is relative to other points, + its value can be greater than `1.0`. + + .. method:: NormalDist.cdf(x) + + Using a `cumulative distribution function (cdf) + `_, + compute the probability that a random variable *X* will be less than or + equal to *x*. Mathematically, it is written ``P(X <= x)``. + + .. method:: NormalDist.inv_cdf(p) + + Compute the inverse cumulative distribution function, also known as the + `quantile function `_ + or the `percent-point + `_ + function. Mathematically, it is written ``x : P(X <= x) = p``. + + Finds the value *x* of the random variable *X* such that the + probability of the variable being less than or equal to that value + equals the given probability *p*. + + .. method:: NormalDist.overlap(other) + + Measures the agreement between two normal probability distributions. + Returns a value between 0.0 and 1.0 giving `the overlapping area for + the two probability density functions + `_. + + .. method:: NormalDist.quantiles(n=4) + + Divide the normal distribution into *n* continuous intervals with + equal probability. Returns a list of (n - 1) cut points separating + the intervals. + + Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. + Set *n* to 100 for percentiles which gives the 99 cuts points that + separate the normal distribution into 100 equal sized groups. + + .. method:: NormalDist.zscore(x) + + Compute the + `Standard Score `_ + describing *x* in terms of the number of standard deviations + above or below the mean of the normal distribution: + ``(x - mean) / stdev``. + + .. versionadded:: 3.9 + + Instances of :class:`NormalDist` support addition, subtraction, + multiplication and division by a constant. These operations + are used for translation and scaling. For example: + + .. doctest:: + + >>> temperature_february = NormalDist(5, 2.5) # Celsius + >>> temperature_february * (9/5) + 32 # Fahrenheit + NormalDist(mu=41.0, sigma=4.5) + + Dividing a constant by an instance of :class:`NormalDist` is not supported + because the result wouldn't be normally distributed. + + Since normal distributions arise from additive effects of independent + variables, it is possible to `add and subtract two independent normally + distributed random variables + `_ + represented as instances of :class:`NormalDist`. For example: + + .. doctest:: + + >>> birth_weights = NormalDist.from_samples([2.5, 3.1, 2.1, 2.4, 2.7, 3.5]) + >>> drug_effects = NormalDist(0.4, 0.15) + >>> combined = birth_weights + drug_effects + >>> round(combined.mean, 1) + 3.1 + >>> round(combined.stdev, 1) + 0.5 + + .. versionadded:: 3.8 + + +:class:`NormalDist` Examples and Recipes +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +:class:`NormalDist` readily solves classic probability problems. + +For example, given `historical data for SAT exams +`_ showing +that scores are normally distributed with a mean of 1060 and a standard +deviation of 195, determine the percentage of students with test scores +between 1100 and 1200, after rounding to the nearest whole number: + +.. doctest:: + + >>> sat = NormalDist(1060, 195) + >>> fraction = sat.cdf(1200 + 0.5) - sat.cdf(1100 - 0.5) + >>> round(fraction * 100.0, 1) + 18.4 + +Find the `quartiles `_ and `deciles +`_ for the SAT scores: + +.. doctest:: + + >>> list(map(round, sat.quantiles())) + [928, 1060, 1192] + >>> list(map(round, sat.quantiles(n=10))) + [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] + +To estimate the distribution for a model than isn't easy to solve +analytically, :class:`NormalDist` can generate input samples for a `Monte +Carlo simulation `_: + +.. doctest:: + + >>> def model(x, y, z): + ... return (3*x + 7*x*y - 5*y) / (11 * z) + ... + >>> n = 100_000 + >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728) + >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471) + >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453) + >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP + [1.4591308524824727, 1.8035946855390597, 2.175091447274739] + +Normal distributions can be used to approximate `Binomial +distributions `_ +when the sample size is large and when the probability of a successful +trial is near 50%. + +For example, an open source conference has 750 attendees and two rooms with a +500 person capacity. There is a talk about Python and another about Ruby. +In previous conferences, 65% of the attendees preferred to listen to Python +talks. Assuming the population preferences haven't changed, what is the +probability that the Python room will stay within its capacity limits? + +.. doctest:: + + >>> n = 750 # Sample size + >>> p = 0.65 # Preference for Python + >>> q = 1.0 - p # Preference for Ruby + >>> k = 500 # Room capacity + + >>> # Approximation using the cumulative normal distribution + >>> from math import sqrt + >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) + 0.8402 + + >>> # Solution using the cumulative binomial distribution + >>> from math import comb, fsum + >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) + 0.8402 + + >>> # Approximation using a simulation + >>> from random import seed, choices + >>> seed(8675309) + >>> def trial(): + ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') + >>> mean(trial() <= k for i in range(10_000)) + 0.8398 + +Normal distributions commonly arise in machine learning problems. + +Wikipedia has a `nice example of a Naive Bayesian Classifier +`_. +The challenge is to predict a person's gender from measurements of normally +distributed features including height, weight, and foot size. + +We're given a training dataset with measurements for eight people. The +measurements are assumed to be normally distributed, so we summarize the data +with :class:`NormalDist`: + +.. doctest:: + + >>> height_male = NormalDist.from_samples([6, 5.92, 5.58, 5.92]) + >>> height_female = NormalDist.from_samples([5, 5.5, 5.42, 5.75]) + >>> weight_male = NormalDist.from_samples([180, 190, 170, 165]) + >>> weight_female = NormalDist.from_samples([100, 150, 130, 150]) + >>> foot_size_male = NormalDist.from_samples([12, 11, 12, 10]) + >>> foot_size_female = NormalDist.from_samples([6, 8, 7, 9]) + +Next, we encounter a new person whose feature measurements are known but whose +gender is unknown: + +.. doctest:: + + >>> ht = 6.0 # height + >>> wt = 130 # weight + >>> fs = 8 # foot size + +Starting with a 50% `prior probability +`_ of being male or female, +we compute the posterior as the prior times the product of likelihoods for the +feature measurements given the gender: + +.. doctest:: + + >>> prior_male = 0.5 + >>> prior_female = 0.5 + >>> posterior_male = (prior_male * height_male.pdf(ht) * + ... weight_male.pdf(wt) * foot_size_male.pdf(fs)) + + >>> posterior_female = (prior_female * height_female.pdf(ht) * + ... weight_female.pdf(wt) * foot_size_female.pdf(fs)) + +The final prediction goes to the largest posterior. This is known as the +`maximum a posteriori +`_ or MAP: + +.. doctest:: + + >>> 'male' if posterior_male > posterior_female else 'female' + 'female' + + +.. + # This modelines must appear within the last ten lines of the file. + kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/tempfile.rst.bak b/Doc/library/tempfile.rst.bak new file mode 100644 index 00000000000000..2b8a35e2651e73 --- /dev/null +++ b/Doc/library/tempfile.rst.bak @@ -0,0 +1,374 @@ +:mod:`tempfile` --- Generate temporary files and directories +============================================================ + +.. module:: tempfile + :synopsis: Generate temporary files and directories. + +.. sectionauthor:: Zack Weinberg + +**Source code:** :source:`Lib/tempfile.py` + +.. index:: + pair: temporary; file name + pair: temporary; file + +-------------- + +This module creates temporary files and directories. It works on all +supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, +:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level +interfaces which provide automatic cleanup and can be used as +context managers. :func:`mkstemp` and +:func:`mkdtemp` are lower-level functions which require manual cleanup. + +All the user-callable functions and constructors take additional arguments which +allow direct control over the location and name of temporary files and +directories. Files names used by this module include a string of +random characters which allows those files to be securely created in +shared temporary directories. +To maintain backward compatibility, the argument order is somewhat odd; it +is recommended to use keyword arguments for clarity. + +The module defines the following user-callable items: + +.. function:: TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) + + Return a :term:`file-like object` that can be used as a temporary storage area. + The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon + as it is closed (including an implicit close when the object is garbage + collected). Under Unix, the directory entry for the file is either not created at all or is removed + immediately after the file is created. Other platforms do not support + this; your code should not rely on a temporary file created using this + function having or not having a visible name in the file system. + + The resulting object can be used as a context manager (see + :ref:`tempfile-examples`). On completion of the context or + destruction of the file object the temporary file will be removed + from the filesystem. + + The *mode* parameter defaults to ``'w+b'`` so that the file created can + be read and written without being closed. Binary mode is used so that it + behaves consistently on all platforms without regard for the data that is + stored. *buffering*, *encoding*, *errors* and *newline* are interpreted as for + :func:`open`. + + The *dir*, *prefix* and *suffix* parameters have the same meaning and + defaults as with :func:`mkstemp`. + + The returned object is a true file object on POSIX platforms. On other + platforms, it is a file-like object whose :attr:`!file` attribute is the + underlying true file object. + + The :py:data:`os.O_TMPFILE` flag is used if it is available and works + (Linux-specific, requires Linux kernel 3.11 or later). + + .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile + + .. versionchanged:: 3.5 + + The :py:data:`os.O_TMPFILE` flag is now used if available. + + .. versionchanged:: 3.8 + Added *errors* parameter. + + +.. function:: NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) + + This function operates exactly as :func:`TemporaryFile` does, except that + the file is guaranteed to have a visible name in the file system (on + Unix, the directory entry is not unlinked). That name can be retrieved + from the :attr:`name` attribute of the returned + file-like object. Whether the name can be + used to open the file a second time, while the named temporary file is + still open, varies across platforms (it can be so used on Unix; it cannot + on Windows NT or later). If *delete* is true (the default), the file is + deleted as soon as it is closed. + The returned object is always a file-like object whose :attr:`!file` + attribute is the underlying true file object. This file-like object can + be used in a :keyword:`with` statement, just like a normal file. + + .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile + + .. versionchanged:: 3.8 + Added *errors* parameter. + + +.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) + + This function operates exactly as :func:`TemporaryFile` does, except that + data is spooled in memory until the file size exceeds *max_size*, or + until the file's :func:`fileno` method is called, at which point the + contents are written to disk and operation proceeds as with + :func:`TemporaryFile`. + + The resulting file has one additional method, :func:`rollover`, which + causes the file to roll over to an on-disk file regardless of its size. + + The returned object is a file-like object whose :attr:`_file` attribute + is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object + (depending on whether binary or text *mode* was specified) or a true file + object, depending on whether :func:`rollover` has been called. This + file-like object can be used in a :keyword:`with` statement, just like + a normal file. + + .. versionchanged:: 3.3 + the truncate method now accepts a ``size`` argument. + + .. versionchanged:: 3.8 + Added *errors* parameter. + + +.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) + + This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. + The resulting object can be used as a context manager (see + :ref:`tempfile-examples`). On completion of the context or destruction + of the temporary directory object the newly created temporary directory + and all its contents are removed from the filesystem. + + The directory name can be retrieved from the :attr:`name` attribute of the + returned object. When the returned object is used as a context manager, the + :attr:`name` will be assigned to the target of the :keyword:`!as` clause in + the :keyword:`with` statement, if there is one. + + The directory can be explicitly cleaned up by calling the + :func:`cleanup` method. + + .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory + + .. versionadded:: 3.2 + + +.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) + + Creates a temporary file in the most secure manner possible. There are + no race conditions in the file's creation, assuming that the platform + properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The + file is readable and writable only by the creating user ID. If the + platform uses permission bits to indicate whether a file is executable, + the file is executable by no one. The file descriptor is not inherited + by child processes. + + Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible + for deleting the temporary file when done with it. + + If *suffix* is not ``None``, the file name will end with that suffix, + otherwise there will be no suffix. :func:`mkstemp` does not put a dot + between the file name and the suffix; if you need one, put it at the + beginning of *suffix*. + + If *prefix* is not ``None``, the file name will begin with that prefix; + otherwise, a default prefix is used. The default is the return value of + :func:`gettempprefix` or :func:`gettempprefixb`, as appropriate. + + If *dir* is not ``None``, the file will be created in that directory; + otherwise, a default directory is used. The default directory is chosen + from a platform-dependent list, but the user of the application can + control the directory location by setting the *TMPDIR*, *TEMP* or *TMP* + environment variables. There is thus no guarantee that the generated + filename will have any nice properties, such as not requiring quoting + when passed to external commands via ``os.popen()``. + + If any of *suffix*, *prefix*, and *dir* are not + ``None``, they must be the same type. + If they are bytes, the returned name will be bytes instead of str. + If you want to force a bytes return value with otherwise default behavior, + pass ``suffix=b''``. + + If *text* is specified and true, the file is opened in text mode. + Otherwise, (the default) the file is opened in binary mode. + + :func:`mkstemp` returns a tuple containing an OS-level handle to an open + file (as would be returned by :func:`os.open`) and the absolute pathname + of that file, in that order. + + .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp + + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + *suffix* and *prefix* now accept and default to ``None`` to cause + an appropriate default value to be used. + + .. versionchanged:: 3.6 + The *dir* parameter now accepts a :term:`path-like object`. + + +.. function:: mkdtemp(suffix=None, prefix=None, dir=None) + + Creates a temporary directory in the most secure manner possible. There + are no race conditions in the directory's creation. The directory is + readable, writable, and searchable only by the creating user ID. + + The user of :func:`mkdtemp` is responsible for deleting the temporary + directory and its contents when done with it. + + The *prefix*, *suffix*, and *dir* arguments are the same as for + :func:`mkstemp`. + + :func:`mkdtemp` returns the absolute pathname of the new directory. + + .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp + + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + *suffix* and *prefix* now accept and default to ``None`` to cause + an appropriate default value to be used. + + .. versionchanged:: 3.6 + The *dir* parameter now accepts a :term:`path-like object`. + + +.. function:: gettempdir() + + Return the name of the directory used for temporary files. This + defines the default value for the *dir* argument to all functions + in this module. + + Python searches a standard list of directories to find one which + the calling user can create files in. The list is: + + #. The directory named by the :envvar:`TMPDIR` environment variable. + + #. The directory named by the :envvar:`TEMP` environment variable. + + #. The directory named by the :envvar:`TMP` environment variable. + + #. A platform-specific location: + + * On Windows, the directories :file:`C:\\TEMP`, :file:`C:\\TMP`, + :file:`\\TEMP`, and :file:`\\TMP`, in that order. + + * On all other platforms, the directories :file:`/tmp`, :file:`/var/tmp`, and + :file:`/usr/tmp`, in that order. + + #. As a last resort, the current working directory. + + The result of this search is cached, see the description of + :data:`tempdir` below. + + .. versionchanged:: 3.10 + + Always returns a str. Previously it would return any :data:`tempdir` + value regardless of type so long as it was not ``None``. + +.. function:: gettempdirb() + + Same as :func:`gettempdir` but the return value is in bytes. + + .. versionadded:: 3.5 + +.. function:: gettempprefix() + + Return the filename prefix used to create temporary files. This does not + contain the directory component. + +.. function:: gettempprefixb() + + Same as :func:`gettempprefix` but the return value is in bytes. + + .. versionadded:: 3.5 + +The module uses a global variable to store the name of the directory +used for temporary files returned by :func:`gettempdir`. It can be +set directly to override the selection process, but this is discouraged. +All functions in this module take a *dir* argument which can be used +to specify the directory. This is the recommended approach that does +not surprise other unsuspecting code by changing global API behavior. + +.. data:: tempdir + + When set to a value other than ``None``, this variable defines the + default value for the *dir* argument to the functions defined in this + module, including its type, bytes or str. It cannot be a + :term:`path-like object`. + + If ``tempdir`` is ``None`` (the default) at any call to any of the above + functions except :func:`gettempprefix` it is initialized following the + algorithm described in :func:`gettempdir`. + + .. note:: + + Beware that if you set ``tempdir`` to a bytes value, there is a + nasty side effect: The global default return type of + :func:`mkstemp` and :func:`mkdtemp` changes to bytes when no + explicit ``prefix``, ``suffix``, or ``dir`` arguments of type + str are supplied. Please do not write code expecting or + depending on this. This awkward behavior is maintained for + compatibility with the historcal implementation. + +.. _tempfile-examples: + +Examples +-------- + +Here are some examples of typical usage of the :mod:`tempfile` module:: + + >>> import tempfile + + # create a temporary file and write some data to it + >>> fp = tempfile.TemporaryFile() + >>> fp.write(b'Hello world!') + # read data from file + >>> fp.seek(0) + >>> fp.read() + b'Hello world!' + # close the file, it will be removed + >>> fp.close() + + # create a temporary file using a context manager + >>> with tempfile.TemporaryFile() as fp: + ... fp.write(b'Hello world!') + ... fp.seek(0) + ... fp.read() + b'Hello world!' + >>> + # file is now closed and removed + + # create a temporary directory using the context manager + >>> with tempfile.TemporaryDirectory() as tmpdirname: + ... print('created temporary directory', tmpdirname) + >>> + # directory and contents have been removed + + +Deprecated functions and variables +---------------------------------- + +A historical way to create temporary files was to first generate a +file name with the :func:`mktemp` function and then create a file +using this name. Unfortunately this is not secure, because a different +process may create a file with this name in the time between the call +to :func:`mktemp` and the subsequent attempt to create the file by the +first process. The solution is to combine the two steps and create the +file immediately. This approach is used by :func:`mkstemp` and the +other functions described above. + +.. function:: mktemp(suffix='', prefix='tmp', dir=None) + + .. deprecated:: 2.3 + Use :func:`mkstemp` instead. + + Return an absolute pathname of a file that did not exist at the time the + call is made. The *prefix*, *suffix*, and *dir* arguments are similar + to those of :func:`mkstemp`, except that bytes file names, ``suffix=None`` + and ``prefix=None`` are not supported. + + .. warning:: + + Use of this function may introduce a security hole in your program. By + the time you get around to doing anything with the file name it returns, + someone else may have beaten you to the punch. :func:`mktemp` usage can + be replaced easily with :func:`NamedTemporaryFile`, passing it the + ``delete=False`` parameter:: + + >>> f = NamedTemporaryFile(delete=False) + >>> f.name + '/tmp/tmptjujjt' + >>> f.write(b"Hello World!\n") + 13 + >>> f.close() + >>> os.unlink(f.name) + >>> os.path.exists(f.name) + False diff --git a/Doc/library/venv.rst.bak b/Doc/library/venv.rst.bak new file mode 100644 index 00000000000000..5d4a36481f1dcc --- /dev/null +++ b/Doc/library/venv.rst.bak @@ -0,0 +1,496 @@ +:mod:`venv` --- Creation of virtual environments +================================================ + +.. module:: venv + :synopsis: Creation of virtual environments. + +.. moduleauthor:: Vinay Sajip +.. sectionauthor:: Vinay Sajip + +.. versionadded:: 3.3 + +**Source code:** :source:`Lib/venv/` + +.. index:: pair: Environments; virtual + +-------------- + +The :mod:`venv` module provides support for creating lightweight "virtual +environments" with their own site directories, optionally isolated from system +site directories. Each virtual environment has its own Python binary (which +matches the version of the binary that was used to create this environment) and +can have its own independent set of installed Python packages in its site +directories. + +See :pep:`405` for more information about Python virtual environments. + +.. seealso:: + + `Python Packaging User Guide: Creating and using virtual environments + `__ + + +Creating virtual environments +----------------------------- + +.. include:: /using/venv-create.inc + + +.. _venv-def: + +.. note:: A virtual environment is a Python environment such that the Python + interpreter, libraries and scripts installed into it are isolated from those + installed in other virtual environments, and (by default) any libraries + installed in a "system" Python, i.e., one which is installed as part of your + operating system. + + A virtual environment is a directory tree which contains Python executable + files and other files which indicate that it is a virtual environment. + + Common installation tools such as setuptools_ and pip_ work as + expected with virtual environments. In other words, when a virtual + environment is active, they install Python packages into the virtual + environment without needing to be told to do so explicitly. + + When a virtual environment is active (i.e., the virtual environment's Python + interpreter is running), the attributes :attr:`sys.prefix` and + :attr:`sys.exec_prefix` point to the base directory of the virtual + environment, whereas :attr:`sys.base_prefix` and + :attr:`sys.base_exec_prefix` point to the non-virtual environment Python + installation which was used to create the virtual environment. If a virtual + environment is not active, then :attr:`sys.prefix` is the same as + :attr:`sys.base_prefix` and :attr:`sys.exec_prefix` is the same as + :attr:`sys.base_exec_prefix` (they all point to a non-virtual environment + Python installation). + + When a virtual environment is active, any options that change the + installation path will be ignored from all :mod:`distutils` configuration + files to prevent projects being inadvertently installed outside of the + virtual environment. + + When working in a command shell, users can make a virtual environment active + by running an ``activate`` script in the virtual environment's executables + directory (the precise filename and command to use the file is + shell-dependent), which prepends the virtual environment's directory for + executables to the ``PATH`` environment variable for the running shell. There + should be no need in other circumstances to activate a virtual + environment; scripts installed into virtual environments have a "shebang" + line which points to the virtual environment's Python interpreter. This means + that the script will run with that interpreter regardless of the value of + ``PATH``. On Windows, "shebang" line processing is supported if you have the + Python Launcher for Windows installed (this was added to Python in 3.3 - see + :pep:`397` for more details). Thus, double-clicking an installed script in a + Windows Explorer window should run the script with the correct interpreter + without there needing to be any reference to its virtual environment in + ``PATH``. + + +.. _venv-api: + +API +--- + +.. highlight:: python + +The high-level method described above makes use of a simple API which provides +mechanisms for third-party virtual environment creators to customize environment +creation according to their needs, the :class:`EnvBuilder` class. + +.. class:: EnvBuilder(system_site_packages=False, clear=False, \ + symlinks=False, upgrade=False, with_pip=False, \ + prompt=None, upgrade_deps=False) + + The :class:`EnvBuilder` class accepts the following keyword arguments on + instantiation: + + * ``system_site_packages`` -- a Boolean value indicating that the system Python + site-packages should be available to the environment (defaults to ``False``). + + * ``clear`` -- a Boolean value which, if true, will delete the contents of + any existing target directory, before creating the environment. + + * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the + Python binary rather than copying. + + * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing + environment with the running Python - for use when that Python has been + upgraded in-place (defaults to ``False``). + + * ``with_pip`` -- a Boolean value which, if true, ensures pip is + installed in the virtual environment. This uses :mod:`ensurepip` with + the ``--default-pip`` option. + + * ``prompt`` -- a String to be used after virtual environment is activated + (defaults to ``None`` which means directory name of the environment would + be used). If the special string ``"."`` is provided, the basename of the + current directory is used as the prompt. + + * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI + + .. versionchanged:: 3.4 + Added the ``with_pip`` parameter + + .. versionadded:: 3.6 + Added the ``prompt`` parameter + + .. versionadded:: 3.9 + Added the ``upgrade_deps`` parameter + + Creators of third-party virtual environment tools will be free to use the + provided :class:`EnvBuilder` class as a base class. + + The returned env-builder is an object which has a method, ``create``: + + .. method:: create(env_dir) + + Create a virtual environment by specifying the target directory + (absolute or relative to the current directory) which is to contain the + virtual environment. The ``create`` method will either create the + environment in the specified directory, or raise an appropriate + exception. + + The ``create`` method of the :class:`EnvBuilder` class illustrates the + hooks available for subclass customization:: + + def create(self, env_dir): + """ + Create a virtualized Python environment in a directory. + env_dir is the target directory to create an environment in. + """ + env_dir = os.path.abspath(env_dir) + context = self.ensure_directories(env_dir) + self.create_configuration(context) + self.setup_python(context) + self.setup_scripts(context) + self.post_setup(context) + + Each of the methods :meth:`ensure_directories`, + :meth:`create_configuration`, :meth:`setup_python`, + :meth:`setup_scripts` and :meth:`post_setup` can be overridden. + + .. method:: ensure_directories(env_dir) + + Creates the environment directory and all necessary directories, and + returns a context object. This is just a holder for attributes (such as + paths), for use by the other methods. The directories are allowed to + exist already, as long as either ``clear`` or ``upgrade`` were + specified to allow operating on an existing environment directory. + + .. method:: create_configuration(context) + + Creates the ``pyvenv.cfg`` configuration file in the environment. + + .. method:: setup_python(context) + + Creates a copy or symlink to the Python executable in the environment. + On POSIX systems, if a specific executable ``python3.x`` was used, + symlinks to ``python`` and ``python3`` will be created pointing to that + executable, unless files with those names already exist. + + .. method:: setup_scripts(context) + + Installs activation scripts appropriate to the platform into the virtual + environment. + + .. method:: upgrade_dependencies(context) + + Upgrades the core venv dependency packages (currently ``pip`` and + ``setuptools``) in the environment. This is done by shelling out to the + ``pip`` executable in the environment. + + .. versionadded:: 3.9 + + .. method:: post_setup(context) + + A placeholder method which can be overridden in third party + implementations to pre-install packages in the virtual environment or + perform other post-creation steps. + + .. versionchanged:: 3.7.2 + Windows now uses redirector scripts for ``python[w].exe`` instead of + copying the actual binaries. In 3.7.2 only :meth:`setup_python` does + nothing unless running from a build in the source tree. + + .. versionchanged:: 3.7.3 + Windows copies the redirector scripts as part of :meth:`setup_python` + instead of :meth:`setup_scripts`. This was not the case in 3.7.2. + When using symlinks, the original executables will be linked. + + In addition, :class:`EnvBuilder` provides this utility method that can be + called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to + assist in installing custom scripts into the virtual environment. + + .. method:: install_scripts(context, path) + + *path* is the path to a directory that should contain subdirectories + "common", "posix", "nt", each containing scripts destined for the bin + directory in the environment. The contents of "common" and the + directory corresponding to :data:`os.name` are copied after some text + replacement of placeholders: + + * ``__VENV_DIR__`` is replaced with the absolute path of the environment + directory. + + * ``__VENV_NAME__`` is replaced with the environment name (final path + segment of environment directory). + + * ``__VENV_PROMPT__`` is replaced with the prompt (the environment + name surrounded by parentheses and with a following space) + + * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory + (either ``bin`` or ``Scripts``). + + * ``__VENV_PYTHON__`` is replaced with the absolute path of the + environment's executable. + + The directories are allowed to exist (for when an existing environment + is being upgraded). + +There is also a module-level convenience function: + +.. function:: create(env_dir, system_site_packages=False, clear=False, \ + symlinks=False, with_pip=False, prompt=None, \ + upgrade_deps=False) + + Create an :class:`EnvBuilder` with the given keyword arguments, and call its + :meth:`~EnvBuilder.create` method with the *env_dir* argument. + + .. versionadded:: 3.3 + + .. versionchanged:: 3.4 + Added the ``with_pip`` parameter + + .. versionchanged:: 3.6 + Added the ``prompt`` parameter + + .. versionchanged:: 3.9 + Added the ``upgrade_deps`` parameter + +An example of extending ``EnvBuilder`` +-------------------------------------- + +The following script shows how to extend :class:`EnvBuilder` by implementing a +subclass which installs setuptools and pip into a created virtual environment:: + + import os + import os.path + from subprocess import Popen, PIPE + import sys + from threading import Thread + from urllib.parse import urlparse + from urllib.request import urlretrieve + import venv + + class ExtendedEnvBuilder(venv.EnvBuilder): + """ + This builder installs setuptools and pip so that you can pip or + easy_install other packages into the created virtual environment. + + :param nodist: If true, setuptools and pip are not installed into the + created virtual environment. + :param nopip: If true, pip is not installed into the created + virtual environment. + :param progress: If setuptools or pip are installed, the progress of the + installation can be monitored by passing a progress + callable. If specified, it is called with two + arguments: a string indicating some progress, and a + context indicating where the string is coming from. + The context argument can have one of three values: + 'main', indicating that it is called from virtualize() + itself, and 'stdout' and 'stderr', which are obtained + by reading lines from the output streams of a subprocess + which is used to install the app. + + If a callable is not specified, default progress + information is output to sys.stderr. + """ + + def __init__(self, *args, **kwargs): + self.nodist = kwargs.pop('nodist', False) + self.nopip = kwargs.pop('nopip', False) + self.progress = kwargs.pop('progress', None) + self.verbose = kwargs.pop('verbose', False) + super().__init__(*args, **kwargs) + + def post_setup(self, context): + """ + Set up any packages which need to be pre-installed into the + virtual environment being created. + + :param context: The information for the virtual environment + creation request being processed. + """ + os.environ['VIRTUAL_ENV'] = context.env_dir + if not self.nodist: + self.install_setuptools(context) + # Can't install pip without setuptools + if not self.nopip and not self.nodist: + self.install_pip(context) + + def reader(self, stream, context): + """ + Read lines from a subprocess' output stream and either pass to a progress + callable (if specified) or write progress information to sys.stderr. + """ + progress = self.progress + while True: + s = stream.readline() + if not s: + break + if progress is not None: + progress(s, context) + else: + if not self.verbose: + sys.stderr.write('.') + else: + sys.stderr.write(s.decode('utf-8')) + sys.stderr.flush() + stream.close() + + def install_script(self, context, name, url): + _, _, path, _, _, _ = urlparse(url) + fn = os.path.split(path)[-1] + binpath = context.bin_path + distpath = os.path.join(binpath, fn) + # Download script into the virtual environment's binaries folder + urlretrieve(url, distpath) + progress = self.progress + if self.verbose: + term = '\n' + else: + term = '' + if progress is not None: + progress('Installing %s ...%s' % (name, term), 'main') + else: + sys.stderr.write('Installing %s ...%s' % (name, term)) + sys.stderr.flush() + # Install in the virtual environment + args = [context.env_exe, fn] + p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath) + t1 = Thread(target=self.reader, args=(p.stdout, 'stdout')) + t1.start() + t2 = Thread(target=self.reader, args=(p.stderr, 'stderr')) + t2.start() + p.wait() + t1.join() + t2.join() + if progress is not None: + progress('done.', 'main') + else: + sys.stderr.write('done.\n') + # Clean up - no longer needed + os.unlink(distpath) + + def install_setuptools(self, context): + """ + Install setuptools in the virtual environment. + + :param context: The information for the virtual environment + creation request being processed. + """ + url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' + self.install_script(context, 'setuptools', url) + # clear up the setuptools archive which gets downloaded + pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') + files = filter(pred, os.listdir(context.bin_path)) + for f in files: + f = os.path.join(context.bin_path, f) + os.unlink(f) + + def install_pip(self, context): + """ + Install pip in the virtual environment. + + :param context: The information for the virtual environment + creation request being processed. + """ + url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' + self.install_script(context, 'pip', url) + + def main(args=None): + compatible = True + if sys.version_info < (3, 3): + compatible = False + elif not hasattr(sys, 'base_prefix'): + compatible = False + if not compatible: + raise ValueError('This script is only for use with ' + 'Python 3.3 or later') + else: + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory in which to create the + 'virtual environment.') + parser.add_argument('--no-setuptools', default=False, + action='store_true', dest='nodist', + help="Don't install setuptools or pip in the " + "virtual environment.") + parser.add_argument('--no-pip', default=False, + action='store_true', dest='nopip', + help="Don't install pip in the virtual " + "environment.") + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False + else: + use_symlinks = True + parser.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'virtual environment ' + 'directory if it already ' + 'exists, before virtual ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the virtual ' + 'environment directory to ' + 'use this version of ' + 'Python, assuming Python ' + 'has been upgraded ' + 'in-place.') + parser.add_argument('--verbose', default=False, action='store_true', + dest='verbose', help='Display the output ' + 'from the scripts which ' + 'install setuptools and pip.') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = ExtendedEnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + nodist=options.nodist, + nopip=options.nopip, + verbose=options.verbose) + for d in options.dirs: + builder.create(d) + + if __name__ == '__main__': + rc = 1 + try: + main() + rc = 0 + except Exception as e: + print('Error: %s' % e, file=sys.stderr) + sys.exit(rc) + + +This script is also available for download `online +`_. + + +.. _setuptools: https://pypi.org/project/setuptools/ +.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/xml.sax.handler.rst.bak b/Doc/library/xml.sax.handler.rst.bak new file mode 100644 index 00000000000000..3746a58c9b9558 --- /dev/null +++ b/Doc/library/xml.sax.handler.rst.bak @@ -0,0 +1,463 @@ +:mod:`xml.sax.handler` --- Base classes for SAX handlers +======================================================== + +.. module:: xml.sax.handler + :synopsis: Base classes for SAX event handlers. + +.. moduleauthor:: Lars Marius Garshol +.. sectionauthor:: Martin v. Löwis + +**Source code:** :source:`Lib/xml/sax/handler.py` + +-------------- + +The SAX API defines five kinds of handlers: content handlers, DTD handlers, +error handlers, entity resolvers and lexical handlers. Applications normally +only need to implement those interfaces whose events they are interested in; +they can implement the interfaces in a single object or in multiple objects. +Handler implementations should inherit from the base classes provided in the +module :mod:`xml.sax.handler`, so that all methods get default implementations. + + +.. class:: ContentHandler + + This is the main callback interface in SAX, and the one most important to + applications. The order of events in this interface mirrors the order of the + information in the document. + + +.. class:: DTDHandler + + Handle DTD events. + + This interface specifies only those DTD events required for basic parsing + (unparsed entities and attributes). + + +.. class:: EntityResolver + + Basic interface for resolving entities. If you create an object implementing + this interface, then register the object with your Parser, the parser will call + the method in your object to resolve all external entities. + + +.. class:: ErrorHandler + + Interface used by the parser to present error and warning messages to the + application. The methods of this object control whether errors are immediately + converted to exceptions or are handled in some other way. + + +.. class:: LexicalHandler + + Interface used by the parser to represent low freqency events which may not + be of interest to many applications. + +In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants +for the feature and property names. + + +.. data:: feature_namespaces + + | value: ``"http://xml.org/sax/features/namespaces"`` + | true: Perform Namespace processing. + | false: Optionally do not perform Namespace processing (implies + namespace-prefixes; default). + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_namespace_prefixes + + | value: ``"http://xml.org/sax/features/namespace-prefixes"`` + | true: Report the original prefixed names and attributes used for Namespace + declarations. + | false: Do not report attributes used for Namespace declarations, and + optionally do not report original prefixed names (default). + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_string_interning + + | value: ``"http://xml.org/sax/features/string-interning"`` + | true: All element names, prefixes, attribute names, Namespace URIs, and + local names are interned using the built-in intern function. + | false: Names are not necessarily interned, although they may be (default). + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_validation + + | value: ``"http://xml.org/sax/features/validation"`` + | true: Report all validation errors (implies external-general-entities and + external-parameter-entities). + | false: Do not report validation errors. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_external_ges + + | value: ``"http://xml.org/sax/features/external-general-entities"`` + | true: Include all external general (text) entities. + | false: Do not include external general entities. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: feature_external_pes + + | value: ``"http://xml.org/sax/features/external-parameter-entities"`` + | true: Include all external parameter entities, including the external DTD + subset. + | false: Do not include any external parameter entities, even the external + DTD subset. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: all_features + + List of all features. + + +.. data:: property_lexical_handler + + | value: ``"http://xml.org/sax/properties/lexical-handler"`` + | data type: xml.sax.handler.LexicalHandler (not supported in Python 2) + | description: An optional extension handler for lexical events like + comments. + | access: read/write + + +.. data:: property_declaration_handler + + | value: ``"http://xml.org/sax/properties/declaration-handler"`` + | data type: xml.sax.sax2lib.DeclHandler (not supported in Python 2) + | description: An optional extension handler for DTD-related events other + than notations and unparsed entities. + | access: read/write + + +.. data:: property_dom_node + + | value: ``"http://xml.org/sax/properties/dom-node"`` + | data type: org.w3c.dom.Node (not supported in Python 2) + | description: When parsing, the current DOM node being visited if this is + a DOM iterator; when not parsing, the root DOM node for iteration. + | access: (parsing) read-only; (not parsing) read/write + + +.. data:: property_xml_string + + | value: ``"http://xml.org/sax/properties/xml-string"`` + | data type: String + | description: The literal string of characters that was the source for the + current event. + | access: read-only + + +.. data:: all_properties + + List of all known property names. + + +.. _content-handler-objects: + +ContentHandler Objects +---------------------- + +Users are expected to subclass :class:`ContentHandler` to support their +application. The following methods are called by the parser on the appropriate +events in the input document: + + +.. method:: ContentHandler.setDocumentLocator(locator) + + Called by the parser to give the application a locator for locating the origin + of document events. + + SAX parsers are strongly encouraged (though not absolutely required) to supply a + locator: if it does so, it must supply the locator to the application by + invoking this method before invoking any of the other methods in the + DocumentHandler interface. + + The locator allows the application to determine the end position of any + document-related event, even if the parser is not reporting an error. Typically, + the application will use this information for reporting its own errors (such as + character content that does not match an application's business rules). The + information returned by the locator is probably not sufficient for use with a + search engine. + + Note that the locator will return correct information only during the invocation + of the events in this interface. The application should not attempt to use it at + any other time. + + +.. method:: ContentHandler.startDocument() + + Receive notification of the beginning of a document. + + The SAX parser will invoke this method only once, before any other methods in + this interface or in DTDHandler (except for :meth:`setDocumentLocator`). + + +.. method:: ContentHandler.endDocument() + + Receive notification of the end of a document. + + The SAX parser will invoke this method only once, and it will be the last method + invoked during the parse. The parser shall not invoke this method until it has + either abandoned parsing (because of an unrecoverable error) or reached the end + of input. + + +.. method:: ContentHandler.startPrefixMapping(prefix, uri) + + Begin the scope of a prefix-URI Namespace mapping. + + The information from this event is not necessary for normal Namespace + processing: the SAX XML reader will automatically replace prefixes for element + and attribute names when the ``feature_namespaces`` feature is enabled (the + default). + + There are cases, however, when applications need to use prefixes in character + data or in attribute values, where they cannot safely be expanded automatically; + the :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events supply the + information to the application to expand prefixes in those contexts itself, if + necessary. + + .. XXX This is not really the default, is it? MvL + + Note that :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events are not + guaranteed to be properly nested relative to each-other: all + :meth:`startPrefixMapping` events will occur before the corresponding + :meth:`startElement` event, and all :meth:`endPrefixMapping` events will occur + after the corresponding :meth:`endElement` event, but their order is not + guaranteed. + + +.. method:: ContentHandler.endPrefixMapping(prefix) + + End the scope of a prefix-URI mapping. + + See :meth:`startPrefixMapping` for details. This event will always occur after + the corresponding :meth:`endElement` event, but the order of + :meth:`endPrefixMapping` events is not otherwise guaranteed. + + +.. method:: ContentHandler.startElement(name, attrs) + + Signals the start of an element in non-namespace mode. + + The *name* parameter contains the raw XML 1.0 name of the element type as a + string and the *attrs* parameter holds an object of the + :class:`~xml.sax.xmlreader.Attributes` + interface (see :ref:`attributes-objects`) containing the attributes of + the element. The object passed as *attrs* may be re-used by the parser; holding + on to a reference to it is not a reliable way to keep a copy of the attributes. + To keep a copy of the attributes, use the :meth:`copy` method of the *attrs* + object. + + +.. method:: ContentHandler.endElement(name) + + Signals the end of an element in non-namespace mode. + + The *name* parameter contains the name of the element type, just as with the + :meth:`startElement` event. + + +.. method:: ContentHandler.startElementNS(name, qname, attrs) + + Signals the start of an element in namespace mode. + + The *name* parameter contains the name of the element type as a ``(uri, + localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in + the source document, and the *attrs* parameter holds an instance of the + :class:`~xml.sax.xmlreader.AttributesNS` interface (see + :ref:`attributes-ns-objects`) + containing the attributes of the element. If no namespace is associated with + the element, the *uri* component of *name* will be ``None``. The object passed + as *attrs* may be re-used by the parser; holding on to a reference to it is not + a reliable way to keep a copy of the attributes. To keep a copy of the + attributes, use the :meth:`copy` method of the *attrs* object. + + Parsers may set the *qname* parameter to ``None``, unless the + ``feature_namespace_prefixes`` feature is activated. + + +.. method:: ContentHandler.endElementNS(name, qname) + + Signals the end of an element in namespace mode. + + The *name* parameter contains the name of the element type, just as with the + :meth:`startElementNS` method, likewise the *qname* parameter. + + +.. method:: ContentHandler.characters(content) + + Receive notification of character data. + + The Parser will call this method to report each chunk of character data. SAX + parsers may return all contiguous character data in a single chunk, or they may + split it into several chunks; however, all of the characters in any single event + must come from the same external entity so that the Locator provides useful + information. + + *content* may be a string or bytes instance; the ``expat`` reader module + always produces strings. + + .. note:: + + The earlier SAX 1 interface provided by the Python XML Special Interest Group + used a more Java-like interface for this method. Since most parsers used from + Python did not take advantage of the older interface, the simpler signature was + chosen to replace it. To convert old code to the new interface, use *content* + instead of slicing content with the old *offset* and *length* parameters. + + +.. method:: ContentHandler.ignorableWhitespace(whitespace) + + Receive notification of ignorable whitespace in element content. + + Validating Parsers must use this method to report each chunk of ignorable + whitespace (see the W3C XML 1.0 recommendation, section 2.10): non-validating + parsers may also use this method if they are capable of parsing and using + content models. + + SAX parsers may return all contiguous whitespace in a single chunk, or they may + split it into several chunks; however, all of the characters in any single event + must come from the same external entity, so that the Locator provides useful + information. + + +.. method:: ContentHandler.processingInstruction(target, data) + + Receive notification of a processing instruction. + + The Parser will invoke this method once for each processing instruction found: + note that processing instructions may occur before or after the main document + element. + + A SAX parser should never report an XML declaration (XML 1.0, section 2.8) or a + text declaration (XML 1.0, section 4.3.1) using this method. + + +.. method:: ContentHandler.skippedEntity(name) + + Receive notification of a skipped entity. + + The Parser will invoke this method once for each entity skipped. Non-validating + processors may skip entities if they have not seen the declarations (because, + for example, the entity was declared in an external DTD subset). All processors + may skip external entities, depending on the values of the + ``feature_external_ges`` and the ``feature_external_pes`` properties. + + +.. _dtd-handler-objects: + +DTDHandler Objects +------------------ + +:class:`DTDHandler` instances provide the following methods: + + +.. method:: DTDHandler.notationDecl(name, publicId, systemId) + + Handle a notation declaration event. + + +.. method:: DTDHandler.unparsedEntityDecl(name, publicId, systemId, ndata) + + Handle an unparsed entity declaration event. + + +.. _entity-resolver-objects: + +EntityResolver Objects +---------------------- + + +.. method:: EntityResolver.resolveEntity(publicId, systemId) + + Resolve the system identifier of an entity and return either the system + identifier to read from as a string, or an InputSource to read from. The default + implementation returns *systemId*. + + +.. _sax-error-handler: + +ErrorHandler Objects +-------------------- + +Objects with this interface are used to receive error and warning information +from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that +implements this interface, then register the object with your +:class:`~xml.sax.xmlreader.XMLReader`, the parser +will call the methods in your object to report all warnings and errors. There +are three levels of errors available: warnings, (possibly) recoverable errors, +and unrecoverable errors. All methods take a :exc:`SAXParseException` as the +only parameter. Errors and warnings may be converted to an exception by raising +the passed-in exception object. + + +.. method:: ErrorHandler.error(exception) + + Called when the parser encounters a recoverable error. If this method does not + raise an exception, parsing may continue, but further document information + should not be expected by the application. Allowing the parser to continue may + allow additional errors to be discovered in the input document. + + +.. method:: ErrorHandler.fatalError(exception) + + Called when the parser encounters an error it cannot recover from; parsing is + expected to terminate when this method returns. + + +.. method:: ErrorHandler.warning(exception) + + Called when the parser presents minor warning information to the application. + Parsing is expected to continue when this method returns, and document + information will continue to be passed to the application. Raising an exception + in this method will cause parsing to end. + + +.. _lexical-handler-objects: + +LexicalHandler Objects +---------------------- +Optional SAX2 handler for lexical events. + +This handler is used to obtain lexical information about an XML +document. Lexical information includes information describing the +document encoding used and XML comments embedded in the document, as +well as section boundaries for the DTD and for any CDATA sections. +The lexical handlers are used in the same manner as content handlers. + +Set the LexicalHandler of an XMLReader by using the setProperty method +with the property identifier +``'http://xml.org/sax/properties/lexical-handler'``. + + +.. method:: LexicalHandler.comment(content) + + Reports a comment anywhere in the document (including the DTD and + outside the document element). + +.. method:: LexicalHandler.startDTD(name, public_id, system_id) + + Reports the start of the DTD declarations if the document has an + associated DTD. + +.. method:: LexicalHandler.endDTD() + + Reports the end of DTD declaration. + +.. method:: LexicalHandler.startCDATA() + + Reports the start of a CDATA marked section. + + The contents of the CDATA marked section will be reported through + the characters handler. + +.. method:: LexicalHandler.endCDATA() + + Reports the end of a CDATA marked section. diff --git a/Doc/library/zipimport.rst.bak b/Doc/library/zipimport.rst.bak new file mode 100644 index 00000000000000..62e1e47e980e17 --- /dev/null +++ b/Doc/library/zipimport.rst.bak @@ -0,0 +1,209 @@ +:mod:`zipimport` --- Import modules from Zip archives +===================================================== + +.. module:: zipimport + :synopsis: Support for importing Python modules from ZIP archives. + +.. moduleauthor:: Just van Rossum + +**Source code:** :source:`Lib/zipimport.py` + +-------------- + +This module adds the ability to import Python modules (:file:`\*.py`, +:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not +needed to use the :mod:`zipimport` module explicitly; it is automatically used +by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths +to ZIP archives. + +Typically, :data:`sys.path` is a list of directory names as strings. This module +also allows an item of :data:`sys.path` to be a string naming a ZIP file archive. +The ZIP archive can contain a subdirectory structure to support package imports, +and a path within the archive can be specified to only import from a +subdirectory. For example, the path :file:`example.zip/lib/` would only +import from the :file:`lib/` subdirectory within the archive. + +Any files may be present in the ZIP archive, but only files :file:`.py` and +:file:`.pyc` are available for import. ZIP import of dynamic modules +(:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains +:file:`.py` files, Python will not attempt to modify the archive by adding the +corresponding :file:`.pyc` file, meaning that if a ZIP archive +doesn't contain :file:`.pyc` files, importing may be rather slow. + +.. versionchanged:: 3.8 + Previously, ZIP archives with an archive comment were not supported. + +.. seealso:: + + `PKZIP Application Note `_ + Documentation on the ZIP file format by Phil Katz, the creator of the format and + algorithms used. + + :pep:`273` - Import Modules from Zip Archives + Written by James C. Ahlstrom, who also provided an implementation. Python 2.3 + follows the specification in :pep:`273`, but uses an implementation written by Just + van Rossum that uses the import hooks described in :pep:`302`. + + :mod:`importlib` - The implementation of the import machinery + Package providing the relevant protocols for all importers to + implement. + + +This module defines an exception: + +.. exception:: ZipImportError + + Exception raised by zipimporter objects. It's a subclass of :exc:`ImportError`, + so it can be caught as :exc:`ImportError`, too. + + +.. _zipimporter-objects: + +zipimporter Objects +------------------- + +:class:`zipimporter` is the class for importing ZIP files. + +.. class:: zipimporter(archivepath) + + Create a new zipimporter instance. *archivepath* must be a path to a ZIP + file, or to a specific path within a ZIP file. For example, an *archivepath* + of :file:`foo/bar.zip/lib` will look for modules in the :file:`lib` directory + inside the ZIP file :file:`foo/bar.zip` (provided that it exists). + + :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP + archive. + + .. method:: create_module(spec) + + Implementation of :meth:`importlib.abc.Loader.create_module` that returns + :const:`None` to explicitly request the default semantics. + + .. versionadded:: 3.10 + + + .. method:: exec_module(module) + + Implementation of :meth:`importlib.abc.Loader.exec_module`. + + .. versionadded:: 3.10 + + + .. method:: find_loader(fullname, path=None) + + An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. + + .. deprecated:: 3.10 + + Use :meth:`find_spec` instead. + + + .. method:: find_module(fullname, path=None) + + Search for a module specified by *fullname*. *fullname* must be the fully + qualified (dotted) module name. It returns the zipimporter instance itself + if the module was found, or :const:`None` if it wasn't. The optional + *path* argument is ignored---it's there for compatibility with the + importer protocol. + + .. deprecated:: 3.10 + + Use :meth:`find_spec` instead. + + + .. method:: find_spec(fullname, target=None) + + An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. + + .. versionadded:: 3.10 + + + .. method:: get_code(fullname) + + Return the code object for the specified module. Raise + :exc:`ZipImportError` if the module couldn't be imported. + + + .. method:: get_data(pathname) + + Return the data associated with *pathname*. Raise :exc:`OSError` if the + file wasn't found. + + .. versionchanged:: 3.3 + :exc:`IOError` used to be raised instead of :exc:`OSError`. + + + .. method:: get_filename(fullname) + + Return the value ``__file__`` would be set to if the specified module + was imported. Raise :exc:`ZipImportError` if the module couldn't be + imported. + + .. versionadded:: 3.1 + + + .. method:: get_source(fullname) + + Return the source code for the specified module. Raise + :exc:`ZipImportError` if the module couldn't be found, return + :const:`None` if the archive does contain the module, but has no source + for it. + + + .. method:: is_package(fullname) + + Return ``True`` if the module specified by *fullname* is a package. Raise + :exc:`ZipImportError` if the module couldn't be found. + + + .. method:: load_module(fullname) + + Load the module specified by *fullname*. *fullname* must be the fully + qualified (dotted) module name. Returns the imported module on success, + raises :exc:`ZipImportError` on failure. + + .. deprecated:: 3.10 + + Use :meth:`exec_module` instead. + + .. attribute:: archive + + The file name of the importer's associated ZIP file, without a possible + subpath. + + + .. attribute:: prefix + + The subpath within the ZIP file where modules are searched. This is the + empty string for zipimporter objects which point to the root of the ZIP + file. + + The :attr:`archive` and :attr:`prefix` attributes, when combined with a + slash, equal the original *archivepath* argument given to the + :class:`zipimporter` constructor. + + +.. _zipimport-examples: + +Examples +-------- + +Here is an example that imports a module from a ZIP archive - note that the +:mod:`zipimport` module is not explicitly used. + +.. code-block:: shell-session + + $ unzip -l example.zip + Archive: example.zip + Length Date Time Name + -------- ---- ---- ---- + 8467 11-26-02 22:30 jwzthreading.py + -------- ------- + 8467 1 file + $ ./python + Python 2.3 (#1, Aug 1 2003, 19:54:32) + >>> import sys + >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path + >>> import jwzthreading + >>> jwzthreading.__file__ + 'example.zip/jwzthreading.py' diff --git a/Doc/whatsnew/3.10.rst.bak b/Doc/whatsnew/3.10.rst.bak new file mode 100644 index 00000000000000..db71f061f14df4 --- /dev/null +++ b/Doc/whatsnew/3.10.rst.bak @@ -0,0 +1,1315 @@ +**************************** + What's New In Python 3.10 +**************************** + +:Release: |release| +:Date: |today| + +.. Rules for maintenance: + + * Anyone can add text to this document. Do not spend very much time + on the wording of your changes, because your text will probably + get rewritten to some degree. + + * The maintainer will go through Misc/NEWS periodically and add + changes; it's therefore more important to add your changes to + Misc/NEWS than to this file. + + * This is not a complete list of every single change; completeness + is the purpose of Misc/NEWS. Some changes I consider too small + or esoteric to include. If such a change is added to the text, + I'll just remove it. (This is another reason you shouldn't spend + too much time on writing your addition.) + + * If you want to draw your new text to the attention of the + maintainer, add 'XXX' to the beginning of the paragraph or + section. + + * It's OK to just add a fragmentary note about a change. For + example: "XXX Describe the transmogrify() function added to the + socket module." The maintainer will research the change and + write the necessary text. + + * You can comment out your additions if you like, but it's not + necessary (especially when a final release is some months away). + + * Credit the author of a patch or bugfix. Just the name is + sufficient; the e-mail address isn't necessary. + + * It's helpful to add the bug/patch number as a comment: + + XXX Describe the transmogrify() function added to the socket + module. + (Contributed by P.Y. Developer in :issue:`12345`.) + + This saves the maintainer the effort of going through the Mercurial log + when researching a change. + +This article explains the new features in Python 3.10, compared to 3.9. + +For full details, see the :ref:`changelog `. + +.. note:: + + Prerelease users should be aware that this document is currently in draft + form. It will be updated substantially as Python 3.10 moves towards release, + so it's worth checking back even after reading earlier versions. + + +Summary -- Release highlights +============================= + +.. This section singles out the most important changes in Python 3.10. + Brevity is key. + + +.. PEP-sized items next. + + + +New Features +============ + +.. _whatsnew310-pep563: + +Parenthesized context managers +------------------------------ + +Using enclosing parentheses for continuation across multiple lines +in context managers is now supported. This allows formatting a long +collection of context managers in multiple lines in a similar way +as it was previously possible with import statements. For instance, +all these examples are now valid: + +.. code-block:: python + + with (CtxManager() as example): + ... + + with ( + CtxManager1(), + CtxManager2() + ): + ... + + with (CtxManager1() as example, + CtxManager2()): + ... + + with (CtxManager1(), + CtxManager2() as example): + ... + + with ( + CtxManager1() as example1, + CtxManager2() as example2 + ): + ... + +it is also possible to use a trailing comma at the end of the +enclosed group: + +.. code-block:: python + + with ( + CtxManager1() as example1, + CtxManager2() as example2, + CtxManager3() as example3, + ): + ... + +This new syntax uses the non LL(1) capacities of the new parser. +Check :pep:`617` for more details. + +(Contributed by Guido van Rossum, Pablo Galindo and Lysandros Nikolaou +in :issue:`12782` and :issue:`40334`.) + + +Better error messages in the parser +----------------------------------- + +When parsing code that contains unclosed parentheses or brackets the interpreter +now includes the location of the unclosed bracket of parentheses instead of displaying +*SyntaxError: unexpected EOF while parsing* or pointing to some incorrect location. +For instance, consider the following code (notice the unclosed '{'): + +.. code-block:: python + + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + 38: 4, 39: 4, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 54: 6, + some_other_code = foo() + +previous versions of the interpreter reported confusing places as the location of +the syntax error: + +.. code-block:: text + + File "example.py", line 3 + some_other_code = foo() + ^ + SyntaxError: invalid syntax + +but in Python3.10 a more informative error is emitted: + +.. code-block:: text + + File "example.py", line 1 + expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, + ^ + SyntaxError: '{' was never closed + + +In a similar way, errors involving unclosed string literals (single and triple +quoted) now point to the start of the string instead of reporting EOF/EOL. + +These improvements are inspired by previous work in the PyPy interpreter. + +(Contributed by Pablo Galindo in :issue:`42864` and Batuhan Taskaya in +:issue:`40176`.) + + +PEP 634: Structural Pattern Matching +------------------------------------ + +Structural pattern matching has been added in the form of a *match statement* +and *case statements* of patterns with associated actions. Patterns +consist of sequences, mappings, primitive data types as well as class instances. +Pattern matching enables programs to extract information from complex data types, +branch on the structure of data, and apply specific actions based on different +forms of data. + +Syntax and operations +~~~~~~~~~~~~~~~~~~~~~ + +The generic syntax of pattern matching is:: + + match subject: + case : + + case : + + case : + + case _: + + +A match statement takes an expression and compares its value to successive +patterns given as one or more case blocks. Specifically, pattern matching +operates by: + + 1. using data with type and shape (the ``subject``) + 2. evaluating the ``subject`` in the ``match`` statement + 3. comparing the subject with each pattern in a ``case`` statement + from top to bottom until a match is confirmed. + 4. executing the action associated with the pattern of the confirmed + match + 5. If an exact match is not confirmed, the last case, a wildcard ``_``, + if provided, will be used as the matching case. If an exact match is + not confirmed and a wildcard case does not exists, the entire match + block is a no-op. + +Declarative approach +~~~~~~~~~~~~~~~~~~~~ + +Readers may be aware of pattern matching through the simple example of matching +a subject (data object) to a literal (pattern) with the switch statement found +in C, Java or JavaScript (and many other languages). Often the switch statement +is used for comparison of an object/expression with case statements containing +literals. + +More powerful examples of pattern matching can be found in languages, such as +Scala and Elixir. With structural pattern matching, the approach is "declarative" and +explicitly states the conditions (the patterns) for data to match. + +While an "imperative" series of instructions using nested "if" statements +could be used to accomplish something similar to structural pattern matching, +it is less clear than the "declarative" approach. Instead the "declarative" +approach states the conditions to meet for a match and is more readable through +its explicit patterns. While structural pattern matching can be used in its +simplest form comparing a variable to a literal in a case statement, its +true value for Python lies in its handling of the subject's type and shape. + +Simple pattern: match to a literal +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's look at this example as pattern matching in its simplest form: a value, +the subject, being matched to several literals, the patterns. In the example +below, ``status`` is the subject of the match statement. The patterns are +each of the case statements, where literals represent request status codes. +The associated action to the case is executed after a match:: + + def http_error(status): + match status: + case 400: + return "Bad request" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + case _: + return "Something's wrong with the Internet" + +If the above function is passed a ``status`` of 418, "I'm a teapot" is returned. +If the above function is passed a ``status`` of 500, the case statement with +``_`` will match as a wildcard, and "Something's wrong with the Internet" is +returned. +Note the last block: the variable name, ``_``, acts as a *wildcard* and insures +the subject will always match. The use of ``_`` is optional. + +You can combine several literals in a single pattern using ``|`` ("or"):: + + case 401 | 403 | 404: + return "Not allowed" + +Behavior without the wildcard +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If we modify the above example by removing the last case block, the example +becomes:: + + def http_error(status): + match status: + case 400: + return "Bad request" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + +Without the use of ``_`` in a case statement, a match may not exist. If no +match exists, the behavior is a no-op. For example, if ``status`` of 500 is +passed, a no-op occurs. + +Patterns with a literal and variable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Patterns can look like unpacking assignments, and a pattern may be used to bind +variables. In this example, a data point can be unpacked to its x-coordinate +and y-coordinate:: + + # point is an (x, y) tuple + match point: + case (0, 0): + print("Origin") + case (0, y): + print(f"Y={y}") + case (x, 0): + print(f"X={x}") + case (x, y): + print(f"X={x}, Y={y}") + case _: + raise ValueError("Not a point") + +The first pattern has two literals, ``(0, 0)``, and may be thought of as an +extension of the literal pattern shown above. The next two patterns combine a +literal and a variable, and the variable *binds* a value from the subject +(``point``). The fourth pattern captures two values, which makes it +conceptually similar to the unpacking assignment ``(x, y) = point``. + +Patterns and classes +~~~~~~~~~~~~~~~~~~~~ + +If you are using classes to structure your data, you can use as a pattern +the class name followed by an argument list resembling a constructor. This +pattern has the ability to capture class attributes into variables:: + + class Point: + x: int + y: int + + def location(point): + match point: + case Point(x=0, y=0): + print("Origin is the point's location.") + case Point(x=0, y=y): + print(f"Y={y} and the point is on the y-axis.") + case Point(x=x, y=0): + print(f"X={x} and the point is on the x-axis.") + case Point(): + print("The point is located somewhere else on the plane.") + case _: + print("Not a point") + +Patterns with positional parameters +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +You can use positional parameters with some builtin classes that provide an +ordering for their attributes (e.g. dataclasses). You can also define a specific +position for attributes in patterns by setting the ``__match_args__`` special +attribute in your classes. If it's set to ("x", "y"), the following patterns +are all equivalent (and all bind the ``y`` attribute to the ``var`` variable):: + + Point(1, var) + Point(1, y=var) + Point(x=1, y=var) + Point(y=var, x=1) + +Nested patterns +~~~~~~~~~~~~~~~ + +Patterns can be arbitrarily nested. For example, if our data is a short +list of points, it could be matched like this:: + + match points: + case []: + print("No points in the list.") + case [Point(0, 0)]: + print("The origin is the only point in the list.") + case [Point(x, y)]: + print(f"A single point {x}, {y} is in the list.") + case [Point(0, y1), Point(0, y2)]: + print(f"Two points on the Y axis at {y1}, {y2} are in the list.") + case _: + print("Something else is found in the list.") + +Complex patterns and the wildcard +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To this point, the examples have used ``_`` alone in the last case statement. +A wildcard can be used in more complex patterns, such as ``('error', code, _)``. +For example:: + + match test_variable: + case ('warning', code, 40): + print("A warning has been received.") + case ('error', code, _): + print(f"An error {code} occured.") + +In the above case, ``test_variable`` will match for ('error', code, 100) and +('error', code, 800). + +Guard +~~~~~ + +We can add an ``if`` clause to a pattern, known as a "guard". If the +guard is false, ``match`` goes on to try the next case block. Note +that value capture happens before the guard is evaluated:: + + match point: + case Point(x, y) if x == y: + print(f"The point is located on the diagonal Y=X at {x}.") + case Point(x, y): + print(f"Point is not on the diagonal.") + +Other Key Features +~~~~~~~~~~~~~~~~~~ + +Several other key features: + +- Like unpacking assignments, tuple and list patterns have exactly the + same meaning and actually match arbitrary sequences. Technically, + the subject must be an instance of ``collections.abc.Sequence``. + Therefore, an important exception is that patterns don't match iterators. + Also, to prevent a common mistake, sequence patterns don't match strings. + +- Sequence patterns support wildcards: ``[x, y, *rest]`` and ``(x, y, + *rest)`` work similar to wildcards in unpacking assignments. The + name after ``*`` may also be ``_``, so ``(x, y, *_)`` matches a sequence + of at least two items without binding the remaining items. + +- Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the + ``"bandwidth"`` and ``"latency"`` values from a dict. Unlike sequence + patterns, extra keys are ignored. A wildcard ``**rest`` is also + supported. (But ``**_`` would be redundant, so it not allowed.) + +- Subpatterns may be captured using the ``as`` keyword:: + + case (Point(x1, y1), Point(x2, y2) as p2): ... + + This binds x1, y1, x2, y2 like you would expect without the ``as`` clause, + and p2 to the entire second item of the subject. + +- Most literals are compared by equality. However, the singletons ``True``, + ``False`` and ``None`` are compared by identity. + +- Named constants may be used in patterns. These named constants must be + dotted names to prevent the constant from being interpreted as a capture + variable:: + + from enum import Enum + class Color(Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + + match color: + case Color.RED: + print("I see red!") + case Color.GREEN: + print("Grass is green") + case Color.BLUE: + print("I'm feeling the blues :(") + +For the full specification see :pep:`634`. Motivation and rationale +are in :pep:`635`, and a longer tutorial is in :pep:`636`. + + +New Features Related to Type Annotations +======================================== + +This section covers major changes affecting :pep:`484` type annotations and +the :mod:`typing` module. + + +PEP 563: Postponed Evaluation of Annotations Becomes Default +------------------------------------------------------------ + +In Python 3.7, postponed evaluation of annotations was added, +to be enabled with a ``from __future__ import annotations`` +directive. In 3.10 this became the default behavior, even +without that future directive. With this being default, all +annotations stored in :attr:`__annotations__` will be strings. +If needed, annotations can be resolved at runtime using +:func:`typing.get_type_hints`. See :pep:`563` for a full +description. Also, the :func:`inspect.signature` will try to +resolve types from now on, and when it fails it will fall back to +showing the string annotations. (Contributed by Batuhan Taskaya +in :issue:`38605`.) + + +PEP 604: New Type Union Operator +-------------------------------- + +A new type union operator was introduced which enables the syntax ``X | Y``. +This provides a cleaner way of expressing 'either type X or type Y' instead of +using :data:`typing.Union`, especially in type hints (annotations). + +In previous versions of Python, to apply a type hint for functions accepting +arguments of multiple types, :data:`typing.Union` was used:: + + def square(number: Union[int, float]) -> Union[int, float]: + return number ** 2 + + +Type hints can now be written in a more succinct manner:: + + def square(number: int | float) -> int | float: + return number ** 2 + + +This new syntax is also accepted as the second argument to :func:`isinstance` +and :func:`issubclass`:: + + >>> isinstance(1, int | str) + True + +See :ref:`types-union` and :pep:`604` for more details. + +(Contributed by Maggie Moss and Philippe Prados in :issue:`41428`.) + + +PEP 612: Parameter Specification Variables +------------------------------------------ + +Two new options to improve the information provided to static type checkers for +:pep:`484`\ 's ``Callable`` have been added to the :mod:`typing` module. + +The first is the parameter specification variable. They are used to forward the +parameter types of one callable to another callable -- a pattern commonly +found in higher order functions and decorators. Examples of usage can be found +in :class:`typing.ParamSpec`. Previously, there was no easy way to type annotate +dependency of parameter types in such a precise manner. + +The second option is the new ``Concatenate`` operator. It's used in conjunction +with parameter specification variables to type annotate a higher order callable +which adds or removes parameters of another callable. Examples of usage can +be found in :class:`typing.Concatenate`. + +See :class:`typing.Callable`, :class:`typing.ParamSpec`, +:class:`typing.Concatenate` and :pep:`612` for more details. + +(Contributed by Ken Jin in :issue:`41559`.) + + +PEP 613: TypeAlias Annotation +----------------------------- + +:pep:`484` introduced the concept of type aliases, only requiring them to be +top-level unannotated assignments. This simplicity sometimes made it difficult +for type checkers to distinguish between type aliases and ordinary assignments, +especially when forward references or invalid types were involved. Compare:: + + StrCache = 'Cache[str]' # a type alias + LOG_PREFIX = 'LOG[DEBUG]' # a module constant + +Now the :mod:`typing` module has a special annotation :data:`TypeAlias` to +declare type aliases more explicitly:: + + StrCache: TypeAlias = 'Cache[str]' # a type alias + LOG_PREFIX = 'LOG[DEBUG]' # a module constant + +See :pep:`613` for more details. + +(Contributed by Mikhail Golubev in :issue:`41923`.) + + +Other Language Changes +====================== + +* The :class:`int` type has a new method :meth:`int.bit_count`, returning the + number of ones in the binary expansion of a given integer, also known + as the population count. (Contributed by Niklas Fiekas in :issue:`29882`.) + +* The views returned by :meth:`dict.keys`, :meth:`dict.values` and + :meth:`dict.items` now all have a ``mapping`` attribute that gives a + :class:`types.MappingProxyType` object wrapping the original + dictionary. (Contributed by Dennis Sweeney in :issue:`40890`.) + +* :pep:`618`: The :func:`zip` function now has an optional ``strict`` flag, used + to require that all the iterables have an equal length. + +* Builtin and extension functions that take integer arguments no longer accept + :class:`~decimal.Decimal`\ s, :class:`~fractions.Fraction`\ s and other + objects that can be converted to integers only with a loss (e.g. that have + the :meth:`~object.__int__` method but do not have the + :meth:`~object.__index__` method). + (Contributed by Serhiy Storchaka in :issue:`37999`.) + +* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will + correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. + (Contributed by Alex Shkop in :issue:`38302`.) + +* Assignment expressions can now be used unparenthesized within set literals + and set comprehensions, as well as in sequence indexes (but not slices). + +* Functions have a new ``__builtins__`` attribute which is used to look for + builtin symbols when a function is executed, instead of looking into + ``__globals__['__builtins__']``. The attribute is initialized from + ``__globals__["__builtins__"]`` if it exists, else from the current builtins. + (Contributed by Mark Shannon in :issue:`42990`.) + + +New Modules +=========== + +* None yet. + + +Improved Modules +================ + +argparse +-------- + +Misleading phrase "optional arguments" was replaced with "options" in argparse help. Some tests might require adaptation if they rely on exact output match. +(Contributed by Raymond Hettinger in :issue:`9694`.) + +base64 +------ + +Add :func:`base64.b32hexencode` and :func:`base64.b32hexdecode` to support the +Base32 Encoding with Extended Hex Alphabet. + +codecs +------ + +Add a :func:`codecs.unregister` function to unregister a codec search function. +(Contributed by Hai Shi in :issue:`41842`.) + +collections.abc +--------------- + +The ``__args__`` of the :ref:`parameterized generic ` for +:class:`collections.abc.Callable` are now consistent with :data:`typing.Callable`. +:class:`collections.abc.Callable` generic now flattens type parameters, similar +to what :data:`typing.Callable` currently does. This means that +``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of +``(int, str, str)``; previously this was ``([int, str], str)``. To allow this +change, :class:`types.GenericAlias` can now be subclassed, and a subclass will +be returned when subscripting the :class:`collections.abc.Callable` type. Note +that a :exc:`TypeError` may be raised for invalid forms of parameterizing +:class:`collections.abc.Callable` which may have passed silently in Python 3.9. +(Contributed by Ken Jin in :issue:`42195`.) + +contextlib +---------- + +Add a :func:`contextlib.aclosing` context manager to safely close async generators +and objects representing asynchronously released resources. +(Contributed by Joongi Kim and John Belmonte in :issue:`41229`.) + +Add asynchronous context manager support to :func:`contextlib.nullcontext`. +(Contributed by Tom Gringauz in :issue:`41543`.) + +curses +------ + +The extended color functions added in ncurses 6.1 will be used transparently +by :func:`curses.color_content`, :func:`curses.init_color`, +:func:`curses.init_pair`, and :func:`curses.pair_content`. A new function, +:func:`curses.has_extended_color_support`, indicates whether extended color +support is provided by the underlying ncurses library. +(Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) + +The ``BUTTON5_*`` constants are now exposed in the :mod:`curses` module if +they are provided by the underlying curses library. +(Contributed by Zackery Spytz in :issue:`39273`.) + +.. _distutils-deprecated: + +distutils +--------- + +The entire ``distutils`` package is deprecated, to be removed in Python +3.12. Its functionality for specifying package builds has already been +completely replaced by third-party packages ``setuptools`` and +``packaging``, and most other commonly used APIs are available elsewhere +in the standard library (such as :mod:`platform`, :mod:`shutil`, +:mod:`subprocess` or :mod:`sysconfig`). There are no plans to migrate +any other functionality from ``distutils``, and applications that are +using other functions should plan to make private copies of the code. +Refer to :pep:`632` for discussion. + +The ``bdist_wininst`` command deprecated in Python 3.8 has been removed. +The ``bdist_wheel`` command is now recommended to distribute binary packages +on Windows. +(Contributed by Victor Stinner in :issue:`42802`.) + +doctest +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +encodings +--------- +:func:`encodings.normalize_encoding` now ignores non-ASCII characters. +(Contributed by Hai Shi in :issue:`39337`.) + +glob +---- + +Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and +:func:`~glob.iglob` which allow to specify the root directory for searching. +(Contributed by Serhiy Storchaka in :issue:`38144`.) + +inspect +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +Added *globalns* and *localns* parameters in :func:`~inspect.signature` and +:meth:`inspect.Signature.from_callable` to retrieve the annotations in given +local and global namespaces. +(Contributed by Batuhan Taskaya in :issue:`41960`.) + +linecache +--------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +os +-- + +Added :func:`os.cpu_count()` support for VxWorks RTOS. +(Contributed by Peixing Xin in :issue:`41440`.) + +Added a new function :func:`os.eventfd` and related helpers to wrap the +``eventfd2`` syscall on Linux. +(Contributed by Christian Heimes in :issue:`41001`.) + +Added :func:`os.splice()` that allows to move data between two file +descriptors without copying between kernel address space and user +address space, where one of the file descriptors must refer to a +pipe. (Contributed by Pablo Galindo in :issue:`41625`.) + +Added :data:`~os.O_EVTONLY`, :data:`~os.O_FSYNC`, :data:`~os.O_SYMLINK` +and :data:`~os.O_NOFOLLOW_ANY` for macOS. +(Contributed by Dong-hee Na in :issue:`43106`.) + +pathlib +------- + +Added slice support to :attr:`PurePath.parents `. +(Contributed by Joshua Cannon in :issue:`35498`) + +Added negative indexing support to :attr:`PurePath.parents +`. +(Contributed by Yaroslav Pankovych in :issue:`21041`) + +platform +-------- + +Added :func:`platform.freedesktop_os_release()` to retrieve operation system +identification from `freedesktop.org os-release +`_ standard file. +(Contributed by Christian Heimes in :issue:`28468`) + +py_compile +---------- + +Added ``--quiet`` option to command-line interface of :mod:`py_compile`. +(Contributed by Gregory Schevchenko in :issue:`38731`.) + +pyclbr +------ + +Added an ``end_lineno`` attribute to the ``Function`` and ``Class`` +objects in the tree returned by :func:`pyclbr.readline` and +:func:`pyclbr.readline_ex`. It matches the existing (start) ``lineno``. +(Contributed by Aviral Srivastava in :issue:`38307`.) + +shelve +------ + +The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default +instead of :mod:`pickle` protocol ``3`` when creating shelves. +(Contributed by Zackery Spytz in :issue:`34204`.) + +site +---- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +socket +------ + +The exception :exc:`socket.timeout` is now an alias of :exc:`TimeoutError`. +(Contributed by Christian Heimes in :issue:`42413`.) + +sys +--- + +Add :data:`sys.orig_argv` attribute: the list of the original command line +arguments passed to the Python executable. +(Contributed by Victor Stinner in :issue:`23427`.) + +Add :data:`sys.stdlib_module_names`, containing the list of the standard library +module names. +(Contributed by Victor Stinner in :issue:`42955`.) + +threading +--------- + +Added :func:`threading.gettrace` and :func:`threading.getprofile` to +retrieve the functions set by :func:`threading.settrace` and +:func:`threading.setprofile` respectively. +(Contributed by Mario Corchero in :issue:`42251`.) + +Add :data:`threading.__excepthook__` to allow retrieving the original value +of :func:`threading.excepthook` in case it is set to a broken or a different +value. +(Contributed by Mario Corchero in :issue:`42308`.) + +traceback +--------- + +The :func:`~traceback.format_exception`, +:func:`~traceback.format_exception_only`, and +:func:`~traceback.print_exception` functions can now take an exception object +as a positional-only argument. +(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + +types +----- + +Reintroduced the :data:`types.EllipsisType`, :data:`types.NoneType` +and :data:`types.NotImplementedType` classes, providing a new set +of types readily interpretable by type checkers. +(Contributed by Bas van Beek in :issue:`41810`.) + +typing +------ + +For major changes, see `New Features Related to Type Annotations`_. + +The behavior of :class:`typing.Literal` was changed to conform with :pep:`586` +and to match the behavior of static type checkers specified in the PEP. + +1. ``Literal`` now de-duplicates parameters. +2. Equality comparisons between ``Literal`` objects are now order independent. +3. ``Literal`` comparisons now respects types. For example, + ``Literal[0] == Literal[False]`` previously evaluated to ``True``. It is + now ``False``. To support this change, the internally used type cache now + supports differentiating types. +4. ``Literal`` objects will now raise a :exc:`TypeError` exception during + equality comparisons if one of their parameters are not :term:`immutable`. + Note that declaring ``Literal`` with mutable parameters will not throw + an error:: + + >>> from typing import Literal + >>> Literal[{0}] + >>> Literal[{0}] == Literal[{False}] + Traceback (most recent call last): + File "", line 1, in + TypeError: unhashable type: 'set' + +(Contributed by Yurii Karabas in :issue:`42345`.) + +unittest +-------- + +Add new method :meth:`~unittest.TestCase.assertNoLogs` to complement the +existing :meth:`~unittest.TestCase.assertLogs`. (Contributed by Kit Yan Choi +in :issue:`39385`.) + +urllib.parse +------------ + +Python versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +xml +--- + +Add a :class:`~xml.sax.handler.LexicalHandler` class to the +:mod:`xml.sax.handler` module. +(Contributed by Jonathan Gossage and Zackery Spytz in :issue:`35018`.) + +zipimport +--------- +Add methods related to :pep:`451`: :meth:`~zipimport.zipimporter.find_spec`, +:meth:`zipimport.zipimporter.create_module`, and +:meth:`zipimport.zipimporter.exec_module`. +(Contributed by Brett Cannon in :issue:`42131`. + + +Optimizations +============= + +* Constructors :func:`str`, :func:`bytes` and :func:`bytearray` are now faster + (around 30--40% for small objects). + (Contributed by Serhiy Storchaka in :issue:`41334`.) + +* The :mod:`runpy` module now imports fewer modules. + The ``python3 -m module-name`` command startup time is 1.3x faster in + average. + (Contributed by Victor Stinner in :issue:`41006`.) + +* The ``LOAD_ATTR`` instruction now uses new "per opcode cache" mechanism. It + is about 36% faster now for regular attributes and 44% faster for slots. + (Contributed by Pablo Galindo and Yury Selivanov in :issue:`42093` and Guido + van Rossum in :issue:`42927`, based on ideas implemented originally in PyPy + and MicroPython.) + +* When building Python with ``--enable-optimizations`` now + ``-fno-semantic-interposition`` is added to both the compile and link line. + This speeds builds of the Python interpreter created with ``--enable-shared`` + with ``gcc`` by up to 30%. See `this article + `_ + for more details. (Contributed by Victor Stinner and Pablo Galindo in + :issue:`38980`.) + + +* Function parameters and their annotations are no longer computed at runtime, + but rather at compilation time. They are stored as a tuple of strings at the + bytecode level. It is now around 2 times faster to create a function with + parameter annotations. (Contributed by Yurii Karabas and Inada Naoki + in :issue:`42202`) + +* Substring search functions such as ``str1 in str2`` and ``str2.find(str1)`` + now sometimes use Crochemore & Perrin's "Two-Way" string searching + algorithm to avoid quadratic behavior on long strings. (Contributed + by Dennis Sweeney in :issue:`41972`) + +Deprecated +========== + +* Starting in this release, there will be a concerted effort to begin + cleaning up old import semantics that were kept for Python 2.7 + compatibility. Specifically, + :meth:`~importlib.abc.PathEntryFinder.find_loader`/:meth:`~importlib.abc.Finder.find_module` + (superseded by :meth:`~importlib.abc.Finder.find_spec`), + :meth:`~importlib.abc.Loader.load_module` + (superseded by :meth:`~importlib.abc.Loader.exec_module`), + :meth:`~importlib.abc.Loader.module_repr` (which the import system + takes care of for you), the ``__package__`` attribute + (superseded by ``__spec__.parent``), the ``__loader__`` attribute + (superseded by ``__spec__.loader``), and the ``__cached__`` attribute + (superseded by ``__spec__.cached``) will slowly be removed (as well + as other classes and methods in :mod:`importlib`). + :exc:`ImportWarning` and/or :exc:`DeprecationWarning` will be raised + as appropriate to help identify code which needs updating during + this transition. + +* The entire ``distutils`` namespace is deprecated, to be removed in + Python 3.12. Refer to the :ref:`module changes ` + section for more information. + +* Non-integer arguments to :func:`random.randrange` are deprecated. + The :exc:`ValueError` is deprecated in favor of a :exc:`TypeError`. + (Contributed by Serhiy Storchaka and Raymond Hettinger in :issue:`37319`.) + +* The various ``load_module()`` methods of :mod:`importlib` have been + documented as deprecated since Python 3.6, but will now also trigger + a :exc:`DeprecationWarning`. Use + :meth:`~importlib.abc.Loader.exec_module` instead. + (Contributed by Brett Cannon in :issue:`26131`.) + +* :meth:`zimport.zipimporter.load_module` has been deprecated in + preference for :meth:`~zipimport.zipimporter.exec_module`. + (Contributed by Brett Cannon in :issue:`26131`.) + +* The use of :meth:`~importlib.abc.Loader.load_module` by the import + system now triggers an :exc:`ImportWarning` as + :meth:`~importlib.abc.Loader.exec_module` is preferred. + (Contributed by Brett Cannon in :issue:`26131`.) + +* ``sqlite3.OptimizedUnicode`` has been undocumented and obsolete since Python + 3.3, when it was made an alias to :class:`str`. It is now deprecated, + scheduled for removal in Python 3.12. + (Contributed by Erlend E. Aasland in :issue:`42264`.) + +* The undocumented built-in function ``sqlite3.enable_shared_cache`` is now + deprecated, scheduled for removal in Python 3.12. Its use is strongly + discouraged by the SQLite3 documentation. See `the SQLite3 docs + `_ for more details. + If shared cache must be used, open the database in URI mode using the + ``cache=shared`` query parameter. + (Contributed by Erlend E. Aasland in :issue:`24464`.) + + +Removed +======= + +* Removed special methods ``__int__``, ``__float__``, ``__floordiv__``, + ``__mod__``, ``__divmod__``, ``__rfloordiv__``, ``__rmod__`` and + ``__rdivmod__`` of the :class:`complex` class. They always raised + a :exc:`TypeError`. + (Contributed by Serhiy Storchaka in :issue:`41974`.) + +* The ``ParserBase.error()`` method from the private and undocumented ``_markupbase`` + module has been removed. :class:`html.parser.HTMLParser` is the only subclass of + ``ParserBase`` and its ``error()`` implementation has already been removed in + Python 3.5. + (Contributed by Berker Peksag in :issue:`31844`.) + +* Removed the ``unicodedata.ucnhash_CAPI`` attribute which was an internal + PyCapsule object. The related private ``_PyUnicode_Name_CAPI`` structure was + moved to the internal C API. + (Contributed by Victor Stinner in :issue:`42157`.) + +* Removed the ``parser`` module, which was deprecated in 3.9 due to the + switch to the new PEG parser, as well as all the C source and header files + that were only being used by the old parser, including ``node.h``, ``parser.h``, + ``graminit.h`` and ``grammar.h``. + +* Removed the Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, + :c:func:`PyParser_SimpleParseStringFlagsFilename`, + :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` + that were deprecated in 3.9 due to the switch to the new PEG parser. + +* Removed the ``formatter`` module, which was deprecated in Python 3.4. + It is somewhat obsolete, little used, and not tested. It was originally + scheduled to be removed in Python 3.6, but such removals were delayed until + after Python 2.7 EOL. Existing users should copy whatever classes they use + into their code. + (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) + +* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless + now due to the _warnings module was converted to a builtin module in 2.6. + (Contributed by Hai Shi in :issue:`42599`.) + +* Remove deprecated aliases to :ref:`collections-abstract-base-classes` from + the :mod:`collections` module. + (Contributed by Victor Stinner in :issue:`37324`.) + +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. The motivation behind this change is multifold: + + 1. This simplifies the high-level API. + 2. The functions in the high-level API have been implicitly getting the + current thread's running event loop since Python 3.7. There isn't a need to + pass the event loop to the API in most normal use cases. + 3. Event loop passing is error-prone especially when dealing with loops + running in different threads. + + Note that the low-level API will still accept ``loop``. + See `Changes in the Python API`_ for examples of how to replace existing code. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) + + +Porting to Python 3.10 +====================== + +This section lists previously described changes and other bugfixes +that may require changes to your code. + + +Changes in the Python API +------------------------- + +* The *etype* parameters of the :func:`~traceback.format_exception`, + :func:`~traceback.format_exception_only`, and + :func:`~traceback.print_exception` functions in the :mod:`traceback` module + have been renamed to *exc*. + (Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + +* :mod:`atexit`: At Python exit, if a callback registered with + :func:`atexit.register` fails, its exception is now logged. Previously, only + some exceptions were logged, and the last exception was always silently + ignored. + (Contributed by Victor Stinner in :issue:`42639`.) + +* :class:`collections.abc.Callable` generic now flattens type parameters, similar + to what :data:`typing.Callable` currently does. This means that + ``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of + ``(int, str, str)``; previously this was ``([int, str], str)``. Code which + accesses the arguments via :func:`typing.get_args` or ``__args__`` need to account + for this change. Furthermore, :exc:`TypeError` may be raised for invalid forms + of parameterizing :class:`collections.abc.Callable` which may have passed + silently in Python 3.9. + (Contributed by Ken Jin in :issue:`42195`.) + +* :meth:`socket.htons` and :meth:`socket.ntohs` now raise :exc:`OverflowError` + instead of :exc:`DeprecationWarning` if the given parameter will not fit in + a 16-bit unsigned integer. + (Contributed by Erlend E. Aasland in :issue:`42393`.) + +* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's + :doc:`high-level API <../library/asyncio-api-index>` following deprecation + in Python 3.8. + + A coroutine that currently look like this:: + + async def foo(loop): + await asyncio.sleep(1, loop=loop) + + Should be replaced with this:: + + async def foo(): + await asyncio.sleep(1) + + If ``foo()`` was specifically designed *not* to run in the current thread's + running event loop (e.g. running in another thread's event loop), consider + using :func:`asyncio.run_coroutine_threadsafe` instead. + + (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley + in :issue:`42392`.) + +* The :data:`types.FunctionType` constructor now inherits the current builtins + if the *globals* dictionary has no ``"__builtins__"`` key, rather than using + ``{"None": None}`` as builtins: same behavior as :func:`eval` and + :func:`exec` functions. Defining a function with ``def function(...): ...`` + in Python is not affected, globals cannot be overriden with this syntax: it + also inherits the current builtins. + (Contributed by Victor Stinner in :issue:`42990`.) + +CPython bytecode changes +======================== + +* The ``MAKE_FUNCTION`` instruction accepts tuple of strings as annotations + instead of dictionary. + (Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`) + +Build Changes +============= + +* The C99 functions :c:func:`snprintf` and :c:func:`vsnprintf` are now required + to build Python. + (Contributed by Victor Stinner in :issue:`36020`.) + +* :mod:`sqlite3` requires SQLite 3.7.15 or higher. (Contributed by Sergey Fedoseev + and Erlend E. Aasland :issue:`40744` and :issue:`40810`.) + +* The :mod:`atexit` module must now always be built as a built-in module. + (Contributed by Victor Stinner in :issue:`42639`.) + +* Added ``--disable-test-modules`` option to the ``configure`` script: + don't build nor install test modules. + (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) + +* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If + specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` + wheel packages in this directory: if both are present, these wheel packages + are used instead of ensurepip bundled wheel packages. + + Some Linux distribution packaging policies recommend against bundling + dependencies. For example, Fedora installs wheel packages in the + ``/usr/share/python-wheels/`` directory and don't install the + ``ensurepip._bundled`` package. + + (Contributed by Victor Stinner in :issue:`42856`.) + +* Add a new configure ``--without-static-libpython`` option to not build the + ``libpythonMAJOR.MINOR.a`` static library and not install the ``python.o`` + object file. + + (Contributed by Victor Stinner in :issue:`43103`.) + +* The ``configure`` script now uses the ``pkg-config`` utility, if available, + to detect the location of Tcl/Tk headers and libraries. As before, those + locations can be explicitly specified with the ``--with-tcltk-includes`` + and ``--with-tcltk-libs`` configuration options. + (Contributed by Manolis Stamatogiannakis in :issue:`42603`.) + + +C API Changes +============= + +New Features +------------ + +* The result of :c:func:`PyNumber_Index` now always has exact type :class:`int`. + Previously, the result could have been an instance of a subclass of ``int``. + (Contributed by Serhiy Storchaka in :issue:`40792`.) + +* Add a new :c:member:`~PyConfig.orig_argv` member to the :c:type:`PyConfig` + structure: the list of the original command line arguments passed to the + Python executable. + (Contributed by Victor Stinner in :issue:`23427`.) + +* The :c:func:`PyDateTime_DATE_GET_TZINFO` and + :c:func:`PyDateTime_TIME_GET_TZINFO` macros have been added for accessing + the ``tzinfo`` attributes of :class:`datetime.datetime` and + :class:`datetime.time` objects. + (Contributed by Zackery Spytz in :issue:`30155`.) + +* Add a :c:func:`PyCodec_Unregister` function to unregister a codec + search function. + (Contributed by Hai Shi in :issue:`41842`.) + +* The :c:func:`PyIter_Send` function was added to allow + sending value into iterator without raising ``StopIteration`` exception. + (Contributed by Vladimir Matveev in :issue:`41756`.) + +* Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API. + (Contributed by Alex Gaynor in :issue:`41784`.) + +* Added :c:func:`PyModule_AddObjectRef` function: similar to + :c:func:`PyModule_AddObject` but don't steal a reference to the value on + success. + (Contributed by Victor Stinner in :issue:`1635741`.) + +* Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the + reference count of an object and return the object. + (Contributed by Victor Stinner in :issue:`42262`.) + +* The :c:func:`PyType_FromSpecWithBases` and :c:func:`PyType_FromModuleAndSpec` + functions now accept a single class as the *bases* argument. + (Contributed by Serhiy Storchaka in :issue:`42423`.) + +* The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc`` + slot. + (Contributed by Hai Shi in :issue:`41832`.) + +* The :c:func:`PyType_GetSlot` function can accept static types. + (Contributed by Hai Shi and Petr Viktorin in :issue:`41073`.) + +* Add a new :c:func:`PySet_CheckExact` function to the C-API to check if an + object is an instance of :class:`set` but not an instance of a subtype. + (Contributed by Pablo Galindo in :issue:`43277`.) + +Porting to Python 3.10 +---------------------- + +* The ``PY_SSIZE_T_CLEAN`` macro must now be defined to use + :c:func:`PyArg_ParseTuple` and :c:func:`Py_BuildValue` formats which use + ``#``: ``es#``, ``et#``, ``s#``, ``u#``, ``y#``, ``z#``, ``U#`` and ``Z#``. + See :ref:`Parsing arguments and building values + ` and the :pep:`353`. + (Contributed by Victor Stinner in :issue:`40943`.) + +* Since :c:func:`Py_REFCNT()` is changed to the inline static function, + ``Py_REFCNT(obj) = new_refcnt`` must be replaced with ``Py_SET_REFCNT(obj, new_refcnt)``: + see :c:func:`Py_SET_REFCNT()` (available since Python 3.9). For backward + compatibility, this macro can be used:: + + #if PY_VERSION_HEX < 0x030900A4 + # define Py_SET_REFCNT(obj, refcnt) ((Py_REFCNT(obj) = (refcnt)), (void)0) + #endif + + (Contributed by Victor Stinner in :issue:`39573`.) + +* Calling :c:func:`PyDict_GetItem` without :term:`GIL` held had been allowed + for historical reason. It is no longer allowed. + (Contributed by Victor Stinner in :issue:`40839`.) + +* ``PyUnicode_FromUnicode(NULL, size)`` and ``PyUnicode_FromStringAndSize(NULL, size)`` + raise ``DeprecationWarning`` now. Use :c:func:`PyUnicode_New` to allocate + Unicode object without initial data. + (Contributed by Inada Naoki in :issue:`36346`.) + +* The private ``_PyUnicode_Name_CAPI`` structure of the PyCapsule API + ``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. + (Contributed by Victor Stinner in :issue:`42157`.) + +* :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, + :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and + :c:func:`Py_GetProgramName` functions now return ``NULL`` if called before + :c:func:`Py_Initialize` (before Python is initialized). Use the new + :ref:`Python Initialization Configuration API ` to get the + :ref:`Python Path Configuration. `. + (Contributed by Victor Stinner in :issue:`42260`.) + +* :c:func:`PyList_SET_ITEM`, :c:func:`PyTuple_SET_ITEM` and + :c:func:`PyCell_SET` macros can no longer be used as l-value or r-value. + For example, ``x = PyList_SET_ITEM(a, b, c)`` and + ``PyList_SET_ITEM(a, b, c) = x`` now fail with a compiler error. It prevents + bugs like ``if (PyList_SET_ITEM (a, b, c) < 0) ...`` test. + (Contributed by Zackery Spytz and Victor Stinner in :issue:`30459`.) + +* The non-limited API files ``odictobject.h``, ``parser_interface.h``, + ``picklebufobject.h``, ``pyarena.h``, ``pyctype.h``, ``pydebug.h``, + ``pyfpe.h``, and ``pytime.h`` have been moved to the ``Include/cpython`` + directory. These files must not be included directly, as they are already + included in ``Python.h``: :ref:`Include Files `. If they have + been included directly, consider including ``Python.h`` instead. + (Contributed by Nicholas Sim in :issue:`35134`) + +Deprecated +---------- + +* The ``PyUnicode_InternImmortal()`` function is now deprecated + and will be removed in Python 3.12: use :c:func:`PyUnicode_InternInPlace` + instead. + (Contributed by Victor Stinner in :issue:`41692`.) + +Removed +------- + +* ``PyObject_AsCharBuffer()``, ``PyObject_AsReadBuffer()``, ``PyObject_CheckReadBuffer()``, + and ``PyObject_AsWriteBuffer()`` are removed. Please migrate to new buffer protocol; + :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release`. + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``Py_UNICODE_str*`` functions manipulating ``Py_UNICODE*`` strings. + (Contributed by Inada Naoki in :issue:`41123`.) + + * ``Py_UNICODE_strlen``: use :c:func:`PyUnicode_GetLength` or + :c:macro:`PyUnicode_GET_LENGTH` + * ``Py_UNICODE_strcat``: use :c:func:`PyUnicode_CopyCharacters` or + :c:func:`PyUnicode_FromFormat` + * ``Py_UNICODE_strcpy``, ``Py_UNICODE_strncpy``: use + :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` + * ``Py_UNICODE_strcmp``: use :c:func:`PyUnicode_Compare` + * ``Py_UNICODE_strncmp``: use :c:func:`PyUnicode_Tailmatch` + * ``Py_UNICODE_strchr``, ``Py_UNICODE_strrchr``: use + :c:func:`PyUnicode_FindChar` + +* Removed ``PyUnicode_GetMax()``. Please migrate to new (:pep:`393`) APIs. + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``PyLong_FromUnicode()``. Please migrate to :c:func:`PyLong_FromUnicodeObject`. + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``PyUnicode_AsUnicodeCopy()``. Please use :c:func:`PyUnicode_AsUCS4Copy` or + :c:func:`PyUnicode_AsWideCharString` + (Contributed by Inada Naoki in :issue:`41103`.) + +* Removed ``_Py_CheckRecursionLimit`` variable: it has been replaced by + ``ceval.recursion_limit`` of the :c:type:`PyInterpreterState` structure. + (Contributed by Victor Stinner in :issue:`41834`.) + +* Removed undocumented macros ``Py_ALLOW_RECURSION`` and + ``Py_END_ALLOW_RECURSION`` and the ``recursion_critical`` field of the + :c:type:`PyInterpreterState` structure. + (Contributed by Serhiy Storchaka in :issue:`41936`.) + +* Removed the undocumented ``PyOS_InitInterrupts()`` function. Initializing + Python already implicitly installs signal handlers: see + :c:member:`PyConfig.install_signal_handlers`. + (Contributed by Victor Stinner in :issue:`41713`.) From 32cbefb60cd9dfa25b480f7659c01576c03af072 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 10:37:20 +0100 Subject: [PATCH 26/93] Update test_locks.py --- Lib/test/test_asyncio/test_locks.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 42ef44db2d3758..26c16b3850d296 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -11,8 +11,8 @@ STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' - r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?(, count:\d+/\d+)?(, state:^-?\d)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier object + r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -71,7 +71,7 @@ def test_lock_doesnt_accept_loop_parameter(self): cls(loop=self.loop) # Barrier object has a positional paramater - # so check alone + # so test alone cls = asyncio.Barrier with self.assertRaisesRegex( TypeError, From e5002bc7d677f8add141268a926bf9ee45da9dfd Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 16:09:54 +0100 Subject: [PATCH 27/93] Update locks.py Add comment when testing self._state Change if to while instruction in _block method --- Lib/asyncio/locks.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 4bc69100ed225d..e7f2561468e8cf 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -452,7 +452,7 @@ def __init__(self, parties, action=None, *, loop=mixins._marker): raise ValueError('parties must be > 0') self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block tasks while wainting tasks are draining or broken + self._blocking = Event() # used block new tasks while waiting tasks are draining or broken self._action = action self._parties = parties self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken @@ -491,7 +491,7 @@ async def wait(self): # Block until the barrier is ready for us, or raise an exception # if it is broken. async def _block (self): - if self._state in (-1, 1): + while self._state in (-1, 1): # It is draining or resetting, wait until done await self._blocking.wait() @@ -522,7 +522,7 @@ def _release(self): async def _wait(self): await self._waiting.wait() # no timeout so - if self._state < 0: + if self._state < 0: # resetting or broken raise BrokenBarrierError assert self._state == 1, repr(self) @@ -530,9 +530,9 @@ async def _wait(self): # waiting for the barrier to drain. def _exit(self): if self._count == 0: - if self._state == 1: + if self._state == 1: # draining self._state = 0 - elif self._state == -1: + elif self._state == -1: # resetting self._state = 0 self._waiting.clear() self._blocking.set() @@ -544,12 +544,12 @@ def reset(self): raised. """ if self._count > 0: - if self._state in (0, 1): - #reset the barrier, waking up tasks + if self._state in (0, 1): # filling or draining + # reset the barrier, waking up tasks self._state = -1 elif self._state == -2: - #was broken, set it to reset state - #which clears when the last tasks exits + # was broken, set it to reset state + # which clears when the last tasks exits self._state = -1 self._waiting.set() self._blocking.clear() From b12599c21518b32da57605a32d12d71ff5d0b936 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 16:13:24 +0100 Subject: [PATCH 28/93] Update test_locks.py Refactoring all test names of BarrierTests to be more readable Add a method cancel_coros Refactoring and simplifyiing some tests Correction of RGX_REPR --- Lib/test/test_asyncio/test_locks.py | 253 +++++++++++++++++----------- 1 file changed, 155 insertions(+), 98 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 26c16b3850d296..ec90698b6155aa 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -12,7 +12,7 @@ r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # this line to repr barrier + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # new line dedicated to repr barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -71,7 +71,7 @@ def test_lock_doesnt_accept_loop_parameter(self): cls(loop=self.loop) # Barrier object has a positional paramater - # so test alone + # so tests alone cls = asyncio.Barrier with self.assertRaisesRegex( TypeError, @@ -954,6 +954,11 @@ async def run_coros(self, n, coro): res = await asyncio.gather(*tasks) return res, tasks + def cancel_coros(self): + for task in asyncio.all_tasks(self.loop): + task.cancel() + test_utils.run_briefly(self.loop) + def test_repr(self): async def coro(): try: @@ -963,7 +968,7 @@ async def coro(): else: return await asyncio.sleep(1, True) - self.N = 999 + self.N = 1011001 barrier = asyncio.Barrier(self.N) self.assertTrue("[unset," in repr(barrier)) self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) @@ -998,23 +1003,24 @@ async def coro(): self.assertTrue(f"0/{self.N}" in repr(barrier)) self.assertTrue(repr(barrier).endswith(':-2]>')) + self.cancel_coros() + def test_init(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - def test_wait_more(self): - async def coro(result): - r = await barrier.wait() - result.append(r) - + def test_wait_task_once(self): + self.N = 1 barrier = asyncio.Barrier(self.N) - results = [] - _ = [self.loop.create_task(coro(results)) for _ in range(self.N)] - test_utils.run_briefly(self.loop) - self.assertEqual(sum(results), sum(range(self.N))) + self.assertEqual(barrier.n_waiting, 0) + r1 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + r2 = self.loop.run_until_complete(barrier.wait()) + self.assertEqual(barrier.n_waiting, 0) + self.assertEqual(r1, r2) self.assertFalse(barrier.broken) - def test_wait_task_multi(self): + def test_wait_task_by_task(self): self.N = 3 barrier = asyncio.Barrier(self.N) self.assertEqual(barrier.n_waiting, 0) @@ -1029,144 +1035,181 @@ def test_wait_task_multi(self): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_wait_task_one(self): - self.N = 1 - barrier = asyncio.Barrier(self.N) - r1 = self.loop.run_until_complete(barrier.wait()) - self.assertEqual(barrier.n_waiting, 0) - r2 = self.loop.run_until_complete(barrier.wait()) - self.assertEqual(barrier.n_waiting, 0) - self.assertEqual(r1, r2) - self.assertFalse(barrier.broken) - - def test_wait_step_by_step(self): - async def coro(result, value): + def test_wait_tasks_twice(self): + results = [] + async def coro(): await barrier.wait() - result.append(value) + results.append(True) + await barrier.wait() + results.append(False) - results = [] barrier = asyncio.Barrier(self.N) - value = 1 - tasks = [] - for n in range(self.N-1): - tasks.append(self.loop.create_task(coro(results, value))) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, n+1) - - tasks.append(self.loop.create_task(coro(results, value))) - test_utils.run_briefly(self.loop) - self.assertEqual(results, [value]* self.N) - self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(all(t.done() for t in tasks)) - self.assertTrue(all(t.result() is None for t in tasks)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertEqual(len(results), self.N*2) + self.assertEqual(results.count(True), self.N) + self.assertEqual(results.count(False), self.N) self.assertFalse(barrier.broken) + self.assertEqual(barrier.n_waiting, 0) - def test_wait_multi_return(self): + def test_wait_return_from_wait(self): results1 = [] results2 = [] - results3 = [] async def coro(): await barrier.wait() results1.append(True) i = await barrier.wait() results2.append(True) - results3.append(i) return i barrier = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + res = self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertEqual(len(results1), self.N) self.assertTrue(all(results1)) self.assertEqual(len(results2), self.N) self.assertTrue(all(results2)) - self.assertEqual(sum(res), sum(results3)) + self.assertEqual(sum(res[0]), sum(range(self.N))) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) - def test_barrier(self, multipass=1, nn=5): - results = [[] for _ in range(multipass)] + + def test_wait_repeat_more(self, more=5): async def coro(result, value): ret = await barrier.wait() result.append(value) - return ret - barrier = asyncio.Barrier(nn) - for i in range(multipass): - _ = [self.loop.create_task(coro(results[i], value)) for value in range(nn)] + results = [[] for _ in range(more)] + barrier = asyncio.Barrier(self.N) + for i in range(more): + _ = [self.loop.create_task(coro(results[i], value)) for value in range(self.N)] test_utils.run_briefly(self.loop) - self.assertEqual(len(results[i]), nn) - self.assertEqual(sum(results[i]), sum(range(nn))) + self.assertEqual(len(results[i]), self.N) + self.assertEqual(sum(results[i]), sum(range(self.N))) if i > 0: self.assertEqual(sum(results[i]), sum(results[i-1])) self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_wait_state_draining(self): + result = [] + state = [] + async def coro(): + i = await barrier.wait() + if i == 0: + result.append(True) + # Do we need to add a Barrie.draining() property ? + state.append(repr(barrier).endswith("state:1]>")) + rest = self.N-1 + nb_tasks = self.N + rest + barrier = asyncio.Barrier(self.N) + for _ in range(nb_tasks): + self.loop.create_task(coro()) + for _ in range((nb_tasks//self.N)+1): + test_utils.run_briefly(self.loop) + self.assertEqual(len(result), nb_tasks//self.N) + self.assertTrue(all(result)) + self.assertTrue(all(state)) + self.assertEqual(barrier.n_waiting, rest) self.assertFalse(barrier.broken) - def test_barrier_multipass(self): - self.test_barrier(10) + self.cancel_coros() - def test_action_callback(self): - async def coro(result, value): - ret = await barrier.wait() - result.append(value) - return ret + def test_wait_tasks_cancel_one_task(self): + results = [] + async def coro(): + await barrier.wait() + results.append(True) - result = [] - result1 = [] - value = 1 - barrier = asyncio.Barrier(1, action=lambda: result1.append(True)) - _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + self.N=3 + barrier = asyncio.Barrier(self.N) + t1 = self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 2) - self.assertEqual(len(result1), self.N) - self.assertTrue(all(result1)) - self.assertEqual(len(result), self.N) - self.assertEqual(sum(result), self.N*value) + t1.cancel() + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 2) + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(len(results), self.N) + self.assertTrue(all(results)) + self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_action_callback_n(self): - async def coro(result, value): + def test_wait_action_callback(self): + result = [] + async def coro(): ret = await barrier.wait() - result.append(value) - return ret + result.append(True) - result = [] result1 = [] - value = 1 + self.N = 2 barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) - _ = [self.loop.create_task(coro(result, value)) for _ in range(self.N)] + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) self.assertEqual(len(result1), 1) self.assertTrue(result1[0]) - self.assertEqual(len(result), self.N) - self.assertEqual(sum(result), value*self.N) + self.assertEqual(len(result), 2) + self.assertTrue(all(result)) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_wait_action_callback_more(self): + result = [] + async def coro(): + ret = await barrier.wait() + result.append(True) + + result1 = [] + + self.N = 2 + more = 3 + barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) + for _ in range(more): + _ = [self.loop.create_task(coro()) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(result1), more) + self.assertTrue(all(result1)) + self.assertEqual(len(result), self.N*more) + self.assertTrue(all(result)) + self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_action_callback_error(self): + def test_wait_action_callback_error_broken(self): + ERROR = ZeroDivisionError results1 = [] results2 = [] - results3 = [] - def raise_except(e): - raise e - - async def coro(e): + async def coro(): try: ret = await barrier.wait() - except e: + except ERROR: results1.append(False) except: results2.append(True) - else: - results3.append(None) - barrier = asyncio.Barrier(self.N, lambda: raise_except(ZeroDivisionError)) - _ = [self.loop.create_task(coro(ZeroDivisionError)) for _ in range(self.N)] + def raise_except(): + raise ERROR + + barrier = asyncio.Barrier(self.N, lambda: raise_except()) + _ = [self.loop.create_task(coro()) for _ in range(self.N)] test_utils.run_briefly(self.loop) self.assertEqual(len(results1), 1) self.assertFalse(results1[0]) self.assertEqual(len(results2), self.N-1) self.assertTrue(all(results2)) - self.assertEqual(len(results3), 0) + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(barrier.broken) def test_reset(self): @@ -1196,16 +1239,21 @@ async def coro2(): self.assertFalse(barrier.broken) self.assertEqual(len(results1), self.N-1) self.assertEqual(len(results2), self.N) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) - def test_reset_multi_wait(self): + def test_reset_and_wait(self): results1 = [] results2 = [] results3 = [] - alls = [] + count = 0 async def coro(): + nonlocal count + i = await barrier1.wait() - alls.append(i) - if len(alls) == self.N: # i == self.N//2: + count += 1 + if count == self.N: # i == self.N//2: barrier.reset() else: try: @@ -1220,13 +1268,16 @@ async def coro(): barrier = asyncio.Barrier(self.N) barrier1 = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertFalse(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertEqual(len(results3), self.N) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) - def test_abort(self): + def test_abort_broken(self): results1 = [] results2 = [] async def coro(): @@ -1242,10 +1293,13 @@ async def coro(): barrier.abort() barrier = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertTrue(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) + self.assertEqual(barrier.n_waiting, 0) + + self.assertTrue(barrier.broken) def test_abort_and_reset(self): results1 = [] @@ -1275,11 +1329,14 @@ async def coro(): barrier = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) - res, tasks = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertFalse(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertEqual(len(results3), self.N) + self.assertEqual(barrier.n_waiting, 0) + + self.assertFalse(barrier.broken) if __name__ == '__main__': From f12123e8c7bd56ca1835f042861272e11e4c5885 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 16 Mar 2021 16:14:05 +0100 Subject: [PATCH 29/93] Update from patch after run python.bat Tools\scripts\patchcheck.py --- Doc/whatsnew/3.10.rst.bak | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/Doc/whatsnew/3.10.rst.bak b/Doc/whatsnew/3.10.rst.bak index db71f061f14df4..c4c282e5a04eae 100644 --- a/Doc/whatsnew/3.10.rst.bak +++ b/Doc/whatsnew/3.10.rst.bak @@ -673,9 +673,16 @@ When a module does not define ``__loader__``, fall back to ``__spec__.loader``. encodings --------- + :func:`encodings.normalize_encoding` now ignores non-ASCII characters. (Contributed by Hai Shi in :issue:`39337`.) +gc +-- + +Added audit hooks for :func:`gc.get_objects`, :func:`gc.get_referrers` and +:func:`gc.get_referents`. (Contributed by Pablo Galindo in :issue:`43439`.) + glob ---- @@ -683,6 +690,19 @@ Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and :func:`~glob.iglob` which allow to specify the root directory for searching. (Contributed by Serhiy Storchaka in :issue:`38144`.) +importlib.metadata +------------------ + +Feature parity with ``importlib_metadata`` 3.7. + +:func:`importlib.metadata.entry_points` now provides a nicer experience +for selecting entry points by group and name through a new +:class:`importlib.metadata.EntryPoints` class. + +Added :func:`importlib.metadata.packages_distributions` for resolving +top-level Python modules and packages to their +:class:`importlib.metadata.Distribution`. + inspect ------- @@ -781,6 +801,13 @@ Add :data:`sys.stdlib_module_names`, containing the list of the standard library module names. (Contributed by Victor Stinner in :issue:`42955`.) +_thread +------- + +:func:`_thread.interrupt_main` now takes an optional signal number to +simulate (the default is still :data:`signal.SIGINT`). +(Contributed by Antoine Pitrou in :issue:`43356`.) + threading --------- @@ -1203,6 +1230,11 @@ New Features object is an instance of :class:`set` but not an instance of a subtype. (Contributed by Pablo Galindo in :issue:`43277`.) +* Added :c:func:`PyErr_SetInterruptEx` which allows passing a signal number + to simulate. + (Contributed by Antoine Pitrou in :issue:`43356`.) + + Porting to Python 3.10 ---------------------- From 9ef71090b995c6013f9d7ed586d1862552bf4727 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 31 Mar 2021 16:24:52 +0200 Subject: [PATCH 30/93] remove cpython/doc/*.rst.bak --- Doc/c-api/exceptions.rst.bak | 1054 ------------- Doc/c-api/memory.rst.bak | 642 -------- Doc/faq/design.rst.bak | 759 --------- Doc/howto/descriptor.rst.bak | 1575 ------------------- Doc/library/_thread.rst.bak | 215 --- Doc/library/ast.rst.bak | 1993 ------------------------ Doc/library/asyncio-api-index.rst.bak | 221 --- Doc/library/collections.rst.bak | 1278 --------------- Doc/library/dataclasses.rst.bak | 595 ------- Doc/library/gc.rst.bak | 322 ---- Doc/library/importlib.metadata.rst.bak | 262 ---- Doc/library/logging.rst.bak | 1350 ---------------- Doc/library/readline.rst.bak | 361 ----- Doc/library/sqlite3.rst.bak | 1094 ------------- Doc/library/statistics.rst.bak | 879 ----------- Doc/library/tempfile.rst.bak | 374 ----- Doc/library/venv.rst.bak | 496 ------ Doc/library/xml.sax.handler.rst.bak | 463 ------ Doc/library/zipimport.rst.bak | 209 --- Doc/whatsnew/3.10.rst.bak | 1347 ---------------- 20 files changed, 15489 deletions(-) delete mode 100644 Doc/c-api/exceptions.rst.bak delete mode 100644 Doc/c-api/memory.rst.bak delete mode 100644 Doc/faq/design.rst.bak delete mode 100644 Doc/howto/descriptor.rst.bak delete mode 100644 Doc/library/_thread.rst.bak delete mode 100644 Doc/library/ast.rst.bak delete mode 100644 Doc/library/asyncio-api-index.rst.bak delete mode 100644 Doc/library/collections.rst.bak delete mode 100644 Doc/library/dataclasses.rst.bak delete mode 100644 Doc/library/gc.rst.bak delete mode 100644 Doc/library/importlib.metadata.rst.bak delete mode 100644 Doc/library/logging.rst.bak delete mode 100644 Doc/library/readline.rst.bak delete mode 100644 Doc/library/sqlite3.rst.bak delete mode 100644 Doc/library/statistics.rst.bak delete mode 100644 Doc/library/tempfile.rst.bak delete mode 100644 Doc/library/venv.rst.bak delete mode 100644 Doc/library/xml.sax.handler.rst.bak delete mode 100644 Doc/library/zipimport.rst.bak delete mode 100644 Doc/whatsnew/3.10.rst.bak diff --git a/Doc/c-api/exceptions.rst.bak b/Doc/c-api/exceptions.rst.bak deleted file mode 100644 index 4e99a0167a632d..00000000000000 --- a/Doc/c-api/exceptions.rst.bak +++ /dev/null @@ -1,1054 +0,0 @@ -.. highlight:: c - - -.. _exceptionhandling: - -****************** -Exception Handling -****************** - -The functions described in this chapter will let you handle and raise Python -exceptions. It is important to understand some of the basics of Python -exception handling. It works somewhat like the POSIX :c:data:`errno` variable: -there is a global indicator (per thread) of the last error that occurred. Most -C API functions don't clear this on success, but will set it to indicate the -cause of the error on failure. Most C API functions also return an error -indicator, usually ``NULL`` if they are supposed to return a pointer, or ``-1`` -if they return an integer (exception: the :c:func:`PyArg_\*` functions -return ``1`` for success and ``0`` for failure). - -Concretely, the error indicator consists of three object pointers: the -exception's type, the exception's value, and the traceback object. Any -of those pointers can be ``NULL`` if non-set (although some combinations are -forbidden, for example you can't have a non-``NULL`` traceback if the exception -type is ``NULL``). - -When a function must fail because some function it called failed, it generally -doesn't set the error indicator; the function it called already set it. It is -responsible for either handling the error and clearing the exception or -returning after cleaning up any resources it holds (such as object references or -memory allocations); it should *not* continue normally if it is not prepared to -handle the error. If returning due to an error, it is important to indicate to -the caller that an error has been set. If the error is not handled or carefully -propagated, additional calls into the Python/C API may not behave as intended -and may fail in mysterious ways. - -.. note:: - The error indicator is **not** the result of :func:`sys.exc_info()`. - The former corresponds to an exception that is not yet caught (and is - therefore still propagating), while the latter returns an exception after - it is caught (and has therefore stopped propagating). - - -Printing and clearing -===================== - - -.. c:function:: void PyErr_Clear() - - Clear the error indicator. If the error indicator is not set, there is no - effect. - - -.. c:function:: void PyErr_PrintEx(int set_sys_last_vars) - - Print a standard traceback to ``sys.stderr`` and clear the error indicator. - **Unless** the error is a ``SystemExit``, in that case no traceback is - printed and the Python process will exit with the error code specified by - the ``SystemExit`` instance. - - Call this function **only** when the error indicator is set. Otherwise it - will cause a fatal error! - - If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, - :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the - type, value and traceback of the printed exception, respectively. - - -.. c:function:: void PyErr_Print() - - Alias for ``PyErr_PrintEx(1)``. - - -.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) - - Call :func:`sys.unraisablehook` using the current exception and *obj* - argument. - - This utility function prints a warning message to ``sys.stderr`` when an - exception has been set but it is impossible for the interpreter to actually - raise the exception. It is used, for example, when an exception occurs in an - :meth:`__del__` method. - - The function is called with a single argument *obj* that identifies the context - in which the unraisable exception occurred. If possible, - the repr of *obj* will be printed in the warning message. - - An exception must be set when calling this function. - - -Raising exceptions -================== - -These functions help you set the current thread's error indicator. -For convenience, some of these functions will always return a -``NULL`` pointer for use in a ``return`` statement. - - -.. c:function:: void PyErr_SetString(PyObject *type, const char *message) - - This is the most common way to set the error indicator. The first argument - specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not increment its reference count. - The second argument is an error message; it is decoded from ``'utf-8``'. - - -.. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) - - This function is similar to :c:func:`PyErr_SetString` but lets you specify an - arbitrary Python object for the "value" of the exception. - - -.. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) - - This function sets the error indicator and returns ``NULL``. *exception* - should be a Python exception class. The *format* and subsequent - parameters help format the error message; they have the same meaning and - values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded - string. - - -.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs) - - Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather - than a variable number of arguments. - - .. versionadded:: 3.5 - - -.. c:function:: void PyErr_SetNone(PyObject *type) - - This is a shorthand for ``PyErr_SetObject(type, Py_None)``. - - -.. c:function:: int PyErr_BadArgument() - - This is a shorthand for ``PyErr_SetString(PyExc_TypeError, message)``, where - *message* indicates that a built-in operation was invoked with an illegal - argument. It is mostly for internal use. - - -.. c:function:: PyObject* PyErr_NoMemory() - - This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns ``NULL`` - so an object allocation function can write ``return PyErr_NoMemory();`` when it - runs out of memory. - - -.. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) - - .. index:: single: strerror() - - This is a convenience function to raise an exception when a C library function - has returned an error and set the C variable :c:data:`errno`. It constructs a - tuple object whose first item is the integer :c:data:`errno` value and whose - second item is the corresponding error message (gotten from :c:func:`strerror`), - and then calls ``PyErr_SetObject(type, object)``. On Unix, when the - :c:data:`errno` value is :const:`EINTR`, indicating an interrupted system call, - this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, - leaves it set to that. The function always returns ``NULL``, so a wrapper - function around a system call can write ``return PyErr_SetFromErrno(type);`` - when the system call returns an error. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) - - Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as - a third parameter. In the case of :exc:`OSError` exception, - this is used to define the :attr:`filename` attribute of the - exception instance. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second - filename object, for raising errors when a function that takes two filenames - fails. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename - is given as a C string. *filename* is decoded from the :term:`filesystem - encoding and error handler`. - - -.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) - - This is a convenience function to raise :exc:`WindowsError`. If called with - *ierr* of :c:data:`0`, the error code returned by a call to :c:func:`GetLastError` - is used instead. It calls the Win32 function :c:func:`FormatMessage` to retrieve - the Windows description of error code given by *ierr* or :c:func:`GetLastError`, - then it constructs a tuple object whose first item is the *ierr* value and whose - second item is the corresponding error message (gotten from - :c:func:`FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, - object)``. This function always returns ``NULL``. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr) - - Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter - specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, but the - filename is given as a C string. *filename* is decoded from the filesystem - encoding (:func:`os.fsdecode`). - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilenameObject`, with an - additional parameter specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) - - Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, - but accepts a second filename object. - - .. availability:: Windows. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional - parameter specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path) - - This is a convenience function to raise :exc:`ImportError`. *msg* will be - set as the exception's message string. *name* and *path*, both of which can - be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name`` - and ``path`` attributes. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) - - Set file, line, and offset information for the current exception. If the - current exception is not a :exc:`SyntaxError`, then it sets additional - attributes, which make the exception printing subsystem think the exception - is a :exc:`SyntaxError`. - - .. versionadded:: 3.4 - - -.. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) - - Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string - decoded from the :term:`filesystem encoding and error handler`. - - .. versionadded:: 3.2 - - -.. c:function:: void PyErr_SyntaxLocation(const char *filename, int lineno) - - Like :c:func:`PyErr_SyntaxLocationEx`, but the col_offset parameter is - omitted. - - -.. c:function:: void PyErr_BadInternalCall() - - This is a shorthand for ``PyErr_SetString(PyExc_SystemError, message)``, - where *message* indicates that an internal operation (e.g. a Python/C API - function) was invoked with an illegal argument. It is mostly for internal - use. - - -Issuing warnings -================ - -Use these functions to issue warnings from C code. They mirror similar -functions exported by the Python :mod:`warnings` module. They normally -print a warning message to *sys.stderr*; however, it is -also possible that the user has specified that warnings are to be turned into -errors, and in that case they will raise an exception. It is also possible that -the functions raise an exception because of a problem with the warning machinery. -The return value is ``0`` if no exception is raised, or ``-1`` if an exception -is raised. (It is not possible to determine whether a warning message is -actually printed, nor what the reason is for the exception; this is -intentional.) If an exception is raised, the caller should do its normal -exception handling (for example, :c:func:`Py_DECREF` owned references and return -an error value). - -.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) - - Issue a warning message. The *category* argument is a warning category (see - below) or ``NULL``; the *message* argument is a UTF-8 encoded string. *stack_level* is a - positive number giving a number of stack frames; the warning will be issued from - the currently executing line of code in that stack frame. A *stack_level* of 1 - is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, - and so forth. - - Warning categories must be subclasses of :c:data:`PyExc_Warning`; - :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; - the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard - Python warning categories are available as global variables whose names are - enumerated at :ref:`standardwarningcategories`. - - For information about warning control, see the documentation for the - :mod:`warnings` module and the :option:`-W` option in the command line - documentation. There is no C API for warning control. - -.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) - - Much like :c:func:`PyErr_SetImportError` but this function allows for - specifying a subclass of :exc:`ImportError` to raise. - - .. versionadded:: 3.6 - - -.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) - - Issue a warning message with explicit control over all warning attributes. This - is a straightforward wrapper around the Python function - :func:`warnings.warn_explicit`, see there for more information. The *module* - and *registry* arguments may be set to ``NULL`` to get the default effect - described there. - - .. versionadded:: 3.4 - - -.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) - - Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and - *module* are UTF-8 encoded strings, and *filename* is decoded from the - :term:`filesystem encoding and error handler`. - - -.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnEx`, but use - :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is - an ASCII-encoded string. - - .. versionadded:: 3.2 - - -.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. - - .. versionadded:: 3.6 - - -Querying the error indicator -============================ - -.. c:function:: PyObject* PyErr_Occurred() - - Test whether the error indicator is set. If set, return the exception *type* - (the first argument to the last call to one of the :c:func:`PyErr_Set\*` - functions or to :c:func:`PyErr_Restore`). If not set, return ``NULL``. You do not - own a reference to the return value, so you do not need to :c:func:`Py_DECREF` - it. - - The caller must hold the GIL. - - .. note:: - - Do not compare the return value to a specific exception; use - :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could - easily fail since the exception may be an instance instead of a class, in the - case of a class exception, or it may be a subclass of the expected exception.) - - -.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) - - Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This - should only be called when an exception is actually set; a memory access - violation will occur if no exception has been raised. - - -.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) - - Return true if the *given* exception matches the exception type in *exc*. If - *exc* is a class object, this also returns true when *given* is an instance - of a subclass. If *exc* is a tuple, all exception types in the tuple (and - recursively in subtuples) are searched for a match. - - -.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the error indicator into three variables whose addresses are passed. - If the error indicator is not set, set all three variables to ``NULL``. If it is - set, it will be cleared and you own a reference to each object retrieved. The - value and traceback object may be ``NULL`` even when the type object is not. - - .. note:: - - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: - - { - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - - /* ... code that might produce other errors ... */ - - PyErr_Restore(type, value, traceback); - } - - -.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) - - Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are ``NULL``, the error - indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or - traceback. The exception type should be a class. Do not pass an invalid - exception type or value. (Violating these rules will cause subtle problems - later.) This call takes away a reference to each object: you must own a - reference to each object before the call and after the call you no longer own - these references. (If you don't understand this, don't use this function. I - warned you.) - - .. note:: - - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current - error indicator. - - -.. c:function:: void PyErr_NormalizeException(PyObject**exc, PyObject**val, PyObject**tb) - - Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below - can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance. - - .. note:: - - This function *does not* implicitly set the ``__traceback__`` - attribute on the exception value. If setting the traceback - appropriately is desired, the following additional snippet is needed:: - - if (tb != NULL) { - PyException_SetTraceback(val, tb); - } - - -.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. Returns new references for the three objects, any of which - may be ``NULL``. Does not modify the exception info state. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the - exception state. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback) - - Set the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. This function steals the references of the arguments. - To clear the exception state, pass ``NULL`` for all three arguments. - For general rules about the three arguments, see :c:func:`PyErr_Restore`. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception - state. - - .. versionadded:: 3.3 - - -Signal Handling -=============== - - -.. c:function:: int PyErr_CheckSignals() - - .. index:: - module: signal - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - This function interacts with Python's signal handling. It checks whether a - signal has been sent to the processes and if so, invokes the corresponding - signal handler. If the :mod:`signal` module is supported, this can invoke a - signal handler written in Python. In all cases, the default effect for - :const:`SIGINT` is to raise the :exc:`KeyboardInterrupt` exception. If an - exception is raised the error indicator is set and the function returns ``-1``; - otherwise the function returns ``0``. The error indicator may or may not be - cleared if it was previously set. - - -.. c:function:: void PyErr_SetInterrupt() - - .. index:: - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - Simulate the effect of a :const:`SIGINT` signal arriving. The next time - :c:func:`PyErr_CheckSignals` is called, the Python signal handler for - :const:`SIGINT` will be called. - - If :const:`SIGINT` isn't handled by Python (it was set to - :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does - nothing. - -.. c:function:: int PySignal_SetWakeupFd(int fd) - - This utility function specifies a file descriptor to which the signal number - is written as a single byte whenever a signal is received. *fd* must be - non-blocking. It returns the previous such file descriptor. - - The value ``-1`` disables the feature; this is the initial state. - This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any - error checking. *fd* should be a valid file descriptor. The function should - only be called from the main thread. - - .. versionchanged:: 3.5 - On Windows, the function now also supports socket handles. - - -Exception Classes -================= - -.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict) - - This utility function creates and returns a new exception class. The *name* - argument must be the name of the new exception, a C string of the form - ``module.classname``. The *base* and *dict* arguments are normally ``NULL``. - This creates a class object derived from :exc:`Exception` (accessible in C as - :c:data:`PyExc_Exception`). - - The :attr:`__module__` attribute of the new class is set to the first part (up - to the last dot) of the *name* argument, and the class name is set to the last - part (after the last dot). The *base* argument can be used to specify alternate - base classes; it can either be only one class or a tuple of classes. The *dict* - argument can be used to specify a dictionary of class variables and methods. - - -.. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) - - Same as :c:func:`PyErr_NewException`, except that the new exception class can - easily be given a docstring: If *doc* is non-``NULL``, it will be used as the - docstring for the exception class. - - .. versionadded:: 3.2 - - -Exception Objects -================= - -.. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) - - Return the traceback associated with the exception as a new reference, as - accessible from Python through :attr:`__traceback__`. If there is no - traceback associated, this returns ``NULL``. - - -.. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) - - Set the traceback associated with the exception to *tb*. Use ``Py_None`` to - clear it. - - -.. c:function:: PyObject* PyException_GetContext(PyObject *ex) - - Return the context (another exception instance during whose handling *ex* was - raised) associated with the exception as a new reference, as accessible from - Python through :attr:`__context__`. If there is no context associated, this - returns ``NULL``. - - -.. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) - - Set the context associated with the exception to *ctx*. Use ``NULL`` to clear - it. There is no type check to make sure that *ctx* is an exception instance. - This steals a reference to *ctx*. - - -.. c:function:: PyObject* PyException_GetCause(PyObject *ex) - - Return the cause (either an exception instance, or :const:`None`, - set by ``raise ... from ...``) associated with the exception as a new - reference, as accessible from Python through :attr:`__cause__`. - - -.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) - - Set the cause associated with the exception to *cause*. Use ``NULL`` to clear - it. There is no type check to make sure that *cause* is either an exception - instance or :const:`None`. This steals a reference to *cause*. - - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. - - -.. _unicodeexceptions: - -Unicode Exception Objects -========================= - -The following functions are used to create and modify Unicode exceptions from C. - -.. c:function:: PyObject* PyUnicodeDecodeError_Create(const char *encoding, const char *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeDecodeError` object with the attributes *encoding*, - *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are - UTF-8 encoded strings. - -.. c:function:: PyObject* PyUnicodeEncodeError_Create(const char *encoding, const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeEncodeError` object with the attributes *encoding*, - *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are - UTF-8 encoded strings. - - .. deprecated:: 3.3 3.11 - - ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to - ``PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", ...)``. - -.. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeTranslateError` object with the attributes *object*, - *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. - - .. deprecated:: 3.3 3.11 - - ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to - ``PyObject_CallFunction(PyExc_UnicodeTranslateError, "Onns", ...)``. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) - - Return the *encoding* attribute of the given exception object. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetObject(PyObject *exc) - - Return the *object* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) - - Get the *start* attribute of the given exception object and place it into - *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. - -.. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) - - Get the *end* attribute of the given exception object and place it into - *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) - - Set the *end* attribute of the given exception object to *end*. Return ``0`` - on success, ``-1`` on failure. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) - - Return the *reason* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) - - Set the *reason* attribute of the given exception object to *reason*. Return - ``0`` on success, ``-1`` on failure. - - -.. _recursion: - -Recursion Control -================= - -These two functions provide a way to perform safe recursive calls at the C -level, both in the core and in extension modules. They are needed if the -recursive code does not necessarily invoke Python code (which tracks its -recursion depth automatically). -They are also not needed for *tp_call* implementations -because the :ref:`call protocol ` takes care of recursion handling. - -.. c:function:: int Py_EnterRecursiveCall(const char *where) - - Marks a point where a recursive C-level call is about to be performed. - - If :const:`USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it - sets a :exc:`MemoryError` and returns a nonzero value. - - The function then checks if the recursion limit is reached. If this is the - case, a :exc:`RecursionError` is set and a nonzero value is returned. - Otherwise, zero is returned. - - *where* should be a UTF-8 encoded string such as ``" in instance check"`` to - be concatenated to the :exc:`RecursionError` message caused by the recursion - depth limit. - - .. versionchanged:: 3.9 - This function is now also available in the limited API. - -.. c:function:: void Py_LeaveRecursiveCall(void) - - Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each - *successful* invocation of :c:func:`Py_EnterRecursiveCall`. - - .. versionchanged:: 3.9 - This function is now also available in the limited API. - -Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires -special recursion handling. In addition to protecting the stack, -:c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The -following two functions facilitate this functionality. Effectively, -these are the C equivalent to :func:`reprlib.recursive_repr`. - -.. c:function:: int Py_ReprEnter(PyObject *object) - - Called at the beginning of the :c:member:`~PyTypeObject.tp_repr` implementation to - detect cycles. - - If the object has already been processed, the function returns a - positive integer. In that case the :c:member:`~PyTypeObject.tp_repr` implementation - should return a string object indicating a cycle. As examples, - :class:`dict` objects return ``{...}`` and :class:`list` objects - return ``[...]``. - - The function will return a negative integer if the recursion limit - is reached. In that case the :c:member:`~PyTypeObject.tp_repr` implementation should - typically return ``NULL``. - - Otherwise, the function returns zero and the :c:member:`~PyTypeObject.tp_repr` - implementation can continue normally. - -.. c:function:: void Py_ReprLeave(PyObject *object) - - Ends a :c:func:`Py_ReprEnter`. Must be called once for each - invocation of :c:func:`Py_ReprEnter` that returns zero. - - -.. _standardexceptions: - -Standard Exceptions -=================== - -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_BufferError - single: PyExc_ChildProcessError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_GeneratorExit - single: PyExc_ImportError - single: PyExc_IndentationError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_LookupError - single: PyExc_MemoryError - single: PyExc_ModuleNotFoundError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_StopAsyncIteration - single: PyExc_StopIteration - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TabError - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_UnboundLocalError - single: PyExc_UnicodeDecodeError - single: PyExc_UnicodeEncodeError - single: PyExc_UnicodeError - single: PyExc_UnicodeTranslateError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | \(1) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | \(2) | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.3 - :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, - :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`, - :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`, - :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`, - :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`, - :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`, - :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` - and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. - -.. versionadded:: 3.5 - :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. - -.. versionadded:: 3.6 - :c:data:`PyExc_ModuleNotFoundError`. - -These are compatibility aliases to :c:data:`PyExc_OSError`: - -.. index:: - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | \(3) | -+-------------------------------------+----------+ - -.. versionchanged:: 3.3 - These aliases used to be separate exception types. - -Notes: - -(1) - This is a base class for other standard exceptions. - -(2) - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. - -.. _standardwarningcategories: - -Standard Warning Categories -=========================== - -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_Warning - single: PyExc_BytesWarning - single: PyExc_DeprecationWarning - single: PyExc_FutureWarning - single: PyExc_ImportWarning - single: PyExc_PendingDeprecationWarning - single: PyExc_ResourceWarning - single: PyExc_RuntimeWarning - single: PyExc_SyntaxWarning - single: PyExc_UnicodeWarning - single: PyExc_UserWarning - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | \(1) | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.2 - :c:data:`PyExc_ResourceWarning`. - -Notes: - -(1) - This is a base class for other standard warning categories. diff --git a/Doc/c-api/memory.rst.bak b/Doc/c-api/memory.rst.bak deleted file mode 100644 index 588668ee853c65..00000000000000 --- a/Doc/c-api/memory.rst.bak +++ /dev/null @@ -1,642 +0,0 @@ -.. highlight:: c - - -.. _memory: - -***************** -Memory Management -***************** - -.. sectionauthor:: Vladimir Marangozov - - - -.. _memoryoverview: - -Overview -======== - -Memory management in Python involves a private heap containing all Python -objects and data structures. The management of this private heap is ensured -internally by the *Python memory manager*. The Python memory manager has -different components which deal with various dynamic storage management aspects, -like sharing, segmentation, preallocation or caching. - -At the lowest level, a raw memory allocator ensures that there is enough room in -the private heap for storing all Python-related data by interacting with the -memory manager of the operating system. On top of the raw memory allocator, -several object-specific allocators operate on the same heap and implement -distinct memory management policies adapted to the peculiarities of every object -type. For example, integer objects are managed differently within the heap than -strings, tuples or dictionaries because integers imply different storage -requirements and speed/space tradeoffs. The Python memory manager thus delegates -some of the work to the object-specific allocators, but ensures that the latter -operate within the bounds of the private heap. - -It is important to understand that the management of the Python heap is -performed by the interpreter itself and that the user has no control over it, -even if they regularly manipulate object pointers to memory blocks inside that -heap. The allocation of heap space for Python objects and other internal -buffers is performed on demand by the Python memory manager through the Python/C -API functions listed in this document. - -.. index:: - single: malloc() - single: calloc() - single: realloc() - single: free() - -To avoid memory corruption, extension writers should never try to operate on -Python objects with the functions exported by the C library: :c:func:`malloc`, -:c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. This will result in mixed -calls between the C allocator and the Python memory manager with fatal -consequences, because they implement different algorithms and operate on -different heaps. However, one may safely allocate and release memory blocks -with the C library allocator for individual purposes, as shown in the following -example:: - - PyObject *res; - char *buf = (char *) malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - ...Do some I/O operation involving buf... - res = PyBytes_FromString(buf); - free(buf); /* malloc'ed */ - return res; - -In this example, the memory request for the I/O buffer is handled by the C -library allocator. The Python memory manager is involved only in the allocation -of the bytes object returned as a result. - -In most situations, however, it is recommended to allocate memory from the -Python heap specifically because the latter is under control of the Python -memory manager. For example, this is required when the interpreter is extended -with new object types written in C. Another reason for using the Python heap is -the desire to *inform* the Python memory manager about the memory needs of the -extension module. Even when the requested memory is used exclusively for -internal, highly-specific purposes, delegating all memory requests to the Python -memory manager causes the interpreter to have a more accurate image of its -memory footprint as a whole. Consequently, under certain circumstances, the -Python memory manager may or may not trigger appropriate actions, like garbage -collection, memory compaction or other preventive procedures. Note that by using -the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. - -.. seealso:: - - The :envvar:`PYTHONMALLOC` environment variable can be used to configure - the memory allocators used by Python. - - The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print - statistics of the :ref:`pymalloc memory allocator ` every time a - new pymalloc object arena is created, and on shutdown. - -Allocator Domains -================= - -All allocating functions belong to one of three different "domains" (see also -:c:type:`PyMemAllocatorDomain`). These domains represent different allocation -strategies and are optimized for different purposes. The specific details on -how every domain allocates memory or what internal functions each domain calls -is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. There is no hard -requirement to use the memory returned by the allocation functions belonging to -a given domain for only the purposes hinted by that domain (although this is the -recommended practice). For example, one could use the memory returned by -:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned -by :c:func:`PyObject_Malloc` for allocating memory for buffers. - -The three allocation domains are: - -* Raw domain: intended for allocating memory for general-purpose memory - buffers where the allocation *must* go to the system allocator or where the - allocator can operate without the :term:`GIL`. The memory is requested directly - to the system. - -* "Mem" domain: intended for allocating memory for Python buffers and - general-purpose memory buffers where the allocation must be performed with - the :term:`GIL` held. The memory is taken from the Python private heap. - -* Object domain: intended for allocating memory belonging to Python objects. The - memory is taken from the Python private heap. - -When freeing memory previously allocated by the allocating functions belonging to a -given domain,the matching specific deallocating functions must be used. For example, -:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. - -Raw Memory Interface -==================== - -The following function sets are wrappers to the system allocator. These -functions are thread-safe, the :term:`GIL ` does not -need to be held. - -The :ref:`default raw memory allocator ` uses -the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` -and :c:func:`free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting -zero bytes. - -.. versionadded:: 3.4 - -.. c:function:: void* PyMem_RawMalloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been - called instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will - be unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyMem_RawMalloc(n)``; else if - *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. - - If the request fails, :c:func:`PyMem_RawRealloc` returns ``NULL`` and *p* - remains a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_RawFree(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been - called before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - - -.. _memoryinterface: - -Memory Interface -================ - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -The :ref:`default memory allocator ` uses the -:ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. versionchanged:: 3.6 - - The default allocator is now pymalloc instead of system :c:func:`malloc`. - -.. c:function:: void* PyMem_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyMem_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. - - If the request fails, :c:func:`PyMem_Realloc` returns ``NULL`` and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or - :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - -The following type-oriented macros are provided for convenience. Note that -*TYPE* refers to any C type. - - -.. c:function:: TYPE* PyMem_New(TYPE, size_t n) - - Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:type:`TYPE*`. The memory will not have - been initialized in any way. - - -.. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) - - Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE*`. On return, - *p* will be a pointer to the new memory area, or ``NULL`` in the event of - failure. - - This is a C preprocessor macro; *p* is always reassigned. Save the original - value of *p* to avoid losing memory when handling errors. - - -.. c:function:: void PyMem_Del(void *p) - - Same as :c:func:`PyMem_Free`. - -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. - -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` - - -Object allocators -================= - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -.. note:: - There is no guarantee that the memory returned by these allocators can be - succesfully casted to a Python object when intercepting the allocating - functions in this domain by the methods described in - the :ref:`Customize Memory Allocators ` section. - -The :ref:`default object allocator ` uses the -:ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. c:function:: void* PyObject_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyObject_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyObject_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. - - If the request fails, :c:func:`PyObject_Realloc` returns ``NULL`` and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyObject_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or - :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - - -.. _default-memory-allocators: - -Default Memory Allocators -========================= - -Default memory allocators: - -=============================== ==================== ================== ===================== ==================== -Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc -=============================== ==================== ================== ===================== ==================== -Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` -Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug -Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` -Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug -=============================== ==================== ================== ===================== ==================== - -Legend: - -* Name: value for :envvar:`PYTHONMALLOC` environment variable -* ``malloc``: system allocators from the standard C library, C functions: - :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free` -* ``pymalloc``: :ref:`pymalloc memory allocator ` -* "+ debug": with debug hooks installed by :c:func:`PyMem_SetupDebugHooks` - -.. _customize-memory-allocators: - -Customize Memory Allocators -=========================== - -.. versionadded:: 3.4 - -.. c:type:: PyMemAllocatorEx - - Structure used to describe a memory block allocator. The structure has - four fields: - - +----------------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==========================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +----------------------------------------------------------+---------------------------------------+ - | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized | - | | with zeros | - +----------------------------------------------------------+---------------------------------------+ - | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, void *ptr)`` | free a memory block | - +----------------------------------------------------------+---------------------------------------+ - - .. versionchanged:: 3.5 - The :c:type:`PyMemAllocator` structure was renamed to - :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. - - -.. c:type:: PyMemAllocatorDomain - - Enum used to identify an allocator domain. Domains: - - .. c:macro:: PYMEM_DOMAIN_RAW - - Functions: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - - .. c:macro:: PYMEM_DOMAIN_MEM - - Functions: - - * :c:func:`PyMem_Malloc`, - * :c:func:`PyMem_Realloc` - * :c:func:`PyMem_Calloc` - * :c:func:`PyMem_Free` - - .. c:macro:: PYMEM_DOMAIN_OBJ - - Functions: - - * :c:func:`PyObject_Malloc` - * :c:func:`PyObject_Realloc` - * :c:func:`PyObject_Calloc` - * :c:func:`PyObject_Free` - -.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Get the memory block allocator of the specified domain. - - -.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Set the memory block allocator of the specified domain. - - The new allocator must return a distinct non-``NULL`` pointer when requesting - zero bytes. - - For the :c:data:`PYMEM_DOMAIN_RAW` domain, the allocator must be - thread-safe: the :term:`GIL ` is not held when the - allocator is called. - - If the new allocator is not a hook (does not call the previous allocator), - the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the - debug hooks on top on the new allocator. - - -.. c:function:: void PyMem_SetupDebugHooks(void) - - Setup hooks to detect bugs in the Python memory allocator functions. - - Newly allocated memory is filled with the byte ``0xCD`` (``CLEANBYTE``), - freed memory is filled with the byte ``0xDD`` (``DEADBYTE``). Memory blocks - are surrounded by "forbidden bytes" (``FORBIDDENBYTE``: byte ``0xFD``). - - Runtime checks: - - - Detect API violations, ex: :c:func:`PyObject_Free` called on a buffer - allocated by :c:func:`PyMem_Malloc` - - Detect write before the start of the buffer (buffer underflow) - - Detect write after the end of the buffer (buffer overflow) - - Check that the :term:`GIL ` is held when - allocator functions of :c:data:`PYMEM_DOMAIN_OBJ` (ex: - :c:func:`PyObject_Malloc`) and :c:data:`PYMEM_DOMAIN_MEM` (ex: - :c:func:`PyMem_Malloc`) domains are called - - On error, the debug hooks use the :mod:`tracemalloc` module to get the - traceback where a memory block was allocated. The traceback is only - displayed if :mod:`tracemalloc` is tracing Python memory allocations and the - memory block was traced. - - These hooks are :ref:`installed by default ` if - Python is compiled in debug - mode. The :envvar:`PYTHONMALLOC` environment variable can be used to install - debug hooks on a Python compiled in release mode. - - .. versionchanged:: 3.6 - This function now also works on Python compiled in release mode. - On error, the debug hooks now use :mod:`tracemalloc` to get the traceback - where a memory block was allocated. The debug hooks now also check - if the GIL is held when functions of :c:data:`PYMEM_DOMAIN_OBJ` and - :c:data:`PYMEM_DOMAIN_MEM` domains are called. - - .. versionchanged:: 3.8 - Byte patterns ``0xCB`` (``CLEANBYTE``), ``0xDB`` (``DEADBYTE``) and - ``0xFB`` (``FORBIDDENBYTE``) have been replaced with ``0xCD``, ``0xDD`` - and ``0xFD`` to use the same values than Windows CRT debug ``malloc()`` - and ``free()``. - - -.. _pymalloc: - -The pymalloc allocator -====================== - -Python has a *pymalloc* allocator optimized for small objects (smaller or equal -to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" -with a fixed size of 256 KiB. It falls back to :c:func:`PyMem_RawMalloc` and -:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. - -*pymalloc* is the :ref:`default allocator ` of the -:c:data:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and -:c:data:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. - -The arena allocator uses the following functions: - -* :c:func:`VirtualAlloc` and :c:func:`VirtualFree` on Windows, -* :c:func:`mmap` and :c:func:`munmap` if available, -* :c:func:`malloc` and :c:func:`free` otherwise. - -Customize pymalloc Arena Allocator ----------------------------------- - -.. versionadded:: 3.4 - -.. c:type:: PyObjectArenaAllocator - - Structure used to describe an arena allocator. The structure has - three fields: - - +--------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +--------------------------------------------------+---------------------------------------+ - | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | - +--------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | - +--------------------------------------------------+---------------------------------------+ - -.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) - - Get the arena allocator. - -.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) - - Set the arena allocator. - - -tracemalloc C API -================= - -.. versionadded:: 3.7 - -.. c:function:: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) - - Track an allocated memory block in the :mod:`tracemalloc` module. - - Return ``0`` on success, return ``-1`` on error (failed to allocate memory to - store the trace). Return ``-2`` if tracemalloc is disabled. - - If memory block is already tracked, update the existing trace. - -.. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) - - Untrack an allocated memory block in the :mod:`tracemalloc` module. - Do nothing if the block was not tracked. - - Return ``-2`` if tracemalloc is disabled, otherwise return ``0``. - - -.. _memoryexamples: - -Examples -======== - -Here is the example from section :ref:`memoryoverview`, rewritten so that the -I/O buffer is allocated from the Python heap by using the first function set:: - - PyObject *res; - char *buf = (char *) PyMem_Malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Free(buf); /* allocated with PyMem_Malloc */ - return res; - -The same code using the type-oriented function set:: - - PyObject *res; - char *buf = PyMem_New(char, BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Del(buf); /* allocated with PyMem_New */ - return res; - -Note that in the two examples above, the buffer is always manipulated via -functions belonging to the same set. Indeed, it is required to use the same -memory API family for a given memory block, so that the risk of mixing different -allocators is reduced to a minimum. The following code sequence contains two -errors, one of which is labeled as *fatal* because it mixes two different -allocators operating on different heaps. :: - - char *buf1 = PyMem_New(char, BUFSIZ); - char *buf2 = (char *) malloc(BUFSIZ); - char *buf3 = (char *) PyMem_Malloc(BUFSIZ); - ... - PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ - free(buf2); /* Right -- allocated via malloc() */ - free(buf1); /* Fatal -- should be PyMem_Del() */ - -In addition to the functions aimed at handling raw memory blocks from the Python -heap, objects in Python are allocated and released with :c:func:`PyObject_New`, -:c:func:`PyObject_NewVar` and :c:func:`PyObject_Del`. - -These will be explained in the next chapter on defining and implementing new -object types in C. diff --git a/Doc/faq/design.rst.bak b/Doc/faq/design.rst.bak deleted file mode 100644 index 7fe1c6d58f58a1..00000000000000 --- a/Doc/faq/design.rst.bak +++ /dev/null @@ -1,759 +0,0 @@ -====================== -Design and History FAQ -====================== - -.. only:: html - - .. contents:: - - -Why does Python use indentation for grouping of statements? ------------------------------------------------------------ - -Guido van Rossum believes that using indentation for grouping is extremely -elegant and contributes a lot to the clarity of the average Python program. -Most people learn to love this feature after a while. - -Since there are no begin/end brackets there cannot be a disagreement between -grouping perceived by the parser and the human reader. Occasionally C -programmers will encounter a fragment of code like this:: - - if (x <= y) - x++; - y--; - z++; - -Only the ``x++`` statement is executed if the condition is true, but the -indentation leads many to believe otherwise. Even experienced C programmers will -sometimes stare at it a long time wondering as to why ``y`` is being decremented even -for ``x > y``. - -Because there are no begin/end brackets, Python is much less prone to -coding-style conflicts. In C there are many different ways to place the braces. -After becoming used to reading and writing code using a particular style, -it is normal to feel somewhat uneasy when reading (or being required to write) -in a different one. - - -Many coding styles place begin/end brackets on a line by themselves. This makes -programs considerably longer and wastes valuable screen space, making it harder -to get a good overview of a program. Ideally, a function should fit on one -screen (say, 20--30 lines). 20 lines of Python can do a lot more work than 20 -lines of C. This is not solely due to the lack of begin/end brackets -- the -lack of declarations and the high-level data types are also responsible -- but -the indentation-based syntax certainly helps. - - -Why am I getting strange results with simple arithmetic operations? -------------------------------------------------------------------- - -See the next question. - - -Why are floating-point calculations so inaccurate? --------------------------------------------------- - -Users are often surprised by results like this:: - - >>> 1.2 - 1.0 - 0.19999999999999996 - -and think it is a bug in Python. It's not. This has little to do with Python, -and much more to do with how the underlying platform handles floating-point -numbers. - -The :class:`float` type in CPython uses a C ``double`` for storage. A -:class:`float` object's value is stored in binary floating-point with a fixed -precision (typically 53 bits) and Python uses C operations, which in turn rely -on the hardware implementation in the processor, to perform floating-point -operations. This means that as far as floating-point operations are concerned, -Python behaves like many popular languages including C and Java. - -Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: - - >>> x = 1.2 - -the value stored for ``x`` is a (very good) approximation to the decimal value -``1.2``, but is not exactly equal to it. On a typical machine, the actual -stored value is:: - - 1.0011001100110011001100110011001100110011001100110011 (binary) - -which is exactly:: - - 1.1999999999999999555910790149937383830547332763671875 (decimal) - -The typical precision of 53 bits provides Python floats with 15--16 -decimal digits of accuracy. - -For a fuller explanation, please see the :ref:`floating point arithmetic -` chapter in the Python tutorial. - - -Why are Python strings immutable? ---------------------------------- - -There are several advantages. - -One is performance: knowing that a string is immutable means we can allocate -space for it at creation time, and the storage requirements are fixed and -unchanging. This is also one of the reasons for the distinction between tuples -and lists. - -Another advantage is that strings in Python are considered as "elemental" as -numbers. No amount of activity will change the value 8 to anything else, and in -Python, no amount of activity will change the string "eight" to anything else. - - -.. _why-self: - -Why must 'self' be used explicitly in method definitions and calls? -------------------------------------------------------------------- - -The idea was borrowed from Modula-3. It turns out to be very useful, for a -variety of reasons. - -First, it's more obvious that you are using a method or instance attribute -instead of a local variable. Reading ``self.x`` or ``self.meth()`` makes it -absolutely clear that an instance variable or method is used even if you don't -know the class definition by heart. In C++, you can sort of tell by the lack of -a local variable declaration (assuming globals are rare or easily recognizable) --- but in Python, there are no local variable declarations, so you'd have to -look up the class definition to be sure. Some C++ and Java coding standards -call for instance attributes to have an ``m_`` prefix, so this explicitness is -still useful in those languages, too. - -Second, it means that no special syntax is necessary if you want to explicitly -reference or call the method from a particular class. In C++, if you want to -use a method from a base class which is overridden in a derived class, you have -to use the ``::`` operator -- in Python you can write -``baseclass.methodname(self, )``. This is particularly useful -for :meth:`__init__` methods, and in general in cases where a derived class -method wants to extend the base class method of the same name and thus has to -call the base class method somehow. - -Finally, for instance variables it solves a syntactic problem with assignment: -since local variables in Python are (by definition!) those variables to which a -value is assigned in a function body (and that aren't explicitly declared -global), there has to be some way to tell the interpreter that an assignment was -meant to assign to an instance variable instead of to a local variable, and it -should preferably be syntactic (for efficiency reasons). C++ does this through -declarations, but Python doesn't have declarations and it would be a pity having -to introduce them just for this purpose. Using the explicit ``self.var`` solves -this nicely. Similarly, for using instance variables, having to write -``self.var`` means that references to unqualified names inside a method don't -have to search the instance's directories. To put it another way, local -variables and instance variables live in two different namespaces, and you need -to tell Python which namespace to use. - - -.. _why-can-t-i-use-an-assignment-in-an-expression: - -Why can't I use an assignment in an expression? ------------------------------------------------ - -Starting in Python 3.8, you can! - -Assignment expressions using the walrus operator `:=` assign a variable in an -expression:: - - while chunk := fp.read(200): - print(chunk) - -See :pep:`572` for more information. - - - -Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))? ----------------------------------------------------------------------------------------------------------------- - -As Guido said: - - (a) For some operations, prefix notation just reads better than - postfix -- prefix (and infix!) operations have a long tradition in - mathematics which likes notations where the visuals help the - mathematician thinking about a problem. Compare the easy with which we - rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of - doing the same thing using a raw OO notation. - - (b) When I read code that says len(x) I *know* that it is asking for - the length of something. This tells me two things: the result is an - integer, and the argument is some kind of container. To the contrary, - when I read x.len(), I have to already know that x is some kind of - container implementing an interface or inheriting from a class that - has a standard len(). Witness the confusion we occasionally have when - a class that is not implementing a mapping has a get() or keys() - method, or something that isn't a file has a write() method. - - -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html - - -Why is join() a string method instead of a list or tuple method? ----------------------------------------------------------------- - -Strings became much more like other standard types starting in Python 1.6, when -methods were added which give the same functionality that has always been -available using the functions of the string module. Most of these new methods -have been widely accepted, but the one which appears to make some programmers -feel uncomfortable is:: - - ", ".join(['1', '2', '4', '8', '16']) - -which gives the result:: - - "1, 2, 4, 8, 16" - -There are two common arguments against this usage. - -The first runs along the lines of: "It looks really ugly using a method of a -string literal (string constant)", to which the answer is that it might, but a -string literal is just a fixed value. If the methods are to be allowed on names -bound to strings there is no logical reason to make them unavailable on -literals. - -The second objection is typically cast as: "I am really telling a sequence to -join its members together with a string constant". Sadly, you aren't. For some -reason there seems to be much less difficulty with having :meth:`~str.split` as -a string method, since in that case it is easy to see that :: - - "1, 2, 4, 8, 16".split(", ") - -is an instruction to a string literal to return the substrings delimited by the -given separator (or, by default, arbitrary runs of white space). - -:meth:`~str.join` is a string method because in using it you are telling the -separator string to iterate over a sequence of strings and insert itself between -adjacent elements. This method can be used with any argument which obeys the -rules for sequence objects, including any new classes you might define yourself. -Similar methods exist for bytes and bytearray objects. - - -How fast are exceptions? ------------------------- - -A try/except block is extremely efficient if no exceptions are raised. Actually -catching an exception is expensive. In versions of Python prior to 2.0 it was -common to use this idiom:: - - try: - value = mydict[key] - except KeyError: - mydict[key] = getvalue(key) - value = mydict[key] - -This only made sense when you expected the dict to have the key almost all the -time. If that wasn't the case, you coded it like this:: - - if key in mydict: - value = mydict[key] - else: - value = mydict[key] = getvalue(key) - -For this specific case, you could also use ``value = dict.setdefault(key, -getvalue(key))``, but only if the ``getvalue()`` call is cheap enough because it -is evaluated in all cases. - - -Why isn't there a switch or case statement in Python? ------------------------------------------------------ - -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. - -For cases where you need to choose from a very large number of possibilities, -you can create a dictionary mapping case values to functions to call. For -example:: - - def function_1(...): - ... - - functions = {'a': function_1, - 'b': function_2, - 'c': self.method_1, ...} - - func = functions[value] - func() - -For calling methods on objects, you can simplify yet further by using the -:func:`getattr` built-in to retrieve methods with a particular name:: - - def visit_a(self, ...): - ... - ... - - def dispatch(self, value): - method_name = 'visit_' + str(value) - method = getattr(self, method_name) - method() - -It's suggested that you use a prefix for the method names, such as ``visit_`` in -this example. Without such a prefix, if values are coming from an untrusted -source, an attacker would be able to call any method on your object. - - -Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? --------------------------------------------------------------------------------------------------------- - -Answer 1: Unfortunately, the interpreter pushes at least one C stack frame for -each Python stack frame. Also, extensions can call back into Python at almost -random moments. Therefore, a complete threads implementation requires thread -support for C. - -Answer 2: Fortunately, there is `Stackless Python `_, -which has a completely redesigned interpreter loop that avoids the C stack. - - -Why can't lambda expressions contain statements? ------------------------------------------------- - -Python lambda expressions cannot contain statements because Python's syntactic -framework can't handle statements nested inside expressions. However, in -Python, this is not a serious problem. Unlike lambda forms in other languages, -where they add functionality, Python lambdas are only a shorthand notation if -you're too lazy to define a function. - -Functions are already first class objects in Python, and can be declared in a -local scope. Therefore the only advantage of using a lambda instead of a -locally-defined function is that you don't need to invent a name for the -function -- but that's just a local variable to which the function object (which -is exactly the same type of object that a lambda expression yields) is assigned! - - -Can Python be compiled to machine code, C or some other language? ------------------------------------------------------------------ - -`Cython `_ compiles a modified version of Python with -optional annotations into C extensions. `Nuitka `_ is -an up-and-coming compiler of Python into C++ code, aiming to support the full -Python language. For compiling to Java you can consider -`VOC `_. - - -How does Python manage memory? ------------------------------- - -The details of Python memory management depend on the implementation. The -standard implementation of Python, :term:`CPython`, uses reference counting to -detect inaccessible objects, and another mechanism to collect reference cycles, -periodically executing a cycle detection algorithm which looks for inaccessible -cycles and deletes the objects involved. The :mod:`gc` module provides functions -to perform a garbage collection, obtain debugging statistics, and tune the -collector's parameters. - -Other implementations (such as `Jython `_ or -`PyPy `_), however, can rely on a different mechanism -such as a full-blown garbage collector. This difference can cause some -subtle porting problems if your Python code depends on the behavior of the -reference counting implementation. - -In some Python implementations, the following code (which is fine in CPython) -will probably run out of file descriptors:: - - for file in very_long_list_of_files: - f = open(file) - c = f.read(1) - -Indeed, using CPython's reference counting and destructor scheme, each new -assignment to *f* closes the previous file. With a traditional GC, however, -those file objects will only get collected (and closed) at varying and possibly -long intervals. - -If you want to write code that will work with any Python implementation, -you should explicitly close the file or use the :keyword:`with` statement; -this will work regardless of memory management scheme:: - - for file in very_long_list_of_files: - with open(file) as f: - c = f.read(1) - - -Why doesn't CPython use a more traditional garbage collection scheme? ---------------------------------------------------------------------- - -For one thing, this is not a C standard feature and hence it's not portable. -(Yes, we know about the Boehm GC library. It has bits of assembler code for -*most* common platforms, not for all of them, and although it is mostly -transparent, it isn't completely transparent; patches are required to get -Python to work with it.) - -Traditional GC also becomes a problem when Python is embedded into other -applications. While in a standalone Python it's fine to replace the standard -malloc() and free() with versions provided by the GC library, an application -embedding Python may want to have its *own* substitute for malloc() and free(), -and may not want Python's. Right now, CPython works with anything that -implements malloc() and free() properly. - - -Why isn't all memory freed when CPython exits? ----------------------------------------------- - -Objects referenced from the global namespaces of Python modules are not always -deallocated when Python exits. This may happen if there are circular -references. There are also certain bits of memory that are allocated by the C -library that are impossible to free (e.g. a tool like Purify will complain about -these). Python is, however, aggressive about cleaning up memory on exit and -does try to destroy every single object. - -If you want to force Python to delete certain things on deallocation use the -:mod:`atexit` module to run a function that will force those deletions. - - -Why are there separate tuple and list data types? -------------------------------------------------- - -Lists and tuples, while similar in many respects, are generally used in -fundamentally different ways. Tuples can be thought of as being similar to -Pascal records or C structs; they're small collections of related data which may -be of different types which are operated on as a group. For example, a -Cartesian coordinate is appropriately represented as a tuple of two or three -numbers. - -Lists, on the other hand, are more like arrays in other languages. They tend to -hold a varying number of objects all of which have the same type and which are -operated on one-by-one. For example, ``os.listdir('.')`` returns a list of -strings representing the files in the current directory. Functions which -operate on this output would generally not break if you added another file or -two to the directory. - -Tuples are immutable, meaning that once a tuple has been created, you can't -replace any of its elements with a new value. Lists are mutable, meaning that -you can always change a list's elements. Only immutable elements can be used as -dictionary keys, and hence only tuples and not lists can be used as keys. - - -How are lists implemented in CPython? -------------------------------------- - -CPython's lists are really variable-length arrays, not Lisp-style linked lists. -The implementation uses a contiguous array of references to other objects, and -keeps a pointer to this array and the array's length in a list head structure. - -This makes indexing a list ``a[i]`` an operation whose cost is independent of -the size of the list or the value of the index. - -When items are appended or inserted, the array of references is resized. Some -cleverness is applied to improve the performance of appending items repeatedly; -when the array must be grown, some extra space is allocated so the next few -times don't require an actual resize. - - -How are dictionaries implemented in CPython? --------------------------------------------- - -CPython's dictionaries are implemented as resizable hash tables. Compared to -B-trees, this gives better performance for lookup (the most common operation by -far) under most circumstances, and the implementation is simpler. - -Dictionaries work by computing a hash code for each key stored in the dictionary -using the :func:`hash` built-in function. The hash code varies widely depending -on the key and a per-process seed; for example, "Python" could hash to --539294296 while "python", a string that differs by a single bit, could hash -to 1142331976. The hash code is then used to calculate a location in an -internal array where the value will be stored. Assuming that you're storing -keys that all have different hash values, this means that dictionaries take -constant time -- O(1), in Big-O notation -- to retrieve a key. - - -Why must dictionary keys be immutable? --------------------------------------- - -The hash table implementation of dictionaries uses a hash value calculated from -the key value to find the key. If the key were a mutable object, its value -could change, and thus its hash could also change. But since whoever changes -the key object can't tell that it was being used as a dictionary key, it can't -move the entry around in the dictionary. Then, when you try to look up the same -object in the dictionary it won't be found because its hash value is different. -If you tried to look up the old value it wouldn't be found either, because the -value of the object found in that hash bin would be different. - -If you want a dictionary indexed with a list, simply convert the list to a tuple -first; the function ``tuple(L)`` creates a tuple with the same entries as the -list ``L``. Tuples are immutable and can therefore be used as dictionary keys. - -Some unacceptable solutions that have been proposed: - -- Hash lists by their address (object ID). This doesn't work because if you - construct a new list with the same value it won't be found; e.g.:: - - mydict = {[1, 2]: '12'} - print(mydict[[1, 2]]) - - would raise a :exc:`KeyError` exception because the id of the ``[1, 2]`` used in the - second line differs from that in the first line. In other words, dictionary - keys should be compared using ``==``, not using :keyword:`is`. - -- Make a copy when using a list as a key. This doesn't work because the list, - being a mutable object, could contain a reference to itself, and then the - copying code would run into an infinite loop. - -- Allow lists as keys but tell the user not to modify them. This would allow a - class of hard-to-track bugs in programs when you forgot or modified a list by - accident. It also invalidates an important invariant of dictionaries: every - value in ``d.keys()`` is usable as a key of the dictionary. - -- Mark lists as read-only once they are used as a dictionary key. The problem - is that it's not just the top-level object that could change its value; you - could use a tuple containing a list as a key. Entering anything as a key into - a dictionary would require marking all objects reachable from there as - read-only -- and again, self-referential objects could cause an infinite loop. - -There is a trick to get around this if you need to, but use it at your own risk: -You can wrap a mutable structure inside a class instance which has both a -:meth:`__eq__` and a :meth:`__hash__` method. You must then make sure that the -hash value for all such wrapper objects that reside in a dictionary (or other -hash based structure), remain fixed while the object is in the dictionary (or -other structure). :: - - class ListWrapper: - def __init__(self, the_list): - self.the_list = the_list - - def __eq__(self, other): - return self.the_list == other.the_list - - def __hash__(self): - l = self.the_list - result = 98767 - len(l)*555 - for i, el in enumerate(l): - try: - result = result + (hash(el) % 9999999) * 1001 + i - except Exception: - result = (result % 7777777) + i * 333 - return result - -Note that the hash computation is complicated by the possibility that some -members of the list may be unhashable and also by the possibility of arithmetic -overflow. - -Furthermore it must always be the case that if ``o1 == o2`` (ie ``o1.__eq__(o2) -is True``) then ``hash(o1) == hash(o2)`` (ie, ``o1.__hash__() == o2.__hash__()``), -regardless of whether the object is in a dictionary or not. If you fail to meet -these restrictions dictionaries and other hash based structures will misbehave. - -In the case of ListWrapper, whenever the wrapper object is in a dictionary the -wrapped list must not change to avoid anomalies. Don't do this unless you are -prepared to think hard about the requirements and the consequences of not -meeting them correctly. Consider yourself warned. - - -Why doesn't list.sort() return the sorted list? ------------------------------------------------ - -In situations where performance matters, making a copy of the list just to sort -it would be wasteful. Therefore, :meth:`list.sort` sorts the list in place. In -order to remind you of that fact, it does not return the sorted list. This way, -you won't be fooled into accidentally overwriting a list when you need a sorted -copy but also need to keep the unsorted version around. - -If you want to return a new list, use the built-in :func:`sorted` function -instead. This function creates a new list from a provided iterable, sorts -it and returns it. For example, here's how to iterate over the keys of a -dictionary in sorted order:: - - for key in sorted(mydict): - ... # do whatever with mydict[key]... - - -How do you specify and enforce an interface spec in Python? ------------------------------------------------------------ - -An interface specification for a module as provided by languages such as C++ and -Java describes the prototypes for the methods and functions of the module. Many -feel that compile-time enforcement of interface specifications helps in the -construction of large programs. - -Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes -(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check -whether an instance or a class implements a particular ABC. The -:mod:`collections.abc` module defines a set of useful ABCs such as -:class:`~collections.abc.Iterable`, :class:`~collections.abc.Container`, and -:class:`~collections.abc.MutableMapping`. - -For Python, many of the advantages of interface specifications can be obtained -by an appropriate test discipline for components. - -A good test suite for a module can both provide a regression test and serve as a -module interface specification and a set of examples. Many Python modules can -be run as a script to provide a simple "self test." Even modules which use -complex external interfaces can often be tested in isolation using trivial -"stub" emulations of the external interface. The :mod:`doctest` and -:mod:`unittest` modules or third-party test frameworks can be used to construct -exhaustive test suites that exercise every line of code in a module. - -An appropriate testing discipline can help build large complex applications in -Python as well as having interface specifications would. In fact, it can be -better because an interface specification cannot test certain properties of a -program. For example, the :meth:`append` method is expected to add new elements -to the end of some internal list; an interface specification cannot test that -your :meth:`append` implementation will actually do this correctly, but it's -trivial to check this property in a test suite. - -Writing test suites is very helpful, and you might want to design your code to -make it easily tested. One increasingly popular technique, test-driven -development, calls for writing parts of the test suite first, before you write -any of the actual code. Of course Python allows you to be sloppy and not write -test cases at all. - - -Why is there no goto? ---------------------- - -You can use exceptions to provide a "structured goto" that even works across -function calls. Many feel that exceptions can conveniently emulate all -reasonable uses of the "go" or "goto" constructs of C, Fortran, and other -languages. For example:: - - class label(Exception): pass # declare a label - - try: - ... - if condition: raise label() # goto label - ... - except label: # where to goto - pass - ... - -This doesn't allow you to jump into the middle of a loop, but that's usually -considered an abuse of goto anyway. Use sparingly. - - -Why can't raw strings (r-strings) end with a backslash? -------------------------------------------------------- - -More precisely, they can't end with an odd number of backslashes: the unpaired -backslash at the end escapes the closing quote character, leaving an -unterminated string. - -Raw strings were designed to ease creating input for processors (chiefly regular -expression engines) that want to do their own backslash escape processing. Such -processors consider an unmatched trailing backslash to be an error anyway, so -raw strings disallow that. In return, they allow you to pass on the string -quote character by escaping it with a backslash. These rules work well when -r-strings are used for their intended purpose. - -If you're trying to build Windows pathnames, note that all Windows system calls -accept forward slashes too:: - - f = open("/mydir/file.txt") # works fine! - -If you're trying to build a pathname for a DOS command, try e.g. one of :: - - dir = r"\this\is\my\dos\dir" "\\" - dir = r"\this\is\my\dos\dir\ "[:-1] - dir = "\\this\\is\\my\\dos\\dir\\" - - -Why doesn't Python have a "with" statement for attribute assignments? ---------------------------------------------------------------------- - -Python has a 'with' statement that wraps the execution of a block, calling code -on the entrance and exit from the block. Some languages have a construct that -looks like this:: - - with obj: - a = 1 # equivalent to obj.a = 1 - total = total + 1 # obj.total = obj.total + 1 - -In Python, such a construct would be ambiguous. - -Other languages, such as Object Pascal, Delphi, and C++, use static types, so -it's possible to know, in an unambiguous way, what member is being assigned -to. This is the main point of static typing -- the compiler *always* knows the -scope of every variable at compile time. - -Python uses dynamic types. It is impossible to know in advance which attribute -will be referenced at runtime. Member attributes may be added or removed from -objects on the fly. This makes it impossible to know, from a simple reading, -what attribute is being referenced: a local one, a global one, or a member -attribute? - -For instance, take the following incomplete snippet:: - - def foo(a): - with a: - print(x) - -The snippet assumes that "a" must have a member attribute called "x". However, -there is nothing in Python that tells the interpreter this. What should happen -if "a" is, let us say, an integer? If there is a global variable named "x", -will it be used inside the with block? As you see, the dynamic nature of Python -makes such choices much harder. - -The primary benefit of "with" and similar language features (reduction of code -volume) can, however, easily be achieved in Python by assignment. Instead of:: - - function(args).mydict[index][index].a = 21 - function(args).mydict[index][index].b = 42 - function(args).mydict[index][index].c = 63 - -write this:: - - ref = function(args).mydict[index][index] - ref.a = 21 - ref.b = 42 - ref.c = 63 - -This also has the side-effect of increasing execution speed because name -bindings are resolved at run-time in Python, and the second version only needs -to perform the resolution once. - - -Why are colons required for the if/while/def/class statements? --------------------------------------------------------------- - -The colon is required primarily to enhance readability (one of the results of -the experimental ABC language). Consider this:: - - if a == b - print(a) - -versus :: - - if a == b: - print(a) - -Notice how the second one is slightly easier to read. Notice further how a -colon sets off the example in this FAQ answer; it's a standard usage in English. - -Another minor reason is that the colon makes it easier for editors with syntax -highlighting; they can look for colons to decide when indentation needs to be -increased instead of having to do a more elaborate parsing of the program text. - - -Why does Python allow commas at the end of lists and tuples? ------------------------------------------------------------- - -Python lets you add a trailing comma at the end of lists, tuples, and -dictionaries:: - - [1, 2, 3,] - ('a', 'b', 'c',) - d = { - "A": [1, 5], - "B": [6, 7], # last trailing comma is optional but good style - } - - -There are several reasons to allow this. - -When you have a literal value for a list, tuple, or dictionary spread across -multiple lines, it's easier to add more elements because you don't have to -remember to add a comma to the previous line. The lines can also be reordered -without creating a syntax error. - -Accidentally omitting the comma can lead to errors that are hard to diagnose. -For example:: - - x = [ - "fee", - "fie" - "foo", - "fum" - ] - -This list looks like it has four elements, but it actually contains three: -"fee", "fiefoo" and "fum". Always adding the comma avoids this source of error. - -Allowing the trailing comma may also make programmatic code generation easier. diff --git a/Doc/howto/descriptor.rst.bak b/Doc/howto/descriptor.rst.bak deleted file mode 100644 index 94a8b4e6b40b96..00000000000000 --- a/Doc/howto/descriptor.rst.bak +++ /dev/null @@ -1,1575 +0,0 @@ -.. _descriptorhowto: - -====================== -Descriptor HowTo Guide -====================== - -:Author: Raymond Hettinger -:Contact: - -.. Contents:: - - -:term:`Descriptors ` let objects customize attribute lookup, -storage, and deletion. - -This guide has four major sections: - -1) The "primer" gives a basic overview, moving gently from simple examples, - adding one feature at a time. Start here if you're new to descriptors. - -2) The second section shows a complete, practical descriptor example. If you - already know the basics, start there. - -3) The third section provides a more technical tutorial that goes into the - detailed mechanics of how descriptors work. Most people don't need this - level of detail. - -4) The last section has pure Python equivalents for built-in descriptors that - are written in C. Read this if you're curious about how functions turn - into bound methods or about the implementation of common tools like - :func:`classmethod`, :func:`staticmethod`, :func:`property`, and - :term:`__slots__`. - - -Primer -^^^^^^ - -In this primer, we start with the most basic possible example and then we'll -add new capabilities one by one. - - -Simple example: A descriptor that returns a constant ----------------------------------------------------- - -The :class:`Ten` class is a descriptor whose :meth:`__get__` method always -returns the constant ``10``: - -.. testcode:: - - class Ten: - def __get__(self, obj, objtype=None): - return 10 - -To use the descriptor, it must be stored as a class variable in another class: - -.. testcode:: - - class A: - x = 5 # Regular class attribute - y = Ten() # Descriptor instance - -An interactive session shows the difference between normal attribute lookup -and descriptor lookup: - -.. doctest:: - - >>> a = A() # Make an instance of class A - >>> a.x # Normal attribute lookup - 5 - >>> a.y # Descriptor lookup - 10 - -In the ``a.x`` attribute lookup, the dot operator finds ``'x': 5`` -in the class dictionary. In the ``a.y`` lookup, the dot operator -finds a descriptor instance, recognized by its ``__get__`` method. -Calling that method returns ``10``. - -Note that the value ``10`` is not stored in either the class dictionary or the -instance dictionary. Instead, the value ``10`` is computed on demand. - -This example shows how a simple descriptor works, but it isn't very useful. -For retrieving constants, normal attribute lookup would be better. - -In the next section, we'll create something more useful, a dynamic lookup. - - -Dynamic lookups ---------------- - -Interesting descriptors typically run computations instead of returning -constants: - -.. testcode:: - - import os - - class DirectorySize: - - def __get__(self, obj, objtype=None): - return len(os.listdir(obj.dirname)) - - class Directory: - - size = DirectorySize() # Descriptor instance - - def __init__(self, dirname): - self.dirname = dirname # Regular instance attribute - -An interactive session shows that the lookup is dynamic — it computes -different, updated answers each time:: - - >>> s = Directory('songs') - >>> g = Directory('games') - >>> s.size # The songs directory has twenty files - 20 - >>> g.size # The games directory has three files - 3 - >>> open('games/newfile').close() # Add a fourth file to the directory - >>> g.size # File count is automatically updated - 4 - -Besides showing how descriptors can run computations, this example also -reveals the purpose of the parameters to :meth:`__get__`. The *self* -parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is -either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that -lets the :meth:`__get__` method learn the target directory. The *objtype* -parameter is the class *Directory*. - - -Managed attributes ------------------- - -A popular use for descriptors is managing access to instance data. The -descriptor is assigned to a public attribute in the class dictionary while the -actual data is stored as a private attribute in the instance dictionary. The -descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when -the public attribute is accessed. - -In the following example, *age* is the public attribute and *_age* is the -private attribute. When the public attribute is accessed, the descriptor logs -the lookup or update: - -.. testcode:: - - import logging - - logging.basicConfig(level=logging.INFO) - - class LoggedAgeAccess: - - def __get__(self, obj, objtype=None): - value = obj._age - logging.info('Accessing %r giving %r', 'age', value) - return value - - def __set__(self, obj, value): - logging.info('Updating %r to %r', 'age', value) - obj._age = value - - class Person: - - age = LoggedAgeAccess() # Descriptor instance - - def __init__(self, name, age): - self.name = name # Regular instance attribute - self.age = age # Calls __set__() - - def birthday(self): - self.age += 1 # Calls both __get__() and __set__() - - -An interactive session shows that all access to the managed attribute *age* is -logged, but that the regular attribute *name* is not logged: - -.. testcode:: - :hide: - - import logging, sys - logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) - -.. doctest:: - - >>> mary = Person('Mary M', 30) # The initial age update is logged - INFO:root:Updating 'age' to 30 - >>> dave = Person('David D', 40) - INFO:root:Updating 'age' to 40 - - >>> vars(mary) # The actual data is in a private attribute - {'name': 'Mary M', '_age': 30} - >>> vars(dave) - {'name': 'David D', '_age': 40} - - >>> mary.age # Access the data and log the lookup - INFO:root:Accessing 'age' giving 30 - 30 - >>> mary.birthday() # Updates are logged as well - INFO:root:Accessing 'age' giving 30 - INFO:root:Updating 'age' to 31 - - >>> dave.name # Regular attribute lookup isn't logged - 'David D' - >>> dave.age # Only the managed attribute is logged - INFO:root:Accessing 'age' giving 40 - 40 - -One major issue with this example is that the private name *_age* is hardwired in -the *LoggedAgeAccess* class. That means that each instance can only have one -logged attribute and that its name is unchangeable. In the next example, -we'll fix that problem. - - -Customized names ----------------- - -When a class uses descriptors, it can inform each descriptor about which -variable name was used. - -In this example, the :class:`Person` class has two descriptor instances, -*name* and *age*. When the :class:`Person` class is defined, it makes a -callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can -be recorded, giving each descriptor its own *public_name* and *private_name*: - -.. testcode:: - - import logging - - logging.basicConfig(level=logging.INFO) - - class LoggedAccess: - - def __set_name__(self, owner, name): - self.public_name = name - self.private_name = '_' + name - - def __get__(self, obj, objtype=None): - value = getattr(obj, self.private_name) - logging.info('Accessing %r giving %r', self.public_name, value) - return value - - def __set__(self, obj, value): - logging.info('Updating %r to %r', self.public_name, value) - setattr(obj, self.private_name, value) - - class Person: - - name = LoggedAccess() # First descriptor instance - age = LoggedAccess() # Second descriptor instance - - def __init__(self, name, age): - self.name = name # Calls the first descriptor - self.age = age # Calls the second descriptor - - def birthday(self): - self.age += 1 - -An interactive session shows that the :class:`Person` class has called -:meth:`__set_name__` so that the field names would be recorded. Here -we call :func:`vars` to look up the descriptor without triggering it: - -.. doctest:: - - >>> vars(vars(Person)['name']) - {'public_name': 'name', 'private_name': '_name'} - >>> vars(vars(Person)['age']) - {'public_name': 'age', 'private_name': '_age'} - -The new class now logs access to both *name* and *age*: - -.. testcode:: - :hide: - - import logging, sys - logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) - -.. doctest:: - - >>> pete = Person('Peter P', 10) - INFO:root:Updating 'name' to 'Peter P' - INFO:root:Updating 'age' to 10 - >>> kate = Person('Catherine C', 20) - INFO:root:Updating 'name' to 'Catherine C' - INFO:root:Updating 'age' to 20 - -The two *Person* instances contain only the private names:: - - >>> vars(pete) - {'_name': 'Peter P', '_age': 10} - >>> vars(kate) - {'_name': 'Catherine C', '_age': 20} - - -Closing thoughts ----------------- - -A :term:`descriptor` is what we call any object that defines :meth:`__get__`, -:meth:`__set__`, or :meth:`__delete__`. - -Optionally, descriptors can have a :meth:`__set_name__` method. This is only -used in cases where a descriptor needs to know either the class where it was -created or the name of class variable it was assigned to. (This method, if -present, is called even if the class is not a descriptor.) - -Descriptors get invoked by the dot operator during attribute lookup. If a -descriptor is accessed indirectly with ``vars(some_class)[descriptor_name]``, -the descriptor instance is returned without invoking it. - -Descriptors only work when used as class variables. When put in instances, -they have no effect. - -The main motivation for descriptors is to provide a hook allowing objects -stored in class variables to control what happens during attribute lookup. - -Traditionally, the calling class controls what happens during lookup. -Descriptors invert that relationship and allow the data being looked-up to -have a say in the matter. - -Descriptors are used throughout the language. It is how functions turn into -bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, -:func:`property`, and :func:`functools.cached_property` are all implemented as -descriptors. - - -Complete Practical Example -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In this example, we create a practical and powerful tool for locating -notoriously hard to find data corruption bugs. - - -Validator class ---------------- - -A validator is a descriptor for managed attribute access. Prior to storing -any data, it verifies that the new value meets various type and range -restrictions. If those restrictions aren't met, it raises an exception to -prevent data corruption at its source. - -This :class:`Validator` class is both an :term:`abstract base class` and a -managed attribute descriptor: - -.. testcode:: - - from abc import ABC, abstractmethod - - class Validator(ABC): - - def __set_name__(self, owner, name): - self.private_name = '_' + name - - def __get__(self, obj, objtype=None): - return getattr(obj, self.private_name) - - def __set__(self, obj, value): - self.validate(value) - setattr(obj, self.private_name, value) - - @abstractmethod - def validate(self, value): - pass - -Custom validators need to inherit from :class:`Validator` and must supply a -:meth:`validate` method to test various restrictions as needed. - - -Custom validators ------------------ - -Here are three practical data validation utilities: - -1) :class:`OneOf` verifies that a value is one of a restricted set of options. - -2) :class:`Number` verifies that a value is either an :class:`int` or - :class:`float`. Optionally, it verifies that a value is between a given - minimum or maximum. - -3) :class:`String` verifies that a value is a :class:`str`. Optionally, it - validates a given minimum or maximum length. It can validate a - user-defined `predicate - `_ as well. - -.. testcode:: - - class OneOf(Validator): - - def __init__(self, *options): - self.options = set(options) - - def validate(self, value): - if value not in self.options: - raise ValueError(f'Expected {value!r} to be one of {self.options!r}') - - class Number(Validator): - - def __init__(self, minvalue=None, maxvalue=None): - self.minvalue = minvalue - self.maxvalue = maxvalue - - def validate(self, value): - if not isinstance(value, (int, float)): - raise TypeError(f'Expected {value!r} to be an int or float') - if self.minvalue is not None and value < self.minvalue: - raise ValueError( - f'Expected {value!r} to be at least {self.minvalue!r}' - ) - if self.maxvalue is not None and value > self.maxvalue: - raise ValueError( - f'Expected {value!r} to be no more than {self.maxvalue!r}' - ) - - class String(Validator): - - def __init__(self, minsize=None, maxsize=None, predicate=None): - self.minsize = minsize - self.maxsize = maxsize - self.predicate = predicate - - def validate(self, value): - if not isinstance(value, str): - raise TypeError(f'Expected {value!r} to be an str') - if self.minsize is not None and len(value) < self.minsize: - raise ValueError( - f'Expected {value!r} to be no smaller than {self.minsize!r}' - ) - if self.maxsize is not None and len(value) > self.maxsize: - raise ValueError( - f'Expected {value!r} to be no bigger than {self.maxsize!r}' - ) - if self.predicate is not None and not self.predicate(value): - raise ValueError( - f'Expected {self.predicate} to be true for {value!r}' - ) - - -Practical application ---------------------- - -Here's how the data validators can be used in a real class: - -.. testcode:: - - class Component: - - name = String(minsize=3, maxsize=10, predicate=str.isupper) - kind = OneOf('wood', 'metal', 'plastic') - quantity = Number(minvalue=0) - - def __init__(self, name, kind, quantity): - self.name = name - self.kind = kind - self.quantity = quantity - -The descriptors prevent invalid instances from being created: - -.. doctest:: - - >>> Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase - Traceback (most recent call last): - ... - ValueError: Expected to be true for 'Widget' - - >>> Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled - Traceback (most recent call last): - ... - ValueError: Expected 'metle' to be one of {'metal', 'plastic', 'wood'} - - >>> Component('WIDGET', 'metal', -5) # Blocked: -5 is negative - Traceback (most recent call last): - ... - ValueError: Expected -5 to be at least 0 - >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number - Traceback (most recent call last): - ... - TypeError: Expected 'V' to be an int or float - - >>> c = Component('WIDGET', 'metal', 5) # Allowed: The inputs are valid - - -Technical Tutorial -^^^^^^^^^^^^^^^^^^ - -What follows is a more technical tutorial for the mechanics and details of how -descriptors work. - - -Abstract --------- - -Defines descriptors, summarizes the protocol, and shows how descriptors are -called. Provides an example showing how object relational mappings work. - -Learning about descriptors not only provides access to a larger toolset, it -creates a deeper understanding of how Python works. - - -Definition and introduction ---------------------------- - -In general, a descriptor is an attribute value that has one of the methods in -the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, -and :meth:`__delete__`. If any of those methods are defined for an -attribute, it is said to be a :term:`descriptor`. - -The default behavior for attribute access is to get, set, or delete the -attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain -starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and -continuing through the method resolution order of ``type(a)``. If the -looked-up value is an object defining one of the descriptor methods, then Python -may override the default behavior and invoke the descriptor method instead. -Where this occurs in the precedence chain depends on which descriptor methods -were defined. - -Descriptors are a powerful, general purpose protocol. They are the mechanism -behind properties, methods, static methods, class methods, and -:func:`super()`. They are used throughout Python itself. Descriptors -simplify the underlying C code and offer a flexible set of new tools for -everyday Python programs. - - -Descriptor protocol -------------------- - -``descr.__get__(self, obj, type=None) -> value`` - -``descr.__set__(self, obj, value) -> None`` - -``descr.__delete__(self, obj) -> None`` - -That is all there is to it. Define any of these methods and an object is -considered a descriptor and can override default behavior upon being looked up -as an attribute. - -If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered -a data descriptor. Descriptors that only define :meth:`__get__` are called -non-data descriptors (they are often used for methods but other uses are -possible). - -Data and non-data descriptors differ in how overrides are calculated with -respect to entries in an instance's dictionary. If an instance's dictionary -has an entry with the same name as a data descriptor, the data descriptor -takes precedence. If an instance's dictionary has an entry with the same -name as a non-data descriptor, the dictionary entry takes precedence. - -To make a read-only data descriptor, define both :meth:`__get__` and -:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when -called. Defining the :meth:`__set__` method with an exception raising -placeholder is enough to make it a data descriptor. - - -Overview of descriptor invocation ---------------------------------- - -A descriptor can be called directly with ``desc.__get__(obj)`` or -``desc.__get__(None, cls)``. - -But it is more common for a descriptor to be invoked automatically from -attribute access. - -The expression ``obj.x`` looks up the attribute ``x`` in the chain of -namespaces for ``obj``. If the search finds a descriptor outside of the -instance ``__dict__``, its :meth:`__get__` method is invoked according to the -precedence rules listed below. - -The details of invocation depend on whether ``obj`` is an object, class, or -instance of super. - - -Invocation from an instance ---------------------------- - -Instance lookup scans through a chain of namespaces giving data descriptors -the highest priority, followed by instance variables, then non-data -descriptors, then class variables, and lastly :meth:`__getattr__` if it is -provided. - -If a descriptor is found for ``a.x``, then it is invoked with: -``desc.__get__(a, type(a))``. - -The logic for a dotted lookup is in :meth:`object.__getattribute__`. Here is -a pure Python equivalent: - -.. testcode:: - - def object_getattribute(obj, name): - "Emulate PyObject_GenericGetAttr() in Objects/object.c" - null = object() - objtype = type(obj) - cls_var = getattr(objtype, name, null) - descr_get = getattr(type(cls_var), '__get__', null) - if descr_get is not null: - if (hasattr(type(cls_var), '__set__') - or hasattr(type(cls_var), '__delete__')): - return descr_get(cls_var, obj, objtype) # data descriptor - if hasattr(obj, '__dict__') and name in vars(obj): - return vars(obj)[name] # instance variable - if descr_get is not null: - return descr_get(cls_var, obj, objtype) # non-data descriptor - if cls_var is not null: - return cls_var # class variable - raise AttributeError(name) - - -.. testcode:: - :hide: - - # Test the fidelity of object_getattribute() by comparing it with the - # normal object.__getattribute__(). The former will be accessed by - # square brackets and the latter by the dot operator. - - class Object: - - def __getitem__(obj, name): - try: - return object_getattribute(obj, name) - except AttributeError: - if not hasattr(type(obj), '__getattr__'): - raise - return type(obj).__getattr__(obj, name) # __getattr__ - - class DualOperator(Object): - - x = 10 - - def __init__(self, z): - self.z = z - - @property - def p2(self): - return 2 * self.x - - @property - def p3(self): - return 3 * self.x - - def m5(self, y): - return 5 * y - - def m7(self, y): - return 7 * y - - def __getattr__(self, name): - return ('getattr_hook', self, name) - - class DualOperatorWithSlots: - - __getitem__ = Object.__getitem__ - - __slots__ = ['z'] - - x = 15 - - def __init__(self, z): - self.z = z - - @property - def p2(self): - return 2 * self.x - - def m5(self, y): - return 5 * y - - def __getattr__(self, name): - return ('getattr_hook', self, name) - - -.. doctest:: - :hide: - - >>> a = DualOperator(11) - >>> vars(a).update(p3 = '_p3', m7 = '_m7') - >>> a.x == a['x'] == 10 - True - >>> a.z == a['z'] == 11 - True - >>> a.p2 == a['p2'] == 20 - True - >>> a.p3 == a['p3'] == 30 - True - >>> a.m5(100) == a.m5(100) == 500 - True - >>> a.m7 == a['m7'] == '_m7' - True - >>> a.g == a['g'] == ('getattr_hook', a, 'g') - True - - >>> b = DualOperatorWithSlots(22) - >>> b.x == b['x'] == 15 - True - >>> b.z == b['z'] == 22 - True - >>> b.p2 == b['p2'] == 30 - True - >>> b.m5(200) == b['m5'](200) == 1000 - True - >>> b.g == b['g'] == ('getattr_hook', b, 'g') - True - - -Interestingly, attribute lookup doesn't call :meth:`object.__getattribute__` -directly. Instead, both the dot operator and the :func:`getattr` function -perform attribute lookup by way of a helper function: - -.. testcode:: - - def getattr_hook(obj, name): - "Emulate slot_tp_getattr_hook() in Objects/typeobject.c" - try: - return obj.__getattribute__(name) - except AttributeError: - if not hasattr(type(obj), '__getattr__'): - raise - return type(obj).__getattr__(obj, name) # __getattr__ - -So if :meth:`__getattr__` exists, it is called whenever :meth:`__getattribute__` -raises :exc:`AttributeError` (either directly or in one of the descriptor calls). - -Also, if a user calls :meth:`object.__getattribute__` directly, the -:meth:`__getattr__` hook is bypassed entirely. - - -Invocation from a class ------------------------ - -The logic for a dotted lookup such as ``A.x`` is in -:meth:`type.__getattribute__`. The steps are similar to those for -:meth:`object.__getattribute__` but the instance dictionary lookup is replaced -by a search through the class's :term:`method resolution order`. - -If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. - -The full C implementation can be found in :c:func:`type_getattro()` and -:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`. - - -Invocation from super ---------------------- - -The logic for super's dotted lookup is in the :meth:`__getattribute__` method for -object returned by :class:`super()`. - -A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` -for the base class ``B`` immediately following ``A`` and then returns -``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned -unchanged. - -The full C implementation can be found in :c:func:`super_getattro()` in -:source:`Objects/typeobject.c`. A pure Python equivalent can be found in -`Guido's Tutorial -`_. - - -Summary of invocation logic ---------------------------- - -The mechanism for descriptors is embedded in the :meth:`__getattribute__()` -methods for :class:`object`, :class:`type`, and :func:`super`. - -The important points to remember are: - -* Descriptors are invoked by the :meth:`__getattribute__` method. - -* Classes inherit this machinery from :class:`object`, :class:`type`, or - :func:`super`. - -* Overriding :meth:`__getattribute__` prevents automatic descriptor calls - because all the descriptor logic is in that method. - -* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make - different calls to :meth:`__get__`. The first includes the instance and may - include the class. The second puts in ``None`` for the instance and always - includes the class. - -* Data descriptors always override instance dictionaries. - -* Non-data descriptors may be overridden by instance dictionaries. - - -Automatic name notification ---------------------------- - -Sometimes it is desirable for a descriptor to know what class variable name it -was assigned to. When a new class is created, the :class:`type` metaclass -scans the dictionary of the new class. If any of the entries are descriptors -and if they define :meth:`__set_name__`, that method is called with two -arguments. The *owner* is the class where the descriptor is used, and the -*name* is the class variable the descriptor was assigned to. - -The implementation details are in :c:func:`type_new()` and -:c:func:`set_names()` in :source:`Objects/typeobject.c`. - -Since the update logic is in :meth:`type.__new__`, notifications only take -place at the time of class creation. If descriptors are added to the class -afterwards, :meth:`__set_name__` will need to be called manually. - - -ORM example ------------ - -The following code is simplified skeleton showing how data descriptors could -be used to implement an `object relational mapping -`_. - -The essential idea is that the data is stored in an external database. The -Python instances only hold keys to the database's tables. Descriptors take -care of lookups or updates: - -.. testcode:: - - class Field: - - def __set_name__(self, owner, name): - self.fetch = f'SELECT {name} FROM {owner.table} WHERE {owner.key}=?;' - self.store = f'UPDATE {owner.table} SET {name}=? WHERE {owner.key}=?;' - - def __get__(self, obj, objtype=None): - return conn.execute(self.fetch, [obj.key]).fetchone()[0] - - def __set__(self, obj, value): - conn.execute(self.store, [value, obj.key]) - conn.commit() - -We can use the :class:`Field` class to define `models -`_ that describe the schema for -each table in a database: - -.. testcode:: - - class Movie: - table = 'Movies' # Table name - key = 'title' # Primary key - director = Field() - year = Field() - - def __init__(self, key): - self.key = key - - class Song: - table = 'Music' - key = 'title' - artist = Field() - year = Field() - genre = Field() - - def __init__(self, key): - self.key = key - -To use the models, first connect to the database:: - - >>> import sqlite3 - >>> conn = sqlite3.connect('entertainment.db') - -An interactive session shows how data is retrieved from the database and how -it can be updated: - -.. testsetup:: - - song_data = [ - ('Country Roads', 'John Denver', 1972), - ('Me and Bobby McGee', 'Janice Joplin', 1971), - ('Coal Miners Daughter', 'Loretta Lynn', 1970), - ] - - movie_data = [ - ('Star Wars', 'George Lucas', 1977), - ('Jaws', 'Steven Spielberg', 1975), - ('Aliens', 'James Cameron', 1986), - ] - - import sqlite3 - - conn = sqlite3.connect(':memory:') - conn.execute('CREATE TABLE Music (title text, artist text, year integer);') - conn.execute('CREATE INDEX MusicNdx ON Music (title);') - conn.executemany('INSERT INTO Music VALUES (?, ?, ?);', song_data) - conn.execute('CREATE TABLE Movies (title text, director text, year integer);') - conn.execute('CREATE INDEX MovieNdx ON Music (title);') - conn.executemany('INSERT INTO Movies VALUES (?, ?, ?);', movie_data) - conn.commit() - -.. doctest:: - - >>> Movie('Star Wars').director - 'George Lucas' - >>> jaws = Movie('Jaws') - >>> f'Released in {jaws.year} by {jaws.director}' - 'Released in 1975 by Steven Spielberg' - - >>> Song('Country Roads').artist - 'John Denver' - - >>> Movie('Star Wars').director = 'J.J. Abrams' - >>> Movie('Star Wars').director - 'J.J. Abrams' - - -Pure Python Equivalents -^^^^^^^^^^^^^^^^^^^^^^^ - -The descriptor protocol is simple and offers exciting possibilities. Several -use cases are so common that they have been prepackaged into built-in tools. -Properties, bound methods, static methods, class methods, and \_\_slots\_\_ are -all based on the descriptor protocol. - - -Properties ----------- - -Calling :func:`property` is a succinct way of building a data descriptor that -triggers a function call upon access to an attribute. Its signature is:: - - property(fget=None, fset=None, fdel=None, doc=None) -> property - -The documentation shows a typical use to define a managed attribute ``x``: - -.. testcode:: - - class C: - def getx(self): return self.__x - def setx(self, value): self.__x = value - def delx(self): del self.__x - x = property(getx, setx, delx, "I'm the 'x' property.") - -To see how :func:`property` is implemented in terms of the descriptor protocol, -here is a pure Python equivalent: - -.. testcode:: - - class Property: - "Emulate PyProperty_Type() in Objects/descrobject.c" - - def __init__(self, fget=None, fset=None, fdel=None, doc=None): - self.fget = fget - self.fset = fset - self.fdel = fdel - if doc is None and fget is not None: - doc = fget.__doc__ - self.__doc__ = doc - self._name = '' - - def __set_name__(self, owner, name): - self._name = name - - def __get__(self, obj, objtype=None): - if obj is None: - return self - if self.fget is None: - raise AttributeError(f'unreadable attribute {self._name}') - return self.fget(obj) - - def __set__(self, obj, value): - if self.fset is None: - raise AttributeError(f"can't set attribute {self._name}") - self.fset(obj, value) - - def __delete__(self, obj): - if self.fdel is None: - raise AttributeError(f"can't delete attribute {self._name}") - self.fdel(obj) - - def getter(self, fget): - prop = type(self)(fget, self.fset, self.fdel, self.__doc__) - prop._name = self._name - return prop - - def setter(self, fset): - prop = type(self)(self.fget, fset, self.fdel, self.__doc__) - prop._name = self._name - return prop - - def deleter(self, fdel): - prop = type(self)(self.fget, self.fset, fdel, self.__doc__) - prop._name = self._name - return prop - -.. testcode:: - :hide: - - # Verify the Property() emulation - - class CC: - def getx(self): - return self.__x - def setx(self, value): - self.__x = value - def delx(self): - del self.__x - x = Property(getx, setx, delx, "I'm the 'x' property.") - - # Now do it again but use the decorator style - - class CCC: - @Property - def x(self): - return self.__x - @x.setter - def x(self, value): - self.__x = value - @x.deleter - def x(self): - del self.__x - - -.. doctest:: - :hide: - - >>> cc = CC() - >>> hasattr(cc, 'x') - False - >>> cc.x = 33 - >>> cc.x - 33 - >>> del cc.x - >>> hasattr(cc, 'x') - False - - >>> ccc = CCC() - >>> hasattr(ccc, 'x') - False - >>> ccc.x = 333 - >>> ccc.x == 333 - True - >>> del ccc.x - >>> hasattr(ccc, 'x') - False - -The :func:`property` builtin helps whenever a user interface has granted -attribute access and then subsequent changes require the intervention of a -method. - -For instance, a spreadsheet class may grant access to a cell value through -``Cell('b10').value``. Subsequent improvements to the program require the cell -to be recalculated on every access; however, the programmer does not want to -affect existing client code accessing the attribute directly. The solution is -to wrap access to the value attribute in a property data descriptor: - -.. testcode:: - - class Cell: - ... - - @property - def value(self): - "Recalculate the cell before returning value" - self.recalc() - return self._value - -Either the built-in :func:`property` or our :func:`Property` equivalent would -work in this example. - - -Functions and methods ---------------------- - -Python's object oriented features are built upon a function based environment. -Using non-data descriptors, the two are merged seamlessly. - -Functions stored in class dictionaries get turned into methods when invoked. -Methods only differ from regular functions in that the object instance is -prepended to the other arguments. By convention, the instance is called -*self* but could be called *this* or any other variable name. - -Methods can be created manually with :class:`types.MethodType` which is -roughly equivalent to: - -.. testcode:: - - class MethodType: - "Emulate Py_MethodType in Objects/classobject.c" - - def __init__(self, func, obj): - self.__func__ = func - self.__self__ = obj - - def __call__(self, *args, **kwargs): - func = self.__func__ - obj = self.__self__ - return func(obj, *args, **kwargs) - -To support automatic creation of methods, functions include the -:meth:`__get__` method for binding methods during attribute access. This -means that functions are non-data descriptors that return bound methods -during dotted lookup from an instance. Here's how it works: - -.. testcode:: - - class Function: - ... - - def __get__(self, obj, objtype=None): - "Simulate func_descr_get() in Objects/funcobject.c" - if obj is None: - return self - return MethodType(self, obj) - -Running the following class in the interpreter shows how the function -descriptor works in practice: - -.. testcode:: - - class D: - def f(self, x): - return x - -The function has a :term:`qualified name` attribute to support introspection: - -.. doctest:: - - >>> D.f.__qualname__ - 'D.f' - -Accessing the function through the class dictionary does not invoke -:meth:`__get__`. Instead, it just returns the underlying function object:: - - >>> D.__dict__['f'] - - -Dotted access from a class calls :meth:`__get__` which just returns the -underlying function unchanged:: - - >>> D.f - - -The interesting behavior occurs during dotted access from an instance. The -dotted lookup calls :meth:`__get__` which returns a bound method object:: - - >>> d = D() - >>> d.f - > - -Internally, the bound method stores the underlying function and the bound -instance:: - - >>> d.f.__func__ - - - >>> d.f.__self__ - <__main__.D object at 0x1012e1f98> - -If you have ever wondered where *self* comes from in regular methods or where -*cls* comes from in class methods, this is it! - - -Static methods --------------- - -Non-data descriptors provide a simple mechanism for variations on the usual -patterns of binding functions into methods. - -To recap, functions have a :meth:`__get__` method so that they can be converted -to a method when accessed as attributes. The non-data descriptor transforms an -``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` -becomes ``f(*args)``. - -This chart summarizes the binding and its two most useful variants: - - +-----------------+----------------------+------------------+ - | Transformation | Called from an | Called from a | - | | object | class | - +=================+======================+==================+ - | function | f(obj, \*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | staticmethod | f(\*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | classmethod | f(type(obj), \*args) | f(cls, \*args) | - +-----------------+----------------------+------------------+ - -Static methods return the underlying function without changes. Calling either -``c.f`` or ``C.f`` is the equivalent of a direct lookup into -``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a -result, the function becomes identically accessible from either an object or a -class. - -Good candidates for static methods are methods that do not reference the -``self`` variable. - -For instance, a statistics package may include a container class for -experimental data. The class provides normal methods for computing the average, -mean, median, and other descriptive statistics that depend on the data. However, -there may be useful functions which are conceptually related but do not depend -on the data. For instance, ``erf(x)`` is handy conversion routine that comes up -in statistical work but does not directly depend on a particular dataset. -It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or -``Sample.erf(1.5) --> .9332``. - -Since static methods return the underlying function with no changes, the -example calls are unexciting: - -.. testcode:: - - class E: - @staticmethod - def f(x): - print(x) - -.. doctest:: - - >>> E.f(3) - 3 - >>> E().f(3) - 3 - -Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this: - -.. doctest:: - - class StaticMethod: - "Emulate PyStaticMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - - def __get__(self, obj, objtype=None): - return self.f - - -Class methods -------------- - -Unlike static methods, class methods prepend the class reference to the -argument list before calling the function. This format is the same -for whether the caller is an object or a class: - -.. testcode:: - - class F: - @classmethod - def f(cls, x): - return cls.__name__, x - -.. doctest:: - - >>> F.f(3) - ('F', 3) - >>> F().f(3) - ('F', 3) - -This behavior is useful whenever the method only needs to have a class -reference and does not rely on data stored in a specific instance. One use for -class methods is to create alternate class constructors. For example, the -classmethod :func:`dict.fromkeys` creates a new dictionary from a list of -keys. The pure Python equivalent is: - -.. testcode:: - - class Dict(dict): - @classmethod - def fromkeys(cls, iterable, value=None): - "Emulate dict_fromkeys() in Objects/dictobject.c" - d = cls() - for key in iterable: - d[key] = value - return d - -Now a new dictionary of unique keys can be constructed like this: - -.. doctest:: - - >>> d = Dict.fromkeys('abracadabra') - >>> type(d) is Dict - True - >>> d - {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} - -Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this: - -.. testcode:: - - class ClassMethod: - "Emulate PyClassMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - - def __get__(self, obj, cls=None): - if cls is None: - cls = type(obj) - if hasattr(obj, '__get__'): - return self.f.__get__(cls) - return MethodType(self.f, cls) - -.. testcode:: - :hide: - - # Verify the emulation works - class T: - @ClassMethod - def cm(cls, x, y): - return (cls, x, y) - -.. doctest:: - :hide: - - >>> T.cm(11, 22) - (, 11, 22) - - # Also call it from an instance - >>> t = T() - >>> t.cm(11, 22) - (, 11, 22) - -The code path for ``hasattr(obj, '__get__')`` was added in Python 3.9 and -makes it possible for :func:`classmethod` to support chained decorators. -For example, a classmethod and property could be chained together: - -.. testcode:: - - class G: - @classmethod - @property - def __doc__(cls): - return f'A doc for {cls.__name__!r}' - -.. doctest:: - - >>> G.__doc__ - "A doc for 'G'" - - -Member objects and __slots__ ----------------------------- - -When a class defines ``__slots__``, it replaces instance dictionaries with a -fixed-length array of slot values. From a user point of view that has -several effects: - -1. Provides immediate detection of bugs due to misspelled attribute -assignments. Only attribute names specified in ``__slots__`` are allowed: - -.. testcode:: - - class Vehicle: - __slots__ = ('id_number', 'make', 'model') - -.. doctest:: - - >>> auto = Vehicle() - >>> auto.id_nubmer = 'VYE483814LQEX' - Traceback (most recent call last): - ... - AttributeError: 'Vehicle' object has no attribute 'id_nubmer' - -2. Helps create immutable objects where descriptors manage access to private -attributes stored in ``__slots__``: - -.. testcode:: - - class Immutable: - - __slots__ = ('_dept', '_name') # Replace the instance dictionary - - def __init__(self, dept, name): - self._dept = dept # Store to private attribute - self._name = name # Store to private attribute - - @property # Read-only descriptor - def dept(self): - return self._dept - - @property - def name(self): # Read-only descriptor - return self._name - -.. doctest:: - - >>> mark = Immutable('Botany', 'Mark Watney') - >>> mark.dept - 'Botany' - >>> mark.dept = 'Space Pirate' - Traceback (most recent call last): - ... - AttributeError: can't set attribute - >>> mark.location = 'Mars' - Traceback (most recent call last): - ... - AttributeError: 'Immutable' object has no attribute 'location' - -3. Saves memory. On a 64-bit Linux build, an instance with two attributes -takes 48 bytes with ``__slots__`` and 152 bytes without. This `flyweight -design pattern `_ likely only -matters when a large number of instances are going to be created. - -4. Improves speed. Reading instance variables is 35% faster with -``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). - -5. Blocks tools like :func:`functools.cached_property` which require an -instance dictionary to function correctly: - -.. testcode:: - - from functools import cached_property - - class CP: - __slots__ = () # Eliminates the instance dict - - @cached_property # Requires an instance dict - def pi(self): - return 4 * sum((-1.0)**n / (2.0*n + 1.0) - for n in reversed(range(100_000))) - -.. doctest:: - - >>> CP().pi - Traceback (most recent call last): - ... - TypeError: No '__dict__' attribute on 'CP' instance to cache 'pi' property. - -It is not possible to create an exact drop-in pure Python version of -``__slots__`` because it requires direct access to C structures and control -over object memory allocation. However, we can build a mostly faithful -simulation where the actual C structure for slots is emulated by a private -``_slotvalues`` list. Reads and writes to that private structure are managed -by member descriptors: - -.. testcode:: - - null = object() - - class Member: - - def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' - # Also see descr_new() in Objects/descrobject.c - self.name = name - self.clsname = clsname - self.offset = offset - - def __get__(self, obj, objtype=None): - 'Emulate member_get() in Objects/descrobject.c' - # Also see PyMember_GetOne() in Python/structmember.c - value = obj._slotvalues[self.offset] - if value is null: - raise AttributeError(self.name) - return value - - def __set__(self, obj, value): - 'Emulate member_set() in Objects/descrobject.c' - obj._slotvalues[self.offset] = value - - def __delete__(self, obj): - 'Emulate member_delete() in Objects/descrobject.c' - value = obj._slotvalues[self.offset] - if value is null: - raise AttributeError(self.name) - obj._slotvalues[self.offset] = null - - def __repr__(self): - 'Emulate member_repr() in Objects/descrobject.c' - return f'' - -The :meth:`type.__new__` method takes care of adding member objects to class -variables: - -.. testcode:: - - class Type(type): - 'Simulate how the type metaclass adds member objects for slots' - - def __new__(mcls, clsname, bases, mapping): - 'Emuluate type_new() in Objects/typeobject.c' - # type_new() calls PyTypeReady() which calls add_methods() - slot_names = mapping.get('slot_names', []) - for offset, name in enumerate(slot_names): - mapping[name] = Member(name, clsname, offset) - return type.__new__(mcls, clsname, bases, mapping) - -The :meth:`object.__new__` method takes care of creating instances that have -slots instead of an instance dictionary. Here is a rough simulation in pure -Python: - -.. testcode:: - - class Object: - 'Simulate how object.__new__() allocates memory for __slots__' - - def __new__(cls, *args): - 'Emulate object_new() in Objects/typeobject.c' - inst = super().__new__(cls) - if hasattr(cls, 'slot_names'): - empty_slots = [null] * len(cls.slot_names) - object.__setattr__(inst, '_slotvalues', empty_slots) - return inst - - def __setattr__(self, name, value): - 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' - cls = type(self) - if hasattr(cls, 'slot_names') and name not in cls.slot_names: - raise AttributeError( - f'{type(self).__name__!r} object has no attribute {name!r}' - ) - super().__setattr__(name, value) - - def __delattr__(self, name): - 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' - cls = type(self) - if hasattr(cls, 'slot_names') and name not in cls.slot_names: - raise AttributeError( - f'{type(self).__name__!r} object has no attribute {name!r}' - ) - super().__delattr__(name) - -To use the simulation in a real class, just inherit from :class:`Object` and -set the :term:`metaclass` to :class:`Type`: - -.. testcode:: - - class H(Object, metaclass=Type): - 'Instance variables stored in slots' - - slot_names = ['x', 'y'] - - def __init__(self, x, y): - self.x = x - self.y = y - -At this point, the metaclass has loaded member objects for *x* and *y*:: - - >>> from pprint import pp - >>> pp(dict(vars(H))) - {'__module__': '__main__', - '__doc__': 'Instance variables stored in slots', - 'slot_names': ['x', 'y'], - '__init__': , - 'x': , - 'y': } - -.. doctest:: - :hide: - - # We test this separately because the preceding section is not - # doctestable due to the hex memory address for the __init__ function - >>> isinstance(vars(H)['x'], Member) - True - >>> isinstance(vars(H)['y'], Member) - True - -When instances are created, they have a ``slot_values`` list where the -attributes are stored: - -.. doctest:: - - >>> h = H(10, 20) - >>> vars(h) - {'_slotvalues': [10, 20]} - >>> h.x = 55 - >>> vars(h) - {'_slotvalues': [55, 20]} - -Misspelled or unassigned attributes will raise an exception: - -.. doctest:: - - >>> h.xz - Traceback (most recent call last): - ... - AttributeError: 'H' object has no attribute 'xz' - -.. doctest:: - :hide: - - # Examples for deleted attributes are not shown because this section - # is already a bit lengthy. We still test that code here. - >>> del h.x - >>> hasattr(h, 'x') - False - - # Also test the code for uninitialized slots - >>> class HU(Object, metaclass=Type): - ... slot_names = ['x', 'y'] - ... - >>> hu = HU() - >>> hasattr(hu, 'x') - False - >>> hasattr(hu, 'y') - False diff --git a/Doc/library/_thread.rst.bak b/Doc/library/_thread.rst.bak deleted file mode 100644 index bd653ab32bb9c4..00000000000000 --- a/Doc/library/_thread.rst.bak +++ /dev/null @@ -1,215 +0,0 @@ -:mod:`_thread` --- Low-level threading API -========================================== - -.. module:: _thread - :synopsis: Low-level threading API. - -.. index:: - single: light-weight processes - single: processes, light-weight - single: binary semaphores - single: semaphores, binary - --------------- - -This module provides low-level primitives for working with multiple threads -(also called :dfn:`light-weight processes` or :dfn:`tasks`) --- multiple threads of -control sharing their global data space. For synchronization, simple locks -(also called :dfn:`mutexes` or :dfn:`binary semaphores`) are provided. -The :mod:`threading` module provides an easier to use and higher-level -threading API built on top of this module. - -.. index:: - single: pthreads - pair: threads; POSIX - -.. versionchanged:: 3.7 - This module used to be optional, it is now always available. - -This module defines the following constants and functions: - -.. exception:: error - - Raised on thread-specific errors. - - .. versionchanged:: 3.3 - This is now a synonym of the built-in :exc:`RuntimeError`. - - -.. data:: LockType - - This is the type of lock objects. - - -.. function:: start_new_thread(function, args[, kwargs]) - - Start a new thread and return its identifier. The thread executes the - function *function* with the argument list *args* (which must be a tuple). - The optional *kwargs* argument specifies a dictionary of keyword arguments. - - When the function returns, the thread silently exits. - - When the function terminates with an unhandled exception, - :func:`sys.unraisablehook` is called to handle the exception. The *object* - attribute of the hook argument is *function*. By default, a stack trace is - printed and then the thread exits (but other threads continue to run). - - When the function raises a :exc:`SystemExit` exception, it is silently - ignored. - - .. versionchanged:: 3.8 - :func:`sys.unraisablehook` is now used to handle unhandled exceptions. - - -.. function:: interrupt_main() - - Simulate the effect of a :data:`signal.SIGINT` signal arriving in the main - thread. A thread can use this function to interrupt the main thread. - - If :data:`signal.SIGINT` isn't handled by Python (it was set to - :data:`signal.SIG_DFL` or :data:`signal.SIG_IGN`), this function does - nothing. - - -.. function:: exit() - - Raise the :exc:`SystemExit` exception. When not caught, this will cause the - thread to exit silently. - -.. - function:: exit_prog(status) - - Exit all threads and report the value of the integer argument - *status* as the exit status of the entire program. - **Caveat:** code in pending :keyword:`finally` clauses, in this thread - or in other threads, is not executed. - - -.. function:: allocate_lock() - - Return a new lock object. Methods of locks are described below. The lock is - initially unlocked. - - -.. function:: get_ident() - - Return the 'thread identifier' of the current thread. This is a nonzero - integer. Its value has no direct meaning; it is intended as a magic cookie to - be used e.g. to index a dictionary of thread-specific data. Thread identifiers - may be recycled when a thread exits and another thread is created. - - -.. function:: get_native_id() - - Return the native integral Thread ID of the current thread assigned by the kernel. - This is a non-negative integer. - Its value may be used to uniquely identify this particular thread system-wide - (until the thread terminates, after which the value may be recycled by the OS). - - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. - - .. versionadded:: 3.8 - - -.. function:: stack_size([size]) - - Return the thread stack size used when creating new threads. The optional - *size* argument specifies the stack size to be used for subsequently created - threads, and must be 0 (use platform or configured default) or a positive - integer value of at least 32,768 (32 KiB). If *size* is not specified, - 0 is used. If changing the thread stack size is - unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is - invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB - is currently the minimum supported stack size value to guarantee sufficient - stack space for the interpreter itself. Note that some platforms may have - particular restrictions on values for the stack size, such as requiring a - minimum stack size > 32 KiB or requiring allocation in multiples of the system - memory page size - platform documentation should be referred to for more - information (4 KiB pages are common; using multiples of 4096 for the stack size is - the suggested approach in the absence of more specific information). - - .. availability:: Windows, systems with POSIX threads. - - -.. data:: TIMEOUT_MAX - - The maximum value allowed for the *timeout* parameter of - :meth:`Lock.acquire`. Specifying a timeout greater than this value will - raise an :exc:`OverflowError`. - - .. versionadded:: 3.2 - - -Lock objects have the following methods: - - -.. method:: lock.acquire(waitflag=1, timeout=-1) - - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). - - If the integer *waitflag* argument is present, the action depends on its - value: if it is zero, the lock is only acquired if it can be acquired - immediately without waiting, while if it is nonzero, the lock is acquired - unconditionally as above. - - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *waitflag* is zero. - - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. - - -.. method:: lock.release() - - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. - - -.. method:: lock.locked() - - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. - -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: - - import _thread - - a_lock = _thread.allocate_lock() - - with a_lock: - print("a_lock is locked while this executes") - -**Caveats:** - - .. index:: module: signal - -* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` - exception will be received by an arbitrary thread. (When the :mod:`signal` - module is available, interrupts always go to the main thread.) - -* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is - equivalent to calling :func:`_thread.exit`. - -* It is not possible to interrupt the :meth:`acquire` method on a lock --- the - :exc:`KeyboardInterrupt` exception will happen after the lock has been acquired. - -* When the main thread exits, it is system defined whether the other threads - survive. On most systems, they are killed without executing - :keyword:`try` ... :keyword:`finally` clauses or executing object - destructors. - -* When the main thread exits, it does not do any of its usual cleanup (except - that :keyword:`try` ... :keyword:`finally` clauses are honored), and the - standard I/O files are not flushed. - diff --git a/Doc/library/ast.rst.bak b/Doc/library/ast.rst.bak deleted file mode 100644 index b5e1be82691157..00000000000000 --- a/Doc/library/ast.rst.bak +++ /dev/null @@ -1,1993 +0,0 @@ -:mod:`ast` --- Abstract Syntax Trees -==================================== - -.. module:: ast - :synopsis: Abstract Syntax Tree classes and manipulation. - -.. sectionauthor:: Martin v. Löwis -.. sectionauthor:: Georg Brandl - -.. testsetup:: - - import ast - -**Source code:** :source:`Lib/ast.py` - --------------- - -The :mod:`ast` module helps Python applications to process trees of the Python -abstract syntax grammar. The abstract syntax itself might change with each -Python release; this module helps to find out programmatically what the current -grammar looks like. - -An abstract syntax tree can be generated by passing :data:`ast.PyCF_ONLY_AST` as -a flag to the :func:`compile` built-in function, or using the :func:`parse` -helper provided in this module. The result will be a tree of objects whose -classes all inherit from :class:`ast.AST`. An abstract syntax tree can be -compiled into a Python code object using the built-in :func:`compile` function. - - -.. _abstract-grammar: - -Abstract Grammar ----------------- - -The abstract grammar is currently defined as follows: - -.. literalinclude:: ../../Parser/Python.asdl - :language: asdl - - -Node classes ------------- - -.. class:: AST - - This is the base of all AST node classes. The actual node classes are - derived from the :file:`Parser/Python.asdl` file, which is reproduced - :ref:`below `. They are defined in the :mod:`_ast` C - module and re-exported in :mod:`ast`. - - There is one class defined for each left-hand side symbol in the abstract - grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, - there is one class defined for each constructor on the right-hand side; these - classes inherit from the classes for the left-hand side trees. For example, - :class:`ast.BinOp` inherits from :class:`ast.expr`. For production rules - with alternatives (aka "sums"), the left-hand side class is abstract: only - instances of specific constructor nodes are ever created. - - .. index:: single: ? (question mark); in AST grammar - .. index:: single: * (asterisk); in AST grammar - - .. attribute:: _fields - - Each concrete class has an attribute :attr:`_fields` which gives the names - of all child nodes. - - Each instance of a concrete class has one attribute for each child node, - of the type as defined in the grammar. For example, :class:`ast.BinOp` - instances have an attribute :attr:`left` of type :class:`ast.expr`. - - If these attributes are marked as optional in the grammar (using a - question mark), the value might be ``None``. If the attributes can have - zero-or-more values (marked with an asterisk), the values are represented - as Python lists. All possible attributes must be present and have valid - values when compiling an AST with :func:`compile`. - - .. attribute:: lineno - col_offset - end_lineno - end_col_offset - - Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have - :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and - :attr:`end_col_offset` attributes. The :attr:`lineno` and :attr:`end_lineno` - are the first and last line numbers of source text span (1-indexed so the - first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset` - are the corresponding UTF-8 byte offsets of the first and last tokens that - generated the node. The UTF-8 offset is recorded because the parser uses - UTF-8 internally. - - Note that the end positions are not required by the compiler and are - therefore optional. The end offset is *after* the last symbol, for example - one can get the source segment of a one-line expression node using - ``source_line[node.col_offset : node.end_col_offset]``. - - The constructor of a class :class:`ast.T` parses its arguments as follows: - - * If there are positional arguments, there must be as many as there are items - in :attr:`T._fields`; they will be assigned as attributes of these names. - * If there are keyword arguments, they will set the attributes of the same - names to the given values. - - For example, to create and populate an :class:`ast.UnaryOp` node, you could - use :: - - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Constant() - node.operand.value = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - - node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), - lineno=0, col_offset=0) - -.. versionchanged:: 3.8 - - Class :class:`ast.Constant` is now used for all constants. - -.. versionchanged:: 3.9 - - Simple indices are represented by their value, extended slices are - represented as tuples. - -.. deprecated:: 3.8 - - Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, - :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meantime, - instantiating them will return an instance of a different class. - -.. deprecated:: 3.9 - - Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still - available, but they will be removed in future Python releases. - In the meantime, instantiating them will return an instance of - a different class. - -.. note:: - The descriptions of the specific node classes displayed here - were initially adapted from the fantastic `Green Tree - Snakes `__ project and - all its contributors. - -Literals -^^^^^^^^ - -.. class:: Constant(value) - - A constant value. The ``value`` attribute of the ``Constant`` literal contains the - Python object it represents. The values represented can be simple types - such as a number, string or ``None``, but also immutable container types - (tuples and frozensets) if all of their elements are constant. - - .. doctest:: - - >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) - Expression( - body=Constant(value=123)) - - -.. class:: FormattedValue(value, conversion, format_spec) - - Node representing a single formatting field in an f-string. If the string - contains a single formatting field and nothing else the node can be - isolated otherwise it appears in :class:`JoinedStr`. - - * ``value`` is any expression node (such as a literal, a variable, or a - function call). - * ``conversion`` is an integer: - - * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting - - * ``format_spec`` is a :class:`JoinedStr` node representing the formatting - of the value, or ``None`` if no format was specified. Both - ``conversion`` and ``format_spec`` can be set at the same time. - - -.. class:: JoinedStr(values) - - An f-string, comprising a series of :class:`FormattedValue` and :class:`Constant` - nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('f"sin({a}) is {sin(a):.3}"', mode='eval'), indent=4)) - Expression( - body=JoinedStr( - values=[ - Constant(value='sin('), - FormattedValue( - value=Name(id='a', ctx=Load()), - conversion=-1), - Constant(value=') is '), - FormattedValue( - value=Call( - func=Name(id='sin', ctx=Load()), - args=[ - Name(id='a', ctx=Load())], - keywords=[]), - conversion=-1, - format_spec=JoinedStr( - values=[ - Constant(value='.3')]))])) - - -.. class:: List(elts, ctx) - Tuple(elts, ctx) - - A list or tuple. ``elts`` holds a list of nodes representing the elements. - ``ctx`` is :class:`Store` if the container is an assignment target (i.e. - ``(x,y)=something``), and :class:`Load` otherwise. - - .. doctest:: - - >>> print(ast.dump(ast.parse('[1, 2, 3]', mode='eval'), indent=4)) - Expression( - body=List( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)], - ctx=Load())) - >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) - Expression( - body=Tuple( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)], - ctx=Load())) - - -.. class:: Set(elts) - - A set. ``elts`` holds a list of nodes representing the set's elements. - - .. doctest:: - - >>> print(ast.dump(ast.parse('{1, 2, 3}', mode='eval'), indent=4)) - Expression( - body=Set( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)])) - - -.. class:: Dict(keys, values) - - A dictionary. ``keys`` and ``values`` hold lists of nodes representing the - keys and the values respectively, in matching order (what would be returned - when calling :code:`dictionary.keys()` and :code:`dictionary.values()`). - - When doing dictionary unpacking using dictionary literals the expression to be - expanded goes in the ``values`` list, with a ``None`` at the corresponding - position in ``keys``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('{"a":1, **d}', mode='eval'), indent=4)) - Expression( - body=Dict( - keys=[ - Constant(value='a'), - None], - values=[ - Constant(value=1), - Name(id='d', ctx=Load())])) - - -Variables -^^^^^^^^^ - -.. class:: Name(id, ctx) - - A variable name. ``id`` holds the name as a string, and ``ctx`` is one of - the following types. - - -.. class:: Load() - Store() - Del() - - Variable references can be used to load the value of a variable, to assign - a new value to it, or to delete it. Variable references are given a context - to distinguish these cases. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a'), indent=4)) - Module( - body=[ - Expr( - value=Name(id='a', ctx=Load()))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a = 1'), indent=4)) - Module( - body=[ - Assign( - targets=[ - Name(id='a', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('del a'), indent=4)) - Module( - body=[ - Delete( - targets=[ - Name(id='a', ctx=Del())])], - type_ignores=[]) - - -.. class:: Starred(value, ctx) - - A ``*var`` variable reference. ``value`` holds the variable, typically a - :class:`Name` node. This type must be used when building a :class:`Call` - node with ``*args``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a, *b = it'), indent=4)) - Module( - body=[ - Assign( - targets=[ - Tuple( - elts=[ - Name(id='a', ctx=Store()), - Starred( - value=Name(id='b', ctx=Store()), - ctx=Store())], - ctx=Store())], - value=Name(id='it', ctx=Load()))], - type_ignores=[]) - - -Expressions -^^^^^^^^^^^ - -.. class:: Expr(value) - - When an expression, such as a function call, appears as a statement by itself - with its return value not used or stored, it is wrapped in this container. - ``value`` holds one of the other nodes in this section, a :class:`Constant`, a - :class:`Name`, a :class:`Lambda`, a :class:`Yield` or :class:`YieldFrom` node. - - .. doctest:: - - >>> print(ast.dump(ast.parse('-a'), indent=4)) - Module( - body=[ - Expr( - value=UnaryOp( - op=USub(), - operand=Name(id='a', ctx=Load())))], - type_ignores=[]) - - -.. class:: UnaryOp(op, operand) - - A unary operation. ``op`` is the operator, and ``operand`` any expression - node. - - -.. class:: UAdd - USub - Not - Invert - - Unary operator tokens. :class:`Not` is the ``not`` keyword, :class:`Invert` - is the ``~`` operator. - - .. doctest:: - - >>> print(ast.dump(ast.parse('not x', mode='eval'), indent=4)) - Expression( - body=UnaryOp( - op=Not(), - operand=Name(id='x', ctx=Load()))) - - -.. class:: BinOp(left, op, right) - - A binary operation (like addition or division). ``op`` is the operator, and - ``left`` and ``right`` are any expression nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) - Expression( - body=BinOp( - left=Name(id='x', ctx=Load()), - op=Add(), - right=Name(id='y', ctx=Load()))) - - -.. class:: Add - Sub - Mult - Div - FloorDiv - Mod - Pow - LShift - RShift - BitOr - BitXor - BitAnd - MatMult - - Binary operator tokens. - - -.. class:: BoolOp(op, values) - - A boolean operation, 'or' or 'and'. ``op`` is :class:`Or` or :class:`And`. - ``values`` are the values involved. Consecutive operations with the same - operator, such as ``a or b or c``, are collapsed into one node with several - values. - - This doesn't include ``not``, which is a :class:`UnaryOp`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x or y', mode='eval'), indent=4)) - Expression( - body=BoolOp( - op=Or(), - values=[ - Name(id='x', ctx=Load()), - Name(id='y', ctx=Load())])) - - -.. class:: And - Or - - Boolean operator tokens. - - -.. class:: Compare(left, ops, comparators) - - A comparison of two or more values. ``left`` is the first value in the - comparison, ``ops`` the list of operators, and ``comparators`` the list - of values after the first element in the comparison. - - .. doctest:: - - >>> print(ast.dump(ast.parse('1 <= a < 10', mode='eval'), indent=4)) - Expression( - body=Compare( - left=Constant(value=1), - ops=[ - LtE(), - Lt()], - comparators=[ - Name(id='a', ctx=Load()), - Constant(value=10)])) - - -.. class:: Eq - NotEq - Lt - LtE - Gt - GtE - Is - IsNot - In - NotIn - - Comparison operator tokens. - - -.. class:: Call(func, args, keywords, starargs, kwargs) - - A function call. ``func`` is the function, which will often be a - :class:`Name` or :class:`Attribute` object. Of the arguments: - - * ``args`` holds a list of the arguments passed by position. - * ``keywords`` holds a list of :class:`keyword` objects representing - arguments passed by keyword. - - When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. ``starargs`` and ``kwargs`` are optional. - - .. doctest:: - - >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) - Expression( - body=Call( - func=Name(id='func', ctx=Load()), - args=[ - Name(id='a', ctx=Load()), - Starred( - value=Name(id='d', ctx=Load()), - ctx=Load())], - keywords=[ - keyword( - arg='b', - value=Name(id='c', ctx=Load())), - keyword( - value=Name(id='e', ctx=Load()))])) - - -.. class:: keyword(arg, value) - - A keyword argument to a function call or class definition. ``arg`` is a raw - string of the parameter name, ``value`` is a node to pass in. - - -.. class:: IfExp(test, body, orelse) - - An expression such as ``a if b else c``. Each field holds a single node, so - in the following example, all three are :class:`Name` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) - Expression( - body=IfExp( - test=Name(id='b', ctx=Load()), - body=Name(id='a', ctx=Load()), - orelse=Name(id='c', ctx=Load()))) - - -.. class:: Attribute(value, attr, ctx) - - Attribute access, e.g. ``d.keys``. ``value`` is a node, typically a - :class:`Name`. ``attr`` is a bare string giving the name of the attribute, - and ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` according to how - the attribute is acted on. - - .. doctest:: - - >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) - Expression( - body=Attribute( - value=Name(id='snake', ctx=Load()), - attr='colour', - ctx=Load())) - - -.. class:: NamedExpr(target, value) - - A named expression. This AST node is produced by the assignment expressions - operator (also known as the walrus operator). As opposed to the :class:`Assign` - node in which the first argument can be multiple nodes, in this case both - ``target`` and ``value`` must be single nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('(x := 4)', mode='eval'), indent=4)) - Expression( - body=NamedExpr( - target=Name(id='x', ctx=Store()), - value=Constant(value=4))) - - -Subscripting -~~~~~~~~~~~~ - -.. class:: Subscript(value, slice, ctx) - - A subscript, such as ``l[1]``. ``value`` is the subscripted object - (usually sequence or mapping). ``slice`` is an index, slice or key. - It can be a :class:`Tuple` and contain a :class:`Slice`. - ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` - according to the action performed with the subscript. - - .. doctest:: - - >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) - Expression( - body=Subscript( - value=Name(id='l', ctx=Load()), - slice=Tuple( - elts=[ - Slice( - lower=Constant(value=1), - upper=Constant(value=2)), - Constant(value=3)], - ctx=Load()), - ctx=Load())) - - -.. class:: Slice(lower, upper, step) - - Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``). - Can occur only inside the *slice* field of :class:`Subscript`, either - directly or as an element of :class:`Tuple`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) - Expression( - body=Subscript( - value=Name(id='l', ctx=Load()), - slice=Slice( - lower=Constant(value=1), - upper=Constant(value=2)), - ctx=Load())) - - -Comprehensions -~~~~~~~~~~~~~~ - -.. class:: ListComp(elt, generators) - SetComp(elt, generators) - GeneratorExp(elt, generators) - DictComp(key, value, generators) - - List and set comprehensions, generator expressions, and dictionary - comprehensions. ``elt`` (or ``key`` and ``value``) is a single node - representing the part that will be evaluated for each item. - - ``generators`` is a list of :class:`comprehension` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) - Expression( - body=ListComp( - elt=Name(id='x', ctx=Load()), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) - Expression( - body=DictComp( - key=Name(id='x', ctx=Load()), - value=BinOp( - left=Name(id='x', ctx=Load()), - op=Pow(), - right=Constant(value=2)), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) - Expression( - body=SetComp( - elt=Name(id='x', ctx=Load()), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - - -.. class:: comprehension(target, iter, ifs, is_async) - - One ``for`` clause in a comprehension. ``target`` is the reference to use for - each element - typically a :class:`Name` or :class:`Tuple` node. ``iter`` - is the object to iterate over. ``ifs`` is a list of test expressions: each - ``for`` clause can have multiple ``ifs``. - - ``is_async`` indicates a comprehension is asynchronous (using an - ``async for`` instead of ``for``). The value is an integer (0 or 1). - - .. doctest:: - - >>> print(ast.dump(ast.parse('[ord(c) for line in file for c in line]', mode='eval'), - ... indent=4)) # Multiple comprehensions in one. - Expression( - body=ListComp( - elt=Call( - func=Name(id='ord', ctx=Load()), - args=[ - Name(id='c', ctx=Load())], - keywords=[]), - generators=[ - comprehension( - target=Name(id='line', ctx=Store()), - iter=Name(id='file', ctx=Load()), - ifs=[], - is_async=0), - comprehension( - target=Name(id='c', ctx=Store()), - iter=Name(id='line', ctx=Load()), - ifs=[], - is_async=0)])) - - >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), - ... indent=4)) # generator comprehension - Expression( - body=GeneratorExp( - elt=BinOp( - left=Name(id='n', ctx=Load()), - op=Pow(), - right=Constant(value=2)), - generators=[ - comprehension( - target=Name(id='n', ctx=Store()), - iter=Name(id='it', ctx=Load()), - ifs=[ - Compare( - left=Name(id='n', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=5)]), - Compare( - left=Name(id='n', ctx=Load()), - ops=[ - Lt()], - comparators=[ - Constant(value=10)])], - is_async=0)])) - - >>> print(ast.dump(ast.parse('[i async for i in soc]', mode='eval'), - ... indent=4)) # Async comprehension - Expression( - body=ListComp( - elt=Name(id='i', ctx=Load()), - generators=[ - comprehension( - target=Name(id='i', ctx=Store()), - iter=Name(id='soc', ctx=Load()), - ifs=[], - is_async=1)])) - -Statements -^^^^^^^^^^ - -.. class:: Assign(targets, value, type_comment) - - An assignment. ``targets`` is a list of nodes, and ``value`` is a single node. - - Multiple nodes in ``targets`` represents assigning the same value to each. - Unpacking is represented by putting a :class:`Tuple` or :class:`List` - within ``targets``. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a = b = 1'), indent=4)) # Multiple assignment - Module( - body=[ - Assign( - targets=[ - Name(id='a', ctx=Store()), - Name(id='b', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking - Module( - body=[ - Assign( - targets=[ - Tuple( - elts=[ - Name(id='a', ctx=Store()), - Name(id='b', ctx=Store())], - ctx=Store())], - value=Name(id='c', ctx=Load()))], - type_ignores=[]) - - -.. class:: AnnAssign(target, annotation, value, simple) - - An assignment with a type annotation. ``target`` is a single node and can - be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. - ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` - node. ``value`` is a single optional node. ``simple`` is a boolean integer - set to True for a :class:`Name` node in ``target`` that do not appear in - between parenthesis and are hence pure names and not expressions. - - .. doctest:: - - >>> print(ast.dump(ast.parse('c: int'), indent=4)) - Module( - body=[ - AnnAssign( - target=Name(id='c', ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=1)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis - Module( - body=[ - AnnAssign( - target=Name(id='a', ctx=Store()), - annotation=Name(id='int', ctx=Load()), - value=Constant(value=1), - simple=0)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation - Module( - body=[ - AnnAssign( - target=Attribute( - value=Name(id='a', ctx=Load()), - attr='b', - ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation - Module( - body=[ - AnnAssign( - target=Subscript( - value=Name(id='a', ctx=Load()), - slice=Constant(value=1), - ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) - - -.. class:: AugAssign(target, op, value) - - Augmented assignment, such as ``a += 1``. In the following example, - ``target`` is a :class:`Name` node for ``x`` (with the :class:`Store` - context), ``op`` is :class:`Add`, and ``value`` is a :class:`Constant` with - value for 1. - - The ``target`` attribute connot be of class :class:`Tuple` or :class:`List`, - unlike the targets of :class:`Assign`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x += 2'), indent=4)) - Module( - body=[ - AugAssign( - target=Name(id='x', ctx=Store()), - op=Add(), - value=Constant(value=2))], - type_ignores=[]) - - -.. class:: Raise(exc, cause) - - A ``raise`` statement. ``exc`` is the exception object to be raised, normally a - :class:`Call` or :class:`Name`, or ``None`` for a standalone ``raise``. - ``cause`` is the optional part for ``y`` in ``raise x from y``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('raise x from y'), indent=4)) - Module( - body=[ - Raise( - exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))], - type_ignores=[]) - - -.. class:: Assert(test, msg) - - An assertion. ``test`` holds the condition, such as a :class:`Compare` node. - ``msg`` holds the failure message. - - .. doctest:: - - >>> print(ast.dump(ast.parse('assert x,y'), indent=4)) - Module( - body=[ - Assert( - test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))], - type_ignores=[]) - - -.. class:: Delete(targets) - - Represents a ``del`` statement. ``targets`` is a list of nodes, such as - :class:`Name`, :class:`Attribute` or :class:`Subscript` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('del x,y,z'), indent=4)) - Module( - body=[ - Delete( - targets=[ - Name(id='x', ctx=Del()), - Name(id='y', ctx=Del()), - Name(id='z', ctx=Del())])], - type_ignores=[]) - - -.. class:: Pass() - - A ``pass`` statement. - - .. doctest:: - - >>> print(ast.dump(ast.parse('pass'), indent=4)) - Module( - body=[ - Pass()], - type_ignores=[]) - - -Other statements which are only applicable inside functions or loops are -described in other sections. - -Imports -~~~~~~~ - -.. class:: Import(names) - - An import statement. ``names`` is a list of :class:`alias` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('import x,y,z'), indent=4)) - Module( - body=[ - Import( - names=[ - alias(name='x'), - alias(name='y'), - alias(name='z')])], - type_ignores=[]) - - -.. class:: ImportFrom(module, names, level) - - Represents ``from x import y``. ``module`` is a raw string of the 'from' name, - without any leading dots, or ``None`` for statements such as ``from . import foo``. - ``level`` is an integer holding the level of the relative import (0 means - absolute import). - - .. doctest:: - - >>> print(ast.dump(ast.parse('from y import x,y,z'), indent=4)) - Module( - body=[ - ImportFrom( - module='y', - names=[ - alias(name='x'), - alias(name='y'), - alias(name='z')], - level=0)], - type_ignores=[]) - - -.. class:: alias(name, asname) - - Both parameters are raw strings of the names. ``asname`` can be ``None`` if - the regular name is to be used. - - .. doctest:: - - >>> print(ast.dump(ast.parse('from ..foo.bar import a as b, c'), indent=4)) - Module( - body=[ - ImportFrom( - module='foo.bar', - names=[ - alias(name='a', asname='b'), - alias(name='c')], - level=2)], - type_ignores=[]) - -Control flow -^^^^^^^^^^^^ - -.. note:: - Optional clauses such as ``else`` are stored as an empty list if they're - not present. - -.. class:: If(test, body, orelse) - - An ``if`` statement. ``test`` holds a single node, such as a :class:`Compare` - node. ``body`` and ``orelse`` each hold a list of nodes. - - ``elif`` clauses don't have a special representation in the AST, but rather - appear as extra :class:`If` nodes within the ``orelse`` section of the - previous one. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... if x: - ... ... - ... elif y: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - If( - test=Name(id='x', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - If( - test=Name(id='y', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -.. class:: For(target, iter, body, orelse, type_comment) - - A ``for`` loop. ``target`` holds the variable(s) the loop assigns to, as a - single :class:`Name`, :class:`Tuple` or :class:`List` node. ``iter`` holds - the item to be looped over, again as a single node. ``body`` and ``orelse`` - contain lists of nodes to execute. Those in ``orelse`` are executed if the - loop finishes normally, rather than via a ``break`` statement. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... for x in y: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - For( - target=Name(id='x', ctx=Store()), - iter=Name(id='y', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: While(test, body, orelse) - - A ``while`` loop. ``test`` holds the condition, such as a :class:`Compare` - node. - - .. doctest:: - - >> print(ast.dump(ast.parse(""" - ... while x: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - While( - test=Name(id='x', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: Break - Continue - - The ``break`` and ``continue`` statements. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... for a in b: - ... if a > 5: - ... break - ... else: - ... continue - ... - ... """), indent=4)) - Module( - body=[ - For( - target=Name(id='a', ctx=Store()), - iter=Name(id='b', ctx=Load()), - body=[ - If( - test=Compare( - left=Name(id='a', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=5)]), - body=[ - Break()], - orelse=[ - Continue()])], - orelse=[])], - type_ignores=[]) - - -.. class:: Try(body, handlers, orelse, finalbody) - - ``try`` blocks. All attributes are list of nodes to execute, except for - ``handlers``, which is a list of :class:`ExceptHandler` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... try: - ... ... - ... except Exception: - ... ... - ... except OtherException as e: - ... ... - ... else: - ... ... - ... finally: - ... ... - ... """), indent=4)) - Module( - body=[ - Try( - body=[ - Expr( - value=Constant(value=Ellipsis))], - handlers=[ - ExceptHandler( - type=Name(id='Exception', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - ExceptHandler( - type=Name(id='OtherException', ctx=Load()), - name='e', - body=[ - Expr( - value=Constant(value=Ellipsis))])], - orelse=[ - Expr( - value=Constant(value=Ellipsis))], - finalbody=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: ExceptHandler(type, name, body) - - A single ``except`` clause. ``type`` is the exception type it will match, - typically a :class:`Name` node (or ``None`` for a catch-all ``except:`` clause). - ``name`` is a raw string for the name to hold the exception, or ``None`` if - the clause doesn't have ``as foo``. ``body`` is a list of nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... try: - ... a + 1 - ... except TypeError: - ... pass - ... """), indent=4)) - Module( - body=[ - Try( - body=[ - Expr( - value=BinOp( - left=Name(id='a', ctx=Load()), - op=Add(), - right=Constant(value=1)))], - handlers=[ - ExceptHandler( - type=Name(id='TypeError', ctx=Load()), - body=[ - Pass()])], - orelse=[], - finalbody=[])], - type_ignores=[]) - - -.. class:: With(items, body, type_comment) - - A ``with`` block. ``items`` is a list of :class:`withitem` nodes representing - the context managers, and ``body`` is the indented block inside the context. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - -.. class:: withitem(context_expr, optional_vars) - - A single context manager in a ``with`` block. ``context_expr`` is the context - manager, often a :class:`Call` node. ``optional_vars`` is a :class:`Name`, - :class:`Tuple` or :class:`List` for the ``as foo`` part, or ``None`` if that - isn't used. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... with a as b, c as d: - ... something(b, d) - ... """), indent=4)) - Module( - body=[ - With( - items=[ - withitem( - context_expr=Name(id='a', ctx=Load()), - optional_vars=Name(id='b', ctx=Store())), - withitem( - context_expr=Name(id='c', ctx=Load()), - optional_vars=Name(id='d', ctx=Store()))], - body=[ - Expr( - value=Call( - func=Name(id='something', ctx=Load()), - args=[ - Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())], - keywords=[]))])], - type_ignores=[]) - - -.. class:: Match(subject, cases) - - A ``match`` statement. ``subject`` holds the subject of the match (the object - that is being matched against the cases) and ``cases`` contains an iterable of - :class:`match_case` nodes with the different cases. - - -.. class:: match_case(context_expr, optional_vars) - - A single case pattern in a ``match`` statement. ``pattern`` contains the - match pattern that will be used to match the subject against. Notice that - the meaning of the :class:`AST` nodes in this attribute have a different - meaning than in other places, as they represent patterns to match against. - The ``guard`` attribute contains an expression that will be evaluated if - the pattern matches the subject. If the pattern matches and the ``guard`` condition - is truthy, the body of the case shall be executed. ``body`` contains a list - of nodes to execute if the guard is truthy. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] if x>0: - ... ... - ... case tuple(): - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - guard=Compare( - left=Name(id='x', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=0)]), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - match_case( - pattern=Call( - func=Name(id='tuple', ctx=Load()), - args=[], - keywords=[]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchAs(pattern, name) - - A match "as-pattern". The as-pattern matches whatever pattern is on its - left-hand side, but also binds the value to a name. ``pattern`` contains - the match pattern that will be used to match the subject agsinst. The ``name`` - attribute contains the name that will be binded if the pattern is successful. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] as y: - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchAs( - pattern=List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - name='y'), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -.. class:: MatchOr(patterns) - - A match "or-pattern". An or-pattern matches each of its subpatterns in turn - to the subject, until one succeeds. The or-pattern is then deemed to - succeed. If none of the subpatterns succeed the or-pattern fails. The - ``patterns`` attribute contains a list of match patterns nodes that will be - matched against the subject. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] | (y): - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchOr( - patterns=[ - List( - elts=[ - Name(id='x', ctx=Store())], - ctx=Load()), - Name(id='y', ctx=Store())]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -Function and class definitions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment) - - A function definition. - - * ``name`` is a raw string of the function name. - * ``args`` is a :class:`arguments` node. - * ``body`` is the list of nodes inside the function. - * ``decorator_list`` is the list of decorators to be applied, stored outermost - first (i.e. the first in the list will be applied last). - * ``returns`` is the return annotation. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - -.. class:: Lambda(args, body) - - ``lambda`` is a minimal function definition that can be used inside an - expression. Unlike :class:`FunctionDef`, ``body`` holds a single node. - - .. doctest:: - - >>> print(ast.dump(ast.parse('lambda x,y: ...'), indent=4)) - Module( - body=[ - Expr( - value=Lambda( - args=arguments( - posonlyargs=[], - args=[ - arg(arg='x'), - arg(arg='y')], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=Constant(value=Ellipsis)))], - type_ignores=[]) - - -.. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) - - The arguments for a function. - - * ``posonlyargs``, ``args`` and ``kwonlyargs`` are lists of :class:`arg` nodes. - * ``vararg`` and ``kwarg`` are single :class:`arg` nodes, referring to the - ``*args, **kwargs`` parameters. - * ``kw_defaults`` is a list of default values for keyword-only arguments. If - one is ``None``, the corresponding argument is required. - * ``defaults`` is a list of default values for arguments that can be passed - positionally. If there are fewer defaults, they correspond to the last n - arguments. - - -.. class:: arg(arg, annotation, type_comment) - - A single argument in a list. ``arg`` is a raw string of the argument - name, ``annotation`` is its annotation, such as a :class:`Str` or - :class:`Name` node. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... @decorator1 - ... @decorator2 - ... def f(a: 'annotation', b=1, c=2, *d, e, f=3, **g) -> 'return annotation': - ... pass - ... """), indent=4)) - Module( - body=[ - FunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[ - arg( - arg='a', - annotation=Constant(value='annotation')), - arg(arg='b'), - arg(arg='c')], - vararg=arg(arg='d'), - kwonlyargs=[ - arg(arg='e'), - arg(arg='f')], - kw_defaults=[ - None, - Constant(value=3)], - kwarg=arg(arg='g'), - defaults=[ - Constant(value=1), - Constant(value=2)]), - body=[ - Pass()], - decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], - returns=Constant(value='return annotation'))], - type_ignores=[]) - - -.. class:: Return(value) - - A ``return`` statement. - - .. doctest:: - - >>> print(ast.dump(ast.parse('return 4'), indent=4)) - Module( - body=[ - Return( - value=Constant(value=4))], - type_ignores=[]) - - -.. class:: Yield(value) - YieldFrom(value) - - A ``yield`` or ``yield from`` expression. Because these are expressions, they - must be wrapped in a :class:`Expr` node if the value sent back is not used. - - .. doctest:: - - >>> print(ast.dump(ast.parse('yield x'), indent=4)) - Module( - body=[ - Expr( - value=Yield( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('yield from x'), indent=4)) - Module( - body=[ - Expr( - value=YieldFrom( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) - - -.. class:: Global(names) - Nonlocal(names) - - ``global`` and ``nonlocal`` statements. ``names`` is a list of raw strings. - - .. doctest:: - - >>> print(ast.dump(ast.parse('global x,y,z'), indent=4)) - Module( - body=[ - Global( - names=[ - 'x', - 'y', - 'z'])], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) - Module( - body=[ - Nonlocal( - names=[ - 'x', - 'y', - 'z'])], - type_ignores=[]) - - -.. class:: ClassDef(name, bases, keywords, starargs, kwargs, body, decorator_list) - - A class definition. - - * ``name`` is a raw string for the class name - * ``bases`` is a list of nodes for explicitly specified base classes. - * ``keywords`` is a list of :class:`keyword` nodes, principally for 'metaclass'. - Other keywords will be passed to the metaclass, as per `PEP-3115 - `_. - * ``starargs`` and ``kwargs`` are each a single node, as in a function call. - starargs will be expanded to join the list of base classes, and kwargs will - be passed to the metaclass. - * ``body`` is a list of nodes representing the code within the class - definition. - * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... @decorator1 - ... @decorator2 - ... class Foo(base1, base2, metaclass=meta): - ... pass - ... """), indent=4)) - Module( - body=[ - ClassDef( - name='Foo', - bases=[ - Name(id='base1', ctx=Load()), - Name(id='base2', ctx=Load())], - keywords=[ - keyword( - arg='metaclass', - value=Name(id='meta', ctx=Load()))], - body=[ - Pass()], - decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())])], - type_ignores=[]) - -Async and await -^^^^^^^^^^^^^^^ - -.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment) - - An ``async def`` function definition. Has the same fields as - :class:`FunctionDef`. - - -.. class:: Await(value) - - An ``await`` expression. ``value`` is what it waits for. - Only valid in the body of an :class:`AsyncFunctionDef`. - -.. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... async def f(): - ... await other_func() - ... """), indent=4)) - Module( - body=[ - AsyncFunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=[ - Expr( - value=Await( - value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[])], - type_ignores=[]) - - -.. class:: AsyncFor(target, iter, body, orelse, type_comment) - AsyncWith(items, body, type_comment) - - ``async for`` loops and ``async with`` context managers. They have the same - fields as :class:`For` and :class:`With`, respectively. Only valid in the - body of an :class:`AsyncFunctionDef`. - -.. note:: - When a string is parsed by :func:`ast.parse`, operator nodes (subclasses - of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, - :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree - will be singletons. Changes to one will be reflected in all other - occurrences of the same value (e.g. :class:`ast.Add`). - - -:mod:`ast` Helpers ------------------- - -Apart from the node classes, the :mod:`ast` module defines these utility functions -and classes for traversing abstract syntax trees: - -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) - - Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, ast.PyCF_ONLY_AST)``. - - If ``type_comments=True`` is given, the parser is modified to check - and return type comments as specified by :pep:`484` and :pep:`526`. - This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the - flags passed to :func:`compile()`. This will report syntax errors - for misplaced type comments. Without this flag, type comments will - be ignored, and the ``type_comment`` field on selected AST nodes - will always be ``None``. In addition, the locations of ``# type: - ignore`` comments will be returned as the ``type_ignores`` - attribute of :class:`Module` (otherwise it is always an empty list). - - In addition, if ``mode`` is ``'func_type'``, the input syntax is - modified to correspond to :pep:`484` "signature type comments", - e.g. ``(str, int) -> List[str]``. - - Also, setting ``feature_version`` to a tuple ``(major, minor)`` - will attempt to parse using that Python version's grammar. - Currently ``major`` must equal to ``3``. For example, setting - ``feature_version=(3, 4)`` will allow the use of ``async`` and - ``await`` as variable names. The lowest supported version is - ``(3, 4)``; the highest is ``sys.version_info[0:2]``. - - .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. - - .. versionchanged:: 3.8 - Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. - - -.. function:: unparse(ast_obj) - - Unparse an :class:`ast.AST` object and generate a string with code - that would produce an equivalent :class:`ast.AST` object if parsed - back with :func:`ast.parse`. - - .. warning:: - The produced code string will not necessarily be equal to the original - code that generated the :class:`ast.AST` object (without any compiler - optimizations, such as constant tuples/frozensets). - - .. warning:: - Trying to unparse a highly complex expression would result with - :exc:`RecursionError`. - - .. versionadded:: 3.9 - - -.. function:: literal_eval(node_or_string) - - Safely evaluate an expression node or a string containing a Python literal or - container display. The string or node provided may only consist of the - following Python literal structures: strings, bytes, numbers, tuples, lists, - dicts, sets, booleans, ``None`` and ``Ellipsis``. - - This can be used for safely evaluating strings containing Python values from - untrusted sources without the need to parse the values oneself. It is not - capable of evaluating arbitrarily complex expressions, for example involving - operators or indexing. - - .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. - - It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, - :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed - input. - - .. versionchanged:: 3.2 - Now allows bytes and set literals. - - .. versionchanged:: 3.9 - Now supports creating empty sets with ``'set()'``. - - .. versionchanged:: 3.10 - For string inputs, leading spaces and tabs are now stripped. - - -.. function:: get_docstring(node, clean=True) - - Return the docstring of the given *node* (which must be a - :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`, - or :class:`Module` node), or ``None`` if it has no docstring. - If *clean* is true, clean up the docstring's indentation with - :func:`inspect.cleandoc`. - - .. versionchanged:: 3.5 - :class:`AsyncFunctionDef` is now supported. - - -.. function:: get_source_segment(source, node, *, padded=False) - - Get source code segment of the *source* that generated *node*. - If some location information (:attr:`lineno`, :attr:`end_lineno`, - :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. - - If *padded* is ``True``, the first line of a multi-line statement will - be padded with spaces to match its original position. - - .. versionadded:: 3.8 - - -.. function:: fix_missing_locations(node) - - When you compile a node tree with :func:`compile`, the compiler expects - :attr:`lineno` and :attr:`col_offset` attributes for every node that supports - them. This is rather tedious to fill in for generated nodes, so this helper - adds these attributes recursively where not already set, by setting them to - the values of the parent node. It works recursively starting at *node*. - - -.. function:: increment_lineno(node, n=1) - - Increment the line number and end line number of each node in the tree - starting at *node* by *n*. This is useful to "move code" to a different - location in a file. - - -.. function:: copy_location(new_node, old_node) - - Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, - and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, - and return *new_node*. - - -.. function:: iter_fields(node) - - Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` - that is present on *node*. - - -.. function:: iter_child_nodes(node) - - Yield all direct child nodes of *node*, that is, all fields that are nodes - and all items of fields that are lists of nodes. - - -.. function:: walk(node) - - Recursively yield all descendant nodes in the tree starting at *node* - (including *node* itself), in no specified order. This is useful if you only - want to modify nodes in place and don't care about the context. - - -.. class:: NodeVisitor() - - A node visitor base class that walks the abstract syntax tree and calls a - visitor function for every node found. This function may return a value - which is forwarded by the :meth:`visit` method. - - This class is meant to be subclassed, with the subclass adding visitor - methods. - - .. method:: visit(node) - - Visit a node. The default implementation calls the method called - :samp:`self.visit_{classname}` where *classname* is the name of the node - class, or :meth:`generic_visit` if that method doesn't exist. - - .. method:: generic_visit(node) - - This visitor calls :meth:`visit` on all children of the node. - - Note that child nodes of nodes that have a custom visitor method won't be - visited unless the visitor calls :meth:`generic_visit` or visits them - itself. - - Don't use the :class:`NodeVisitor` if you want to apply changes to nodes - during traversal. For this a special visitor exists - (:class:`NodeTransformer`) that allows modifications. - - .. deprecated:: 3.8 - - Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, - :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated - now and will not be called in future Python versions. Add the - :meth:`visit_Constant` method to handle all constant nodes. - - -.. class:: NodeTransformer() - - A :class:`NodeVisitor` subclass that walks the abstract syntax tree and - allows modification of nodes. - - The :class:`NodeTransformer` will walk the AST and use the return value of - the visitor methods to replace or remove the old node. If the return value - of the visitor method is ``None``, the node will be removed from its - location, otherwise it is replaced with the return value. The return value - may be the original node in which case no replacement takes place. - - Here is an example transformer that rewrites all occurrences of name lookups - (``foo``) to ``data['foo']``:: - - class RewriteName(NodeTransformer): - - def visit_Name(self, node): - return Subscript( - value=Name(id='data', ctx=Load()), - slice=Constant(value=node.id), - ctx=node.ctx - ) - - Keep in mind that if the node you're operating on has child nodes you must - either transform the child nodes yourself or call the :meth:`generic_visit` - method for the node first. - - For nodes that were part of a collection of statements (that applies to all - statement nodes), the visitor may also return a list of nodes rather than - just a single node. - - If :class:`NodeTransformer` introduces new nodes (that weren't part of - original tree) without giving them location information (such as - :attr:`lineno`), :func:`fix_missing_locations` should be called with - the new sub-tree to recalculate the location information:: - - tree = ast.parse('foo', mode='eval') - new_tree = fix_missing_locations(RewriteName().visit(tree)) - - Usually you use the transformer like this:: - - node = YourTransformer().visit(node) - - -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) - - Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. If *annotate_fields* is true (by default), - the returned string will show the names and the values for fields. - If *annotate_fields* is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to true. - - If *indent* is a non-negative integer or string, then the tree will be - pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the single line representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. - - .. versionchanged:: 3.9 - Added the *indent* option. - - -.. _ast-compiler-flags: - -Compiler Flags --------------- - -The following flags may be passed to :func:`compile` in order to change -effects on the compilation of a program: - -.. data:: PyCF_ALLOW_TOP_LEVEL_AWAIT - - Enables support for top-level ``await``, ``async for``, ``async with`` - and async comprehensions. - - .. versionadded:: 3.8 - -.. data:: PyCF_ONLY_AST - - Generates and returns an abstract syntax tree instead of returning a - compiled code object. - -.. data:: PyCF_TYPE_COMMENTS - - Enables support for :pep:`484` and :pep:`526` style type comments - (``# type: ``, ``# type: ignore ``). - - .. versionadded:: 3.8 - - -.. _ast-cli: - -Command-Line Usage ------------------- - -.. versionadded:: 3.9 - -The :mod:`ast` module can be executed as a script from the command line. -It is as simple as: - -.. code-block:: sh - - python -m ast [-m ] [-a] [infile] - -The following options are accepted: - -.. program:: ast - -.. cmdoption:: -h, --help - - Show the help message and exit. - -.. cmdoption:: -m - --mode - - Specify what kind of code must be compiled, like the *mode* argument - in :func:`parse`. - -.. cmdoption:: --no-type-comments - - Don't parse type comments. - -.. cmdoption:: -a, --include-attributes - - Include attributes such as line numbers and column offsets. - -.. cmdoption:: -i - --indent - - Indentation of nodes in AST (number of spaces). - -If :file:`infile` is specified its contents are parsed to AST and dumped -to stdout. Otherwise, the content is read from stdin. - - -.. seealso:: - - `Green Tree Snakes `_, an external - documentation resource, has good details on working with Python ASTs. - - `ASTTokens `_ - annotates Python ASTs with the positions of tokens and text in the source - code that generated them. This is helpful for tools that make source code - transformations. - - `leoAst.py `_ unifies the - token-based and parse-tree-based views of python programs by inserting - two-way links between tokens and ast nodes. - - `LibCST `_ parses code as a Concrete Syntax - Tree that looks like an ast tree and keeps all formatting details. It's - useful for building automated refactoring (codemod) applications and - linters. - - `Parso `_ is a Python parser that supports - error recovery and round-trip parsing for different Python versions (in - multiple Python versions). Parso is also able to list multiple syntax errors - in your python file. diff --git a/Doc/library/asyncio-api-index.rst.bak b/Doc/library/asyncio-api-index.rst.bak deleted file mode 100644 index 047e5bbc58ccad..00000000000000 --- a/Doc/library/asyncio-api-index.rst.bak +++ /dev/null @@ -1,221 +0,0 @@ -.. currentmodule:: asyncio - - -==================== -High-level API Index -==================== - -This page lists all high-level async/await enabled asyncio APIs. - - -Tasks -===== - -Utilities to run asyncio programs, create Tasks, and -await on multiple things with timeouts. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :func:`run` - - Create event loop, run a coroutine, close the loop. - - * - :func:`create_task` - - Start an asyncio Task. - - * - ``await`` :func:`sleep` - - Sleep for a number of seconds. - - * - ``await`` :func:`gather` - - Schedule and wait for things concurrently. - - * - ``await`` :func:`wait_for` - - Run with a timeout. - - * - ``await`` :func:`shield` - - Shield from cancellation. - - * - ``await`` :func:`wait` - - Monitor for completion. - - * - :func:`current_task` - - Return the current Task. - - * - :func:`all_tasks` - - Return all tasks for an event loop. - - * - :class:`Task` - - Task object. - - * - :func:`to_thread` - - Asychronously run a function in a separate OS thread. - - * - :func:`run_coroutine_threadsafe` - - Schedule a coroutine from another OS thread. - - * - ``for in`` :func:`as_completed` - - Monitor for completion with a ``for`` loop. - - -.. rubric:: Examples - -* :ref:`Using asyncio.gather() to run things in parallel - `. - -* :ref:`Using asyncio.wait_for() to enforce a timeout - `. - -* :ref:`Cancellation `. - -* :ref:`Using asyncio.sleep() `. - -* See also the main :ref:`Tasks documentation page `. - - -Queues -====== - -Queues should be used to distribute work amongst multiple asyncio Tasks, -implement connection pools, and pub/sub patterns. - - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :class:`Queue` - - A FIFO queue. - - * - :class:`PriorityQueue` - - A priority queue. - - * - :class:`LifoQueue` - - A LIFO queue. - - -.. rubric:: Examples - -* :ref:`Using asyncio.Queue to distribute workload between several - Tasks `. - -* See also the :ref:`Queues documentation page `. - - -Subprocesses -============ - -Utilities to spawn subprocesses and run shell commands. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :func:`create_subprocess_exec` - - Create a subprocess. - - * - ``await`` :func:`create_subprocess_shell` - - Run a shell command. - - -.. rubric:: Examples - -* :ref:`Executing a shell command `. - -* See also the :ref:`subprocess APIs ` - documentation. - - -Streams -======= - -High-level APIs to work with network IO. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :func:`open_connection` - - Establish a TCP connection. - - * - ``await`` :func:`open_unix_connection` - - Establish a Unix socket connection. - - * - ``await`` :func:`start_server` - - Start a TCP server. - - * - ``await`` :func:`start_unix_server` - - Start a Unix socket server. - - * - :class:`StreamReader` - - High-level async/await object to receive network data. - - * - :class:`StreamWriter` - - High-level async/await object to send network data. - - -.. rubric:: Examples - -* :ref:`Example TCP client `. - -* See also the :ref:`streams APIs ` - documentation. - - -Synchronization -=============== - -Threading-like synchronization primitives that can be used in Tasks. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :class:`Lock` - - A mutex lock. - - * - :class:`Event` - - An event object. - - * - :class:`Condition` - - A condition object. - - * - :class:`Semaphore` - - A semaphore. - - * - :class:`BoundedSemaphore` - - A bounded semaphore. - - -.. rubric:: Examples - -* :ref:`Using asyncio.Event `. - -* See also the documentation of asyncio - :ref:`synchronization primitives `. - - -Exceptions -========== - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - - * - :exc:`asyncio.TimeoutError` - - Raised on timeout by functions like :func:`wait_for`. - Keep in mind that ``asyncio.TimeoutError`` is **unrelated** - to the built-in :exc:`TimeoutError` exception. - - * - :exc:`asyncio.CancelledError` - - Raised when a Task is cancelled. See also :meth:`Task.cancel`. - - -.. rubric:: Examples - -* :ref:`Handling CancelledError to run code on cancellation request - `. - -* See also the full list of - :ref:`asyncio-specific exceptions `. diff --git a/Doc/library/collections.rst.bak b/Doc/library/collections.rst.bak deleted file mode 100644 index aa0acfaae45a5f..00000000000000 --- a/Doc/library/collections.rst.bak +++ /dev/null @@ -1,1278 +0,0 @@ -:mod:`collections` --- Container datatypes -========================================== - -.. module:: collections - :synopsis: Container datatypes - -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - -**Source code:** :source:`Lib/collections/__init__.py` - -.. testsetup:: * - - from collections import * - import itertools - __name__ = '' - --------------- - -This module implements specialized container datatypes providing alternatives to -Python's general purpose built-in containers, :class:`dict`, :class:`list`, -:class:`set`, and :class:`tuple`. - -===================== ==================================================================== -:func:`namedtuple` factory function for creating tuple subclasses with named fields -:class:`deque` list-like container with fast appends and pops on either end -:class:`ChainMap` dict-like class for creating a single view of multiple mappings -:class:`Counter` dict subclass for counting hashable objects -:class:`OrderedDict` dict subclass that remembers the order entries were added -:class:`defaultdict` dict subclass that calls a factory function to supply missing values -:class:`UserDict` wrapper around dictionary objects for easier dict subclassing -:class:`UserList` wrapper around list objects for easier list subclassing -:class:`UserString` wrapper around string objects for easier string subclassing -===================== ==================================================================== - - -:class:`ChainMap` objects -------------------------- - -.. versionadded:: 3.3 - -A :class:`ChainMap` class is provided for quickly linking a number of mappings -so they can be treated as a single unit. It is often much faster than creating -a new dictionary and running multiple :meth:`~dict.update` calls. - -The class can be used to simulate nested scopes and is useful in templating. - -.. class:: ChainMap(*maps) - - A :class:`ChainMap` groups multiple dicts or other mappings together to - create a single, updateable view. If no *maps* are specified, a single empty - dictionary is provided so that a new chain always has at least one mapping. - - The underlying mappings are stored in a list. That list is public and can - be accessed or updated using the *maps* attribute. There is no other state. - - Lookups search the underlying mappings successively until a key is found. In - contrast, writes, updates, and deletions only operate on the first mapping. - - A :class:`ChainMap` incorporates the underlying mappings by reference. So, if - one of the underlying mappings gets updated, those changes will be reflected - in :class:`ChainMap`. - - All of the usual dictionary methods are supported. In addition, there is a - *maps* attribute, a method for creating new subcontexts, and a property for - accessing all but the first mapping: - - .. attribute:: maps - - A user updateable list of mappings. The list is ordered from - first-searched to last-searched. It is the only stored state and can - be modified to change which mappings are searched. The list should - always contain at least one mapping. - - .. method:: new_child(m=None) - - Returns a new :class:`ChainMap` containing a new map followed by - all of the maps in the current instance. If ``m`` is specified, - it becomes the new map at the front of the list of mappings; if not - specified, an empty dict is used, so that a call to ``d.new_child()`` - is equivalent to: ``ChainMap({}, *d.maps)``. This method is used for - creating subcontexts that can be updated without altering values in any - of the parent mappings. - - .. versionchanged:: 3.4 - The optional ``m`` parameter was added. - - .. attribute:: parents - - Property returning a new :class:`ChainMap` containing all of the maps in - the current instance except the first one. This is useful for skipping - the first map in the search. Use cases are similar to those for the - :keyword:`nonlocal` keyword used in :term:`nested scopes `. The use cases also parallel those for the built-in - :func:`super` function. A reference to ``d.parents`` is equivalent to: - ``ChainMap(*d.maps[1:])``. - - Note, the iteration order of a :class:`ChainMap()` is determined by - scanning the mappings last to first:: - - >>> baseline = {'music': 'bach', 'art': 'rembrandt'} - >>> adjustments = {'art': 'van gogh', 'opera': 'carmen'} - >>> list(ChainMap(adjustments, baseline)) - ['music', 'art', 'opera'] - - This gives the same ordering as a series of :meth:`dict.update` calls - starting with the last mapping:: - - >>> combined = baseline.copy() - >>> combined.update(adjustments) - >>> list(combined) - ['music', 'art', 'opera'] - - .. versionchanged:: 3.9 - Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. - -.. seealso:: - - * The `MultiContext class - `_ - in the Enthought `CodeTools package - `_ has options to support - writing to any mapping in the chain. - - * Django's `Context class - `_ - for templating is a read-only chain of mappings. It also features - pushing and popping of contexts similar to the - :meth:`~collections.ChainMap.new_child` method and the - :attr:`~collections.ChainMap.parents` property. - - * The `Nested Contexts recipe - `_ has options to control - whether writes and other mutations apply only to the first mapping or to - any mapping in the chain. - - * A `greatly simplified read-only version of Chainmap - `_. - - -:class:`ChainMap` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with chained maps. - - -Example of simulating Python's internal lookup chain:: - - import builtins - pylookup = ChainMap(locals(), globals(), vars(builtins)) - -Example of letting user specified command-line arguments take precedence over -environment variables which in turn take precedence over default values:: - - import os, argparse - - defaults = {'color': 'red', 'user': 'guest'} - - parser = argparse.ArgumentParser() - parser.add_argument('-u', '--user') - parser.add_argument('-c', '--color') - namespace = parser.parse_args() - command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} - - combined = ChainMap(command_line_args, os.environ, defaults) - print(combined['color']) - print(combined['user']) - -Example patterns for using the :class:`ChainMap` class to simulate nested -contexts:: - - c = ChainMap() # Create root context - d = c.new_child() # Create nested child context - e = c.new_child() # Child of c, independent from d - e.maps[0] # Current context dictionary -- like Python's locals() - e.maps[-1] # Root context -- like Python's globals() - e.parents # Enclosing context chain -- like Python's nonlocals - - d['x'] = 1 # Set value in current context - d['x'] # Get first key in the chain of contexts - del d['x'] # Delete from current context - list(d) # All nested values - k in d # Check all nested values - len(d) # Number of nested values - d.items() # All nested items - dict(d) # Flatten into a regular dictionary - -The :class:`ChainMap` class only makes updates (writes and deletions) to the -first mapping in the chain while lookups will search the full chain. However, -if deep writes and deletions are desired, it is easy to make a subclass that -updates keys found deeper in the chain:: - - class DeepChainMap(ChainMap): - 'Variant of ChainMap that allows direct updates to inner scopes' - - def __setitem__(self, key, value): - for mapping in self.maps: - if key in mapping: - mapping[key] = value - return - self.maps[0][key] = value - - def __delitem__(self, key): - for mapping in self.maps: - if key in mapping: - del mapping[key] - return - raise KeyError(key) - - >>> d = DeepChainMap({'zebra': 'black'}, {'elephant': 'blue'}, {'lion': 'yellow'}) - >>> d['lion'] = 'orange' # update an existing key two levels down - >>> d['snake'] = 'red' # new keys get added to the topmost dict - >>> del d['elephant'] # remove an existing key one level down - >>> d # display result - DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) - - -:class:`Counter` objects ------------------------- - -A counter tool is provided to support convenient and rapid tallies. -For example:: - - >>> # Tally occurrences of words in a list - >>> cnt = Counter() - >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: - ... cnt[word] += 1 - >>> cnt - Counter({'blue': 3, 'red': 2, 'green': 1}) - - >>> # Find the ten most common words in Hamlet - >>> import re - >>> words = re.findall(r'\w+', open('hamlet.txt').read().lower()) - >>> Counter(words).most_common(10) - [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), - ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] - -.. class:: Counter([iterable-or-mapping]) - - A :class:`Counter` is a :class:`dict` subclass for counting hashable objects. - It is a collection where elements are stored as dictionary keys - and their counts are stored as dictionary values. Counts are allowed to be - any integer value including zero or negative counts. The :class:`Counter` - class is similar to bags or multisets in other languages. - - Elements are counted from an *iterable* or initialized from another - *mapping* (or counter): - - >>> c = Counter() # a new, empty counter - >>> c = Counter('gallahad') # a new counter from an iterable - >>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping - >>> c = Counter(cats=4, dogs=8) # a new counter from keyword args - - Counter objects have a dictionary interface except that they return a zero - count for missing items instead of raising a :exc:`KeyError`: - - >>> c = Counter(['eggs', 'ham']) - >>> c['bacon'] # count of a missing element is zero - 0 - - Setting a count to zero does not remove an element from a counter. - Use ``del`` to remove it entirely: - - >>> c['sausage'] = 0 # counter entry with a zero count - >>> del c['sausage'] # del actually removes the entry - - .. versionadded:: 3.1 - - .. versionchanged:: 3.7 As a :class:`dict` subclass, :class:`Counter` - Inherited the capability to remember insertion order. Math operations - on *Counter* objects also preserve order. Results are ordered - according to when an element is first encountered in the left operand - and then by the order encountered in the right operand. - - Counter objects support three methods beyond those available for all - dictionaries: - - .. method:: elements() - - Return an iterator over elements repeating each as many times as its - count. Elements are returned in the order first encountered. If an - element's count is less than one, :meth:`elements` will ignore it. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> sorted(c.elements()) - ['a', 'a', 'a', 'a', 'b', 'b'] - - .. method:: most_common([n]) - - Return a list of the *n* most common elements and their counts from the - most common to the least. If *n* is omitted or ``None``, - :meth:`most_common` returns *all* elements in the counter. - Elements with equal counts are ordered in the order first encountered: - - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('b', 2), ('r', 2)] - - .. method:: subtract([iterable-or-mapping]) - - Elements are subtracted from an *iterable* or from another *mapping* - (or counter). Like :meth:`dict.update` but subtracts counts instead - of replacing them. Both inputs and outputs may be zero or negative. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> d = Counter(a=1, b=2, c=3, d=4) - >>> c.subtract(d) - >>> c - Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6}) - - .. versionadded:: 3.2 - - The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. - - .. method:: fromkeys(iterable) - - This class method is not implemented for :class:`Counter` objects. - - .. method:: update([iterable-or-mapping]) - - Elements are counted from an *iterable* or added-in from another - *mapping* (or counter). Like :meth:`dict.update` but adds counts - instead of replacing them. Also, the *iterable* is expected to be a - sequence of elements, not a sequence of ``(key, value)`` pairs. - -Counters support rich comparison operators for equality, subset, and -superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. -All of those tests treat missing elements as having zero counts so that -``Counter(a=1) == Counter(a=1, b=0)`` returns true. - -.. versionadded:: 3.10 - Rich comparison operations we were added - -.. versionchanged:: 3.10 - In equality tests, missing elements are treated as having zero counts. - Formerly, ``Counter(a=3)`` and ``Counter(a=3, b=0)`` were considered - distinct. - -Common patterns for working with :class:`Counter` objects:: - - sum(c.values()) # total of all counts - c.clear() # reset all counts - list(c) # list unique elements - set(c) # convert to a set - dict(c) # convert to a regular dictionary - c.items() # convert to a list of (elem, cnt) pairs - Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs - c.most_common()[:-n-1:-1] # n least common elements - +c # remove zero and negative counts - -Several mathematical operations are provided for combining :class:`Counter` -objects to produce multisets (counters that have counts greater than zero). -Addition and subtraction combine counters by adding or subtracting the counts -of corresponding elements. Intersection and union return the minimum and -maximum of corresponding counts. Each operation can accept inputs with signed -counts, but the output will exclude results with counts of zero or less. - - >>> c = Counter(a=3, b=1) - >>> d = Counter(a=1, b=2) - >>> c + d # add two counters together: c[x] + d[x] - Counter({'a': 4, 'b': 3}) - >>> c - d # subtract (keeping only positive counts) - Counter({'a': 2}) - >>> c & d # intersection: min(c[x], d[x]) # doctest: +SKIP - Counter({'a': 1, 'b': 1}) - >>> c | d # union: max(c[x], d[x]) - Counter({'a': 3, 'b': 2}) - -Unary addition and subtraction are shortcuts for adding an empty counter -or subtracting from an empty counter. - - >>> c = Counter(a=2, b=-4) - >>> +c - Counter({'a': 2}) - >>> -c - Counter({'b': 4}) - -.. versionadded:: 3.3 - Added support for unary plus, unary minus, and in-place multiset operations. - -.. note:: - - Counters were primarily designed to work with positive integers to represent - running counts; however, care was taken to not unnecessarily preclude use - cases needing other types or negative values. To help with those use cases, - this section documents the minimum range and type restrictions. - - * The :class:`Counter` class itself is a dictionary subclass with no - restrictions on its keys and values. The values are intended to be numbers - representing counts, but you *could* store anything in the value field. - - * The :meth:`~Counter.most_common` method requires only that the values be orderable. - - * For in-place operations such as ``c[key] += 1``, the value type need only - support addition and subtraction. So fractions, floats, and decimals would - work and negative values are supported. The same is also true for - :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values - for both inputs and outputs. - - * The multiset methods are designed only for use cases with positive values. - The inputs may be negative or zero, but only outputs with positive values - are created. There are no type restrictions, but the value type needs to - support addition, subtraction, and comparison. - - * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and - negative counts. - -.. seealso:: - - * `Bag class `_ - in Smalltalk. - - * Wikipedia entry for `Multisets `_. - - * `C++ multisets `_ - tutorial with examples. - - * For mathematical operations on multisets and their use cases, see - *Knuth, Donald. The Art of Computer Programming Volume II, - Section 4.6.3, Exercise 19*. - - * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`:: - - map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC - - -:class:`deque` objects ----------------------- - -.. class:: deque([iterable, [maxlen]]) - - Returns a new deque object initialized left-to-right (using :meth:`append`) with - data from *iterable*. If *iterable* is not specified, the new deque is empty. - - Deques are a generalization of stacks and queues (the name is pronounced "deck" - and is short for "double-ended queue"). Deques support thread-safe, memory - efficient appends and pops from either side of the deque with approximately the - same O(1) performance in either direction. - - Though :class:`list` objects support similar operations, they are optimized for - fast fixed-length operations and incur O(n) memory movement costs for - ``pop(0)`` and ``insert(0, v)`` operations which change both the size and - position of the underlying data representation. - - - If *maxlen* is not specified or is ``None``, deques may grow to an - arbitrary length. Otherwise, the deque is bounded to the specified maximum - length. Once a bounded length deque is full, when new items are added, a - corresponding number of items are discarded from the opposite end. Bounded - length deques provide functionality similar to the ``tail`` filter in - Unix. They are also useful for tracking transactions and other pools of data - where only the most recent activity is of interest. - - - Deque objects support the following methods: - - .. method:: append(x) - - Add *x* to the right side of the deque. - - - .. method:: appendleft(x) - - Add *x* to the left side of the deque. - - - .. method:: clear() - - Remove all elements from the deque leaving it with length 0. - - - .. method:: copy() - - Create a shallow copy of the deque. - - .. versionadded:: 3.5 - - - .. method:: count(x) - - Count the number of deque elements equal to *x*. - - .. versionadded:: 3.2 - - - .. method:: extend(iterable) - - Extend the right side of the deque by appending elements from the iterable - argument. - - - .. method:: extendleft(iterable) - - Extend the left side of the deque by appending elements from *iterable*. - Note, the series of left appends results in reversing the order of - elements in the iterable argument. - - - .. method:: index(x[, start[, stop]]) - - Return the position of *x* in the deque (at or after index *start* - and before index *stop*). Returns the first match or raises - :exc:`ValueError` if not found. - - .. versionadded:: 3.5 - - - .. method:: insert(i, x) - - Insert *x* into the deque at position *i*. - - If the insertion would cause a bounded deque to grow beyond *maxlen*, - an :exc:`IndexError` is raised. - - .. versionadded:: 3.5 - - - .. method:: pop() - - Remove and return an element from the right side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: popleft() - - Remove and return an element from the left side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: remove(value) - - Remove the first occurrence of *value*. If not found, raises a - :exc:`ValueError`. - - - .. method:: reverse() - - Reverse the elements of the deque in-place and then return ``None``. - - .. versionadded:: 3.2 - - - .. method:: rotate(n=1) - - Rotate the deque *n* steps to the right. If *n* is negative, rotate - to the left. - - When the deque is not empty, rotating one step to the right is equivalent - to ``d.appendleft(d.pop())``, and rotating one step to the left is - equivalent to ``d.append(d.popleft())``. - - - Deque objects also provide one read-only attribute: - - .. attribute:: maxlen - - Maximum size of a deque or ``None`` if unbounded. - - .. versionadded:: 3.1 - - -In addition to the above, deques support iteration, pickling, ``len(d)``, -``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with -the :keyword:`in` operator, and subscript references such as ``d[0]`` to access -the first element. Indexed access is O(1) at both ends but slows to O(n) in -the middle. For fast random access, use lists instead. - -Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, -and ``__imul__()``. - -Example: - -.. doctest:: - - >>> from collections import deque - >>> d = deque('ghi') # make a new deque with three items - >>> for elem in d: # iterate over the deque's elements - ... print(elem.upper()) - G - H - I - - >>> d.append('j') # add a new entry to the right side - >>> d.appendleft('f') # add a new entry to the left side - >>> d # show the representation of the deque - deque(['f', 'g', 'h', 'i', 'j']) - - >>> d.pop() # return and remove the rightmost item - 'j' - >>> d.popleft() # return and remove the leftmost item - 'f' - >>> list(d) # list the contents of the deque - ['g', 'h', 'i'] - >>> d[0] # peek at leftmost item - 'g' - >>> d[-1] # peek at rightmost item - 'i' - - >>> list(reversed(d)) # list the contents of a deque in reverse - ['i', 'h', 'g'] - >>> 'h' in d # search the deque - True - >>> d.extend('jkl') # add multiple elements at once - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - >>> d.rotate(1) # right rotation - >>> d - deque(['l', 'g', 'h', 'i', 'j', 'k']) - >>> d.rotate(-1) # left rotation - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - - >>> deque(reversed(d)) # make a new deque in reverse order - deque(['l', 'k', 'j', 'i', 'h', 'g']) - >>> d.clear() # empty the deque - >>> d.pop() # cannot pop from an empty deque - Traceback (most recent call last): - File "", line 1, in -toplevel- - d.pop() - IndexError: pop from an empty deque - - >>> d.extendleft('abc') # extendleft() reverses the input order - >>> d - deque(['c', 'b', 'a']) - - -:class:`deque` Recipes -^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with deques. - -Bounded length deques provide functionality similar to the ``tail`` filter -in Unix:: - - def tail(filename, n=10): - 'Return the last n lines of a file' - with open(filename) as f: - return deque(f, n) - -Another approach to using deques is to maintain a sequence of recently -added elements by appending to the right and popping to the left:: - - def moving_average(iterable, n=3): - # moving_average([40, 30, 50, 46, 39, 44]) --> 40.0 42.0 45.0 43.0 - # http://en.wikipedia.org/wiki/Moving_average - it = iter(iterable) - d = deque(itertools.islice(it, n-1)) - d.appendleft(0) - s = sum(d) - for elem in it: - s += elem - d.popleft() - d.append(elem) - yield s / n - -A `round-robin scheduler -`_ can be implemented with -input iterators stored in a :class:`deque`. Values are yielded from the active -iterator in position zero. If that iterator is exhausted, it can be removed -with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with -the :meth:`~deque.rotate` method:: - - def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" - iterators = deque(map(iter, iterables)) - while iterators: - try: - while True: - yield next(iterators[0]) - iterators.rotate(-1) - except StopIteration: - # Remove an exhausted iterator. - iterators.popleft() - -The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and -deletion. For example, a pure Python implementation of ``del d[n]`` relies on -the ``rotate()`` method to position elements to be popped:: - - def delete_nth(d, n): - d.rotate(-n) - d.popleft() - d.rotate(n) - -To implement :class:`deque` slicing, use a similar approach applying -:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove -old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then -reverse the rotation. -With minor variations on that approach, it is easy to implement Forth style -stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, -``rot``, and ``roll``. - - -:class:`defaultdict` objects ----------------------------- - -.. class:: defaultdict([default_factory[, ...]]) - - Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the - built-in :class:`dict` class. It overrides one method and adds one writable - instance variable. The remaining functionality is the same as for the - :class:`dict` class and is not documented here. - - The first argument provides the initial value for the :attr:`default_factory` - attribute; it defaults to ``None``. All remaining arguments are treated the same - as if they were passed to the :class:`dict` constructor, including keyword - arguments. - - - :class:`defaultdict` objects support the following method in addition to the - standard :class:`dict` operations: - - .. method:: __missing__(key) - - If the :attr:`default_factory` attribute is ``None``, this raises a - :exc:`KeyError` exception with the *key* as argument. - - If :attr:`default_factory` is not ``None``, it is called without arguments - to provide a default value for the given *key*, this value is inserted in - the dictionary for the *key*, and returned. - - If calling :attr:`default_factory` raises an exception this exception is - propagated unchanged. - - This method is called by the :meth:`__getitem__` method of the - :class:`dict` class when the requested key is not found; whatever it - returns or raises is then returned or raised by :meth:`__getitem__`. - - Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`__getitem__`. This means that :meth:`get` will, like normal - dictionaries, return ``None`` as a default rather than using - :attr:`default_factory`. - - - :class:`defaultdict` objects support the following instance variable: - - - .. attribute:: default_factory - - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. - - .. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in - :pep:`584`. - - -:class:`defaultdict` Examples -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a -sequence of key-value pairs into a dictionary of lists: - - >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] - >>> d = defaultdict(list) - >>> for k, v in s: - ... d[k].append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -When each key is encountered for the first time, it is not already in the -mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`list.append` -operation then attaches the value to the new list. When keys are encountered -again, the look-up proceeds normally (returning the list for that key) and the -:meth:`list.append` operation adds another value to the list. This technique is -simpler and faster than an equivalent technique using :meth:`dict.setdefault`: - - >>> d = {} - >>> for k, v in s: - ... d.setdefault(k, []).append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the -:class:`defaultdict` useful for counting (like a bag or multiset in other -languages): - - >>> s = 'mississippi' - >>> d = defaultdict(int) - >>> for k in s: - ... d[k] += 1 - ... - >>> sorted(d.items()) - [('i', 4), ('m', 1), ('p', 2), ('s', 4)] - -When a letter is first encountered, it is missing from the mapping, so the -:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of -zero. The increment operation then builds up the count for each letter. - -The function :func:`int` which always returns zero is just a special case of -constant functions. A faster and more flexible way to create constant functions -is to use a lambda function which can supply any constant value (not just -zero): - - >>> def constant_factory(value): - ... return lambda: value - >>> d = defaultdict(constant_factory('')) - >>> d.update(name='John', action='ran') - >>> '%(name)s %(action)s to %(object)s' % d - 'John ran to ' - -Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the -:class:`defaultdict` useful for building a dictionary of sets: - - >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] - >>> d = defaultdict(set) - >>> for k, v in s: - ... d[k].add(v) - ... - >>> sorted(d.items()) - [('blue', {2, 4}), ('red', {1, 3})] - - -:func:`namedtuple` Factory Function for Tuples with Named Fields ----------------------------------------------------------------- - -Named tuples assign meaning to each position in a tuple and allow for more readable, -self-documenting code. They can be used wherever regular tuples are used, and -they add the ability to access fields by name instead of position index. - -.. function:: namedtuple(typename, field_names, *, rename=False, defaults=None, module=None) - - Returns a new tuple subclass named *typename*. The new subclass is used to - create tuple-like objects that have fields accessible by attribute lookup as - well as being indexable and iterable. Instances of the subclass also have a - helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` - method which lists the tuple contents in a ``name=value`` format. - - The *field_names* are a sequence of strings such as ``['x', 'y']``. - Alternatively, *field_names* can be a single string with each fieldname - separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. - - Any valid Python identifier may be used for a fieldname except for names - starting with an underscore. Valid identifiers consist of letters, digits, - and underscores but do not start with a digit or underscore and cannot be - a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, - or *raise*. - - If *rename* is true, invalid fieldnames are automatically replaced - with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is - converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword - ``def`` and the duplicate fieldname ``abc``. - - *defaults* can be ``None`` or an :term:`iterable` of default values. - Since fields with a default value must come after any fields without a - default, the *defaults* are applied to the rightmost parameters. For - example, if the fieldnames are ``['x', 'y', 'z']`` and the defaults are - ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to - ``1``, and ``z`` will default to ``2``. - - If *module* is defined, the ``__module__`` attribute of the named tuple is - set to that value. - - Named tuple instances do not have per-instance dictionaries, so they are - lightweight and require no more memory than regular tuples. - - To support pickling, the named tuple class should be assigned to a variable - that matches *typename*. - - .. versionchanged:: 3.1 - Added support for *rename*. - - .. versionchanged:: 3.6 - The *verbose* and *rename* parameters became - :ref:`keyword-only arguments `. - - .. versionchanged:: 3.6 - Added the *module* parameter. - - .. versionchanged:: 3.7 - Removed the *verbose* parameter and the :attr:`_source` attribute. - - .. versionchanged:: 3.7 - Added the *defaults* parameter and the :attr:`_field_defaults` - attribute. - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> # Basic example - >>> Point = namedtuple('Point', ['x', 'y']) - >>> p = Point(11, y=22) # instantiate with positional or keyword arguments - >>> p[0] + p[1] # indexable like the plain tuple (11, 22) - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessible by name - 33 - >>> p # readable __repr__ with a name=value style - Point(x=11, y=22) - -Named tuples are especially useful for assigning field names to result tuples returned -by the :mod:`csv` or :mod:`sqlite3` modules:: - - EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') - - import csv - for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): - print(emp.name, emp.title) - - import sqlite3 - conn = sqlite3.connect('/companydata') - cursor = conn.cursor() - cursor.execute('SELECT name, age, title, department, paygrade FROM employees') - for emp in map(EmployeeRecord._make, cursor.fetchall()): - print(emp.name, emp.title) - -In addition to the methods inherited from tuples, named tuples support -three additional methods and two attributes. To prevent conflicts with -field names, the method and attribute names start with an underscore. - -.. classmethod:: somenamedtuple._make(iterable) - - Class method that makes a new instance from an existing sequence or iterable. - - .. doctest:: - - >>> t = [11, 22] - >>> Point._make(t) - Point(x=11, y=22) - -.. method:: somenamedtuple._asdict() - - Return a new :class:`dict` which maps field names to their corresponding - values: - - .. doctest:: - - >>> p = Point(x=11, y=22) - >>> p._asdict() - {'x': 11, 'y': 22} - - .. versionchanged:: 3.1 - Returns an :class:`OrderedDict` instead of a regular :class:`dict`. - - .. versionchanged:: 3.8 - Returns a regular :class:`dict` instead of an :class:`OrderedDict`. - As of Python 3.7, regular dicts are guaranteed to be ordered. If the - extra features of :class:`OrderedDict` are required, the suggested - remediation is to cast the result to the desired type: - ``OrderedDict(nt._asdict())``. - -.. method:: somenamedtuple._replace(**kwargs) - - Return a new instance of the named tuple replacing specified fields with new - values:: - - >>> p = Point(x=11, y=22) - >>> p._replace(x=33) - Point(x=33, y=22) - - >>> for partnum, record in inventory.items(): - ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) - -.. attribute:: somenamedtuple._fields - - Tuple of strings listing the field names. Useful for introspection - and for creating new named tuple types from existing named tuples. - - .. doctest:: - - >>> p._fields # view the field names - ('x', 'y') - - >>> Color = namedtuple('Color', 'red green blue') - >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) - >>> Pixel(11, 22, 128, 255, 0) - Pixel(x=11, y=22, red=128, green=255, blue=0) - -.. attribute:: somenamedtuple._field_defaults - - Dictionary mapping field names to default values. - - .. doctest:: - - >>> Account = namedtuple('Account', ['type', 'balance'], defaults=[0]) - >>> Account._field_defaults - {'balance': 0} - >>> Account('premium') - Account(type='premium', balance=0) - -To retrieve a field whose name is stored in a string, use the :func:`getattr` -function: - - >>> getattr(p, 'x') - 11 - -To convert a dictionary to a named tuple, use the double-star-operator -(as described in :ref:`tut-unpacking-arguments`): - - >>> d = {'x': 11, 'y': 22} - >>> Point(**d) - Point(x=11, y=22) - -Since a named tuple is a regular Python class, it is easy to add or change -functionality with a subclass. Here is how to add a calculated field and -a fixed-width print format: - -.. doctest:: - - >>> class Point(namedtuple('Point', ['x', 'y'])): - ... __slots__ = () - ... @property - ... def hypot(self): - ... return (self.x ** 2 + self.y ** 2) ** 0.5 - ... def __str__(self): - ... return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) - - >>> for p in Point(3, 4), Point(14, 5/7): - ... print(p) - Point: x= 3.000 y= 4.000 hypot= 5.000 - Point: x=14.000 y= 0.714 hypot=14.018 - -The subclass shown above sets ``__slots__`` to an empty tuple. This helps -keep memory requirements low by preventing the creation of instance dictionaries. - -Subclassing is not useful for adding new, stored fields. Instead, simply -create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: - - >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) - -Docstrings can be customized by making direct assignments to the ``__doc__`` -fields: - - >>> Book = namedtuple('Book', ['id', 'title', 'authors']) - >>> Book.__doc__ += ': Hardcover book in active collection' - >>> Book.id.__doc__ = '13-digit ISBN' - >>> Book.title.__doc__ = 'Title of first printing' - >>> Book.authors.__doc__ = 'List of authors sorted by last name' - -.. versionchanged:: 3.5 - Property docstrings became writeable. - -.. seealso:: - - * See :class:`typing.NamedTuple` for a way to add type hints for named - tuples. It also provides an elegant notation using the :keyword:`class` - keyword:: - - class Component(NamedTuple): - part_number: int - weight: float - description: Optional[str] = None - - * See :meth:`types.SimpleNamespace` for a mutable namespace based on an - underlying dictionary instead of a tuple. - - * The :mod:`dataclasses` module provides a decorator and functions for - automatically adding generated special methods to user-defined classes. - - -:class:`OrderedDict` objects ----------------------------- - -Ordered dictionaries are just like regular dictionaries but have some extra -capabilities relating to ordering operations. They have become less -important now that the built-in :class:`dict` class gained the ability -to remember insertion order (this new behavior became guaranteed in -Python 3.7). - -Some differences from :class:`dict` still remain: - -* The regular :class:`dict` was designed to be very good at mapping - operations. Tracking insertion order was secondary. - -* The :class:`OrderedDict` was designed to be good at reordering operations. - Space efficiency, iteration speed, and the performance of update - operations were secondary. - -* Algorithmically, :class:`OrderedDict` can handle frequent reordering - operations better than :class:`dict`. This makes it suitable for tracking - recent accesses (for example in an `LRU cache - `_). - -* The equality operation for :class:`OrderedDict` checks for matching order. - -* The :meth:`popitem` method of :class:`OrderedDict` has a different - signature. It accepts an optional argument to specify which item is popped. - -* :class:`OrderedDict` has a :meth:`move_to_end` method to - efficiently reposition an element to an endpoint. - -* Until Python 3.8, :class:`dict` lacked a :meth:`__reversed__` method. - - -.. class:: OrderedDict([items]) - - Return an instance of a :class:`dict` subclass that has methods - specialized for rearranging dictionary order. - - .. versionadded:: 3.1 - - .. method:: popitem(last=True) - - The :meth:`popitem` method for ordered dictionaries returns and removes a - (key, value) pair. The pairs are returned in - :abbr:`LIFO (last-in, first-out)` order if *last* is true - or :abbr:`FIFO (first-in, first-out)` order if false. - - .. method:: move_to_end(key, last=True) - - Move an existing *key* to either end of an ordered dictionary. The item - is moved to the right end if *last* is true (the default) or to the - beginning if *last* is false. Raises :exc:`KeyError` if the *key* does - not exist:: - - >>> d = OrderedDict.fromkeys('abcde') - >>> d.move_to_end('b') - >>> ''.join(d.keys()) - 'acdeb' - >>> d.move_to_end('b', last=False) - >>> ''.join(d.keys()) - 'bacde' - - .. versionadded:: 3.2 - -In addition to the usual mapping methods, ordered dictionaries also support -reverse iteration using :func:`reversed`. - -Equality tests between :class:`OrderedDict` objects are order-sensitive -and are implemented as ``list(od1.items())==list(od2.items())``. -Equality tests between :class:`OrderedDict` objects and other -:class:`~collections.abc.Mapping` objects are order-insensitive like regular -dictionaries. This allows :class:`OrderedDict` objects to be substituted -anywhere a regular dictionary is used. - -.. versionchanged:: 3.5 - The items, keys, and values :term:`views ` - of :class:`OrderedDict` now support reverse iteration using :func:`reversed`. - -.. versionchanged:: 3.6 - With the acceptance of :pep:`468`, order is retained for keyword arguments - passed to the :class:`OrderedDict` constructor and its :meth:`update` - method. - -.. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. - - -:class:`OrderedDict` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is straightforward to create an ordered dictionary variant -that remembers the order the keys were *last* inserted. -If a new entry overwrites an existing entry, the -original insertion position is changed and moved to the end:: - - class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' - - def __setitem__(self, key, value): - super().__setitem__(key, value) - self.move_to_end(key) - -An :class:`OrderedDict` would also be useful for implementing -variants of :func:`functools.lru_cache`:: - - class LRU(OrderedDict): - 'Limit size, evicting the least recently looked-up key when full' - - def __init__(self, maxsize=128, /, *args, **kwds): - self.maxsize = maxsize - super().__init__(*args, **kwds) - - def __getitem__(self, key): - value = super().__getitem__(key) - self.move_to_end(key) - return value - - def __setitem__(self, key, value): - if key in self: - self.move_to_end(key) - super().__setitem__(key, value) - if len(self) > self.maxsize: - oldest = next(iter(self)) - del self[oldest] - - -:class:`UserDict` objects -------------------------- - -The class, :class:`UserDict` acts as a wrapper around dictionary objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`dict`; however, this class can be easier -to work with because the underlying dictionary is accessible as an -attribute. - -.. class:: UserDict([initialdata]) - - Class that simulates a dictionary. The instance's contents are kept in a - regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it be used for other purposes. - - In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: - - .. attribute:: data - - A real dictionary used to store the contents of the :class:`UserDict` - class. - - - -:class:`UserList` objects -------------------------- - -This class acts as a wrapper around list objects. It is a useful base class -for your own list-like classes which can inherit from them and override -existing methods or add new ones. In this way, one can add new behaviors to -lists. - -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`list`; however, this class can be easier -to work with because the underlying list is accessible as an attribute. - -.. class:: UserList([list]) - - Class that simulates a list. The instance's contents are kept in a regular - list, which is accessible via the :attr:`data` attribute of :class:`UserList` - instances. The instance's contents are initially set to a copy of *list*, - defaulting to the empty list ``[]``. *list* can be any iterable, for - example a real Python list or a :class:`UserList` object. - - In addition to supporting the methods and operations of mutable sequences, - :class:`UserList` instances provide the following attribute: - - .. attribute:: data - - A real :class:`list` object used to store the contents of the - :class:`UserList` class. - -**Subclassing requirements:** Subclasses of :class:`UserList` are expected to -offer a constructor which can be called with either no arguments or one -argument. List operations which return a new sequence attempt to create an -instance of the actual implementation class. To do so, it assumes that the -constructor can be called with a single parameter, which is a sequence object -used as a data source. - -If a derived class does not wish to comply with this requirement, all of the -special methods supported by this class will need to be overridden; please -consult the sources for information about the methods which need to be provided -in that case. - -:class:`UserString` objects ---------------------------- - -The class, :class:`UserString` acts as a wrapper around string objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`str`; however, this class can be easier -to work with because the underlying string is accessible as an -attribute. - -.. class:: UserString(seq) - - Class that simulates a string object. The instance's - content is kept in a regular string object, which is accessible via the - :attr:`data` attribute of :class:`UserString` instances. The instance's - contents are initially set to a copy of *seq*. The *seq* argument can - be any object which can be converted into a string using the built-in - :func:`str` function. - - In addition to supporting the methods and operations of strings, - :class:`UserString` instances provide the following attribute: - - .. attribute:: data - - A real :class:`str` object used to store the contents of the - :class:`UserString` class. - - .. versionchanged:: 3.5 - New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, - ``format_map``, ``isprintable``, and ``maketrans``. diff --git a/Doc/library/dataclasses.rst.bak b/Doc/library/dataclasses.rst.bak deleted file mode 100644 index e706f7fcc566d8..00000000000000 --- a/Doc/library/dataclasses.rst.bak +++ /dev/null @@ -1,595 +0,0 @@ -:mod:`dataclasses` --- Data Classes -=================================== - -.. module:: dataclasses - :synopsis: Generate special methods on user-defined classes. - -.. moduleauthor:: Eric V. Smith -.. sectionauthor:: Eric V. Smith - -**Source code:** :source:`Lib/dataclasses.py` - --------------- - -This module provides a decorator and functions for automatically -adding generated :term:`special method`\s such as :meth:`__init__` and -:meth:`__repr__` to user-defined classes. It was originally described -in :pep:`557`. - -The member variables to use in these generated methods are defined -using :pep:`526` type annotations. For example this code:: - - from dataclasses import dataclass - - @dataclass - class InventoryItem: - """Class for keeping track of an item in inventory.""" - name: str - unit_price: float - quantity_on_hand: int = 0 - - def total_cost(self) -> float: - return self.unit_price * self.quantity_on_hand - -Will add, among other things, a :meth:`__init__` that looks like:: - - def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0): - self.name = name - self.unit_price = unit_price - self.quantity_on_hand = quantity_on_hand - -Note that this method is automatically added to the class: it is not -directly specified in the ``InventoryItem`` definition shown above. - -.. versionadded:: 3.7 - -Module-level decorators, classes, and functions ------------------------------------------------ - -.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) - - This function is a :term:`decorator` that is used to add generated - :term:`special method`\s to classes, as described below. - - The :func:`dataclass` decorator examines the class to find - ``field``\s. A ``field`` is defined as class variable that has a - :term:`type annotation `. With two - exceptions described below, nothing in :func:`dataclass` - examines the type specified in the variable annotation. - - The order of the fields in all of the generated methods is the - order in which they appear in the class definition. - - The :func:`dataclass` decorator will add various "dunder" methods to - the class, described below. If any of the added methods already - exist on the class, the behavior depends on the parameter, as documented - below. The decorator returns the same class that is called on; no new - class is created. - - If :func:`dataclass` is used just as a simple decorator with no parameters, - it acts as if it has the default values documented in this - signature. That is, these three uses of :func:`dataclass` are - equivalent:: - - @dataclass - class C: - ... - - @dataclass() - class C: - ... - - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) - class C: - ... - - The parameters to :func:`dataclass` are: - - - ``init``: If true (the default), a :meth:`__init__` method will be - generated. - - If the class already defines :meth:`__init__`, this parameter is - ignored. - - - ``repr``: If true (the default), a :meth:`__repr__` method will be - generated. The generated repr string will have the class name and - the name and repr of each field, in the order they are defined in - the class. Fields that are marked as being excluded from the repr - are not included. For example: - ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. - - If the class already defines :meth:`__repr__`, this parameter is - ignored. - - - ``eq``: If true (the default), an :meth:`__eq__` method will be - generated. This method compares the class as if it were a tuple - of its fields, in order. Both instances in the comparison must - be of the identical type. - - If the class already defines :meth:`__eq__`, this parameter is - ignored. - - - ``order``: If true (the default is ``False``), :meth:`__lt__`, - :meth:`__le__`, :meth:`__gt__`, and :meth:`__ge__` methods will be - generated. These compare the class as if it were a tuple of its - fields, in order. Both instances in the comparison must be of the - identical type. If ``order`` is true and ``eq`` is false, a - :exc:`ValueError` is raised. - - If the class already defines any of :meth:`__lt__`, - :meth:`__le__`, :meth:`__gt__`, or :meth:`__ge__`, then - :exc:`TypeError` is raised. - - - ``unsafe_hash``: If ``False`` (the default), a :meth:`__hash__` method - is generated according to how ``eq`` and ``frozen`` are set. - - :meth:`__hash__` is used by built-in :meth:`hash()`, and when objects are - added to hashed collections such as dictionaries and sets. Having a - :meth:`__hash__` implies that instances of the class are immutable. - Mutability is a complicated property that depends on the programmer's - intent, the existence and behavior of :meth:`__eq__`, and the values of - the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. - - By default, :func:`dataclass` will not implicitly add a :meth:`__hash__` - method unless it is safe to do so. Neither will it add or change an - existing explicitly defined :meth:`__hash__` method. Setting the class - attribute ``__hash__ = None`` has a specific meaning to Python, as - described in the :meth:`__hash__` documentation. - - If :meth:`__hash__` is not explicit defined, or if it is set to ``None``, - then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. - Although not recommended, you can force :func:`dataclass` to create a - :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case - if your class is logically immutable but can nonetheless be mutated. - This is a specialized use case and should be considered carefully. - - Here are the rules governing implicit creation of a :meth:`__hash__` - method. Note that you cannot both have an explicit :meth:`__hash__` - method in your dataclass and set ``unsafe_hash=True``; this will result - in a :exc:`TypeError`. - - If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will - generate a :meth:`__hash__` method for you. If ``eq`` is true and - ``frozen`` is false, :meth:`__hash__` will be set to ``None``, marking it - unhashable (which it is, since it is mutable). If ``eq`` is false, - :meth:`__hash__` will be left untouched meaning the :meth:`__hash__` - method of the superclass will be used (if the superclass is - :class:`object`, this means it will fall back to id-based hashing). - - - ``frozen``: If true (the default is ``False``), assigning to fields will - generate an exception. This emulates read-only frozen instances. If - :meth:`__setattr__` or :meth:`__delattr__` is defined in the class, then - :exc:`TypeError` is raised. See the discussion below. - - ``field``\s may optionally specify a default value, using normal - Python syntax:: - - @dataclass - class C: - a: int # 'a' has no default value - b: int = 0 # assign a default value for 'b' - - In this example, both ``a`` and ``b`` will be included in the added - :meth:`__init__` method, which will be defined as:: - - def __init__(self, a: int, b: int = 0): - - :exc:`TypeError` will be raised if a field without a default value - follows a field with a default value. This is true either when this - occurs in a single class, or as a result of class inheritance. - -.. function:: field(*, default=MISSING, default_factory=MISSING, repr=True, hash=None, init=True, compare=True, metadata=None) - - For common and simple use cases, no other functionality is - required. There are, however, some dataclass features that - require additional per-field information. To satisfy this need for - additional information, you can replace the default field value - with a call to the provided :func:`field` function. For example:: - - @dataclass - class C: - mylist: list[int] = field(default_factory=list) - - c = C() - c.mylist += [1, 2, 3] - - As shown above, the ``MISSING`` value is a sentinel object used to - detect if the ``default`` and ``default_factory`` parameters are - provided. This sentinel is used because ``None`` is a valid value - for ``default``. No code should directly use the ``MISSING`` - value. - - The parameters to :func:`field` are: - - - ``default``: If provided, this will be the default value for this - field. This is needed because the :meth:`field` call itself - replaces the normal position of the default value. - - - ``default_factory``: If provided, it must be a zero-argument - callable that will be called when a default value is needed for - this field. Among other purposes, this can be used to specify - fields with mutable default values, as discussed below. It is an - error to specify both ``default`` and ``default_factory``. - - - ``init``: If true (the default), this field is included as a - parameter to the generated :meth:`__init__` method. - - - ``repr``: If true (the default), this field is included in the - string returned by the generated :meth:`__repr__` method. - - - ``compare``: If true (the default), this field is included in the - generated equality and comparison methods (:meth:`__eq__`, - :meth:`__gt__`, et al.). - - - ``hash``: This can be a bool or ``None``. If true, this field is - included in the generated :meth:`__hash__` method. If ``None`` (the - default), use the value of ``compare``: this would normally be - the expected behavior. A field should be considered in the hash - if it's used for comparisons. Setting this value to anything - other than ``None`` is discouraged. - - One possible reason to set ``hash=False`` but ``compare=True`` - would be if a field is expensive to compute a hash value for, - that field is needed for equality testing, and there are other - fields that contribute to the type's hash value. Even if a field - is excluded from the hash, it will still be used for comparisons. - - - ``metadata``: This can be a mapping or None. None is treated as - an empty dict. This value is wrapped in - :func:`~types.MappingProxyType` to make it read-only, and exposed - on the :class:`Field` object. It is not used at all by Data - Classes, and is provided as a third-party extension mechanism. - Multiple third-parties can each have their own key, to use as a - namespace in the metadata. - - If the default value of a field is specified by a call to - :func:`field()`, then the class attribute for this field will be - replaced by the specified ``default`` value. If no ``default`` is - provided, then the class attribute will be deleted. The intent is - that after the :func:`dataclass` decorator runs, the class - attributes will all contain the default values for the fields, just - as if the default value itself were specified. For example, - after:: - - @dataclass - class C: - x: int - y: int = field(repr=False) - z: int = field(repr=False, default=10) - t: int = 20 - - The class attribute ``C.z`` will be ``10``, the class attribute - ``C.t`` will be ``20``, and the class attributes ``C.x`` and - ``C.y`` will not be set. - -.. class:: Field - - :class:`Field` objects describe each defined field. These objects - are created internally, and are returned by the :func:`fields` - module-level method (see below). Users should never instantiate a - :class:`Field` object directly. Its documented attributes are: - - - ``name``: The name of the field. - - - ``type``: The type of the field. - - - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, - ``compare``, and ``metadata`` have the identical meaning and - values as they do in the :func:`field` declaration. - - Other attributes may exist, but they are private and must not be - inspected or relied on. - -.. function:: fields(class_or_instance) - - Returns a tuple of :class:`Field` objects that define the fields for this - dataclass. Accepts either a dataclass, or an instance of a dataclass. - Raises :exc:`TypeError` if not passed a dataclass or instance of one. - Does not return pseudo-fields which are ``ClassVar`` or ``InitVar``. - -.. function:: asdict(instance, *, dict_factory=dict) - - Converts the dataclass ``instance`` to a dict (by using the - factory function ``dict_factory``). Each dataclass is converted - to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. For example:: - - @dataclass - class Point: - x: int - y: int - - @dataclass - class C: - mylist: list[Point] - - p = Point(10, 20) - assert asdict(p) == {'x': 10, 'y': 20} - - c = C([Point(0, 0), Point(10, 4)]) - assert asdict(c) == {'mylist': [{'x': 0, 'y': 0}, {'x': 10, 'y': 4}]} - - Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. - -.. function:: astuple(instance, *, tuple_factory=tuple) - - Converts the dataclass ``instance`` to a tuple (by using the - factory function ``tuple_factory``). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. - - Continuing from the previous example:: - - assert astuple(p) == (10, 20) - assert astuple(c) == ([(0, 0), (10, 4)],) - - Raises :exc:`TypeError` if ``instance`` is not a dataclass instance. - -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False) - - Creates a new dataclass with name ``cls_name``, fields as defined - in ``fields``, base classes as given in ``bases``, and initialized - with a namespace as given in ``namespace``. ``fields`` is an - iterable whose elements are each either ``name``, ``(name, type)``, - or ``(name, type, Field)``. If just ``name`` is supplied, - ``typing.Any`` is used for ``type``. The values of ``init``, - ``repr``, ``eq``, ``order``, ``unsafe_hash``, and ``frozen`` have - the same meaning as they do in :func:`dataclass`. - - This function is not strictly required, because any Python - mechanism for creating a new class with ``__annotations__`` can - then apply the :func:`dataclass` function to convert that class to - a dataclass. This function is provided as a convenience. For - example:: - - C = make_dataclass('C', - [('x', int), - 'y', - ('z', int, field(default=5))], - namespace={'add_one': lambda self: self.x + 1}) - - Is equivalent to:: - - @dataclass - class C: - x: int - y: 'typing.Any' - z: int = 5 - - def add_one(self): - return self.x + 1 - -.. function:: replace(instance, /, **changes) - - Creates a new object of the same type of ``instance``, replacing - fields with values from ``changes``. If ``instance`` is not a Data - Class, raises :exc:`TypeError`. If values in ``changes`` do not - specify fields, raises :exc:`TypeError`. - - The newly returned object is created by calling the :meth:`__init__` - method of the dataclass. This ensures that - :meth:`__post_init__`, if present, is also called. - - Init-only variables without default values, if any exist, must be - specified on the call to :func:`replace` so that they can be passed to - :meth:`__init__` and :meth:`__post_init__`. - - It is an error for ``changes`` to contain any fields that are - defined as having ``init=False``. A :exc:`ValueError` will be raised - in this case. - - Be forewarned about how ``init=False`` fields work during a call to - :func:`replace`. They are not copied from the source object, but - rather are initialized in :meth:`__post_init__`, if they're - initialized at all. It is expected that ``init=False`` fields will - be rarely and judiciously used. If they are used, it might be wise - to have alternate class constructors, or perhaps a custom - ``replace()`` (or similarly named) method which handles instance - copying. - -.. function:: is_dataclass(class_or_instance) - - Return ``True`` if its parameter is a dataclass or an instance of one, - otherwise return ``False``. - - If you need to know if a class is an instance of a dataclass (and - not a dataclass itself), then add a further check for ``not - isinstance(obj, type)``:: - - def is_dataclass_instance(obj): - return is_dataclass(obj) and not isinstance(obj, type) - -Post-init processing --------------------- - -The generated :meth:`__init__` code will call a method named -:meth:`__post_init__`, if :meth:`__post_init__` is defined on the -class. It will normally be called as ``self.__post_init__()``. -However, if any ``InitVar`` fields are defined, they will also be -passed to :meth:`__post_init__` in the order they were defined in the -class. If no :meth:`__init__` method is generated, then -:meth:`__post_init__` will not automatically be called. - -Among other uses, this allows for initializing field values that -depend on one or more other fields. For example:: - - @dataclass - class C: - a: float - b: float - c: float = field(init=False) - - def __post_init__(self): - self.c = self.a + self.b - -See the section below on init-only variables for ways to pass -parameters to :meth:`__post_init__`. Also see the warning about how -:func:`replace` handles ``init=False`` fields. - -Class variables ---------------- - -One of two places where :func:`dataclass` actually inspects the type -of a field is to determine if a field is a class variable as defined -in :pep:`526`. It does this by checking if the type of the field is -``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded -from consideration as a field and is ignored by the dataclass -mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the -module-level :func:`fields` function. - -Init-only variables -------------------- - -The other place where :func:`dataclass` inspects a type annotation is to -determine if a field is an init-only variable. It does this by seeing -if the type of a field is of type ``dataclasses.InitVar``. If a field -is an ``InitVar``, it is considered a pseudo-field called an init-only -field. As it is not a true field, it is not returned by the -module-level :func:`fields` function. Init-only fields are added as -parameters to the generated :meth:`__init__` method, and are passed to -the optional :meth:`__post_init__` method. They are not otherwise used -by dataclasses. - -For example, suppose a field will be initialized from a database, if a -value is not provided when creating the class:: - - @dataclass - class C: - i: int - j: int = None - database: InitVar[DatabaseType] = None - - def __post_init__(self, database): - if self.j is None and database is not None: - self.j = database.lookup('j') - - c = C(10, database=my_database) - -In this case, :func:`fields` will return :class:`Field` objects for ``i`` and -``j``, but not for ``database``. - -Frozen instances ----------------- - -It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :meth:`dataclass` decorator you can -emulate immutability. In that case, dataclasses will add -:meth:`__setattr__` and :meth:`__delattr__` methods to the class. These -methods will raise a :exc:`FrozenInstanceError` when invoked. - -There is a tiny performance penalty when using ``frozen=True``: -:meth:`__init__` cannot use simple assignment to initialize fields, and -must use :meth:`object.__setattr__`. - -Inheritance ------------ - -When the dataclass is being created by the :meth:`dataclass` decorator, -it looks through all of the class's base classes in reverse MRO (that -is, starting at :class:`object`) and, for each dataclass that it finds, -adds the fields from that base class to an ordered mapping of fields. -After all of the base class fields are added, it adds its own fields -to the ordered mapping. All of the generated methods will use this -combined, calculated ordered mapping of fields. Because the fields -are in insertion order, derived classes override base classes. An -example:: - - @dataclass - class Base: - x: Any = 15.0 - y: int = 0 - - @dataclass - class C(Base): - z: int = 10 - x: int = 15 - -The final list of fields is, in order, ``x``, ``y``, ``z``. The final -type of ``x`` is ``int``, as specified in class ``C``. - -The generated :meth:`__init__` method for ``C`` will look like:: - - def __init__(self, x: int = 15, y: int = 0, z: int = 10): - -Default factory functions -------------------------- - - If a :func:`field` specifies a ``default_factory``, it is called with - zero arguments when a default value for the field is needed. For - example, to create a new instance of a list, use:: - - mylist: list = field(default_factory=list) - - If a field is excluded from :meth:`__init__` (using ``init=False``) - and the field also specifies ``default_factory``, then the default - factory function will always be called from the generated - :meth:`__init__` function. This happens because there is no other - way to give the field an initial value. - -Mutable default values ----------------------- - - Python stores default member variable values in class attributes. - Consider this example, not using dataclasses:: - - class C: - x = [] - def add(self, element): - self.x.append(element) - - o1 = C() - o2 = C() - o1.add(1) - o2.add(2) - assert o1.x == [1, 2] - assert o1.x is o2.x - - Note that the two instances of class ``C`` share the same class - variable ``x``, as expected. - - Using dataclasses, *if* this code was valid:: - - @dataclass - class D: - x: List = [] - def add(self, element): - self.x += element - - it would generate code similar to:: - - class D: - x = [] - def __init__(self, x=x): - self.x = x - def add(self, element): - self.x += element - - assert D().x is D().x - - This has the same issue as the original example using class ``C``. - That is, two instances of class ``D`` that do not specify a value for - ``x`` when creating a class instance will share the same copy of - ``x``. Because dataclasses just use normal Python class creation - they also share this behavior. There is no general way for Data - Classes to detect this condition. Instead, dataclasses will raise a - :exc:`TypeError` if it detects a default parameter of type ``list``, - ``dict``, or ``set``. This is a partial solution, but it does protect - against many common errors. - - Using default factory functions is a way to create new instances of - mutable types as default values for fields:: - - @dataclass - class D: - x: list = field(default_factory=list) - - assert D().x is not D().x - -Exceptions ----------- - -.. exception:: FrozenInstanceError - - Raised when an implicitly defined :meth:`__setattr__` or - :meth:`__delattr__` is called on a dataclass which was defined with - ``frozen=True``. diff --git a/Doc/library/gc.rst.bak b/Doc/library/gc.rst.bak deleted file mode 100644 index a3d201d5055c8e..00000000000000 --- a/Doc/library/gc.rst.bak +++ /dev/null @@ -1,322 +0,0 @@ -:mod:`gc` --- Garbage Collector interface -========================================= - -.. module:: gc - :synopsis: Interface to the cycle-detecting garbage collector. - -.. moduleauthor:: Neil Schemenauer -.. sectionauthor:: Neil Schemenauer - --------------- - -This module provides an interface to the optional garbage collector. It -provides the ability to disable the collector, tune the collection frequency, -and set debugging options. It also provides access to unreachable objects that -the collector found but cannot free. Since the collector supplements the -reference counting already used in Python, you can disable the collector if you -are sure your program does not create reference cycles. Automatic collection -can be disabled by calling ``gc.disable()``. To debug a leaking program call -``gc.set_debug(gc.DEBUG_LEAK)``. Notice that this includes -``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in -gc.garbage for inspection. - -The :mod:`gc` module provides the following functions: - - -.. function:: enable() - - Enable automatic garbage collection. - - -.. function:: disable() - - Disable automatic garbage collection. - - -.. function:: isenabled() - - Return ``True`` if automatic collection is enabled. - - -.. function:: collect(generation=2) - - With no arguments, run a full collection. The optional argument *generation* - may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The number of - unreachable objects found is returned. - - The free lists maintained for a number of built-in types are cleared - whenever a full collection or collection of the highest generation (2) - is run. Not all items in some free lists may be freed due to the - particular implementation, in particular :class:`float`. - - -.. function:: set_debug(flags) - - Set the garbage collection debugging flags. Debugging information will be - written to ``sys.stderr``. See below for a list of debugging flags which can be - combined using bit operations to control debugging. - - -.. function:: get_debug() - - Return the debugging flags currently set. - - -.. function:: get_objects(generation=None) - - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not None, return only the objects tracked by - the collector that are in that generation. - - .. versionchanged:: 3.8 - New *generation* parameter. - -.. function:: get_stats() - - Return a list of three per-generation dictionaries containing collection - statistics since interpreter start. The number of keys may change - in the future, but currently each dictionary will contain the following - items: - - * ``collections`` is the number of times this generation was collected; - - * ``collected`` is the total number of objects collected inside this - generation; - - * ``uncollectable`` is the total number of objects which were found - to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. - - .. versionadded:: 3.4 - - -.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) - - Set the garbage collection thresholds (the collection frequency). Setting - *threshold0* to zero disables collection. - - The GC classifies objects into three generations depending on how many - collection sweeps they have survived. New objects are placed in the youngest - generation (generation ``0``). If an object survives a collection it is moved - into the next older generation. Since generation ``2`` is the oldest - generation, objects in that generation remain there after a collection. In - order to decide when to run, the collector keeps track of the number object - allocations and deallocations since the last collection. When the number of - allocations minus the number of deallocations exceeds *threshold0*, collection - starts. Initially only generation ``0`` is examined. If generation ``0`` has - been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. - With the third generation, things are a bit more complicated, - see `Collecting the oldest generation `_ for more information. - - -.. function:: get_count() - - Return the current collection counts as a tuple of ``(count0, count1, - count2)``. - - -.. function:: get_threshold() - - Return the current collection thresholds as a tuple of ``(threshold0, - threshold1, threshold2)``. - - -.. function:: get_referrers(*objs) - - Return the list of objects that directly refer to any of objs. This function - will only locate those containers which support garbage collection; extension - types which do refer to other objects but do not support garbage collection will - not be found. - - Note that objects which have already been dereferenced, but which live in cycles - and have not yet been collected by the garbage collector can be listed among the - resulting referrers. To get only currently live objects, call :func:`collect` - before calling :func:`get_referrers`. - - .. warning:: - Care must be taken when using objects returned by :func:`get_referrers` because - some of them could still be under construction and hence in a temporarily - invalid state. Avoid using :func:`get_referrers` for any purpose other than - debugging. - - -.. function:: get_referents(*objs) - - Return a list of objects directly referred to by any of the arguments. The - referents returned are those objects visited by the arguments' C-level - :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually - directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects - that support garbage collection, and are only required to visit objects that may - be involved in a cycle. So, for example, if an integer is directly reachable - from an argument, that integer object may or may not appear in the result list. - - -.. function:: is_tracked(obj) - - Returns ``True`` if the object is currently tracked by the garbage collector, - ``False`` otherwise. As a general rule, instances of atomic types aren't - tracked and instances of non-atomic types (containers, user-defined - objects...) are. However, some type-specific optimizations can be present - in order to suppress the garbage collector footprint of simple instances - (e.g. dicts containing only atomic keys and values):: - - >>> gc.is_tracked(0) - False - >>> gc.is_tracked("a") - False - >>> gc.is_tracked([]) - True - >>> gc.is_tracked({}) - False - >>> gc.is_tracked({"a": 1}) - False - >>> gc.is_tracked({"a": []}) - True - - .. versionadded:: 3.1 - - -.. function:: is_finalized(obj) - - Returns ``True`` if the given object has been finalized by the - garbage collector, ``False`` otherwise. :: - - >>> x = None - >>> class Lazarus: - ... def __del__(self): - ... global x - ... x = self - ... - >>> lazarus = Lazarus() - >>> gc.is_finalized(lazarus) - False - >>> del lazarus - >>> gc.is_finalized(x) - True - - .. versionadded:: 3.9 - - -.. function:: freeze() - - Freeze all the objects tracked by gc - move them to a permanent generation - and ignore all the future collections. This can be used before a POSIX - fork() call to make the gc copy-on-write friendly or to speed up collection. - Also collection before a POSIX fork() call may free pages for future - allocation which can cause copy-on-write too so it's advised to disable gc - in parent process and freeze before fork and enable gc in child process. - - .. versionadded:: 3.7 - - -.. function:: unfreeze() - - Unfreeze the objects in the permanent generation, put them back into the - oldest generation. - - .. versionadded:: 3.7 - - -.. function:: get_freeze_count() - - Return the number of objects in the permanent generation. - - .. versionadded:: 3.7 - - -The following variables are provided for read-only access (you can mutate the -values but should not rebind them): - -.. data:: garbage - - A list of objects which the collector found to be unreachable but could - not be freed (uncollectable objects). Starting with Python 3.4, this - list should be empty most of the time, except when using instances of - C extension types with a non-``NULL`` ``tp_del`` slot. - - If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be - added to this list rather than freed. - - .. versionchanged:: 3.2 - If this list is non-empty at :term:`interpreter shutdown`, a - :exc:`ResourceWarning` is emitted, which is silent by default. If - :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects - are printed. - - .. versionchanged:: 3.4 - Following :pep:`442`, objects with a :meth:`__del__` method don't end - up in :attr:`gc.garbage` anymore. - -.. data:: callbacks - - A list of callbacks that will be invoked by the garbage collector before and - after collection. The callbacks will be called with two arguments, - *phase* and *info*. - - *phase* can be one of two values: - - "start": The garbage collection is about to start. - - "stop": The garbage collection has finished. - - *info* is a dict providing more information for the callback. The following - keys are currently defined: - - "generation": The oldest generation being collected. - - "collected": When *phase* is "stop", the number of objects - successfully collected. - - "uncollectable": When *phase* is "stop", the number of objects - that could not be collected and were put in :data:`garbage`. - - Applications can add their own callbacks to this list. The primary - use cases are: - - Gathering statistics about garbage collection, such as how often - various generations are collected, and how long the collection - takes. - - Allowing applications to identify and clear their own uncollectable - types when they appear in :data:`garbage`. - - .. versionadded:: 3.3 - - -The following constants are provided for use with :func:`set_debug`: - - -.. data:: DEBUG_STATS - - Print statistics during collection. This information can be useful when tuning - the collection frequency. - - -.. data:: DEBUG_COLLECTABLE - - Print information on collectable objects found. - - -.. data:: DEBUG_UNCOLLECTABLE - - Print information of uncollectable objects found (objects which are not - reachable but cannot be freed by the collector). These objects will be added - to the ``garbage`` list. - - .. versionchanged:: 3.2 - Also print the contents of the :data:`garbage` list at - :term:`interpreter shutdown`, if it isn't empty. - -.. data:: DEBUG_SAVEALL - - When set, all unreachable objects found will be appended to *garbage* rather - than being freed. This can be useful for debugging a leaking program. - - -.. data:: DEBUG_LEAK - - The debugging flags necessary for the collector to print information about a - leaking program (equal to ``DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | - DEBUG_SAVEALL``). diff --git a/Doc/library/importlib.metadata.rst.bak b/Doc/library/importlib.metadata.rst.bak deleted file mode 100644 index 7f154ea02cc4f2..00000000000000 --- a/Doc/library/importlib.metadata.rst.bak +++ /dev/null @@ -1,262 +0,0 @@ -.. _using: - -================================= - Using :mod:`!importlib.metadata` -================================= - -.. note:: - This functionality is provisional and may deviate from the usual - version semantics of the standard library. - -``importlib.metadata`` is a library that provides for access to installed -package metadata. Built in part on Python's import system, this library -intends to replace similar functionality in the `entry point -API`_ and `metadata API`_ of ``pkg_resources``. Along with -:mod:`importlib.resources` in Python 3.7 -and newer (backported as `importlib_resources`_ for older versions of -Python), this can eliminate the need to use the older and less efficient -``pkg_resources`` package. - -By "installed package" we generally mean a third-party package installed into -Python's ``site-packages`` directory via tools such as `pip -`_. Specifically, -it means a package with either a discoverable ``dist-info`` or ``egg-info`` -directory, and metadata defined by :pep:`566` or its older specifications. -By default, package metadata can live on the file system or in zip archives on -:data:`sys.path`. Through an extension mechanism, the metadata can live almost -anywhere. - - -Overview -======== - -Let's say you wanted to get the version string for a package you've installed -using ``pip``. We start by creating a virtual environment and installing -something into it: - -.. code-block:: shell-session - - $ python3 -m venv example - $ source example/bin/activate - (example) $ pip install wheel - -You can get the version string for ``wheel`` by running the following: - -.. code-block:: pycon - - (example) $ python - >>> from importlib.metadata import version # doctest: +SKIP - >>> version('wheel') # doctest: +SKIP - '0.32.3' - -You can also get the set of entry points keyed by group, such as -``console_scripts``, ``distutils.commands`` and others. Each group contains a -sequence of :ref:`EntryPoint ` objects. - -You can get the :ref:`metadata for a distribution `:: - - >>> list(metadata('wheel')) # doctest: +SKIP - ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] - -You can also get a :ref:`distribution's version number `, list its -:ref:`constituent files `, and get a list of the distribution's -:ref:`requirements`. - - -Functional API -============== - -This package provides the following functionality via its public API. - - -.. _entry-points: - -Entry points ------------- - -The ``entry_points()`` function returns a dictionary of all entry points, -keyed by group. Entry points are represented by ``EntryPoint`` instances; -each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. There are also ``.module``, -``.attr``, and ``.extras`` attributes for getting the components of the -``.value`` attribute:: - - >>> eps = entry_points() # doctest: +SKIP - >>> list(eps) # doctest: +SKIP - ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] - >>> scripts = eps['console_scripts'] # doctest: +SKIP - >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP - >>> wheel # doctest: +SKIP - EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') - >>> wheel.module # doctest: +SKIP - 'wheel.cli' - >>> wheel.attr # doctest: +SKIP - 'main' - >>> wheel.extras # doctest: +SKIP - [] - >>> main = wheel.load() # doctest: +SKIP - >>> main # doctest: +SKIP - - -The ``group`` and ``name`` are arbitrary values defined by the package author -and usually a client will wish to resolve all entry points for a particular -group. Read `the setuptools docs -`_ -for more information on entry points, their definition, and usage. - - -.. _metadata: - -Distribution metadata ---------------------- - -Every distribution includes some metadata, which you can extract using the -``metadata()`` function:: - - >>> wheel_metadata = metadata('wheel') # doctest: +SKIP - -The keys of the returned data structure, a ``PackageMetadata``, -name the metadata keywords, and -the values are returned unparsed from the distribution metadata:: - - >>> wheel_metadata['Requires-Python'] # doctest: +SKIP - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - - -.. _version: - -Distribution versions ---------------------- - -The ``version()`` function is the quickest way to get a distribution's version -number, as a string:: - - >>> version('wheel') # doctest: +SKIP - '0.32.3' - - -.. _files: - -Distribution files ------------------- - -You can also get the full set of files contained within a distribution. The -``files()`` function takes a distribution package name and returns all of the -files installed by this distribution. Each file object returned is a -``PackagePath``, a :class:`pathlib.Path` derived object with additional ``dist``, -``size``, and ``hash`` properties as indicated by the metadata. For example:: - - >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] # doctest: +SKIP - >>> util # doctest: +SKIP - PackagePath('wheel/util.py') - >>> util.size # doctest: +SKIP - 859 - >>> util.dist # doctest: +SKIP - - >>> util.hash # doctest: +SKIP - - -Once you have the file, you can also read its contents:: - - >>> print(util.read_text()) # doctest: +SKIP - import base64 - import sys - ... - def as_bytes(s): - if isinstance(s, text_type): - return s.encode('utf-8') - return s - -In the case where the metadata file listing files -(RECORD or SOURCES.txt) is missing, ``files()`` will -return ``None``. The caller may wish to wrap calls to -``files()`` in `always_iterable -`_ -or otherwise guard against this condition if the target -distribution is not known to have the metadata present. - -.. _requirements: - -Distribution requirements -------------------------- - -To get the full set of requirements for a distribution, use the ``requires()`` -function:: - - >>> requires('wheel') # doctest: +SKIP - ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] - - -Distributions -============= - -While the above API is the most common and convenient usage, you can get all -of that information from the ``Distribution`` class. A ``Distribution`` is an -abstract object that represents the metadata for a Python package. You can -get the ``Distribution`` instance:: - - >>> from importlib.metadata import distribution # doctest: +SKIP - >>> dist = distribution('wheel') # doctest: +SKIP - -Thus, an alternative way to get the version number is through the -``Distribution`` instance:: - - >>> dist.version # doctest: +SKIP - '0.32.3' - -There are all kinds of additional metadata available on the ``Distribution`` -instance:: - - >>> dist.metadata['Requires-Python'] # doctest: +SKIP - '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> dist.metadata['License'] # doctest: +SKIP - 'MIT' - -The full set of available metadata is not described here. See :pep:`566` -for additional details. - - -Extending the search algorithm -============================== - -Because package metadata is not available through :data:`sys.path` searches, or -package loaders directly, the metadata for a package is found through import -system :ref:`finders `. To find a distribution package's metadata, -``importlib.metadata`` queries the list of :term:`meta path finders ` on -:data:`sys.meta_path`. - -The default ``PathFinder`` for Python includes a hook that calls into -``importlib.metadata.MetadataPathFinder`` for finding distributions -loaded from typical file-system-based paths. - -The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the -interface expected of finders by Python's import system. -``importlib.metadata`` extends this protocol by looking for an optional -``find_distributions`` callable on the finders from -:data:`sys.meta_path` and presents this extended interface as the -``DistributionFinder`` abstract base class, which defines this abstract -method:: - - @abc.abstractmethod - def find_distributions(context=DistributionFinder.Context()): - """Return an iterable of all Distribution instances capable of - loading the metadata for packages for the indicated ``context``. - """ - -The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` -properties indicating the path to search and name to match and may -supply other relevant context. - -What this means in practice is that to support finding distribution package -metadata in locations other than the file system, subclass -``Distribution`` and implement the abstract methods. Then from -a custom finder, return instances of this derived ``Distribution`` in the -``find_distributions()`` method. - - -.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points -.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api -.. _`importlib_resources`: https://importlib-resources.readthedocs.io/en/latest/index.html - - -.. rubric:: Footnotes diff --git a/Doc/library/logging.rst.bak b/Doc/library/logging.rst.bak deleted file mode 100644 index 431a5849fa9bff..00000000000000 --- a/Doc/library/logging.rst.bak +++ /dev/null @@ -1,1350 +0,0 @@ -:mod:`logging` --- Logging facility for Python -============================================== - -.. module:: logging - :synopsis: Flexible event logging system for applications. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -**Source code:** :source:`Lib/logging/__init__.py` - -.. index:: pair: Errors; logging - -.. sidebar:: Important - - This page contains the API reference information. For tutorial - information and discussion of more advanced topics, see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Logging Cookbook ` - --------------- - -This module defines functions and classes which implement a flexible event -logging system for applications and libraries. - -The key benefit of having the logging API provided by a standard library module -is that all Python modules can participate in logging, so your application log -can include your own messages integrated with messages from third-party -modules. - -The module provides a lot of functionality and flexibility. If you are -unfamiliar with logging, the best way to get to grips with it is to see the -tutorials (see the links on the right). - -The basic classes defined by the module, together with their functions, are -listed below. - -* Loggers expose the interface that application code directly uses. -* Handlers send the log records (created by loggers) to the appropriate - destination. -* Filters provide a finer grained facility for determining which log records - to output. -* Formatters specify the layout of log records in the final output. - - -.. _logger: - -Logger Objects --------------- - -Loggers have the following attributes and methods. Note that Loggers should -*NEVER* be instantiated directly, but always through the module-level function -``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same -name will always return a reference to the same Logger object. - -The ``name`` is potentially a period-separated hierarchical value, like -``foo.bar.baz`` (though it could also be just plain ``foo``, for example). -Loggers that are further down in the hierarchical list are children of loggers -higher up in the list. For example, given a logger with a name of ``foo``, -loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. - - -.. class:: Logger - - .. attribute:: Logger.propagate - - If this attribute evaluates to true, events logged to this logger will be - passed to the handlers of higher level (ancestor) loggers, in addition to - any handlers attached to this logger. Messages are passed directly to the - ancestor loggers' handlers - neither the level nor filters of the ancestor - loggers in question are considered. - - If this evaluates to false, logging messages are not passed to the handlers - of ancestor loggers. - - The constructor sets this attribute to ``True``. - - .. note:: If you attach a handler to a logger *and* one or more of its - ancestors, it may emit the same record multiple times. In general, you - should not need to attach a handler to more than one logger - if you just - attach it to the appropriate logger which is highest in the logger - hierarchy, then it will see all events logged by all descendant loggers, - provided that their propagate setting is left set to ``True``. A common - scenario is to attach handlers only to the root logger, and to let - propagation take care of the rest. - - .. method:: Logger.setLevel(level) - - Sets the threshold for this logger to *level*. Logging messages which are less - severe than *level* will be ignored; logging messages which have severity *level* - or higher will be emitted by whichever handler or handlers service this logger, - unless a handler's level has been set to a higher severity level than *level*. - - When a logger is created, the level is set to :const:`NOTSET` (which causes - all messages to be processed when the logger is the root logger, or delegation - to the parent when the logger is a non-root logger). Note that the root logger - is created with level :const:`WARNING`. - - The term 'delegation to the parent' means that if a logger has a level of - NOTSET, its chain of ancestor loggers is traversed until either an ancestor with - a level other than NOTSET is found, or the root is reached. - - If an ancestor is found with a level other than NOTSET, then that ancestor's - level is treated as the effective level of the logger where the ancestor search - began, and is used to determine how a logging event is handled. - - If the root is reached, and it has a level of NOTSET, then all messages will be - processed. Otherwise, the root's level will be used as the effective level. - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. Note, however, that levels are internally stored - as integers, and methods such as e.g. :meth:`getEffectiveLevel` and - :meth:`isEnabledFor` will return/expect to be passed integers. - - - .. method:: Logger.isEnabledFor(level) - - Indicates if a message of severity *level* would be processed by this logger. - This method checks first the module-level level set by - ``logging.disable(level)`` and then the logger's effective level as determined - by :meth:`getEffectiveLevel`. - - - .. method:: Logger.getEffectiveLevel() - - Indicates the effective level for this logger. If a value other than - :const:`NOTSET` has been set using :meth:`setLevel`, it is returned. Otherwise, - the hierarchy is traversed towards the root until a value other than - :const:`NOTSET` is found, and that value is returned. The value returned is - an integer, typically one of :const:`logging.DEBUG`, :const:`logging.INFO` - etc. - - - .. method:: Logger.getChild(suffix) - - Returns a logger which is a descendant to this logger, as determined by the suffix. - Thus, ``logging.getLogger('abc').getChild('def.ghi')`` would return the same - logger as would be returned by ``logging.getLogger('abc.def.ghi')``. This is a - convenience method, useful when the parent logger is named using e.g. ``__name__`` - rather than a literal string. - - .. versionadded:: 3.2 - - - .. method:: Logger.debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on this logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - No % formatting operation is performed on *msg* when no *args* are supplied. - - There are four keyword arguments in *kwargs* which are inspected: - *exc_info*, *stack_info*, *stacklevel* and *extra*. - - If *exc_info* does not evaluate as false, it causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *stacklevel*, which defaults to ``1``. - If greater than 1, the corresponding number of stack frames are skipped - when computing the line number and function name set in the :class:`LogRecord` - created for the logging event. This can be used in logging helpers so that - the function name, filename and line number recorded are not the information - for the helper function/method, but rather its caller. The name of this - parameter mirrors the equivalent one in the :mod:`warnings` module. - - The fourth keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the :class:`LogRecord` - created for the logging event with user-defined attributes. These custom - attributes can then be used as you like. For example, they could be - incorporated into logged messages. For example:: - - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logger = logging.getLogger('tcpserver') - logger.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the :class:`LogRecord`. If these are missing, the message will - not be logged because a string formatting exception will occur. So in this case, - you always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. - - .. versionchanged:: 3.5 - The *exc_info* parameter can now accept exception instances. - - .. versionchanged:: 3.8 - The *stacklevel* parameter was added. - - - .. method:: Logger.info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on this logger. The arguments are - interpreted as for :meth:`debug`. - - .. note:: There is an obsolete method ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - .. method:: Logger.error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.log(level, msg, *args, **kwargs) - - Logs a message with integer level *level* on this logger. The other arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. Exception info is added to the logging - message. This method should only be called from an exception handler. - - - .. method:: Logger.addFilter(filter) - - Adds the specified filter *filter* to this logger. - - - .. method:: Logger.removeFilter(filter) - - Removes the specified filter *filter* from this logger. - - - .. method:: Logger.filter(record) - - Apply this logger's filters to the record and return ``True`` if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be processed (passed to handlers). If one returns a false value, no - further processing of the record occurs. - - - .. method:: Logger.addHandler(hdlr) - - Adds the specified handler *hdlr* to this logger. - - - .. method:: Logger.removeHandler(hdlr) - - Removes the specified handler *hdlr* from this logger. - - - .. method:: Logger.findCaller(stack_info=False, stacklevel=1) - - Finds the caller's source filename and line number. Returns the filename, line - number, function name and stack information as a 4-element tuple. The stack - information is returned as ``None`` unless *stack_info* is ``True``. - - The *stacklevel* parameter is passed from code calling the :meth:`debug` - and other APIs. If greater than 1, the excess is used to skip stack frames - before determining the values to be returned. This will generally be useful - when calling logging APIs from helper/wrapper code, so that the information - in the event log refers not to the helper/wrapper code, but to the code that - calls it. - - - .. method:: Logger.handle(record) - - Handles a record by passing it to all handlers associated with this logger and - its ancestors (until a false value of *propagate* is found). This method is used - for unpickled records received from a socket, as well as those created locally. - Logger-level filtering is applied using :meth:`~Logger.filter`. - - - .. method:: Logger.makeRecord(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) - - This is a factory method which can be overridden in subclasses to create - specialized :class:`LogRecord` instances. - - .. method:: Logger.hasHandlers() - - Checks to see if this logger has any handlers configured. This is done by - looking for handlers in this logger and its parents in the logger hierarchy. - Returns ``True`` if a handler was found, else ``False``. The method stops searching - up the hierarchy whenever a logger with the 'propagate' attribute set to - false is found - that will be the last logger which is checked for the - existence of handlers. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.7 - Loggers can now be pickled and unpickled. - -.. _levels: - -Logging Levels --------------- - -The numeric values of logging levels are given in the following table. These are -primarily of interest if you want to define your own levels, and need them to -have specific values relative to the predefined levels. If you define a level -with the same numeric value, it overwrites the predefined value; the predefined -name is lost. - -+--------------+---------------+ -| Level | Numeric value | -+==============+===============+ -| ``CRITICAL`` | 50 | -+--------------+---------------+ -| ``ERROR`` | 40 | -+--------------+---------------+ -| ``WARNING`` | 30 | -+--------------+---------------+ -| ``INFO`` | 20 | -+--------------+---------------+ -| ``DEBUG`` | 10 | -+--------------+---------------+ -| ``NOTSET`` | 0 | -+--------------+---------------+ - - -.. _handler: - -Handler Objects ---------------- - -Handlers have the following attributes and methods. Note that :class:`Handler` -is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`__init__` method in subclasses needs to call -:meth:`Handler.__init__`. - -.. class:: Handler - - .. method:: Handler.__init__(level=NOTSET) - - Initializes the :class:`Handler` instance by setting its level, setting the list - of filters to the empty list and creating a lock (using :meth:`createLock`) for - serializing access to an I/O mechanism. - - - .. method:: Handler.createLock() - - Initializes a thread lock which can be used to serialize access to underlying - I/O functionality which may not be threadsafe. - - - .. method:: Handler.acquire() - - Acquires the thread lock created with :meth:`createLock`. - - - .. method:: Handler.release() - - Releases the thread lock acquired with :meth:`acquire`. - - - .. method:: Handler.setLevel(level) - - Sets the threshold for this handler to *level*. Logging messages which are - less severe than *level* will be ignored. When a handler is created, the - level is set to :const:`NOTSET` (which causes all messages to be - processed). - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. - - - .. method:: Handler.setFormatter(fmt) - - Sets the :class:`Formatter` for this handler to *fmt*. - - - .. method:: Handler.addFilter(filter) - - Adds the specified filter *filter* to this handler. - - - .. method:: Handler.removeFilter(filter) - - Removes the specified filter *filter* from this handler. - - - .. method:: Handler.filter(record) - - Apply this handler's filters to the record and return ``True`` if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be emitted. If one returns a false value, the handler will not emit the - record. - - - .. method:: Handler.flush() - - Ensure all logging output has been flushed. This version does nothing and is - intended to be implemented by subclasses. - - - .. method:: Handler.close() - - Tidy up any resources used by the handler. This version does no output but - removes the handler from an internal list of handlers which is closed when - :func:`shutdown` is called. Subclasses should ensure that this gets called - from overridden :meth:`close` methods. - - - .. method:: Handler.handle(record) - - Conditionally emits the specified logging record, depending on filters which may - have been added to the handler. Wraps the actual emission of the record with - acquisition/release of the I/O thread lock. - - - .. method:: Handler.handleError(record) - - This method should be called from handlers when an exception is encountered - during an :meth:`emit` call. If the module-level attribute - ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is - what is mostly wanted for a logging system - most users will not care about - errors in the logging system, they are more interested in application - errors. You could, however, replace this with a custom handler if you wish. - The specified record is the one which was being processed when the exception - occurred. (The default value of ``raiseExceptions`` is ``True``, as that is - more useful during development). - - - .. method:: Handler.format(record) - - Do formatting for a record - if a formatter is set, use it. Otherwise, use the - default formatter for the module. - - - .. method:: Handler.emit(record) - - Do whatever it takes to actually log the specified logging record. This version - is intended to be implemented by subclasses and so raises a - :exc:`NotImplementedError`. - -For a list of handlers included as standard, see :mod:`logging.handlers`. - -.. _formatter-objects: - -Formatter Objects ------------------ - -.. currentmodule:: logging - -:class:`Formatter` objects have the following attributes and methods. They are -responsible for converting a :class:`LogRecord` to (usually) a string which can -be interpreted by either a human or an external system. The base -:class:`Formatter` allows a formatting string to be specified. If none is -supplied, the default value of ``'%(message)s'`` is used, which just includes -the message in the logging call. To have additional items of information in the -formatted output (such as a timestamp), keep reading. - -A Formatter can be initialized with a format string which makes use of knowledge -of the :class:`LogRecord` attributes - such as the default value mentioned above -making use of the fact that the user's message and arguments are pre-formatted -into a :class:`LogRecord`'s *message* attribute. This format string contains -standard Python %-style mapping keys. See section :ref:`old-string-formatting` -for more information on string formatting. - -The useful mapping keys in a :class:`LogRecord` are given in the section on -:ref:`logrecord-attributes`. - - -.. class:: Formatter(fmt=None, datefmt=None, style='%', validate=True, *, defaults=None) - - Returns a new instance of the :class:`Formatter` class. The instance is - initialized with a format string for the message as a whole, as well as a - format string for the date/time portion of a message. If no *fmt* is - specified, ``'%(message)s'`` is used. If no *datefmt* is specified, a format - is used which is described in the :meth:`formatTime` documentation. - - The *style* parameter can be one of '%', '{' or '$' and determines how - the format string will be merged with its data: using one of %-formatting, - :meth:`str.format` or :class:`string.Template`. This only applies to the - format string *fmt* (e.g. ``'%(message)s'`` or ``{message}``), not to the - actual log messages passed to ``Logger.debug`` etc; see - :ref:`formatting-styles` for more information on using {- and $-formatting - for log messages. - - The *defaults* parameter can be a dictionary with default values to use in - custom fields. For example: - ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` - - .. versionchanged:: 3.2 - The *style* parameter was added. - - .. versionchanged:: 3.8 - The *validate* parameter was added. Incorrect or mismatched style and fmt - will raise a ``ValueError``. - For example: ``logging.Formatter('%(asctime)s - %(message)s', style='{')``. - - .. versionchanged:: 3.10 - The *defaults* parameter was added. - - .. method:: format(record) - - The record's attribute dictionary is used as the operand to a string - formatting operation. Returns the resulting string. Before formatting the - dictionary, a couple of preparatory steps are carried out. The *message* - attribute of the record is computed using *msg* % *args*. If the - formatting string contains ``'(asctime)'``, :meth:`formatTime` is called - to format the event time. If there is exception information, it is - formatted using :meth:`formatException` and appended to the message. Note - that the formatted exception information is cached in attribute - *exc_text*. This is useful because the exception information can be - pickled and sent across the wire, but you should be careful if you have - more than one :class:`Formatter` subclass which customizes the formatting - of exception information. In this case, you will have to clear the cached - value (by setting the *exc_text* attribute to ``None``) after a formatter - has done its formatting, so that the next formatter to handle the event - doesn't use the cached value, but recalculates it afresh. - - If stack information is available, it's appended after the exception - information, using :meth:`formatStack` to transform it if necessary. - - - .. method:: formatTime(record, datefmt=None) - - This method should be called from :meth:`format` by a formatter which - wants to make use of a formatted time. This method can be overridden in - formatters to provide for any specific requirement, but the basic behavior - is as follows: if *datefmt* (a string) is specified, it is used with - :func:`time.strftime` to format the creation time of the - record. Otherwise, the format '%Y-%m-%d %H:%M:%S,uuu' is used, where the - uuu part is a millisecond value and the other letters are as per the - :func:`time.strftime` documentation. An example time in this format is - ``2003-01-23 00:29:50,411``. The resulting string is returned. - - This function uses a user-configurable function to convert the creation - time to a tuple. By default, :func:`time.localtime` is used; to change - this for a particular formatter instance, set the ``converter`` attribute - to a function with the same signature as :func:`time.localtime` or - :func:`time.gmtime`. To change it for all formatters, for example if you - want all logging times to be shown in GMT, set the ``converter`` - attribute in the ``Formatter`` class. - - .. versionchanged:: 3.3 - Previously, the default format was hard-coded as in this example: - ``2010-09-06 22:38:15,292`` where the part before the comma is - handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the - part after the comma is a millisecond value. Because strptime does not - have a format placeholder for milliseconds, the millisecond value is - appended using another format string, ``'%s,%03d'`` --- and both of these - format strings have been hardcoded into this method. With the change, - these strings are defined as class-level attributes which can be - overridden at the instance level when desired. The names of the - attributes are ``default_time_format`` (for the strptime format string) - and ``default_msec_format`` (for appending the millisecond value). - - .. versionchanged:: 3.9 - The ``default_msec_format`` can be ``None``. - - .. method:: formatException(exc_info) - - Formats the specified exception information (a standard exception tuple as - returned by :func:`sys.exc_info`) as a string. This default implementation - just uses :func:`traceback.print_exception`. The resulting string is - returned. - - .. method:: formatStack(stack_info) - - Formats the specified stack information (a string as returned by - :func:`traceback.print_stack`, but with the last newline removed) as a - string. This default implementation just returns the input value. - -.. _filter: - -Filter Objects --------------- - -``Filters`` can be used by ``Handlers`` and ``Loggers`` for more sophisticated -filtering than is provided by levels. The base filter class only allows events -which are below a certain point in the logger hierarchy. For example, a filter -initialized with 'A.B' will allow events logged by loggers 'A.B', 'A.B.C', -'A.B.C.D', 'A.B.D' etc. but not 'A.BB', 'B.A.B' etc. If initialized with the -empty string, all events are passed. - - -.. class:: Filter(name='') - - Returns an instance of the :class:`Filter` class. If *name* is specified, it - names a logger which, together with its children, will have its events allowed - through the filter. If *name* is the empty string, allows every event. - - - .. method:: filter(record) - - Is the specified record to be logged? Returns zero for no, nonzero for - yes. If deemed appropriate, the record may be modified in-place by this - method. - -Note that filters attached to handlers are consulted before an event is -emitted by the handler, whereas filters attached to loggers are consulted -whenever an event is logged (using :meth:`debug`, :meth:`info`, -etc.), before sending an event to handlers. This means that events which have -been generated by descendant loggers will not be filtered by a logger's filter -setting, unless the filter has also been applied to those descendant loggers. - -You don't actually need to subclass ``Filter``: you can pass any instance -which has a ``filter`` method with the same semantics. - -.. versionchanged:: 3.2 - You don't need to create specialized ``Filter`` classes, or use other - classes with a ``filter`` method: you can use a function (or other - callable) as a filter. The filtering logic will check to see if the filter - object has a ``filter`` attribute: if it does, it's assumed to be a - ``Filter`` and its :meth:`~Filter.filter` method is called. Otherwise, it's - assumed to be a callable and called with the record as the single - parameter. The returned value should conform to that returned by - :meth:`~Filter.filter`. - -Although filters are used primarily to filter records based on more -sophisticated criteria than levels, they get to see every record which is -processed by the handler or logger they're attached to: this can be useful if -you want to do things like counting how many records were processed by a -particular logger or handler, or adding, changing or removing attributes in -the :class:`LogRecord` being processed. Obviously changing the LogRecord needs -to be done with some care, but it does allow the injection of contextual -information into logs (see :ref:`filters-contextual`). - -.. _log-record: - -LogRecord Objects ------------------ - -:class:`LogRecord` instances are created automatically by the :class:`Logger` -every time something is logged, and can be created manually via -:func:`makeLogRecord` (for example, from a pickled event received over the -wire). - - -.. class:: LogRecord(name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None) - - Contains all the information pertinent to the event being logged. - - The primary information is passed in :attr:`msg` and :attr:`args`, which - are combined using ``msg % args`` to create the :attr:`message` field of the - record. - - :param name: The name of the logger used to log the event represented by - this LogRecord. Note that this name will always have this - value, even though it may be emitted by a handler attached to - a different (ancestor) logger. - :param level: The numeric level of the logging event (one of DEBUG, INFO etc.) - Note that this is converted to *two* attributes of the LogRecord: - ``levelno`` for the numeric value and ``levelname`` for the - corresponding level name. - :param pathname: The full pathname of the source file where the logging call - was made. - :param lineno: The line number in the source file where the logging call was - made. - :param msg: The event description message, possibly a format string with - placeholders for variable data. - :param args: Variable data to merge into the *msg* argument to obtain the - event description. - :param exc_info: An exception tuple with the current exception information, - or ``None`` if no exception information is available. - :param func: The name of the function or method from which the logging call - was invoked. - :param sinfo: A text string representing stack information from the base of - the stack in the current thread, up to the logging call. - - .. method:: getMessage() - - Returns the message for this :class:`LogRecord` instance after merging any - user-supplied arguments with the message. If the user-supplied message - argument to the logging call is not a string, :func:`str` is called on it to - convert it to a string. This allows use of user-defined classes as - messages, whose ``__str__`` method can return the actual format string to - be used. - - .. versionchanged:: 3.2 - The creation of a :class:`LogRecord` has been made more configurable by - providing a factory which is used to create the record. The factory can be - set using :func:`getLogRecordFactory` and :func:`setLogRecordFactory` - (see this for the factory's signature). - - This functionality can be used to inject your own values into a - :class:`LogRecord` at creation time. You can use the following pattern:: - - old_factory = logging.getLogRecordFactory() - - def record_factory(*args, **kwargs): - record = old_factory(*args, **kwargs) - record.custom_attribute = 0xdecafbad - return record - - logging.setLogRecordFactory(record_factory) - - With this pattern, multiple factories could be chained, and as long - as they don't overwrite each other's attributes or unintentionally - overwrite the standard attributes listed above, there should be no - surprises. - - -.. _logrecord-attributes: - -LogRecord attributes --------------------- - -The LogRecord has a number of attributes, most of which are derived from the -parameters to the constructor. (Note that the names do not always correspond -exactly between the LogRecord constructor parameters and the LogRecord -attributes.) These attributes can be used to merge data from the record into -the format string. The following table lists (in alphabetical order) the -attribute names, their meanings and the corresponding placeholder in a %-style -format string. - -If you are using {}-formatting (:func:`str.format`), you can use -``{attrname}`` as the placeholder in the format string. If you are using -$-formatting (:class:`string.Template`), use the form ``${attrname}``. In -both cases, of course, replace ``attrname`` with the actual attribute name -you want to use. - -In the case of {}-formatting, you can specify formatting flags by placing them -after the attribute name, separated from it with a colon. For example: a -placeholder of ``{msecs:03d}`` would format a millisecond value of ``4`` as -``004``. Refer to the :meth:`str.format` documentation for full details on -the options available to you. - -+----------------+-------------------------+-----------------------------------------------+ -| Attribute name | Format | Description | -+================+=========================+===============================================+ -| args | You shouldn't need to | The tuple of arguments merged into ``msg`` to | -| | format this yourself. | produce ``message``, or a dict whose values | -| | | are used for the merge (when there is only one| -| | | argument, and it is a dictionary). | -+----------------+-------------------------+-----------------------------------------------+ -| asctime | ``%(asctime)s`` | Human-readable time when the | -| | | :class:`LogRecord` was created. By default | -| | | this is of the form '2003-07-08 16:49:45,896' | -| | | (the numbers after the comma are millisecond | -| | | portion of the time). | -+----------------+-------------------------+-----------------------------------------------+ -| created | ``%(created)f`` | Time when the :class:`LogRecord` was created | -| | | (as returned by :func:`time.time`). | -+----------------+-------------------------+-----------------------------------------------+ -| exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | -| | format this yourself. | if no exception has occurred, ``None``. | -+----------------+-------------------------+-----------------------------------------------+ -| filename | ``%(filename)s`` | Filename portion of ``pathname``. | -+----------------+-------------------------+-----------------------------------------------+ -| funcName | ``%(funcName)s`` | Name of function containing the logging call. | -+----------------+-------------------------+-----------------------------------------------+ -| levelname | ``%(levelname)s`` | Text logging level for the message | -| | | (``'DEBUG'``, ``'INFO'``, ``'WARNING'``, | -| | | ``'ERROR'``, ``'CRITICAL'``). | -+----------------+-------------------------+-----------------------------------------------+ -| levelno | ``%(levelno)s`` | Numeric logging level for the message | -| | | (:const:`DEBUG`, :const:`INFO`, | -| | | :const:`WARNING`, :const:`ERROR`, | -| | | :const:`CRITICAL`). | -+----------------+-------------------------+-----------------------------------------------+ -| lineno | ``%(lineno)d`` | Source line number where the logging call was | -| | | issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| message | ``%(message)s`` | The logged message, computed as ``msg % | -| | | args``. This is set when | -| | | :meth:`Formatter.format` is invoked. | -+----------------+-------------------------+-----------------------------------------------+ -| module | ``%(module)s`` | Module (name portion of ``filename``). | -+----------------+-------------------------+-----------------------------------------------+ -| msecs | ``%(msecs)d`` | Millisecond portion of the time when the | -| | | :class:`LogRecord` was created. | -+----------------+-------------------------+-----------------------------------------------+ -| msg | You shouldn't need to | The format string passed in the original | -| | format this yourself. | logging call. Merged with ``args`` to | -| | | produce ``message``, or an arbitrary object | -| | | (see :ref:`arbitrary-object-messages`). | -+----------------+-------------------------+-----------------------------------------------+ -| name | ``%(name)s`` | Name of the logger used to log the call. | -+----------------+-------------------------+-----------------------------------------------+ -| pathname | ``%(pathname)s`` | Full pathname of the source file where the | -| | | logging call was issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| process | ``%(process)d`` | Process ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| processName | ``%(processName)s`` | Process name (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| relativeCreated| ``%(relativeCreated)d`` | Time in milliseconds when the LogRecord was | -| | | created, relative to the time the logging | -| | | module was loaded. | -+----------------+-------------------------+-----------------------------------------------+ -| stack_info | You shouldn't need to | Stack frame information (where available) | -| | format this yourself. | from the bottom of the stack in the current | -| | | thread, up to and including the stack frame | -| | | of the logging call which resulted in the | -| | | creation of this record. | -+----------------+-------------------------+-----------------------------------------------+ -| thread | ``%(thread)d`` | Thread ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| threadName | ``%(threadName)s`` | Thread name (if available). | -+----------------+-------------------------+-----------------------------------------------+ - -.. versionchanged:: 3.1 - *processName* was added. - - -.. _logger-adapter: - -LoggerAdapter Objects ---------------------- - -:class:`LoggerAdapter` instances are used to conveniently pass contextual -information into logging calls. For a usage example, see the section on -:ref:`adding contextual information to your logging output `. - -.. class:: LoggerAdapter(logger, extra) - - Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance and a dict-like object. - - .. method:: process(msg, kwargs) - - Modifies the message and/or keyword arguments passed to a logging call in - order to insert contextual information. This implementation takes the object - passed as *extra* to the constructor and adds it to *kwargs* using key - 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the - (possibly modified) versions of the arguments passed in. - -In addition to the above, :class:`LoggerAdapter` supports the following -methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, -:meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, -:meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, -:meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and -:meth:`~Logger.hasHandlers`. These methods have the same signatures as their -counterparts in :class:`Logger`, so you can use the two types of instances -interchangeably. - -.. versionchanged:: 3.2 - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. - - -Thread Safety -------------- - -The logging module is intended to be thread-safe without any special work -needing to be done by its clients. It achieves this though using threading -locks; there is one lock to serialize access to the module's shared data, and -each handler also creates a lock to serialize access to its underlying I/O. - -If you are implementing asynchronous signal handlers using the :mod:`signal` -module, you may not be able to use logging from within such handlers. This is -because lock implementations in the :mod:`threading` module are not always -re-entrant, and so cannot be invoked from such signal handlers. - - -Module-Level Functions ----------------------- - -In addition to the classes described above, there are a number of module-level -functions. - - -.. function:: getLogger(name=None) - - Return a logger with the specified name or, if name is ``None``, return a - logger which is the root logger of the hierarchy. If specified, the name is - typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. - Choice of these names is entirely up to the developer who is using logging. - - All calls to this function with a given name return the same logger instance. - This means that logger instances never need to be passed between different parts - of an application. - - -.. function:: getLoggerClass() - - Return either the standard :class:`Logger` class, or the last class passed to - :func:`setLoggerClass`. This function may be called from within a new class - definition, to ensure that installing a customized :class:`Logger` class will - not undo customizations already applied by other code. For example:: - - class MyLogger(logging.getLoggerClass()): - # ... override behaviour here - - -.. function:: getLogRecordFactory() - - Return a callable which is used to create a :class:`LogRecord`. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`setLogRecordFactory`, - to allow developers more control over how the :class:`LogRecord` - representing a logging event is constructed. - - See :func:`setLogRecordFactory` for more information about the how the - factory is called. - -.. function:: debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)-15s %(clientip)s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like: - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. - -.. function:: info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. - - .. note:: There is an obsolete function ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - -.. function:: error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. - - -.. function:: exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging - message. This function should only be called from an exception handler. - -.. function:: log(level, msg, *args, **kwargs) - - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. - - .. note:: The above module-level convenience functions, which delegate to the - root logger, call :func:`basicConfig` to ensure that at least one handler - is available. Because of this, they should *not* be used in threads, - in versions of Python earlier than 2.7.1 and 3.2, unless at least one - handler has been added to the root logger *before* the threads are - started. In earlier versions of Python, due to a thread safety shortcoming - in :func:`basicConfig`, this can (under rare circumstances) lead to - handlers being added multiple times to the root logger, which can in turn - lead to multiple messages for the same event. - -.. function:: disable(level=CRITICAL) - - Provides an overriding level *level* for all loggers which takes precedence over - the logger's own level. When the need arises to temporarily throttle logging - output down across the whole application, this function can be useful. Its - effect is to disable all logging calls of severity *level* and below, so that - if you call it with a value of INFO, then all INFO and DEBUG events would be - discarded, whereas those of severity WARNING and above would be processed - according to the logger's effective level. If - ``logging.disable(logging.NOTSET)`` is called, it effectively removes this - overriding level, so that logging output again depends on the effective - levels of individual loggers. - - Note that if you have defined any custom logging level higher than - ``CRITICAL`` (this is not recommended), you won't be able to rely on the - default value for the *level* parameter, but will have to explicitly supply a - suitable value. - - .. versionchanged:: 3.7 - The *level* parameter was defaulted to level ``CRITICAL``. See - :issue:`28524` for more information about this change. - -.. function:: addLevelName(level, levelName) - - Associates level *level* with text *levelName* in an internal dictionary, which is - used to map numeric levels to a textual representation, for example when a - :class:`Formatter` formats a message. This function can also be used to define - your own levels. The only constraints are that all levels used must be - registered using this function, levels should be positive integers and they - should increase in increasing order of severity. - - .. note:: If you are thinking of defining your own levels, please see the - section on :ref:`custom-levels`. - -.. function:: getLevelName(level) - - Returns the textual representation of logging level *level*. If the level is one - of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, - :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you - have associated levels with names using :func:`addLevelName` then the name you - have associated with *level* is returned. If a numeric value corresponding to one - of the defined levels is passed in, the corresponding string representation is - returned. Otherwise, the string 'Level %s' % level is returned. - - .. note:: Levels are internally integers (as they need to be compared in the - logging logic). This function is used to convert between an integer level - and the level name displayed in the formatted log output by means of the - ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). - - .. versionchanged:: 3.4 - In Python versions earlier than 3.4, this function could also be passed a - text level, and would return the corresponding numeric value of the level. - This undocumented behaviour was considered a mistake, and was removed in - Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. - -.. function:: makeLogRecord(attrdict) - - Creates and returns a new :class:`LogRecord` instance whose attributes are - defined by *attrdict*. This function is useful for taking a pickled - :class:`LogRecord` attribute dictionary, sent over a socket, and reconstituting - it as a :class:`LogRecord` instance at the receiving end. - - -.. function:: basicConfig(**kwargs) - - Does basic configuration for the logging system by creating a - :class:`StreamHandler` with a default :class:`Formatter` and adding it to the - root logger. The functions :func:`debug`, :func:`info`, :func:`warning`, - :func:`error` and :func:`critical` will call :func:`basicConfig` automatically - if no handlers are defined for the root logger. - - This function does nothing if the root logger already has handlers - configured, unless the keyword argument *force* is set to ``True``. - - .. note:: This function should be called from the main thread - before other threads are started. In versions of Python prior to - 2.7.1 and 3.2, if this function is called from multiple threads, - it is possible (in rare circumstances) that a handler will be added - to the root logger more than once, leading to unexpected results - such as messages being duplicated in the log. - - The following keyword arguments are supported. - - .. tabularcolumns:: |l|L| - - +--------------+---------------------------------------------+ - | Format | Description | - +==============+=============================================+ - | *filename* | Specifies that a :class:`FileHandler` be | - | | created, using the specified filename, | - | | rather than a :class:`StreamHandler`. | - +--------------+---------------------------------------------+ - | *filemode* | If *filename* is specified, open the file | - | | in this :ref:`mode `. Defaults | - | | to ``'a'``. | - +--------------+---------------------------------------------+ - | *format* | Use the specified format string for the | - | | handler. Defaults to attributes | - | | ``levelname``, ``name`` and ``message`` | - | | separated by colons. | - +--------------+---------------------------------------------+ - | *datefmt* | Use the specified date/time format, as | - | | accepted by :func:`time.strftime`. | - +--------------+---------------------------------------------+ - | *style* | If *format* is specified, use this style | - | | for the format string. One of ``'%'``, | - | | ``'{'`` or ``'$'`` for :ref:`printf-style | - | | `, | - | | :meth:`str.format` or | - | | :class:`string.Template` respectively. | - | | Defaults to ``'%'``. | - +--------------+---------------------------------------------+ - | *level* | Set the root logger level to the specified | - | | :ref:`level `. | - +--------------+---------------------------------------------+ - | *stream* | Use the specified stream to initialize the | - | | :class:`StreamHandler`. Note that this | - | | argument is incompatible with *filename* - | - | | if both are present, a ``ValueError`` is | - | | raised. | - +--------------+---------------------------------------------+ - | *handlers* | If specified, this should be an iterable of | - | | already created handlers to add to the root | - | | logger. Any handlers which don't already | - | | have a formatter set will be assigned the | - | | default formatter created in this function. | - | | Note that this argument is incompatible | - | | with *filename* or *stream* - if both | - | | are present, a ``ValueError`` is raised. | - +--------------+---------------------------------------------+ - | *force* | If this keyword argument is specified as | - | | true, any existing handlers attached to the | - | | root logger are removed and closed, before | - | | carrying out the configuration as specified | - | | by the other arguments. | - +--------------+---------------------------------------------+ - | *encoding* | If this keyword argument is specified along | - | | with *filename*, its value is used when the | - | | :class:`FileHandler` is created, and thus | - | | used when opening the output file. | - +--------------+---------------------------------------------+ - | *errors* | If this keyword argument is specified along | - | | with *filename*, its value is used when the | - | | :class:`FileHandler` is created, and thus | - | | used when opening the output file. If not | - | | specified, the value 'backslashreplace' is | - | | used. Note that if ``None`` is specified, | - | | it will be passed as such to :func:`open`, | - | | which means that it will be treated the | - | | same as passing 'errors'. | - +--------------+---------------------------------------------+ - - .. versionchanged:: 3.2 - The *style* argument was added. - - .. versionchanged:: 3.3 - The *handlers* argument was added. Additional checks were added to - catch situations where incompatible arguments are specified (e.g. - *handlers* together with *stream* or *filename*, or *stream* - together with *filename*). - - .. versionchanged:: 3.8 - The *force* argument was added. - - .. versionchanged:: 3.9 - The *encoding* and *errors* arguments were added. - -.. function:: shutdown() - - Informs the logging system to perform an orderly shutdown by flushing and - closing all handlers. This should be called at application exit and no - further use of the logging system should be made after this call. - - When the logging module is imported, it registers this function as an exit - handler (see :mod:`atexit`), so normally there's no need to do that - manually. - - -.. function:: setLoggerClass(klass) - - Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`__init__` such that only a name argument is - required, and the :meth:`__init__` should call :meth:`Logger.__init__`. This - function is typically called before any loggers are instantiated by applications - which need to use custom logger behavior. After this call, as at any other - time, do not instantiate loggers directly using the subclass: continue to use - the :func:`logging.getLogger` API to get your loggers. - - -.. function:: setLogRecordFactory(factory) - - Set a callable which is used to create a :class:`LogRecord`. - - :param factory: The factory callable to be used to instantiate a log record. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`getLogRecordFactory`, to - allow developers more control over how the :class:`LogRecord` representing - a logging event is constructed. - - The factory has the following signature: - - ``factory(name, level, fn, lno, msg, args, exc_info, func=None, sinfo=None, **kwargs)`` - - :name: The logger name. - :level: The logging level (numeric). - :fn: The full pathname of the file where the logging call was made. - :lno: The line number in the file where the logging call was made. - :msg: The logging message. - :args: The arguments for the logging message. - :exc_info: An exception tuple, or ``None``. - :func: The name of the function or method which invoked the logging - call. - :sinfo: A stack traceback such as is provided by - :func:`traceback.print_stack`, showing the call hierarchy. - :kwargs: Additional keyword arguments. - - -Module-Level Attributes ------------------------ - -.. attribute:: lastResort - - A "handler of last resort" is available through this attribute. This - is a :class:`StreamHandler` writing to ``sys.stderr`` with a level of - ``WARNING``, and is used to handle logging events in the absence of any - logging configuration. The end result is to just print the message to - ``sys.stderr``. This replaces the earlier error message saying that - "no handlers could be found for logger XYZ". If you need the earlier - behaviour for some reason, ``lastResort`` can be set to ``None``. - - .. versionadded:: 3.2 - -Integration with the warnings module ------------------------------------- - -The :func:`captureWarnings` function can be used to integrate :mod:`logging` -with the :mod:`warnings` module. - -.. function:: captureWarnings(capture) - - This function is used to turn the capture of warnings by logging on and - off. - - If *capture* is ``True``, warnings issued by the :mod:`warnings` module will - be redirected to the logging system. Specifically, a warning will be - formatted using :func:`warnings.formatwarning` and the resulting string - logged to a logger named ``'py.warnings'`` with a severity of :const:`WARNING`. - - If *capture* is ``False``, the redirection of warnings to the logging system - will stop, and warnings will be redirected to their original destinations - (i.e. those in effect before ``captureWarnings(True)`` was called). - - -.. seealso:: - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :pep:`282` - A Logging System - The proposal which described this feature for inclusion in the Python standard - library. - - `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the - package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard - library. diff --git a/Doc/library/readline.rst.bak b/Doc/library/readline.rst.bak deleted file mode 100644 index 3ff64885f7fced..00000000000000 --- a/Doc/library/readline.rst.bak +++ /dev/null @@ -1,361 +0,0 @@ -:mod:`readline` --- GNU readline interface -========================================== - -.. module:: readline - :platform: Unix - :synopsis: GNU readline support for Python. - -.. sectionauthor:: Skip Montanaro - --------------- - -The :mod:`readline` module defines a number of functions to facilitate -completion and reading/writing of history files from the Python interpreter. -This module can be used directly, or via the :mod:`rlcompleter` module, which -supports completion of Python identifiers at the interactive prompt. Settings -made using this module affect the behaviour of both the interpreter's -interactive prompt and the prompts offered by the built-in :func:`input` -function. - -Readline keybindings may be configured via an initialization file, typically -``.inputrc`` in your home directory. See `Readline Init File -`_ -in the GNU Readline manual for information about the format and -allowable constructs of that file, and the capabilities of the -Readline library in general. - -.. note:: - - The underlying Readline library API may be implemented by - the ``libedit`` library instead of GNU readline. - On macOS the :mod:`readline` module detects which library is being used - at run time. - - The configuration file for ``libedit`` is different from that - of GNU readline. If you programmatically load configuration strings - you can check for the text "libedit" in :const:`readline.__doc__` - to differentiate between GNU readline and libedit. - - If you use *editline*/``libedit`` readline emulation on macOS, the - initialization file located in your home directory is named - ``.editrc``. For example, the following content in ``~/.editrc`` will - turn ON *vi* keybindings and TAB completion:: - - python:bind -v - python:bind ^I rl_complete - - -Init file ---------- - -The following functions relate to the init file and user configuration: - - -.. function:: parse_and_bind(string) - - Execute the init line provided in the *string* argument. This calls - :c:func:`rl_parse_and_bind` in the underlying library. - - -.. function:: read_init_file([filename]) - - Execute a readline initialization file. The default filename is the last filename - used. This calls :c:func:`rl_read_init_file` in the underlying library. - - -Line buffer ------------ - -The following functions operate on the line buffer: - - -.. function:: get_line_buffer() - - Return the current contents of the line buffer (:c:data:`rl_line_buffer` - in the underlying library). - - -.. function:: insert_text(string) - - Insert text into the line buffer at the cursor position. This calls - :c:func:`rl_insert_text` in the underlying library, but ignores - the return value. - - -.. function:: redisplay() - - Change what's displayed on the screen to reflect the current contents of the - line buffer. This calls :c:func:`rl_redisplay` in the underlying library. - - -History file ------------- - -The following functions operate on a history file: - - -.. function:: read_history_file([filename]) - - Load a readline history file, and append it to the history list. - The default filename is :file:`~/.history`. This calls - :c:func:`read_history` in the underlying library. - - -.. function:: write_history_file([filename]) - - Save the history list to a readline history file, overwriting any - existing file. The default filename is :file:`~/.history`. This calls - :c:func:`write_history` in the underlying library. - - -.. function:: append_history_file(nelements[, filename]) - - Append the last *nelements* items of history to a file. The default filename is - :file:`~/.history`. The file must already exist. This calls - :c:func:`append_history` in the underlying library. This function - only exists if Python was compiled for a version of the library - that supports it. - - .. versionadded:: 3.5 - - -.. function:: get_history_length() - set_history_length(length) - - Set or return the desired number of lines to save in the history file. - The :func:`write_history_file` function uses this value to truncate - the history file, by calling :c:func:`history_truncate_file` in - the underlying library. Negative values imply - unlimited history file size. - - -History list ------------- - -The following functions operate on a global history list: - - -.. function:: clear_history() - - Clear the current history. This calls :c:func:`clear_history` in the - underlying library. The Python function only exists if Python was - compiled for a version of the library that supports it. - - -.. function:: get_current_history_length() - - Return the number of items currently in the history. (This is different from - :func:`get_history_length`, which returns the maximum number of lines that will - be written to a history file.) - - -.. function:: get_history_item(index) - - Return the current contents of history item at *index*. The item index - is one-based. This calls :c:func:`history_get` in the underlying library. - - -.. function:: remove_history_item(pos) - - Remove history item specified by its position from the history. - The position is zero-based. This calls :c:func:`remove_history` in - the underlying library. - - -.. function:: replace_history_item(pos, line) - - Replace history item specified by its position with *line*. - The position is zero-based. This calls :c:func:`replace_history_entry` - in the underlying library. - - -.. function:: add_history(line) - - Append *line* to the history buffer, as if it was the last line typed. - This calls :c:func:`add_history` in the underlying library. - - -.. function:: set_auto_history(enabled) - - Enable or disable automatic calls to :c:func:`add_history` when reading - input via readline. The *enabled* argument should be a Boolean value - that when true, enables auto history, and that when false, disables - auto history. - - .. versionadded:: 3.6 - - .. impl-detail:: - Auto history is enabled by default, and changes to this do not persist - across multiple sessions. - - -Startup hooks -------------- - - -.. function:: set_startup_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_startup_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any function - already installed is removed. The hook is called with no - arguments just before readline prints the first prompt. - - -.. function:: set_pre_input_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_pre_input_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any - function already installed is removed. The hook is called - with no arguments after the first prompt has been printed and just before - readline starts reading input characters. This function only exists - if Python was compiled for a version of the library that supports it. - - -Completion ----------- - -The following functions relate to implementing a custom word completion -function. This is typically operated by the Tab key, and can suggest and -automatically complete a word being typed. By default, Readline is set up -to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used -with a custom completer, a different set of word delimiters should be set. - - -.. function:: set_completer([function]) - - Set or remove the completer function. If *function* is specified, it will be - used as the new completer function; if omitted or ``None``, any completer - function already installed is removed. The completer function is called as - ``function(text, state)``, for *state* in ``0``, ``1``, ``2``, ..., until it - returns a non-string value. It should return the next possible completion - starting with *text*. - - The installed completer function is invoked by the *entry_func* callback - passed to :c:func:`rl_completion_matches` in the underlying library. - The *text* string comes from the first parameter to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. - - -.. function:: get_completer() - - Get the completer function, or ``None`` if no completer function has been set. - - -.. function:: get_completion_type() - - Get the type of completion being attempted. This returns the - :c:data:`rl_completion_type` variable in the underlying library as - an integer. - - -.. function:: get_begidx() - get_endidx() - - Get the beginning or ending index of the completion scope. - These indexes are the *start* and *end* arguments passed to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. The values may be different in the same - input editing scenario based on the underlying C readline implemtation. - Ex: libedit is known to behave differently than libreadline. - - -.. function:: set_completer_delims(string) - get_completer_delims() - - Set or get the word delimiters for completion. These determine the - start of the word to be considered for completion (the completion scope). - These functions access the :c:data:`rl_completer_word_break_characters` - variable in the underlying library. - - -.. function:: set_completion_display_matches_hook([function]) - - Set or remove the completion display function. If *function* is - specified, it will be used as the new completion display function; - if omitted or ``None``, any completion display function already - installed is removed. This sets or clears the - :c:data:`rl_completion_display_matches_hook` callback in the - underlying library. The completion display function is called as - ``function(substitution, [matches], longest_match_length)`` once - each time matches need to be displayed. - - -.. _readline-example: - -Example -------- - -The following example demonstrates how to use the :mod:`readline` module's -history reading and writing functions to automatically load and save a history -file named :file:`.python_history` from the user's home directory. The code -below would normally be executed automatically during interactive sessions -from the user's :envvar:`PYTHONSTARTUP` file. :: - - import atexit - import os - import readline - - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - try: - readline.read_history_file(histfile) - # default history len is -1 (infinite), which may grow unruly - readline.set_history_length(1000) - except FileNotFoundError: - pass - - atexit.register(readline.write_history_file, histfile) - -This code is actually automatically run when Python is run in -:ref:`interactive mode ` (see :ref:`rlcompleter-config`). - -The following example achieves the same goal but supports concurrent interactive -sessions, by only appending the new history. :: - - import atexit - import os - import readline - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - - try: - readline.read_history_file(histfile) - h_len = readline.get_current_history_length() - except FileNotFoundError: - open(histfile, 'wb').close() - h_len = 0 - - def save(prev_h_len, histfile): - new_h_len = readline.get_current_history_length() - readline.set_history_length(1000) - readline.append_history_file(new_h_len - prev_h_len, histfile) - atexit.register(save, h_len, histfile) - -The following example extends the :class:`code.InteractiveConsole` class to -support history save/restore. :: - - import atexit - import code - import os - import readline - - class HistoryConsole(code.InteractiveConsole): - def __init__(self, locals=None, filename="", - histfile=os.path.expanduser("~/.console-history")): - code.InteractiveConsole.__init__(self, locals, filename) - self.init_history(histfile) - - def init_history(self, histfile): - readline.parse_and_bind("tab: complete") - if hasattr(readline, "read_history_file"): - try: - readline.read_history_file(histfile) - except FileNotFoundError: - pass - atexit.register(self.save_history, histfile) - - def save_history(self, histfile): - readline.set_history_length(1000) - readline.write_history_file(histfile) diff --git a/Doc/library/sqlite3.rst.bak b/Doc/library/sqlite3.rst.bak deleted file mode 100644 index 172ce6c6bb03ba..00000000000000 --- a/Doc/library/sqlite3.rst.bak +++ /dev/null @@ -1,1094 +0,0 @@ -:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases -============================================================ - -.. module:: sqlite3 - :synopsis: A DB-API 2.0 implementation using SQLite 3.x. - -.. sectionauthor:: Gerhard Häring - -**Source code:** :source:`Lib/sqlite3/` - --------------- - -SQLite is a C library that provides a lightweight disk-based database that -doesn't require a separate server process and allows accessing the database -using a nonstandard variant of the SQL query language. Some applications can use -SQLite for internal data storage. It's also possible to prototype an -application using SQLite and then port the code to a larger database such as -PostgreSQL or Oracle. - -The sqlite3 module was written by Gerhard Häring. It provides a SQL interface -compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.7.15 or newer. - -To use the module, you must first create a :class:`Connection` object that -represents the database. Here the data will be stored in the -:file:`example.db` file:: - - import sqlite3 - con = sqlite3.connect('example.db') - -You can also supply the special name ``:memory:`` to create a database in RAM. - -Once you have a :class:`Connection`, you can create a :class:`Cursor` object -and call its :meth:`~Cursor.execute` method to perform SQL commands:: - - cur = con.cursor() - - # Create table - cur.execute('''CREATE TABLE stocks - (date text, trans text, symbol text, qty real, price real)''') - - # Insert a row of data - cur.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") - - # Save (commit) the changes - con.commit() - - # We can also close the connection if we are done with it. - # Just be sure any changes have been committed or they will be lost. - con.close() - -The data you've saved is persistent and is available in subsequent sessions:: - - import sqlite3 - con = sqlite3.connect('example.db') - cur = con.cursor() - -Usually your SQL operations will need to use values from Python variables. You -shouldn't assemble your query using Python's string operations because doing so -is insecure; it makes your program vulnerable to an SQL injection attack -(see https://xkcd.com/327/ for humorous example of what can go wrong). - -Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder -wherever you want to use a value, and then provide a tuple of values as the -second argument to the cursor's :meth:`~Cursor.execute` method. (Other database -modules may use a different placeholder, such as ``%s`` or ``:1``.) For -example:: - - # Never do this -- insecure! - symbol = 'RHAT' - cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) - - # Do this instead - t = ('RHAT',) - cur.execute('SELECT * FROM stocks WHERE symbol=?', t) - print(cur.fetchone()) - - # Larger example that inserts many records at a time - purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), - ('2006-04-06', 'SELL', 'IBM', 500, 53.00), - ] - cur.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) - -To retrieve data after executing a SELECT statement, you can either treat the -cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to -retrieve a single matching row, or call :meth:`~Cursor.fetchall` to get a list of the -matching rows. - -This example uses the iterator form:: - - >>> for row in cur.execute('SELECT * FROM stocks ORDER BY price'): - print(row) - - ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) - ('2006-03-28', 'BUY', 'IBM', 1000, 45.0) - ('2006-04-06', 'SELL', 'IBM', 500, 53.0) - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) - - -.. seealso:: - - https://www.sqlite.org - The SQLite web page; the documentation describes the syntax and the - available data types for the supported SQL dialect. - - https://www.w3schools.com/sql/ - Tutorial, reference and examples for learning SQL syntax. - - :pep:`249` - Database API Specification 2.0 - PEP written by Marc-André Lemburg. - - -.. _sqlite3-module-contents: - -Module functions and constants ------------------------------- - - -.. data:: version - - The version number of this module, as a string. This is not the version of - the SQLite library. - - -.. data:: version_info - - The version number of this module, as a tuple of integers. This is not the - version of the SQLite library. - - -.. data:: sqlite_version - - The version number of the run-time SQLite library, as a string. - - -.. data:: sqlite_version_info - - The version number of the run-time SQLite library, as a tuple of integers. - - -.. data:: PARSE_DECLTYPES - - This constant is meant to be used with the *detect_types* parameter of the - :func:`connect` function. - - Setting it makes the :mod:`sqlite3` module parse the declared type for each - column it returns. It will parse out the first word of the declared type, - i. e. for "integer primary key", it will parse out "integer", or for - "number(10)" it will parse out "number". Then for that column, it will look - into the converters dictionary and use the converter function registered for - that type there. - - -.. data:: PARSE_COLNAMES - - This constant is meant to be used with the *detect_types* parameter of the - :func:`connect` function. - - Setting this makes the SQLite interface parse the column name for each column it - returns. It will look for a string formed [mytype] in there, and then decide - that 'mytype' is the type of the column. It will try to find an entry of - 'mytype' in the converters dictionary and then use the converter function found - there to return the value. The column name found in :attr:`Cursor.description` - does not include the type, i. e. if you use something like - ``'as "Expiration date [datetime]"'`` in your SQL, then we will parse out - everything until the first ``'['`` for the column name and strip - the preceeding space: the column name would simply be "Expiration date". - - -.. function:: connect(database[, timeout, detect_types, isolation_level, check_same_thread, factory, cached_statements, uri]) - - Opens a connection to the SQLite database file *database*. By default returns a - :class:`Connection` object, unless a custom *factory* is given. - - *database* is a :term:`path-like object` giving the pathname (absolute or - relative to the current working directory) of the database file to be opened. - You can use ``":memory:"`` to open a database connection to a database that - resides in RAM instead of on disk. - - When a database is accessed by multiple connections, and one of the processes - modifies the database, the SQLite database is locked until that transaction is - committed. The *timeout* parameter specifies how long the connection should wait - for the lock to go away until raising an exception. The default for the timeout - parameter is 5.0 (five seconds). - - For the *isolation_level* parameter, please see the - :attr:`~Connection.isolation_level` property of :class:`Connection` objects. - - SQLite natively supports only the types TEXT, INTEGER, REAL, BLOB and NULL. If - you want to use other types you must add support for them yourself. The - *detect_types* parameter and the using custom **converters** registered with the - module-level :func:`register_converter` function allow you to easily do that. - - *detect_types* defaults to 0 (i. e. off, no type detection), you can set it to - any combination of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to turn - type detection on. Due to SQLite behaviour, types can't be detected for generated - fields (for example ``max(data)``), even when *detect_types* parameter is set. In - such case, the returned type is :class:`str`. - - By default, *check_same_thread* is :const:`True` and only the creating thread may - use the connection. If set :const:`False`, the returned connection may be shared - across multiple threads. When using multiple threads with the same connection - writing operations should be serialized by the user to avoid data corruption. - - By default, the :mod:`sqlite3` module uses its :class:`Connection` class for the - connect call. You can, however, subclass the :class:`Connection` class and make - :func:`connect` use your class instead by providing your class for the *factory* - parameter. - - Consult the section :ref:`sqlite3-types` of this manual for details. - - The :mod:`sqlite3` module internally uses a statement cache to avoid SQL parsing - overhead. If you want to explicitly set the number of statements that are cached - for the connection, you can set the *cached_statements* parameter. The currently - implemented default is to cache 100 statements. - - If *uri* is true, *database* is interpreted as a URI. This allows you - to specify options. For example, to open a database in read-only mode - you can use:: - - db = sqlite3.connect('file:path/to/database?mode=ro', uri=True) - - More information about this feature, including a list of recognized options, can - be found in the `SQLite URI documentation `_. - - .. audit-event:: sqlite3.connect database sqlite3.connect - - .. versionchanged:: 3.4 - Added the *uri* parameter. - - .. versionchanged:: 3.7 - *database* can now also be a :term:`path-like object`, not only a string. - - -.. function:: register_converter(typename, callable) - - Registers a callable to convert a bytestring from the database into a custom - Python type. The callable will be invoked for all database values that are of - the type *typename*. Confer the parameter *detect_types* of the :func:`connect` - function for how the type detection works. Note that *typename* and the name of - the type in your query are matched in case-insensitive manner. - - -.. function:: register_adapter(type, callable) - - Registers a callable to convert the custom Python type *type* into one of - SQLite's supported types. The callable *callable* accepts as single parameter - the Python value, and must return a value of the following types: int, - float, str or bytes. - - -.. function:: complete_statement(sql) - - Returns :const:`True` if the string *sql* contains one or more complete SQL - statements terminated by semicolons. It does not verify that the SQL is - syntactically correct, only that there are no unclosed string literals and the - statement is terminated by a semicolon. - - This can be used to build a shell for SQLite, as in the following example: - - - .. literalinclude:: ../includes/sqlite3/complete_statement.py - - -.. function:: enable_callback_tracebacks(flag) - - By default you will not get any tracebacks in user-defined functions, - aggregates, converters, authorizer callbacks etc. If you want to debug them, - you can call this function with *flag* set to ``True``. Afterwards, you will - get tracebacks from callbacks on ``sys.stderr``. Use :const:`False` to - disable the feature again. - - -.. _sqlite3-connection-objects: - -Connection Objects ------------------- - -.. class:: Connection - - A SQLite database connection has the following attributes and methods: - - .. attribute:: isolation_level - - Get or set the current default isolation level. :const:`None` for autocommit mode or - one of "DEFERRED", "IMMEDIATE" or "EXCLUSIVE". See section - :ref:`sqlite3-controlling-transactions` for a more detailed explanation. - - .. attribute:: in_transaction - - :const:`True` if a transaction is active (there are uncommitted changes), - :const:`False` otherwise. Read-only attribute. - - .. versionadded:: 3.2 - - .. method:: cursor(factory=Cursor) - - The cursor method accepts a single optional parameter *factory*. If - supplied, this must be a callable returning an instance of :class:`Cursor` - or its subclasses. - - .. method:: commit() - - This method commits the current transaction. If you don't call this method, - anything you did since the last call to ``commit()`` is not visible from - other database connections. If you wonder why you don't see the data you've - written to the database, please check you didn't forget to call this method. - - .. method:: rollback() - - This method rolls back any changes to the database since the last call to - :meth:`commit`. - - .. method:: close() - - This closes the database connection. Note that this does not automatically - call :meth:`commit`. If you just close your database connection without - calling :meth:`commit` first, your changes will be lost! - - .. method:: execute(sql[, parameters]) - - This is a nonstandard shortcut that creates a cursor object by calling - the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.execute` method with the *parameters* given, and returns - the cursor. - - .. method:: executemany(sql[, parameters]) - - This is a nonstandard shortcut that creates a cursor object by - calling the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.executemany` method with the *parameters* given, and - returns the cursor. - - .. method:: executescript(sql_script) - - This is a nonstandard shortcut that creates a cursor object by - calling the :meth:`~Connection.cursor` method, calls the cursor's - :meth:`~Cursor.executescript` method with the given *sql_script*, and - returns the cursor. - - .. method:: create_function(name, num_params, func, *, deterministic=False) - - Creates a user-defined function that you can later use from within SQL - statements under the function name *name*. *num_params* is the number of - parameters the function accepts (if *num_params* is -1, the function may - take any number of arguments), and *func* is a Python callable that is - called as the SQL function. If *deterministic* is true, the created function - is marked as `deterministic `_, which - allows SQLite to perform additional optimizations. This flag is supported by - SQLite 3.8.3 or higher, :exc:`NotSupportedError` will be raised if used - with older versions. - - The function can return any of the types supported by SQLite: bytes, str, int, - float and ``None``. - - .. versionchanged:: 3.8 - The *deterministic* parameter was added. - - Example: - - .. literalinclude:: ../includes/sqlite3/md5func.py - - - .. method:: create_aggregate(name, num_params, aggregate_class) - - Creates a user-defined aggregate function. - - The aggregate class must implement a ``step`` method, which accepts the number - of parameters *num_params* (if *num_params* is -1, the function may take - any number of arguments), and a ``finalize`` method which will return the - final result of the aggregate. - - The ``finalize`` method can return any of the types supported by SQLite: - bytes, str, int, float and ``None``. - - Example: - - .. literalinclude:: ../includes/sqlite3/mysumaggr.py - - - .. method:: create_collation(name, callable) - - Creates a collation with the specified *name* and *callable*. The callable will - be passed two string arguments. It should return -1 if the first is ordered - lower than the second, 0 if they are ordered equal and 1 if the first is ordered - higher than the second. Note that this controls sorting (ORDER BY in SQL) so - your comparisons don't affect other SQL operations. - - Note that the callable will get its parameters as Python bytestrings, which will - normally be encoded in UTF-8. - - The following example shows a custom collation that sorts "the wrong way": - - .. literalinclude:: ../includes/sqlite3/collation_reverse.py - - To remove a collation, call ``create_collation`` with ``None`` as callable:: - - con.create_collation("reverse", None) - - - .. method:: interrupt() - - You can call this method from a different thread to abort any queries that might - be executing on the connection. The query will then abort and the caller will - get an exception. - - - .. method:: set_authorizer(authorizer_callback) - - This routine registers a callback. The callback is invoked for each attempt to - access a column of a table in the database. The callback should return - :const:`SQLITE_OK` if access is allowed, :const:`SQLITE_DENY` if the entire SQL - statement should be aborted with an error and :const:`SQLITE_IGNORE` if the - column should be treated as a NULL value. These constants are available in the - :mod:`sqlite3` module. - - The first argument to the callback signifies what kind of operation is to be - authorized. The second and third argument will be arguments or :const:`None` - depending on the first argument. The 4th argument is the name of the database - ("main", "temp", etc.) if applicable. The 5th argument is the name of the - inner-most trigger or view that is responsible for the access attempt or - :const:`None` if this access attempt is directly from input SQL code. - - Please consult the SQLite documentation about the possible values for the first - argument and the meaning of the second and third argument depending on the first - one. All necessary constants are available in the :mod:`sqlite3` module. - - - .. method:: set_progress_handler(handler, n) - - This routine registers a callback. The callback is invoked for every *n* - instructions of the SQLite virtual machine. This is useful if you want to - get called from SQLite during long-running operations, for example to update - a GUI. - - If you want to clear any previously installed progress handler, call the - method with :const:`None` for *handler*. - - Returning a non-zero value from the handler function will terminate the - currently executing query and cause it to raise an :exc:`OperationalError` - exception. - - - .. method:: set_trace_callback(trace_callback) - - Registers *trace_callback* to be called for each SQL statement that is - actually executed by the SQLite backend. - - The only argument passed to the callback is the statement (as string) that - is being executed. The return value of the callback is ignored. Note that - the backend does not only run statements passed to the :meth:`Cursor.execute` - methods. Other sources include the transaction management of the Python - module and the execution of triggers defined in the current database. - - Passing :const:`None` as *trace_callback* will disable the trace callback. - - .. versionadded:: 3.3 - - - .. method:: enable_load_extension(enabled) - - This routine allows/disallows the SQLite engine to load SQLite extensions - from shared libraries. SQLite extensions can define new functions, - aggregates or whole new virtual table implementations. One well-known - extension is the fulltext-search extension distributed with SQLite. - - Loadable extensions are disabled by default. See [#f1]_. - - .. versionadded:: 3.2 - - .. literalinclude:: ../includes/sqlite3/load_extension.py - - .. method:: load_extension(path) - - This routine loads a SQLite extension from a shared library. You have to - enable extension loading with :meth:`enable_load_extension` before you can - use this routine. - - Loadable extensions are disabled by default. See [#f1]_. - - .. versionadded:: 3.2 - - .. attribute:: row_factory - - You can change this attribute to a callable that accepts the cursor and the - original row as a tuple and will return the real result row. This way, you can - implement more advanced ways of returning results, such as returning an object - that can also access columns by name. - - Example: - - .. literalinclude:: ../includes/sqlite3/row_factory.py - - If returning a tuple doesn't suffice and you want name-based access to - columns, you should consider setting :attr:`row_factory` to the - highly-optimized :class:`sqlite3.Row` type. :class:`Row` provides both - index-based and case-insensitive name-based access to columns with almost no - memory overhead. It will probably be better than your own custom - dictionary-based approach or even a db_row based solution. - - .. XXX what's a db_row-based solution? - - - .. attribute:: text_factory - - Using this attribute you can control what objects are returned for the ``TEXT`` - data type. By default, this attribute is set to :class:`str` and the - :mod:`sqlite3` module will return Unicode objects for ``TEXT``. If you want to - return bytestrings instead, you can set it to :class:`bytes`. - - You can also set it to any other callable that accepts a single bytestring - parameter and returns the resulting object. - - See the following example code for illustration: - - .. literalinclude:: ../includes/sqlite3/text_factory.py - - - .. attribute:: total_changes - - Returns the total number of database rows that have been modified, inserted, or - deleted since the database connection was opened. - - - .. method:: iterdump - - Returns an iterator to dump the database in an SQL text format. Useful when - saving an in-memory database for later restoration. This function provides - the same capabilities as the :kbd:`.dump` command in the :program:`sqlite3` - shell. - - Example:: - - # Convert file existing_db.db to SQL dump file dump.sql - import sqlite3 - - con = sqlite3.connect('existing_db.db') - with open('dump.sql', 'w') as f: - for line in con.iterdump(): - f.write('%s\n' % line) - con.close() - - - .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) - - This method makes a backup of a SQLite database even while it's being accessed - by other clients, or concurrently by the same connection. The copy will be - written into the mandatory argument *target*, that must be another - :class:`Connection` instance. - - By default, or when *pages* is either ``0`` or a negative integer, the entire - database is copied in a single step; otherwise the method performs a loop - copying up to *pages* pages at a time. - - If *progress* is specified, it must either be ``None`` or a callable object that - will be executed at each iteration with three integer arguments, respectively - the *status* of the last iteration, the *remaining* number of pages still to be - copied and the *total* number of pages. - - The *name* argument specifies the database name that will be copied: it must be - a string containing either ``"main"``, the default, to indicate the main - database, ``"temp"`` to indicate the temporary database or the name specified - after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached - database. - - The *sleep* argument specifies the number of seconds to sleep by between - successive attempts to backup remaining pages, can be specified either as an - integer or a floating point value. - - Example 1, copy an existing database into another:: - - import sqlite3 - - def progress(status, remaining, total): - print(f'Copied {total-remaining} of {total} pages...') - - con = sqlite3.connect('existing_db.db') - bck = sqlite3.connect('backup.db') - with bck: - con.backup(bck, pages=1, progress=progress) - bck.close() - con.close() - - Example 2, copy an existing database into a transient copy:: - - import sqlite3 - - source = sqlite3.connect('existing_db.db') - dest = sqlite3.connect(':memory:') - source.backup(dest) - - .. versionadded:: 3.7 - - -.. _sqlite3-cursor-objects: - -Cursor Objects --------------- - -.. class:: Cursor - - A :class:`Cursor` instance has the following attributes and methods. - - .. index:: single: ? (question mark); in SQL statements - .. index:: single: : (colon); in SQL statements - - .. method:: execute(sql[, parameters]) - - Executes an SQL statement. The SQL statement may be parameterized (i. e. - placeholders instead of SQL literals). The :mod:`sqlite3` module supports two - kinds of placeholders: question marks (qmark style) and named placeholders - (named style). - - Here's an example of both styles: - - .. literalinclude:: ../includes/sqlite3/execute_1.py - - :meth:`execute` will only execute a single SQL statement. If you try to execute - more than one statement with it, it will raise a :exc:`.Warning`. Use - :meth:`executescript` if you want to execute multiple SQL statements with one - call. - - - .. method:: executemany(sql, seq_of_parameters) - - Executes an SQL command against all parameter sequences or mappings found in - the sequence *seq_of_parameters*. The :mod:`sqlite3` module also allows - using an :term:`iterator` yielding parameters instead of a sequence. - - .. literalinclude:: ../includes/sqlite3/executemany_1.py - - Here's a shorter example using a :term:`generator`: - - .. literalinclude:: ../includes/sqlite3/executemany_2.py - - - .. method:: executescript(sql_script) - - This is a nonstandard convenience method for executing multiple SQL statements - at once. It issues a ``COMMIT`` statement first, then executes the SQL script it - gets as a parameter. - - *sql_script* can be an instance of :class:`str`. - - Example: - - .. literalinclude:: ../includes/sqlite3/executescript.py - - - .. method:: fetchone() - - Fetches the next row of a query result set, returning a single sequence, - or :const:`None` when no more data is available. - - - .. method:: fetchmany(size=cursor.arraysize) - - Fetches the next set of rows of a query result, returning a list. An empty - list is returned when no more rows are available. - - The number of rows to fetch per call is specified by the *size* parameter. - If it is not given, the cursor's arraysize determines the number of rows - to be fetched. The method should try to fetch as many rows as indicated by - the size parameter. If this is not possible due to the specified number of - rows not being available, fewer rows may be returned. - - Note there are performance considerations involved with the *size* parameter. - For optimal performance, it is usually best to use the arraysize attribute. - If the *size* parameter is used, then it is best for it to retain the same - value from one :meth:`fetchmany` call to the next. - - .. method:: fetchall() - - Fetches all (remaining) rows of a query result, returning a list. Note that - the cursor's arraysize attribute can affect the performance of this operation. - An empty list is returned when no rows are available. - - .. method:: close() - - Close the cursor now (rather than whenever ``__del__`` is called). - - The cursor will be unusable from this point forward; a :exc:`ProgrammingError` - exception will be raised if any operation is attempted with the cursor. - - .. attribute:: rowcount - - Although the :class:`Cursor` class of the :mod:`sqlite3` module implements this - attribute, the database engine's own support for the determination of "rows - affected"/"rows selected" is quirky. - - For :meth:`executemany` statements, the number of modifications are summed up - into :attr:`rowcount`. - - As required by the Python DB API Spec, the :attr:`rowcount` attribute "is -1 in - case no ``executeXX()`` has been performed on the cursor or the rowcount of the - last operation is not determinable by the interface". This includes ``SELECT`` - statements because we cannot determine the number of rows a query produced - until all rows were fetched. - - .. attribute:: lastrowid - - This read-only attribute provides the rowid of the last modified row. It is - only set if you issued an ``INSERT`` or a ``REPLACE`` statement using the - :meth:`execute` method. For operations other than ``INSERT`` or - ``REPLACE`` or when :meth:`executemany` is called, :attr:`lastrowid` is - set to :const:`None`. - - If the ``INSERT`` or ``REPLACE`` statement failed to insert the previous - successful rowid is returned. - - .. versionchanged:: 3.6 - Added support for the ``REPLACE`` statement. - - .. attribute:: arraysize - - Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. - The default value is 1 which means a single row would be fetched per call. - - .. attribute:: description - - This read-only attribute provides the column names of the last query. To - remain compatible with the Python DB API, it returns a 7-tuple for each - column where the last six items of each tuple are :const:`None`. - - It is set for ``SELECT`` statements without any matching rows as well. - - .. attribute:: connection - - This read-only attribute provides the SQLite database :class:`Connection` - used by the :class:`Cursor` object. A :class:`Cursor` object created by - calling :meth:`con.cursor() ` will have a - :attr:`connection` attribute that refers to *con*:: - - >>> con = sqlite3.connect(":memory:") - >>> cur = con.cursor() - >>> cur.connection == con - True - -.. _sqlite3-row-objects: - -Row Objects ------------ - -.. class:: Row - - A :class:`Row` instance serves as a highly optimized - :attr:`~Connection.row_factory` for :class:`Connection` objects. - It tries to mimic a tuple in most of its features. - - It supports mapping access by column name and index, iteration, - representation, equality testing and :func:`len`. - - If two :class:`Row` objects have exactly the same columns and their - members are equal, they compare equal. - - .. method:: keys - - This method returns a list of column names. Immediately after a query, - it is the first member of each tuple in :attr:`Cursor.description`. - - .. versionchanged:: 3.5 - Added support of slicing. - -Let's assume we initialize a table as in the example given above:: - - con = sqlite3.connect(":memory:") - cur = con.cursor() - cur.execute('''create table stocks - (date text, trans text, symbol text, - qty real, price real)''') - cur.execute("""insert into stocks - values ('2006-01-05','BUY','RHAT',100,35.14)""") - con.commit() - cur.close() - -Now we plug :class:`Row` in:: - - >>> con.row_factory = sqlite3.Row - >>> cur = con.cursor() - >>> cur.execute('select * from stocks') - - >>> r = cur.fetchone() - >>> type(r) - - >>> tuple(r) - ('2006-01-05', 'BUY', 'RHAT', 100.0, 35.14) - >>> len(r) - 5 - >>> r[2] - 'RHAT' - >>> r.keys() - ['date', 'trans', 'symbol', 'qty', 'price'] - >>> r['qty'] - 100.0 - >>> for member in r: - ... print(member) - ... - 2006-01-05 - BUY - RHAT - 100.0 - 35.14 - - -.. _sqlite3-exceptions: - -Exceptions ----------- - -.. exception:: Warning - - A subclass of :exc:`Exception`. - -.. exception:: Error - - The base class of the other exceptions in this module. It is a subclass - of :exc:`Exception`. - -.. exception:: DatabaseError - - Exception raised for errors that are related to the database. - -.. exception:: IntegrityError - - Exception raised when the relational integrity of the database is affected, - e.g. a foreign key check fails. It is a subclass of :exc:`DatabaseError`. - -.. exception:: ProgrammingError - - Exception raised for programming errors, e.g. table not found or already - exists, syntax error in the SQL statement, wrong number of parameters - specified, etc. It is a subclass of :exc:`DatabaseError`. - -.. exception:: OperationalError - - Exception raised for errors that are related to the database's operation - and not necessarily under the control of the programmer, e.g. an unexpected - disconnect occurs, the data source name is not found, a transaction could - not be processed, etc. It is a subclass of :exc:`DatabaseError`. - -.. exception:: NotSupportedError - - Exception raised in case a method or database API was used which is not - supported by the database, e.g. calling the :meth:`~Connection.rollback` - method on a connection that does not support transaction or has - transactions turned off. It is a subclass of :exc:`DatabaseError`. - - -.. _sqlite3-types: - -SQLite and Python types ------------------------ - - -Introduction -^^^^^^^^^^^^ - -SQLite natively supports the following types: ``NULL``, ``INTEGER``, -``REAL``, ``TEXT``, ``BLOB``. - -The following Python types can thus be sent to SQLite without any problem: - -+-------------------------------+-------------+ -| Python type | SQLite type | -+===============================+=============+ -| :const:`None` | ``NULL`` | -+-------------------------------+-------------+ -| :class:`int` | ``INTEGER`` | -+-------------------------------+-------------+ -| :class:`float` | ``REAL`` | -+-------------------------------+-------------+ -| :class:`str` | ``TEXT`` | -+-------------------------------+-------------+ -| :class:`bytes` | ``BLOB`` | -+-------------------------------+-------------+ - - -This is how SQLite types are converted to Python types by default: - -+-------------+----------------------------------------------+ -| SQLite type | Python type | -+=============+==============================================+ -| ``NULL`` | :const:`None` | -+-------------+----------------------------------------------+ -| ``INTEGER`` | :class:`int` | -+-------------+----------------------------------------------+ -| ``REAL`` | :class:`float` | -+-------------+----------------------------------------------+ -| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | -| | :class:`str` by default | -+-------------+----------------------------------------------+ -| ``BLOB`` | :class:`bytes` | -+-------------+----------------------------------------------+ - -The type system of the :mod:`sqlite3` module is extensible in two ways: you can -store additional Python types in a SQLite database via object adaptation, and -you can let the :mod:`sqlite3` module convert SQLite types to different Python -types via converters. - - -Using adapters to store additional Python types in SQLite databases -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As described before, SQLite supports only a limited set of types natively. To -use other Python types with SQLite, you must **adapt** them to one of the -sqlite3 module's supported types for SQLite: one of NoneType, int, float, -str, bytes. - -There are two ways to enable the :mod:`sqlite3` module to adapt a custom Python -type to one of the supported ones. - - -Letting your object adapt itself -"""""""""""""""""""""""""""""""" - -This is a good approach if you write the class yourself. Let's suppose you have -a class like this:: - - class Point: - def __init__(self, x, y): - self.x, self.y = x, y - -Now you want to store the point in a single SQLite column. First you'll have to -choose one of the supported types to be used for representing the point. -Let's just use str and separate the coordinates using a semicolon. Then you need -to give your class a method ``__conform__(self, protocol)`` which must return -the converted value. The parameter *protocol* will be :class:`PrepareProtocol`. - -.. literalinclude:: ../includes/sqlite3/adapter_point_1.py - - -Registering an adapter callable -""""""""""""""""""""""""""""""" - -The other possibility is to create a function that converts the type to the -string representation and register the function with :meth:`register_adapter`. - -.. literalinclude:: ../includes/sqlite3/adapter_point_2.py - -The :mod:`sqlite3` module has two default adapters for Python's built-in -:class:`datetime.date` and :class:`datetime.datetime` types. Now let's suppose -we want to store :class:`datetime.datetime` objects not in ISO representation, -but as a Unix timestamp. - -.. literalinclude:: ../includes/sqlite3/adapter_datetime.py - - -Converting SQLite values to custom Python types -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Writing an adapter lets you send custom Python types to SQLite. But to make it -really useful we need to make the Python to SQLite to Python roundtrip work. - -Enter converters. - -Let's go back to the :class:`Point` class. We stored the x and y coordinates -separated via semicolons as strings in SQLite. - -First, we'll define a converter function that accepts the string as a parameter -and constructs a :class:`Point` object from it. - -.. note:: - - Converter functions **always** get called with a :class:`bytes` object, no - matter under which data type you sent the value to SQLite. - -:: - - def convert_point(s): - x, y = map(float, s.split(b";")) - return Point(x, y) - -Now you need to make the :mod:`sqlite3` module know that what you select from -the database is actually a point. There are two ways of doing this: - -* Implicitly via the declared type - -* Explicitly via the column name - -Both ways are described in section :ref:`sqlite3-module-contents`, in the entries -for the constants :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES`. - -The following example illustrates both approaches. - -.. literalinclude:: ../includes/sqlite3/converter_point.py - - -Default adapters and converters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are default adapters for the date and datetime types in the datetime -module. They will be sent as ISO dates/ISO timestamps to SQLite. - -The default converters are registered under the name "date" for -:class:`datetime.date` and under the name "timestamp" for -:class:`datetime.datetime`. - -This way, you can use date/timestamps from Python without any additional -fiddling in most cases. The format of the adapters is also compatible with the -experimental SQLite date/time functions. - -The following example demonstrates this. - -.. literalinclude:: ../includes/sqlite3/pysqlite_datetime.py - -If a timestamp stored in SQLite has a fractional part longer than 6 -numbers, its value will be truncated to microsecond precision by the -timestamp converter. - - -.. _sqlite3-controlling-transactions: - -Controlling Transactions ------------------------- - -The underlying ``sqlite3`` library operates in ``autocommit`` mode by default, -but the Python :mod:`sqlite3` module by default does not. - -``autocommit`` mode means that statements that modify the database take effect -immediately. A ``BEGIN`` or ``SAVEPOINT`` statement disables ``autocommit`` -mode, and a ``COMMIT``, a ``ROLLBACK``, or a ``RELEASE`` that ends the -outermost transaction, turns ``autocommit`` mode back on. - -The Python :mod:`sqlite3` module by default issues a ``BEGIN`` statement -implicitly before a Data Modification Language (DML) statement (i.e. -``INSERT``/``UPDATE``/``DELETE``/``REPLACE``). - -You can control which kind of ``BEGIN`` statements :mod:`sqlite3` implicitly -executes via the *isolation_level* parameter to the :func:`connect` -call, or via the :attr:`isolation_level` property of connections. -If you specify no *isolation_level*, a plain ``BEGIN`` is used, which is -equivalent to specifying ``DEFERRED``. Other possible values are ``IMMEDIATE`` -and ``EXCLUSIVE``. - -You can disable the :mod:`sqlite3` module's implicit transaction management by -setting :attr:`isolation_level` to ``None``. This will leave the underlying -``sqlite3`` library operating in ``autocommit`` mode. You can then completely -control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, -``SAVEPOINT``, and ``RELEASE`` statements in your code. - -.. versionchanged:: 3.6 - :mod:`sqlite3` used to implicitly commit an open transaction before DDL - statements. This is no longer the case. - - -Using :mod:`sqlite3` efficiently --------------------------------- - - -Using shortcut methods -^^^^^^^^^^^^^^^^^^^^^^ - -Using the nonstandard :meth:`execute`, :meth:`executemany` and -:meth:`executescript` methods of the :class:`Connection` object, your code can -be written more concisely because you don't have to create the (often -superfluous) :class:`Cursor` objects explicitly. Instead, the :class:`Cursor` -objects are created implicitly and these shortcut methods return the cursor -objects. This way, you can execute a ``SELECT`` statement and iterate over it -directly using only a single call on the :class:`Connection` object. - -.. literalinclude:: ../includes/sqlite3/shortcut_methods.py - - -Accessing columns by name instead of by index -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -One useful feature of the :mod:`sqlite3` module is the built-in -:class:`sqlite3.Row` class designed to be used as a row factory. - -Rows wrapped with this class can be accessed both by index (like tuples) and -case-insensitively by name: - -.. literalinclude:: ../includes/sqlite3/rowclass.py - - -Using the connection as a context manager -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Connection objects can be used as context managers -that automatically commit or rollback transactions. In the event of an -exception, the transaction is rolled back; otherwise, the transaction is -committed: - -.. literalinclude:: ../includes/sqlite3/ctx_manager.py - - -.. rubric:: Footnotes - -.. [#f1] The sqlite3 module is not built with loadable extension support by - default, because some platforms (notably Mac OS X) have SQLite - libraries which are compiled without this feature. To get loadable - extension support, you must pass --enable-loadable-sqlite-extensions to - configure. diff --git a/Doc/library/statistics.rst.bak b/Doc/library/statistics.rst.bak deleted file mode 100644 index 6b6d3154a28810..00000000000000 --- a/Doc/library/statistics.rst.bak +++ /dev/null @@ -1,879 +0,0 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= - -.. module:: statistics - :synopsis: Mathematical statistics functions - -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/statistics.py` - -.. testsetup:: * - - from statistics import * - __name__ = '' - --------------- - -This module provides functions for calculating mathematical statistics of -numeric (:class:`~numbers.Real`-valued) data. - -The module is not intended to be a competitor to third-party libraries such -as `NumPy `_, `SciPy `_, or -proprietary full-featured statistics packages aimed at professional -statisticians such as Minitab, SAS and Matlab. It is aimed at the level of -graphing and scientific calculators. - -Unless explicitly noted, these functions support :class:`int`, -:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. -Behaviour with other types (whether in the numeric tower or not) is -currently unsupported. Collections with a mix of types are also undefined -and implementation-dependent. If your input data consists of mixed types, -you may be able to use :func:`map` to ensure a consistent result, for -example: ``map(float, input_data)``. - -Averages and measures of central location ------------------------------------------ - -These functions calculate an average or typical value from a population -or sample. - -======================= =============================================================== -:func:`mean` Arithmetic mean ("average") of data. -:func:`fmean` Fast, floating point arithmetic mean. -:func:`geometric_mean` Geometric mean of data. -:func:`harmonic_mean` Harmonic mean of data. -:func:`median` Median (middle value) of data. -:func:`median_low` Low median of data. -:func:`median_high` High median of data. -:func:`median_grouped` Median, or 50th percentile, of grouped data. -:func:`mode` Single mode (most common value) of discrete or nominal data. -:func:`multimode` List of modes (most common values) of discrete or nomimal data. -:func:`quantiles` Divide data into intervals with equal probability. -======================= =============================================================== - -Measures of spread ------------------- - -These functions calculate a measure of how much the population or sample -tends to deviate from the typical or average values. - -======================= ============================================= -:func:`pstdev` Population standard deviation of data. -:func:`pvariance` Population variance of data. -:func:`stdev` Sample standard deviation of data. -:func:`variance` Sample variance of data. -======================= ============================================= - - -Function details ----------------- - -Note: The functions do not require the data given to them to be sorted. -However, for reading convenience, most of the examples show sorted sequences. - -.. function:: mean(data) - - Return the sample arithmetic mean of *data* which can be a sequence or iterable. - - The arithmetic mean is the sum of the data divided by the number of data - points. It is commonly called "the average", although it is only one of many - different mathematical averages. It is a measure of the central location of - the data. - - If *data* is empty, :exc:`StatisticsError` will be raised. - - Some examples of use: - - .. doctest:: - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - >>> mean([-1.0, 2.5, 3.25, 5.75]) - 2.625 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - .. note:: - - The mean is strongly affected by outliers and is not a robust estimator - for central location: the mean is not necessarily a typical example of - the data points. For more robust measures of central location, see - :func:`median` and :func:`mode`. - - The sample mean gives an unbiased estimate of the true population mean, - so that when taken on average over all the possible samples, - ``mean(sample)`` converges on the true mean of the entire population. If - *data* represents the entire population rather than a sample, then - ``mean(data)`` is equivalent to calculating the true population mean μ. - - -.. function:: fmean(data) - - Convert *data* to floats and compute the arithmetic mean. - - This runs faster than the :func:`mean` function and it always returns a - :class:`float`. The *data* may be a sequence or iterable. If the input - dataset is empty, raises a :exc:`StatisticsError`. - - .. doctest:: - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - - .. versionadded:: 3.8 - - -.. function:: geometric_mean(data) - - Convert *data* to floats and compute the geometric mean. - - The geometric mean indicates the central tendency or typical value of the - *data* using the product of the values (as opposed to the arithmetic mean - which uses their sum). - - Raises a :exc:`StatisticsError` if the input dataset is empty, - if it contains a zero, or if it contains a negative value. - The *data* may be a sequence or iterable. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - .. doctest:: - - >>> round(geometric_mean([54, 24, 36]), 1) - 36.0 - - .. versionadded:: 3.8 - - -.. function:: harmonic_mean(data, weights=None) - - Return the harmonic mean of *data*, a sequence or iterable of - real-valued numbers. If *weights* is omitted or *None*, then - equal weighting is assumed. - - The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the - reciprocals of the data. For example, the harmonic mean of three values *a*, - *b* and *c* will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the - values is zero, the result will be zero. - - The harmonic mean is a type of average, a measure of the central - location of the data. It is often appropriate when averaging - ratios or rates, for example speeds. - - Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. - What is the average speed? - - .. doctest:: - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - .. doctest:: - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 - - :exc:`StatisticsError` is raised if *data* is empty, any element - is less than zero, or if the weighted sum isn't positive. - - The current algorithm has an early-out when it encounters a zero - in the input. This means that the subsequent inputs are not tested - for validity. (This behavior may change in the future.) - - .. versionadded:: 3.6 - - .. versionchanged:: 3.10 - Added support for *weights*. - -.. function:: median(data) - - Return the median (middle value) of numeric data, using the common "mean of - middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. - *data* can be a sequence or iterable. - - The median is a robust measure of central location and is less affected by - the presence of outliers. When the number of data points is odd, the - middle data point is returned: - - .. doctest:: - - >>> median([1, 3, 5]) - 3 - - When the number of data points is even, the median is interpolated by taking - the average of the two middle values: - - .. doctest:: - - >>> median([1, 3, 5, 7]) - 4.0 - - This is suited for when your data is discrete, and you don't mind that the - median may not be an actual data point. - - If the data is ordinal (supports order operations) but not numeric (doesn't - support addition), consider using :func:`median_low` or :func:`median_high` - instead. - -.. function:: median_low(data) - - Return the low median of numeric data. If *data* is empty, - :exc:`StatisticsError` is raised. *data* can be a sequence or iterable. - - The low median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the smaller of - the two middle values is returned. - - .. doctest:: - - >>> median_low([1, 3, 5]) - 3 - >>> median_low([1, 3, 5, 7]) - 3 - - Use the low median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_high(data) - - Return the high median of data. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. - - The high median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the larger of - the two middle values is returned. - - .. doctest:: - - >>> median_high([1, 3, 5]) - 3 - >>> median_high([1, 3, 5, 7]) - 5 - - Use the high median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_grouped(data, interval=1) - - Return the median of grouped continuous data, calculated as the 50th - percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. - - .. doctest:: - - >>> median_grouped([52, 52, 53, 54]) - 52.5 - - In the following example, the data are rounded, so that each value represents - the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 - is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data - given, the middle value falls somewhere in the class 3.5--4.5, and - interpolation is used to estimate it: - - .. doctest:: - - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - - Optional argument *interval* represents the class interval, and defaults - to 1. Changing the class interval naturally will change the interpolation: - - .. doctest:: - - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 - - This function does not check whether the data points are at least - *interval* apart. - - .. impl-detail:: - - Under some circumstances, :func:`median_grouped` may coerce data points to - floats. This behaviour is likely to change in the future. - - .. seealso:: - - * "Statistics for the Behavioral Sciences", Frederick J Gravetter and - Larry B Wallnau (8th Edition). - - * The `SSMEDIAN - `_ - function in the Gnome Gnumeric spreadsheet, including `this discussion - `_. - - -.. function:: mode(data) - - Return the single most common data point from discrete or nominal *data*. - The mode (when it exists) is the most typical value and serves as a - measure of central location. - - If there are multiple modes with the same frequency, returns the first one - encountered in the *data*. If the smallest or largest of those is - desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``. - If the input *data* is empty, :exc:`StatisticsError` is raised. - - ``mode`` assumes discrete data and returns a single value. This is the - standard treatment of the mode as commonly taught in schools: - - .. doctest:: - - >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) - 3 - - The mode is unique in that it is the only statistic in this package that - also applies to nominal (non-numeric) data: - - .. doctest:: - - >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) - 'red' - - .. versionchanged:: 3.8 - Now handles multimodal datasets by returning the first mode encountered. - Formerly, it raised :exc:`StatisticsError` when more than one mode was - found. - - -.. function:: multimode(data) - - Return a list of the most frequently occurring values in the order they - were first encountered in the *data*. Will return more than one result if - there are multiple modes or an empty list if the *data* is empty: - - .. doctest:: - - >>> multimode('aabbbbccddddeeffffgg') - ['b', 'd', 'f'] - >>> multimode('') - [] - - .. versionadded:: 3.8 - - -.. function:: pstdev(data, mu=None) - - Return the population standard deviation (the square root of the population - variance). See :func:`pvariance` for arguments and other details. - - .. doctest:: - - >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 0.986893273527251 - - -.. function:: pvariance(data, mu=None) - - Return the population variance of *data*, a non-empty sequence or iterable - of real-valued numbers. Variance, or second moment about the mean, is a - measure of the variability (spread or dispersion) of data. A large - variance indicates that the data is spread out; a small variance indicates - it is clustered closely around the mean. - - If the optional second argument *mu* is given, it is typically the mean of - the *data*. It can also be used to compute the second moment around a - point that is not the mean. If it is missing or ``None`` (the default), - the arithmetic mean is automatically calculated. - - Use this function to calculate the variance from the entire population. To - estimate the variance from a sample, the :func:`variance` function is usually - a better choice. - - Raises :exc:`StatisticsError` if *data* is empty. - - Examples: - - .. doctest:: - - >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] - >>> pvariance(data) - 1.25 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *mu* to avoid recalculation: - - .. doctest:: - - >>> mu = mean(data) - >>> pvariance(data, mu) - 1.25 - - Decimals and Fractions are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('24.815') - - >>> from fractions import Fraction as F - >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) - Fraction(13, 72) - - .. note:: - - When called with the entire population, this gives the population variance - σ². When called on a sample instead, this is the biased sample variance - s², also known as variance with N degrees of freedom. - - If you somehow know the true population mean μ, you may use this - function to calculate the variance of a sample, giving the known - population mean as the second argument. Provided the data points are a - random sample of the population, the result will be an unbiased estimate - of the population variance. - - -.. function:: stdev(data, xbar=None) - - Return the sample standard deviation (the square root of the sample - variance). See :func:`variance` for arguments and other details. - - .. doctest:: - - >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 1.0810874155219827 - - -.. function:: variance(data, xbar=None) - - Return the sample variance of *data*, an iterable of at least two real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. - - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. - - Use this function when your data is a sample from a population. To calculate - the variance from the entire population, see :func:`pvariance`. - - Raises :exc:`StatisticsError` if *data* has fewer than two values. - - Examples: - - .. doctest:: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: - - .. doctest:: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not attempt to verify that you have passed the actual mean - as *xbar*. Using arbitrary values for *xbar* can lead to invalid or - impossible results. - - Decimal and Fraction values are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) - - .. note:: - - This is the sample variance s² with Bessel's correction, also known as - variance with N-1 degrees of freedom. Provided that the data points are - representative (e.g. independent and identically distributed), the result - should be an unbiased estimate of the true population variance. - - If you somehow know the actual population mean μ you should pass it to the - :func:`pvariance` function as the *mu* parameter to get the variance of a - sample. - -.. function:: quantiles(data, *, n=4, method='exclusive') - - Divide *data* into *n* continuous intervals with equal probability. - Returns a list of ``n - 1`` cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. Set - *n* to 100 for percentiles which gives the 99 cuts points that separate - *data* into 100 equal sized groups. Raises :exc:`StatisticsError` if *n* - is not least 1. - - The *data* can be any iterable containing sample data. For meaningful - results, the number of data points in *data* should be larger than *n*. - Raises :exc:`StatisticsError` if there are not at least two data points. - - The cut points are linearly interpolated from the - two nearest data points. For example, if a cut point falls one-third - of the distance between two sample values, ``100`` and ``112``, the - cut-point will evaluate to ``104``. - - The *method* for computing quantiles can be varied depending on - whether the *data* includes or excludes the lowest and - highest possible values from the population. - - The default *method* is "exclusive" and is used for data sampled from - a population that can have more extreme values than found in the - samples. The portion of the population falling below the *i-th* of - *m* sorted data points is computed as ``i / (m + 1)``. Given nine - sample values, the method sorts them and assigns the following - percentiles: 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%. - - Setting the *method* to "inclusive" is used for describing population - data or for samples that are known to include the most extreme values - from the population. The minimum value in *data* is treated as the 0th - percentile and the maximum value is treated as the 100th percentile. - The portion of the population falling below the *i-th* of *m* sorted - data points is computed as ``(i - 1) / (m - 1)``. Given 11 sample - values, the method sorts them and assigns the following percentiles: - 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%. - - .. doctest:: - - # Decile cut points for empirically sampled data - >>> data = [105, 129, 87, 86, 111, 111, 89, 81, 108, 92, 110, - ... 100, 75, 105, 103, 109, 76, 119, 99, 91, 103, 129, - ... 106, 101, 84, 111, 74, 87, 86, 103, 103, 106, 86, - ... 111, 75, 87, 102, 121, 111, 88, 89, 101, 106, 95, - ... 103, 107, 101, 81, 109, 104] - >>> [round(q, 1) for q in quantiles(data, n=10)] - [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] - - .. versionadded:: 3.8 - - -Exceptions ----------- - -A single exception is defined: - -.. exception:: StatisticsError - - Subclass of :exc:`ValueError` for statistics-related exceptions. - - -:class:`NormalDist` objects ---------------------------- - -:class:`NormalDist` is a tool for creating and manipulating normal -distributions of a `random variable -`_. It is a -class that treats the mean and standard deviation of data -measurements as a single entity. - -Normal distributions arise from the `Central Limit Theorem -`_ and have a wide range -of applications in statistics. - -.. class:: NormalDist(mu=0.0, sigma=1.0) - - Returns a new *NormalDist* object where *mu* represents the `arithmetic - mean `_ and *sigma* - represents the `standard deviation - `_. - - If *sigma* is negative, raises :exc:`StatisticsError`. - - .. attribute:: mean - - A read-only property for the `arithmetic mean - `_ of a normal - distribution. - - .. attribute:: median - - A read-only property for the `median - `_ of a normal - distribution. - - .. attribute:: mode - - A read-only property for the `mode - `_ of a normal - distribution. - - .. attribute:: stdev - - A read-only property for the `standard deviation - `_ of a normal - distribution. - - .. attribute:: variance - - A read-only property for the `variance - `_ of a normal - distribution. Equal to the square of the standard deviation. - - .. classmethod:: NormalDist.from_samples(data) - - Makes a normal distribution instance with *mu* and *sigma* parameters - estimated from the *data* using :func:`fmean` and :func:`stdev`. - - The *data* can be any :term:`iterable` and should consist of values - that can be converted to type :class:`float`. If *data* does not - contain at least two elements, raises :exc:`StatisticsError` because it - takes at least one point to estimate a central value and at least two - points to estimate dispersion. - - .. method:: NormalDist.samples(n, *, seed=None) - - Generates *n* random samples for a given mean and standard deviation. - Returns a :class:`list` of :class:`float` values. - - If *seed* is given, creates a new instance of the underlying random - number generator. This is useful for creating reproducible results, - even in a multi-threading context. - - .. method:: NormalDist.pdf(x) - - Using a `probability density function (pdf) - `_, compute - the relative likelihood that a random variable *X* will be near the - given value *x*. Mathematically, it is the limit of the ratio ``P(x <= - X < x+dx) / dx`` as *dx* approaches zero. - - The relative likelihood is computed as the probability of a sample - occurring in a narrow range divided by the width of the range (hence - the word "density"). Since the likelihood is relative to other points, - its value can be greater than `1.0`. - - .. method:: NormalDist.cdf(x) - - Using a `cumulative distribution function (cdf) - `_, - compute the probability that a random variable *X* will be less than or - equal to *x*. Mathematically, it is written ``P(X <= x)``. - - .. method:: NormalDist.inv_cdf(p) - - Compute the inverse cumulative distribution function, also known as the - `quantile function `_ - or the `percent-point - `_ - function. Mathematically, it is written ``x : P(X <= x) = p``. - - Finds the value *x* of the random variable *X* such that the - probability of the variable being less than or equal to that value - equals the given probability *p*. - - .. method:: NormalDist.overlap(other) - - Measures the agreement between two normal probability distributions. - Returns a value between 0.0 and 1.0 giving `the overlapping area for - the two probability density functions - `_. - - .. method:: NormalDist.quantiles(n=4) - - Divide the normal distribution into *n* continuous intervals with - equal probability. Returns a list of (n - 1) cut points separating - the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate the normal distribution into 100 equal sized groups. - - .. method:: NormalDist.zscore(x) - - Compute the - `Standard Score `_ - describing *x* in terms of the number of standard deviations - above or below the mean of the normal distribution: - ``(x - mean) / stdev``. - - .. versionadded:: 3.9 - - Instances of :class:`NormalDist` support addition, subtraction, - multiplication and division by a constant. These operations - are used for translation and scaling. For example: - - .. doctest:: - - >>> temperature_february = NormalDist(5, 2.5) # Celsius - >>> temperature_february * (9/5) + 32 # Fahrenheit - NormalDist(mu=41.0, sigma=4.5) - - Dividing a constant by an instance of :class:`NormalDist` is not supported - because the result wouldn't be normally distributed. - - Since normal distributions arise from additive effects of independent - variables, it is possible to `add and subtract two independent normally - distributed random variables - `_ - represented as instances of :class:`NormalDist`. For example: - - .. doctest:: - - >>> birth_weights = NormalDist.from_samples([2.5, 3.1, 2.1, 2.4, 2.7, 3.5]) - >>> drug_effects = NormalDist(0.4, 0.15) - >>> combined = birth_weights + drug_effects - >>> round(combined.mean, 1) - 3.1 - >>> round(combined.stdev, 1) - 0.5 - - .. versionadded:: 3.8 - - -:class:`NormalDist` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`NormalDist` readily solves classic probability problems. - -For example, given `historical data for SAT exams -`_ showing -that scores are normally distributed with a mean of 1060 and a standard -deviation of 195, determine the percentage of students with test scores -between 1100 and 1200, after rounding to the nearest whole number: - -.. doctest:: - - >>> sat = NormalDist(1060, 195) - >>> fraction = sat.cdf(1200 + 0.5) - sat.cdf(1100 - 0.5) - >>> round(fraction * 100.0, 1) - 18.4 - -Find the `quartiles `_ and `deciles -`_ for the SAT scores: - -.. doctest:: - - >>> list(map(round, sat.quantiles())) - [928, 1060, 1192] - >>> list(map(round, sat.quantiles(n=10))) - [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] - -To estimate the distribution for a model than isn't easy to solve -analytically, :class:`NormalDist` can generate input samples for a `Monte -Carlo simulation `_: - -.. doctest:: - - >>> def model(x, y, z): - ... return (3*x + 7*x*y - 5*y) / (11 * z) - ... - >>> n = 100_000 - >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728) - >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471) - >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453) - >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP - [1.4591308524824727, 1.8035946855390597, 2.175091447274739] - -Normal distributions can be used to approximate `Binomial -distributions `_ -when the sample size is large and when the probability of a successful -trial is near 50%. - -For example, an open source conference has 750 attendees and two rooms with a -500 person capacity. There is a talk about Python and another about Ruby. -In previous conferences, 65% of the attendees preferred to listen to Python -talks. Assuming the population preferences haven't changed, what is the -probability that the Python room will stay within its capacity limits? - -.. doctest:: - - >>> n = 750 # Sample size - >>> p = 0.65 # Preference for Python - >>> q = 1.0 - p # Preference for Ruby - >>> k = 500 # Room capacity - - >>> # Approximation using the cumulative normal distribution - >>> from math import sqrt - >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) - 0.8402 - - >>> # Solution using the cumulative binomial distribution - >>> from math import comb, fsum - >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) - 0.8402 - - >>> # Approximation using a simulation - >>> from random import seed, choices - >>> seed(8675309) - >>> def trial(): - ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') - >>> mean(trial() <= k for i in range(10_000)) - 0.8398 - -Normal distributions commonly arise in machine learning problems. - -Wikipedia has a `nice example of a Naive Bayesian Classifier -`_. -The challenge is to predict a person's gender from measurements of normally -distributed features including height, weight, and foot size. - -We're given a training dataset with measurements for eight people. The -measurements are assumed to be normally distributed, so we summarize the data -with :class:`NormalDist`: - -.. doctest:: - - >>> height_male = NormalDist.from_samples([6, 5.92, 5.58, 5.92]) - >>> height_female = NormalDist.from_samples([5, 5.5, 5.42, 5.75]) - >>> weight_male = NormalDist.from_samples([180, 190, 170, 165]) - >>> weight_female = NormalDist.from_samples([100, 150, 130, 150]) - >>> foot_size_male = NormalDist.from_samples([12, 11, 12, 10]) - >>> foot_size_female = NormalDist.from_samples([6, 8, 7, 9]) - -Next, we encounter a new person whose feature measurements are known but whose -gender is unknown: - -.. doctest:: - - >>> ht = 6.0 # height - >>> wt = 130 # weight - >>> fs = 8 # foot size - -Starting with a 50% `prior probability -`_ of being male or female, -we compute the posterior as the prior times the product of likelihoods for the -feature measurements given the gender: - -.. doctest:: - - >>> prior_male = 0.5 - >>> prior_female = 0.5 - >>> posterior_male = (prior_male * height_male.pdf(ht) * - ... weight_male.pdf(wt) * foot_size_male.pdf(fs)) - - >>> posterior_female = (prior_female * height_female.pdf(ht) * - ... weight_female.pdf(wt) * foot_size_female.pdf(fs)) - -The final prediction goes to the largest posterior. This is known as the -`maximum a posteriori -`_ or MAP: - -.. doctest:: - - >>> 'male' if posterior_male > posterior_female else 'female' - 'female' - - -.. - # This modelines must appear within the last ten lines of the file. - kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/tempfile.rst.bak b/Doc/library/tempfile.rst.bak deleted file mode 100644 index 2b8a35e2651e73..00000000000000 --- a/Doc/library/tempfile.rst.bak +++ /dev/null @@ -1,374 +0,0 @@ -:mod:`tempfile` --- Generate temporary files and directories -============================================================ - -.. module:: tempfile - :synopsis: Generate temporary files and directories. - -.. sectionauthor:: Zack Weinberg - -**Source code:** :source:`Lib/tempfile.py` - -.. index:: - pair: temporary; file name - pair: temporary; file - --------------- - -This module creates temporary files and directories. It works on all -supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, -:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level -interfaces which provide automatic cleanup and can be used as -context managers. :func:`mkstemp` and -:func:`mkdtemp` are lower-level functions which require manual cleanup. - -All the user-callable functions and constructors take additional arguments which -allow direct control over the location and name of temporary files and -directories. Files names used by this module include a string of -random characters which allows those files to be securely created in -shared temporary directories. -To maintain backward compatibility, the argument order is somewhat odd; it -is recommended to use keyword arguments for clarity. - -The module defines the following user-callable items: - -.. function:: TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) - - Return a :term:`file-like object` that can be used as a temporary storage area. - The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon - as it is closed (including an implicit close when the object is garbage - collected). Under Unix, the directory entry for the file is either not created at all or is removed - immediately after the file is created. Other platforms do not support - this; your code should not rely on a temporary file created using this - function having or not having a visible name in the file system. - - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or - destruction of the file object the temporary file will be removed - from the filesystem. - - The *mode* parameter defaults to ``'w+b'`` so that the file created can - be read and written without being closed. Binary mode is used so that it - behaves consistently on all platforms without regard for the data that is - stored. *buffering*, *encoding*, *errors* and *newline* are interpreted as for - :func:`open`. - - The *dir*, *prefix* and *suffix* parameters have the same meaning and - defaults as with :func:`mkstemp`. - - The returned object is a true file object on POSIX platforms. On other - platforms, it is a file-like object whose :attr:`!file` attribute is the - underlying true file object. - - The :py:data:`os.O_TMPFILE` flag is used if it is available and works - (Linux-specific, requires Linux kernel 3.11 or later). - - .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile - - .. versionchanged:: 3.5 - - The :py:data:`os.O_TMPFILE` flag is now used if available. - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) - - This function operates exactly as :func:`TemporaryFile` does, except that - the file is guaranteed to have a visible name in the file system (on - Unix, the directory entry is not unlinked). That name can be retrieved - from the :attr:`name` attribute of the returned - file-like object. Whether the name can be - used to open the file a second time, while the named temporary file is - still open, varies across platforms (it can be so used on Unix; it cannot - on Windows NT or later). If *delete* is true (the default), the file is - deleted as soon as it is closed. - The returned object is always a file-like object whose :attr:`!file` - attribute is the underlying true file object. This file-like object can - be used in a :keyword:`with` statement, just like a normal file. - - .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) - - This function operates exactly as :func:`TemporaryFile` does, except that - data is spooled in memory until the file size exceeds *max_size*, or - until the file's :func:`fileno` method is called, at which point the - contents are written to disk and operation proceeds as with - :func:`TemporaryFile`. - - The resulting file has one additional method, :func:`rollover`, which - causes the file to roll over to an on-disk file regardless of its size. - - The returned object is a file-like object whose :attr:`_file` attribute - is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object - (depending on whether binary or text *mode* was specified) or a true file - object, depending on whether :func:`rollover` has been called. This - file-like object can be used in a :keyword:`with` statement, just like - a normal file. - - .. versionchanged:: 3.3 - the truncate method now accepts a ``size`` argument. - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: TemporaryDirectory(suffix=None, prefix=None, dir=None) - - This function securely creates a temporary directory using the same rules as :func:`mkdtemp`. - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or destruction - of the temporary directory object the newly created temporary directory - and all its contents are removed from the filesystem. - - The directory name can be retrieved from the :attr:`name` attribute of the - returned object. When the returned object is used as a context manager, the - :attr:`name` will be assigned to the target of the :keyword:`!as` clause in - the :keyword:`with` statement, if there is one. - - The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. - - .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory - - .. versionadded:: 3.2 - - -.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) - - Creates a temporary file in the most secure manner possible. There are - no race conditions in the file's creation, assuming that the platform - properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The - file is readable and writable only by the creating user ID. If the - platform uses permission bits to indicate whether a file is executable, - the file is executable by no one. The file descriptor is not inherited - by child processes. - - Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible - for deleting the temporary file when done with it. - - If *suffix* is not ``None``, the file name will end with that suffix, - otherwise there will be no suffix. :func:`mkstemp` does not put a dot - between the file name and the suffix; if you need one, put it at the - beginning of *suffix*. - - If *prefix* is not ``None``, the file name will begin with that prefix; - otherwise, a default prefix is used. The default is the return value of - :func:`gettempprefix` or :func:`gettempprefixb`, as appropriate. - - If *dir* is not ``None``, the file will be created in that directory; - otherwise, a default directory is used. The default directory is chosen - from a platform-dependent list, but the user of the application can - control the directory location by setting the *TMPDIR*, *TEMP* or *TMP* - environment variables. There is thus no guarantee that the generated - filename will have any nice properties, such as not requiring quoting - when passed to external commands via ``os.popen()``. - - If any of *suffix*, *prefix*, and *dir* are not - ``None``, they must be the same type. - If they are bytes, the returned name will be bytes instead of str. - If you want to force a bytes return value with otherwise default behavior, - pass ``suffix=b''``. - - If *text* is specified and true, the file is opened in text mode. - Otherwise, (the default) the file is opened in binary mode. - - :func:`mkstemp` returns a tuple containing an OS-level handle to an open - file (as would be returned by :func:`os.open`) and the absolute pathname - of that file, in that order. - - .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - .. versionchanged:: 3.6 - The *dir* parameter now accepts a :term:`path-like object`. - - -.. function:: mkdtemp(suffix=None, prefix=None, dir=None) - - Creates a temporary directory in the most secure manner possible. There - are no race conditions in the directory's creation. The directory is - readable, writable, and searchable only by the creating user ID. - - The user of :func:`mkdtemp` is responsible for deleting the temporary - directory and its contents when done with it. - - The *prefix*, *suffix*, and *dir* arguments are the same as for - :func:`mkstemp`. - - :func:`mkdtemp` returns the absolute pathname of the new directory. - - .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - .. versionchanged:: 3.6 - The *dir* parameter now accepts a :term:`path-like object`. - - -.. function:: gettempdir() - - Return the name of the directory used for temporary files. This - defines the default value for the *dir* argument to all functions - in this module. - - Python searches a standard list of directories to find one which - the calling user can create files in. The list is: - - #. The directory named by the :envvar:`TMPDIR` environment variable. - - #. The directory named by the :envvar:`TEMP` environment variable. - - #. The directory named by the :envvar:`TMP` environment variable. - - #. A platform-specific location: - - * On Windows, the directories :file:`C:\\TEMP`, :file:`C:\\TMP`, - :file:`\\TEMP`, and :file:`\\TMP`, in that order. - - * On all other platforms, the directories :file:`/tmp`, :file:`/var/tmp`, and - :file:`/usr/tmp`, in that order. - - #. As a last resort, the current working directory. - - The result of this search is cached, see the description of - :data:`tempdir` below. - - .. versionchanged:: 3.10 - - Always returns a str. Previously it would return any :data:`tempdir` - value regardless of type so long as it was not ``None``. - -.. function:: gettempdirb() - - Same as :func:`gettempdir` but the return value is in bytes. - - .. versionadded:: 3.5 - -.. function:: gettempprefix() - - Return the filename prefix used to create temporary files. This does not - contain the directory component. - -.. function:: gettempprefixb() - - Same as :func:`gettempprefix` but the return value is in bytes. - - .. versionadded:: 3.5 - -The module uses a global variable to store the name of the directory -used for temporary files returned by :func:`gettempdir`. It can be -set directly to override the selection process, but this is discouraged. -All functions in this module take a *dir* argument which can be used -to specify the directory. This is the recommended approach that does -not surprise other unsuspecting code by changing global API behavior. - -.. data:: tempdir - - When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to the functions defined in this - module, including its type, bytes or str. It cannot be a - :term:`path-like object`. - - If ``tempdir`` is ``None`` (the default) at any call to any of the above - functions except :func:`gettempprefix` it is initialized following the - algorithm described in :func:`gettempdir`. - - .. note:: - - Beware that if you set ``tempdir`` to a bytes value, there is a - nasty side effect: The global default return type of - :func:`mkstemp` and :func:`mkdtemp` changes to bytes when no - explicit ``prefix``, ``suffix``, or ``dir`` arguments of type - str are supplied. Please do not write code expecting or - depending on this. This awkward behavior is maintained for - compatibility with the historcal implementation. - -.. _tempfile-examples: - -Examples --------- - -Here are some examples of typical usage of the :mod:`tempfile` module:: - - >>> import tempfile - - # create a temporary file and write some data to it - >>> fp = tempfile.TemporaryFile() - >>> fp.write(b'Hello world!') - # read data from file - >>> fp.seek(0) - >>> fp.read() - b'Hello world!' - # close the file, it will be removed - >>> fp.close() - - # create a temporary file using a context manager - >>> with tempfile.TemporaryFile() as fp: - ... fp.write(b'Hello world!') - ... fp.seek(0) - ... fp.read() - b'Hello world!' - >>> - # file is now closed and removed - - # create a temporary directory using the context manager - >>> with tempfile.TemporaryDirectory() as tmpdirname: - ... print('created temporary directory', tmpdirname) - >>> - # directory and contents have been removed - - -Deprecated functions and variables ----------------------------------- - -A historical way to create temporary files was to first generate a -file name with the :func:`mktemp` function and then create a file -using this name. Unfortunately this is not secure, because a different -process may create a file with this name in the time between the call -to :func:`mktemp` and the subsequent attempt to create the file by the -first process. The solution is to combine the two steps and create the -file immediately. This approach is used by :func:`mkstemp` and the -other functions described above. - -.. function:: mktemp(suffix='', prefix='tmp', dir=None) - - .. deprecated:: 2.3 - Use :func:`mkstemp` instead. - - Return an absolute pathname of a file that did not exist at the time the - call is made. The *prefix*, *suffix*, and *dir* arguments are similar - to those of :func:`mkstemp`, except that bytes file names, ``suffix=None`` - and ``prefix=None`` are not supported. - - .. warning:: - - Use of this function may introduce a security hole in your program. By - the time you get around to doing anything with the file name it returns, - someone else may have beaten you to the punch. :func:`mktemp` usage can - be replaced easily with :func:`NamedTemporaryFile`, passing it the - ``delete=False`` parameter:: - - >>> f = NamedTemporaryFile(delete=False) - >>> f.name - '/tmp/tmptjujjt' - >>> f.write(b"Hello World!\n") - 13 - >>> f.close() - >>> os.unlink(f.name) - >>> os.path.exists(f.name) - False diff --git a/Doc/library/venv.rst.bak b/Doc/library/venv.rst.bak deleted file mode 100644 index 5d4a36481f1dcc..00000000000000 --- a/Doc/library/venv.rst.bak +++ /dev/null @@ -1,496 +0,0 @@ -:mod:`venv` --- Creation of virtual environments -================================================ - -.. module:: venv - :synopsis: Creation of virtual environments. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/venv/` - -.. index:: pair: Environments; virtual - --------------- - -The :mod:`venv` module provides support for creating lightweight "virtual -environments" with their own site directories, optionally isolated from system -site directories. Each virtual environment has its own Python binary (which -matches the version of the binary that was used to create this environment) and -can have its own independent set of installed Python packages in its site -directories. - -See :pep:`405` for more information about Python virtual environments. - -.. seealso:: - - `Python Packaging User Guide: Creating and using virtual environments - `__ - - -Creating virtual environments ------------------------------ - -.. include:: /using/venv-create.inc - - -.. _venv-def: - -.. note:: A virtual environment is a Python environment such that the Python - interpreter, libraries and scripts installed into it are isolated from those - installed in other virtual environments, and (by default) any libraries - installed in a "system" Python, i.e., one which is installed as part of your - operating system. - - A virtual environment is a directory tree which contains Python executable - files and other files which indicate that it is a virtual environment. - - Common installation tools such as setuptools_ and pip_ work as - expected with virtual environments. In other words, when a virtual - environment is active, they install Python packages into the virtual - environment without needing to be told to do so explicitly. - - When a virtual environment is active (i.e., the virtual environment's Python - interpreter is running), the attributes :attr:`sys.prefix` and - :attr:`sys.exec_prefix` point to the base directory of the virtual - environment, whereas :attr:`sys.base_prefix` and - :attr:`sys.base_exec_prefix` point to the non-virtual environment Python - installation which was used to create the virtual environment. If a virtual - environment is not active, then :attr:`sys.prefix` is the same as - :attr:`sys.base_prefix` and :attr:`sys.exec_prefix` is the same as - :attr:`sys.base_exec_prefix` (they all point to a non-virtual environment - Python installation). - - When a virtual environment is active, any options that change the - installation path will be ignored from all :mod:`distutils` configuration - files to prevent projects being inadvertently installed outside of the - virtual environment. - - When working in a command shell, users can make a virtual environment active - by running an ``activate`` script in the virtual environment's executables - directory (the precise filename and command to use the file is - shell-dependent), which prepends the virtual environment's directory for - executables to the ``PATH`` environment variable for the running shell. There - should be no need in other circumstances to activate a virtual - environment; scripts installed into virtual environments have a "shebang" - line which points to the virtual environment's Python interpreter. This means - that the script will run with that interpreter regardless of the value of - ``PATH``. On Windows, "shebang" line processing is supported if you have the - Python Launcher for Windows installed (this was added to Python in 3.3 - see - :pep:`397` for more details). Thus, double-clicking an installed script in a - Windows Explorer window should run the script with the correct interpreter - without there needing to be any reference to its virtual environment in - ``PATH``. - - -.. _venv-api: - -API ---- - -.. highlight:: python - -The high-level method described above makes use of a simple API which provides -mechanisms for third-party virtual environment creators to customize environment -creation according to their needs, the :class:`EnvBuilder` class. - -.. class:: EnvBuilder(system_site_packages=False, clear=False, \ - symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False) - - The :class:`EnvBuilder` class accepts the following keyword arguments on - instantiation: - - * ``system_site_packages`` -- a Boolean value indicating that the system Python - site-packages should be available to the environment (defaults to ``False``). - - * ``clear`` -- a Boolean value which, if true, will delete the contents of - any existing target directory, before creating the environment. - - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the - Python binary rather than copying. - - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing - environment with the running Python - for use when that Python has been - upgraded in-place (defaults to ``False``). - - * ``with_pip`` -- a Boolean value which, if true, ensures pip is - installed in the virtual environment. This uses :mod:`ensurepip` with - the ``--default-pip`` option. - - * ``prompt`` -- a String to be used after virtual environment is activated - (defaults to ``None`` which means directory name of the environment would - be used). If the special string ``"."`` is provided, the basename of the - current directory is used as the prompt. - - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - .. versionadded:: 3.6 - Added the ``prompt`` parameter - - .. versionadded:: 3.9 - Added the ``upgrade_deps`` parameter - - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: - - .. method:: create(env_dir) - - Create a virtual environment by specifying the target directory - (absolute or relative to the current directory) which is to contain the - virtual environment. The ``create`` method will either create the - environment in the specified directory, or raise an appropriate - exception. - - The ``create`` method of the :class:`EnvBuilder` class illustrates the - hooks available for subclass customization:: - - def create(self, env_dir): - """ - Create a virtualized Python environment in a directory. - env_dir is the target directory to create an environment in. - """ - env_dir = os.path.abspath(env_dir) - context = self.ensure_directories(env_dir) - self.create_configuration(context) - self.setup_python(context) - self.setup_scripts(context) - self.post_setup(context) - - Each of the methods :meth:`ensure_directories`, - :meth:`create_configuration`, :meth:`setup_python`, - :meth:`setup_scripts` and :meth:`post_setup` can be overridden. - - .. method:: ensure_directories(env_dir) - - Creates the environment directory and all necessary directories, and - returns a context object. This is just a holder for attributes (such as - paths), for use by the other methods. The directories are allowed to - exist already, as long as either ``clear`` or ``upgrade`` were - specified to allow operating on an existing environment directory. - - .. method:: create_configuration(context) - - Creates the ``pyvenv.cfg`` configuration file in the environment. - - .. method:: setup_python(context) - - Creates a copy or symlink to the Python executable in the environment. - On POSIX systems, if a specific executable ``python3.x`` was used, - symlinks to ``python`` and ``python3`` will be created pointing to that - executable, unless files with those names already exist. - - .. method:: setup_scripts(context) - - Installs activation scripts appropriate to the platform into the virtual - environment. - - .. method:: upgrade_dependencies(context) - - Upgrades the core venv dependency packages (currently ``pip`` and - ``setuptools``) in the environment. This is done by shelling out to the - ``pip`` executable in the environment. - - .. versionadded:: 3.9 - - .. method:: post_setup(context) - - A placeholder method which can be overridden in third party - implementations to pre-install packages in the virtual environment or - perform other post-creation steps. - - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - - .. method:: install_scripts(context, path) - - *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the - directory corresponding to :data:`os.name` are copied after some text - replacement of placeholders: - - * ``__VENV_DIR__`` is replaced with the absolute path of the environment - directory. - - * ``__VENV_NAME__`` is replaced with the environment name (final path - segment of environment directory). - - * ``__VENV_PROMPT__`` is replaced with the prompt (the environment - name surrounded by parentheses and with a following space) - - * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory - (either ``bin`` or ``Scripts``). - - * ``__VENV_PYTHON__`` is replaced with the absolute path of the - environment's executable. - - The directories are allowed to exist (for when an existing environment - is being upgraded). - -There is also a module-level convenience function: - -.. function:: create(env_dir, system_site_packages=False, clear=False, \ - symlinks=False, with_pip=False, prompt=None, \ - upgrade_deps=False) - - Create an :class:`EnvBuilder` with the given keyword arguments, and call its - :meth:`~EnvBuilder.create` method with the *env_dir* argument. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - .. versionchanged:: 3.6 - Added the ``prompt`` parameter - - .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter - -An example of extending ``EnvBuilder`` --------------------------------------- - -The following script shows how to extend :class:`EnvBuilder` by implementing a -subclass which installs setuptools and pip into a created virtual environment:: - - import os - import os.path - from subprocess import Popen, PIPE - import sys - from threading import Thread - from urllib.parse import urlparse - from urllib.request import urlretrieve - import venv - - class ExtendedEnvBuilder(venv.EnvBuilder): - """ - This builder installs setuptools and pip so that you can pip or - easy_install other packages into the created virtual environment. - - :param nodist: If true, setuptools and pip are not installed into the - created virtual environment. - :param nopip: If true, pip is not installed into the created - virtual environment. - :param progress: If setuptools or pip are installed, the progress of the - installation can be monitored by passing a progress - callable. If specified, it is called with two - arguments: a string indicating some progress, and a - context indicating where the string is coming from. - The context argument can have one of three values: - 'main', indicating that it is called from virtualize() - itself, and 'stdout' and 'stderr', which are obtained - by reading lines from the output streams of a subprocess - which is used to install the app. - - If a callable is not specified, default progress - information is output to sys.stderr. - """ - - def __init__(self, *args, **kwargs): - self.nodist = kwargs.pop('nodist', False) - self.nopip = kwargs.pop('nopip', False) - self.progress = kwargs.pop('progress', None) - self.verbose = kwargs.pop('verbose', False) - super().__init__(*args, **kwargs) - - def post_setup(self, context): - """ - Set up any packages which need to be pre-installed into the - virtual environment being created. - - :param context: The information for the virtual environment - creation request being processed. - """ - os.environ['VIRTUAL_ENV'] = context.env_dir - if not self.nodist: - self.install_setuptools(context) - # Can't install pip without setuptools - if not self.nopip and not self.nodist: - self.install_pip(context) - - def reader(self, stream, context): - """ - Read lines from a subprocess' output stream and either pass to a progress - callable (if specified) or write progress information to sys.stderr. - """ - progress = self.progress - while True: - s = stream.readline() - if not s: - break - if progress is not None: - progress(s, context) - else: - if not self.verbose: - sys.stderr.write('.') - else: - sys.stderr.write(s.decode('utf-8')) - sys.stderr.flush() - stream.close() - - def install_script(self, context, name, url): - _, _, path, _, _, _ = urlparse(url) - fn = os.path.split(path)[-1] - binpath = context.bin_path - distpath = os.path.join(binpath, fn) - # Download script into the virtual environment's binaries folder - urlretrieve(url, distpath) - progress = self.progress - if self.verbose: - term = '\n' - else: - term = '' - if progress is not None: - progress('Installing %s ...%s' % (name, term), 'main') - else: - sys.stderr.write('Installing %s ...%s' % (name, term)) - sys.stderr.flush() - # Install in the virtual environment - args = [context.env_exe, fn] - p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath) - t1 = Thread(target=self.reader, args=(p.stdout, 'stdout')) - t1.start() - t2 = Thread(target=self.reader, args=(p.stderr, 'stderr')) - t2.start() - p.wait() - t1.join() - t2.join() - if progress is not None: - progress('done.', 'main') - else: - sys.stderr.write('done.\n') - # Clean up - no longer needed - os.unlink(distpath) - - def install_setuptools(self, context): - """ - Install setuptools in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' - self.install_script(context, 'setuptools', url) - # clear up the setuptools archive which gets downloaded - pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') - files = filter(pred, os.listdir(context.bin_path)) - for f in files: - f = os.path.join(context.bin_path, f) - os.unlink(f) - - def install_pip(self, context): - """ - Install pip in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' - self.install_script(context, 'pip', url) - - def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with ' - 'Python 3.3 or later') - else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory in which to create the - 'virtual environment.') - parser.add_argument('--no-setuptools', default=False, - action='store_true', dest='nodist', - help="Don't install setuptools or pip in the " - "virtual environment.") - parser.add_argument('--no-pip', default=False, - action='store_true', dest='nopip', - help="Don't install pip in the virtual " - "environment.") - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - parser.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'virtual environment ' - 'directory if it already ' - 'exists, before virtual ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the virtual ' - 'environment directory to ' - 'use this version of ' - 'Python, assuming Python ' - 'has been upgraded ' - 'in-place.') - parser.add_argument('--verbose', default=False, action='store_true', - dest='verbose', help='Display the output ' - 'from the scripts which ' - 'install setuptools and pip.') - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = ExtendedEnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - nodist=options.nodist, - nopip=options.nopip, - verbose=options.verbose) - for d in options.dirs: - builder.create(d) - - if __name__ == '__main__': - rc = 1 - try: - main() - rc = 0 - except Exception as e: - print('Error: %s' % e, file=sys.stderr) - sys.exit(rc) - - -This script is also available for download `online -`_. - - -.. _setuptools: https://pypi.org/project/setuptools/ -.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/xml.sax.handler.rst.bak b/Doc/library/xml.sax.handler.rst.bak deleted file mode 100644 index 3746a58c9b9558..00000000000000 --- a/Doc/library/xml.sax.handler.rst.bak +++ /dev/null @@ -1,463 +0,0 @@ -:mod:`xml.sax.handler` --- Base classes for SAX handlers -======================================================== - -.. module:: xml.sax.handler - :synopsis: Base classes for SAX event handlers. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/handler.py` - --------------- - -The SAX API defines five kinds of handlers: content handlers, DTD handlers, -error handlers, entity resolvers and lexical handlers. Applications normally -only need to implement those interfaces whose events they are interested in; -they can implement the interfaces in a single object or in multiple objects. -Handler implementations should inherit from the base classes provided in the -module :mod:`xml.sax.handler`, so that all methods get default implementations. - - -.. class:: ContentHandler - - This is the main callback interface in SAX, and the one most important to - applications. The order of events in this interface mirrors the order of the - information in the document. - - -.. class:: DTDHandler - - Handle DTD events. - - This interface specifies only those DTD events required for basic parsing - (unparsed entities and attributes). - - -.. class:: EntityResolver - - Basic interface for resolving entities. If you create an object implementing - this interface, then register the object with your Parser, the parser will call - the method in your object to resolve all external entities. - - -.. class:: ErrorHandler - - Interface used by the parser to present error and warning messages to the - application. The methods of this object control whether errors are immediately - converted to exceptions or are handled in some other way. - - -.. class:: LexicalHandler - - Interface used by the parser to represent low freqency events which may not - be of interest to many applications. - -In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants -for the feature and property names. - - -.. data:: feature_namespaces - - | value: ``"http://xml.org/sax/features/namespaces"`` - | true: Perform Namespace processing. - | false: Optionally do not perform Namespace processing (implies - namespace-prefixes; default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_namespace_prefixes - - | value: ``"http://xml.org/sax/features/namespace-prefixes"`` - | true: Report the original prefixed names and attributes used for Namespace - declarations. - | false: Do not report attributes used for Namespace declarations, and - optionally do not report original prefixed names (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_string_interning - - | value: ``"http://xml.org/sax/features/string-interning"`` - | true: All element names, prefixes, attribute names, Namespace URIs, and - local names are interned using the built-in intern function. - | false: Names are not necessarily interned, although they may be (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_validation - - | value: ``"http://xml.org/sax/features/validation"`` - | true: Report all validation errors (implies external-general-entities and - external-parameter-entities). - | false: Do not report validation errors. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_ges - - | value: ``"http://xml.org/sax/features/external-general-entities"`` - | true: Include all external general (text) entities. - | false: Do not include external general entities. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_pes - - | value: ``"http://xml.org/sax/features/external-parameter-entities"`` - | true: Include all external parameter entities, including the external DTD - subset. - | false: Do not include any external parameter entities, even the external - DTD subset. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: all_features - - List of all features. - - -.. data:: property_lexical_handler - - | value: ``"http://xml.org/sax/properties/lexical-handler"`` - | data type: xml.sax.handler.LexicalHandler (not supported in Python 2) - | description: An optional extension handler for lexical events like - comments. - | access: read/write - - -.. data:: property_declaration_handler - - | value: ``"http://xml.org/sax/properties/declaration-handler"`` - | data type: xml.sax.sax2lib.DeclHandler (not supported in Python 2) - | description: An optional extension handler for DTD-related events other - than notations and unparsed entities. - | access: read/write - - -.. data:: property_dom_node - - | value: ``"http://xml.org/sax/properties/dom-node"`` - | data type: org.w3c.dom.Node (not supported in Python 2) - | description: When parsing, the current DOM node being visited if this is - a DOM iterator; when not parsing, the root DOM node for iteration. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: property_xml_string - - | value: ``"http://xml.org/sax/properties/xml-string"`` - | data type: String - | description: The literal string of characters that was the source for the - current event. - | access: read-only - - -.. data:: all_properties - - List of all known property names. - - -.. _content-handler-objects: - -ContentHandler Objects ----------------------- - -Users are expected to subclass :class:`ContentHandler` to support their -application. The following methods are called by the parser on the appropriate -events in the input document: - - -.. method:: ContentHandler.setDocumentLocator(locator) - - Called by the parser to give the application a locator for locating the origin - of document events. - - SAX parsers are strongly encouraged (though not absolutely required) to supply a - locator: if it does so, it must supply the locator to the application by - invoking this method before invoking any of the other methods in the - DocumentHandler interface. - - The locator allows the application to determine the end position of any - document-related event, even if the parser is not reporting an error. Typically, - the application will use this information for reporting its own errors (such as - character content that does not match an application's business rules). The - information returned by the locator is probably not sufficient for use with a - search engine. - - Note that the locator will return correct information only during the invocation - of the events in this interface. The application should not attempt to use it at - any other time. - - -.. method:: ContentHandler.startDocument() - - Receive notification of the beginning of a document. - - The SAX parser will invoke this method only once, before any other methods in - this interface or in DTDHandler (except for :meth:`setDocumentLocator`). - - -.. method:: ContentHandler.endDocument() - - Receive notification of the end of a document. - - The SAX parser will invoke this method only once, and it will be the last method - invoked during the parse. The parser shall not invoke this method until it has - either abandoned parsing (because of an unrecoverable error) or reached the end - of input. - - -.. method:: ContentHandler.startPrefixMapping(prefix, uri) - - Begin the scope of a prefix-URI Namespace mapping. - - The information from this event is not necessary for normal Namespace - processing: the SAX XML reader will automatically replace prefixes for element - and attribute names when the ``feature_namespaces`` feature is enabled (the - default). - - There are cases, however, when applications need to use prefixes in character - data or in attribute values, where they cannot safely be expanded automatically; - the :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events supply the - information to the application to expand prefixes in those contexts itself, if - necessary. - - .. XXX This is not really the default, is it? MvL - - Note that :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events are not - guaranteed to be properly nested relative to each-other: all - :meth:`startPrefixMapping` events will occur before the corresponding - :meth:`startElement` event, and all :meth:`endPrefixMapping` events will occur - after the corresponding :meth:`endElement` event, but their order is not - guaranteed. - - -.. method:: ContentHandler.endPrefixMapping(prefix) - - End the scope of a prefix-URI mapping. - - See :meth:`startPrefixMapping` for details. This event will always occur after - the corresponding :meth:`endElement` event, but the order of - :meth:`endPrefixMapping` events is not otherwise guaranteed. - - -.. method:: ContentHandler.startElement(name, attrs) - - Signals the start of an element in non-namespace mode. - - The *name* parameter contains the raw XML 1.0 name of the element type as a - string and the *attrs* parameter holds an object of the - :class:`~xml.sax.xmlreader.Attributes` - interface (see :ref:`attributes-objects`) containing the attributes of - the element. The object passed as *attrs* may be re-used by the parser; holding - on to a reference to it is not a reliable way to keep a copy of the attributes. - To keep a copy of the attributes, use the :meth:`copy` method of the *attrs* - object. - - -.. method:: ContentHandler.endElement(name) - - Signals the end of an element in non-namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElement` event. - - -.. method:: ContentHandler.startElementNS(name, qname, attrs) - - Signals the start of an element in namespace mode. - - The *name* parameter contains the name of the element type as a ``(uri, - localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in - the source document, and the *attrs* parameter holds an instance of the - :class:`~xml.sax.xmlreader.AttributesNS` interface (see - :ref:`attributes-ns-objects`) - containing the attributes of the element. If no namespace is associated with - the element, the *uri* component of *name* will be ``None``. The object passed - as *attrs* may be re-used by the parser; holding on to a reference to it is not - a reliable way to keep a copy of the attributes. To keep a copy of the - attributes, use the :meth:`copy` method of the *attrs* object. - - Parsers may set the *qname* parameter to ``None``, unless the - ``feature_namespace_prefixes`` feature is activated. - - -.. method:: ContentHandler.endElementNS(name, qname) - - Signals the end of an element in namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElementNS` method, likewise the *qname* parameter. - - -.. method:: ContentHandler.characters(content) - - Receive notification of character data. - - The Parser will call this method to report each chunk of character data. SAX - parsers may return all contiguous character data in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity so that the Locator provides useful - information. - - *content* may be a string or bytes instance; the ``expat`` reader module - always produces strings. - - .. note:: - - The earlier SAX 1 interface provided by the Python XML Special Interest Group - used a more Java-like interface for this method. Since most parsers used from - Python did not take advantage of the older interface, the simpler signature was - chosen to replace it. To convert old code to the new interface, use *content* - instead of slicing content with the old *offset* and *length* parameters. - - -.. method:: ContentHandler.ignorableWhitespace(whitespace) - - Receive notification of ignorable whitespace in element content. - - Validating Parsers must use this method to report each chunk of ignorable - whitespace (see the W3C XML 1.0 recommendation, section 2.10): non-validating - parsers may also use this method if they are capable of parsing and using - content models. - - SAX parsers may return all contiguous whitespace in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity, so that the Locator provides useful - information. - - -.. method:: ContentHandler.processingInstruction(target, data) - - Receive notification of a processing instruction. - - The Parser will invoke this method once for each processing instruction found: - note that processing instructions may occur before or after the main document - element. - - A SAX parser should never report an XML declaration (XML 1.0, section 2.8) or a - text declaration (XML 1.0, section 4.3.1) using this method. - - -.. method:: ContentHandler.skippedEntity(name) - - Receive notification of a skipped entity. - - The Parser will invoke this method once for each entity skipped. Non-validating - processors may skip entities if they have not seen the declarations (because, - for example, the entity was declared in an external DTD subset). All processors - may skip external entities, depending on the values of the - ``feature_external_ges`` and the ``feature_external_pes`` properties. - - -.. _dtd-handler-objects: - -DTDHandler Objects ------------------- - -:class:`DTDHandler` instances provide the following methods: - - -.. method:: DTDHandler.notationDecl(name, publicId, systemId) - - Handle a notation declaration event. - - -.. method:: DTDHandler.unparsedEntityDecl(name, publicId, systemId, ndata) - - Handle an unparsed entity declaration event. - - -.. _entity-resolver-objects: - -EntityResolver Objects ----------------------- - - -.. method:: EntityResolver.resolveEntity(publicId, systemId) - - Resolve the system identifier of an entity and return either the system - identifier to read from as a string, or an InputSource to read from. The default - implementation returns *systemId*. - - -.. _sax-error-handler: - -ErrorHandler Objects --------------------- - -Objects with this interface are used to receive error and warning information -from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that -implements this interface, then register the object with your -:class:`~xml.sax.xmlreader.XMLReader`, the parser -will call the methods in your object to report all warnings and errors. There -are three levels of errors available: warnings, (possibly) recoverable errors, -and unrecoverable errors. All methods take a :exc:`SAXParseException` as the -only parameter. Errors and warnings may be converted to an exception by raising -the passed-in exception object. - - -.. method:: ErrorHandler.error(exception) - - Called when the parser encounters a recoverable error. If this method does not - raise an exception, parsing may continue, but further document information - should not be expected by the application. Allowing the parser to continue may - allow additional errors to be discovered in the input document. - - -.. method:: ErrorHandler.fatalError(exception) - - Called when the parser encounters an error it cannot recover from; parsing is - expected to terminate when this method returns. - - -.. method:: ErrorHandler.warning(exception) - - Called when the parser presents minor warning information to the application. - Parsing is expected to continue when this method returns, and document - information will continue to be passed to the application. Raising an exception - in this method will cause parsing to end. - - -.. _lexical-handler-objects: - -LexicalHandler Objects ----------------------- -Optional SAX2 handler for lexical events. - -This handler is used to obtain lexical information about an XML -document. Lexical information includes information describing the -document encoding used and XML comments embedded in the document, as -well as section boundaries for the DTD and for any CDATA sections. -The lexical handlers are used in the same manner as content handlers. - -Set the LexicalHandler of an XMLReader by using the setProperty method -with the property identifier -``'http://xml.org/sax/properties/lexical-handler'``. - - -.. method:: LexicalHandler.comment(content) - - Reports a comment anywhere in the document (including the DTD and - outside the document element). - -.. method:: LexicalHandler.startDTD(name, public_id, system_id) - - Reports the start of the DTD declarations if the document has an - associated DTD. - -.. method:: LexicalHandler.endDTD() - - Reports the end of DTD declaration. - -.. method:: LexicalHandler.startCDATA() - - Reports the start of a CDATA marked section. - - The contents of the CDATA marked section will be reported through - the characters handler. - -.. method:: LexicalHandler.endCDATA() - - Reports the end of a CDATA marked section. diff --git a/Doc/library/zipimport.rst.bak b/Doc/library/zipimport.rst.bak deleted file mode 100644 index 62e1e47e980e17..00000000000000 --- a/Doc/library/zipimport.rst.bak +++ /dev/null @@ -1,209 +0,0 @@ -:mod:`zipimport` --- Import modules from Zip archives -===================================================== - -.. module:: zipimport - :synopsis: Support for importing Python modules from ZIP archives. - -.. moduleauthor:: Just van Rossum - -**Source code:** :source:`Lib/zipimport.py` - --------------- - -This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not -needed to use the :mod:`zipimport` module explicitly; it is automatically used -by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths -to ZIP archives. - -Typically, :data:`sys.path` is a list of directory names as strings. This module -also allows an item of :data:`sys.path` to be a string naming a ZIP file archive. -The ZIP archive can contain a subdirectory structure to support package imports, -and a path within the archive can be specified to only import from a -subdirectory. For example, the path :file:`example.zip/lib/` would only -import from the :file:`lib/` subdirectory within the archive. - -Any files may be present in the ZIP archive, but only files :file:`.py` and -:file:`.pyc` are available for import. ZIP import of dynamic modules -(:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains -:file:`.py` files, Python will not attempt to modify the archive by adding the -corresponding :file:`.pyc` file, meaning that if a ZIP archive -doesn't contain :file:`.pyc` files, importing may be rather slow. - -.. versionchanged:: 3.8 - Previously, ZIP archives with an archive comment were not supported. - -.. seealso:: - - `PKZIP Application Note `_ - Documentation on the ZIP file format by Phil Katz, the creator of the format and - algorithms used. - - :pep:`273` - Import Modules from Zip Archives - Written by James C. Ahlstrom, who also provided an implementation. Python 2.3 - follows the specification in :pep:`273`, but uses an implementation written by Just - van Rossum that uses the import hooks described in :pep:`302`. - - :mod:`importlib` - The implementation of the import machinery - Package providing the relevant protocols for all importers to - implement. - - -This module defines an exception: - -.. exception:: ZipImportError - - Exception raised by zipimporter objects. It's a subclass of :exc:`ImportError`, - so it can be caught as :exc:`ImportError`, too. - - -.. _zipimporter-objects: - -zipimporter Objects -------------------- - -:class:`zipimporter` is the class for importing ZIP files. - -.. class:: zipimporter(archivepath) - - Create a new zipimporter instance. *archivepath* must be a path to a ZIP - file, or to a specific path within a ZIP file. For example, an *archivepath* - of :file:`foo/bar.zip/lib` will look for modules in the :file:`lib` directory - inside the ZIP file :file:`foo/bar.zip` (provided that it exists). - - :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP - archive. - - .. method:: create_module(spec) - - Implementation of :meth:`importlib.abc.Loader.create_module` that returns - :const:`None` to explicitly request the default semantics. - - .. versionadded:: 3.10 - - - .. method:: exec_module(module) - - Implementation of :meth:`importlib.abc.Loader.exec_module`. - - .. versionadded:: 3.10 - - - .. method:: find_loader(fullname, path=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_module(fullname, path=None) - - Search for a module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. It returns the zipimporter instance itself - if the module was found, or :const:`None` if it wasn't. The optional - *path* argument is ignored---it's there for compatibility with the - importer protocol. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_spec(fullname, target=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. - - .. versionadded:: 3.10 - - - .. method:: get_code(fullname) - - Return the code object for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be imported. - - - .. method:: get_data(pathname) - - Return the data associated with *pathname*. Raise :exc:`OSError` if the - file wasn't found. - - .. versionchanged:: 3.3 - :exc:`IOError` used to be raised instead of :exc:`OSError`. - - - .. method:: get_filename(fullname) - - Return the value ``__file__`` would be set to if the specified module - was imported. Raise :exc:`ZipImportError` if the module couldn't be - imported. - - .. versionadded:: 3.1 - - - .. method:: get_source(fullname) - - Return the source code for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be found, return - :const:`None` if the archive does contain the module, but has no source - for it. - - - .. method:: is_package(fullname) - - Return ``True`` if the module specified by *fullname* is a package. Raise - :exc:`ZipImportError` if the module couldn't be found. - - - .. method:: load_module(fullname) - - Load the module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. Returns the imported module on success, - raises :exc:`ZipImportError` on failure. - - .. deprecated:: 3.10 - - Use :meth:`exec_module` instead. - - .. attribute:: archive - - The file name of the importer's associated ZIP file, without a possible - subpath. - - - .. attribute:: prefix - - The subpath within the ZIP file where modules are searched. This is the - empty string for zipimporter objects which point to the root of the ZIP - file. - - The :attr:`archive` and :attr:`prefix` attributes, when combined with a - slash, equal the original *archivepath* argument given to the - :class:`zipimporter` constructor. - - -.. _zipimport-examples: - -Examples --------- - -Here is an example that imports a module from a ZIP archive - note that the -:mod:`zipimport` module is not explicitly used. - -.. code-block:: shell-session - - $ unzip -l example.zip - Archive: example.zip - Length Date Time Name - -------- ---- ---- ---- - 8467 11-26-02 22:30 jwzthreading.py - -------- ------- - 8467 1 file - $ ./python - Python 2.3 (#1, Aug 1 2003, 19:54:32) - >>> import sys - >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path - >>> import jwzthreading - >>> jwzthreading.__file__ - 'example.zip/jwzthreading.py' diff --git a/Doc/whatsnew/3.10.rst.bak b/Doc/whatsnew/3.10.rst.bak deleted file mode 100644 index c4c282e5a04eae..00000000000000 --- a/Doc/whatsnew/3.10.rst.bak +++ /dev/null @@ -1,1347 +0,0 @@ -**************************** - What's New In Python 3.10 -**************************** - -:Release: |release| -:Date: |today| - -.. Rules for maintenance: - - * Anyone can add text to this document. Do not spend very much time - on the wording of your changes, because your text will probably - get rewritten to some degree. - - * The maintainer will go through Misc/NEWS periodically and add - changes; it's therefore more important to add your changes to - Misc/NEWS than to this file. - - * This is not a complete list of every single change; completeness - is the purpose of Misc/NEWS. Some changes I consider too small - or esoteric to include. If such a change is added to the text, - I'll just remove it. (This is another reason you shouldn't spend - too much time on writing your addition.) - - * If you want to draw your new text to the attention of the - maintainer, add 'XXX' to the beginning of the paragraph or - section. - - * It's OK to just add a fragmentary note about a change. For - example: "XXX Describe the transmogrify() function added to the - socket module." The maintainer will research the change and - write the necessary text. - - * You can comment out your additions if you like, but it's not - necessary (especially when a final release is some months away). - - * Credit the author of a patch or bugfix. Just the name is - sufficient; the e-mail address isn't necessary. - - * It's helpful to add the bug/patch number as a comment: - - XXX Describe the transmogrify() function added to the socket - module. - (Contributed by P.Y. Developer in :issue:`12345`.) - - This saves the maintainer the effort of going through the Mercurial log - when researching a change. - -This article explains the new features in Python 3.10, compared to 3.9. - -For full details, see the :ref:`changelog `. - -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.10 moves towards release, - so it's worth checking back even after reading earlier versions. - - -Summary -- Release highlights -============================= - -.. This section singles out the most important changes in Python 3.10. - Brevity is key. - - -.. PEP-sized items next. - - - -New Features -============ - -.. _whatsnew310-pep563: - -Parenthesized context managers ------------------------------- - -Using enclosing parentheses for continuation across multiple lines -in context managers is now supported. This allows formatting a long -collection of context managers in multiple lines in a similar way -as it was previously possible with import statements. For instance, -all these examples are now valid: - -.. code-block:: python - - with (CtxManager() as example): - ... - - with ( - CtxManager1(), - CtxManager2() - ): - ... - - with (CtxManager1() as example, - CtxManager2()): - ... - - with (CtxManager1(), - CtxManager2() as example): - ... - - with ( - CtxManager1() as example1, - CtxManager2() as example2 - ): - ... - -it is also possible to use a trailing comma at the end of the -enclosed group: - -.. code-block:: python - - with ( - CtxManager1() as example1, - CtxManager2() as example2, - CtxManager3() as example3, - ): - ... - -This new syntax uses the non LL(1) capacities of the new parser. -Check :pep:`617` for more details. - -(Contributed by Guido van Rossum, Pablo Galindo and Lysandros Nikolaou -in :issue:`12782` and :issue:`40334`.) - - -Better error messages in the parser ------------------------------------ - -When parsing code that contains unclosed parentheses or brackets the interpreter -now includes the location of the unclosed bracket of parentheses instead of displaying -*SyntaxError: unexpected EOF while parsing* or pointing to some incorrect location. -For instance, consider the following code (notice the unclosed '{'): - -.. code-block:: python - - expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, - 38: 4, 39: 4, 45: 5, 46: 5, 47: 5, 48: 5, 49: 5, 54: 6, - some_other_code = foo() - -previous versions of the interpreter reported confusing places as the location of -the syntax error: - -.. code-block:: text - - File "example.py", line 3 - some_other_code = foo() - ^ - SyntaxError: invalid syntax - -but in Python3.10 a more informative error is emitted: - -.. code-block:: text - - File "example.py", line 1 - expected = {9: 1, 18: 2, 19: 2, 27: 3, 28: 3, 29: 3, 36: 4, 37: 4, - ^ - SyntaxError: '{' was never closed - - -In a similar way, errors involving unclosed string literals (single and triple -quoted) now point to the start of the string instead of reporting EOF/EOL. - -These improvements are inspired by previous work in the PyPy interpreter. - -(Contributed by Pablo Galindo in :issue:`42864` and Batuhan Taskaya in -:issue:`40176`.) - - -PEP 634: Structural Pattern Matching ------------------------------------- - -Structural pattern matching has been added in the form of a *match statement* -and *case statements* of patterns with associated actions. Patterns -consist of sequences, mappings, primitive data types as well as class instances. -Pattern matching enables programs to extract information from complex data types, -branch on the structure of data, and apply specific actions based on different -forms of data. - -Syntax and operations -~~~~~~~~~~~~~~~~~~~~~ - -The generic syntax of pattern matching is:: - - match subject: - case : - - case : - - case : - - case _: - - -A match statement takes an expression and compares its value to successive -patterns given as one or more case blocks. Specifically, pattern matching -operates by: - - 1. using data with type and shape (the ``subject``) - 2. evaluating the ``subject`` in the ``match`` statement - 3. comparing the subject with each pattern in a ``case`` statement - from top to bottom until a match is confirmed. - 4. executing the action associated with the pattern of the confirmed - match - 5. If an exact match is not confirmed, the last case, a wildcard ``_``, - if provided, will be used as the matching case. If an exact match is - not confirmed and a wildcard case does not exists, the entire match - block is a no-op. - -Declarative approach -~~~~~~~~~~~~~~~~~~~~ - -Readers may be aware of pattern matching through the simple example of matching -a subject (data object) to a literal (pattern) with the switch statement found -in C, Java or JavaScript (and many other languages). Often the switch statement -is used for comparison of an object/expression with case statements containing -literals. - -More powerful examples of pattern matching can be found in languages, such as -Scala and Elixir. With structural pattern matching, the approach is "declarative" and -explicitly states the conditions (the patterns) for data to match. - -While an "imperative" series of instructions using nested "if" statements -could be used to accomplish something similar to structural pattern matching, -it is less clear than the "declarative" approach. Instead the "declarative" -approach states the conditions to meet for a match and is more readable through -its explicit patterns. While structural pattern matching can be used in its -simplest form comparing a variable to a literal in a case statement, its -true value for Python lies in its handling of the subject's type and shape. - -Simple pattern: match to a literal -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Let's look at this example as pattern matching in its simplest form: a value, -the subject, being matched to several literals, the patterns. In the example -below, ``status`` is the subject of the match statement. The patterns are -each of the case statements, where literals represent request status codes. -The associated action to the case is executed after a match:: - - def http_error(status): - match status: - case 400: - return "Bad request" - case 404: - return "Not found" - case 418: - return "I'm a teapot" - case _: - return "Something's wrong with the Internet" - -If the above function is passed a ``status`` of 418, "I'm a teapot" is returned. -If the above function is passed a ``status`` of 500, the case statement with -``_`` will match as a wildcard, and "Something's wrong with the Internet" is -returned. -Note the last block: the variable name, ``_``, acts as a *wildcard* and insures -the subject will always match. The use of ``_`` is optional. - -You can combine several literals in a single pattern using ``|`` ("or"):: - - case 401 | 403 | 404: - return "Not allowed" - -Behavior without the wildcard -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If we modify the above example by removing the last case block, the example -becomes:: - - def http_error(status): - match status: - case 400: - return "Bad request" - case 404: - return "Not found" - case 418: - return "I'm a teapot" - -Without the use of ``_`` in a case statement, a match may not exist. If no -match exists, the behavior is a no-op. For example, if ``status`` of 500 is -passed, a no-op occurs. - -Patterns with a literal and variable -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Patterns can look like unpacking assignments, and a pattern may be used to bind -variables. In this example, a data point can be unpacked to its x-coordinate -and y-coordinate:: - - # point is an (x, y) tuple - match point: - case (0, 0): - print("Origin") - case (0, y): - print(f"Y={y}") - case (x, 0): - print(f"X={x}") - case (x, y): - print(f"X={x}, Y={y}") - case _: - raise ValueError("Not a point") - -The first pattern has two literals, ``(0, 0)``, and may be thought of as an -extension of the literal pattern shown above. The next two patterns combine a -literal and a variable, and the variable *binds* a value from the subject -(``point``). The fourth pattern captures two values, which makes it -conceptually similar to the unpacking assignment ``(x, y) = point``. - -Patterns and classes -~~~~~~~~~~~~~~~~~~~~ - -If you are using classes to structure your data, you can use as a pattern -the class name followed by an argument list resembling a constructor. This -pattern has the ability to capture class attributes into variables:: - - class Point: - x: int - y: int - - def location(point): - match point: - case Point(x=0, y=0): - print("Origin is the point's location.") - case Point(x=0, y=y): - print(f"Y={y} and the point is on the y-axis.") - case Point(x=x, y=0): - print(f"X={x} and the point is on the x-axis.") - case Point(): - print("The point is located somewhere else on the plane.") - case _: - print("Not a point") - -Patterns with positional parameters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can use positional parameters with some builtin classes that provide an -ordering for their attributes (e.g. dataclasses). You can also define a specific -position for attributes in patterns by setting the ``__match_args__`` special -attribute in your classes. If it's set to ("x", "y"), the following patterns -are all equivalent (and all bind the ``y`` attribute to the ``var`` variable):: - - Point(1, var) - Point(1, y=var) - Point(x=1, y=var) - Point(y=var, x=1) - -Nested patterns -~~~~~~~~~~~~~~~ - -Patterns can be arbitrarily nested. For example, if our data is a short -list of points, it could be matched like this:: - - match points: - case []: - print("No points in the list.") - case [Point(0, 0)]: - print("The origin is the only point in the list.") - case [Point(x, y)]: - print(f"A single point {x}, {y} is in the list.") - case [Point(0, y1), Point(0, y2)]: - print(f"Two points on the Y axis at {y1}, {y2} are in the list.") - case _: - print("Something else is found in the list.") - -Complex patterns and the wildcard -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -To this point, the examples have used ``_`` alone in the last case statement. -A wildcard can be used in more complex patterns, such as ``('error', code, _)``. -For example:: - - match test_variable: - case ('warning', code, 40): - print("A warning has been received.") - case ('error', code, _): - print(f"An error {code} occured.") - -In the above case, ``test_variable`` will match for ('error', code, 100) and -('error', code, 800). - -Guard -~~~~~ - -We can add an ``if`` clause to a pattern, known as a "guard". If the -guard is false, ``match`` goes on to try the next case block. Note -that value capture happens before the guard is evaluated:: - - match point: - case Point(x, y) if x == y: - print(f"The point is located on the diagonal Y=X at {x}.") - case Point(x, y): - print(f"Point is not on the diagonal.") - -Other Key Features -~~~~~~~~~~~~~~~~~~ - -Several other key features: - -- Like unpacking assignments, tuple and list patterns have exactly the - same meaning and actually match arbitrary sequences. Technically, - the subject must be an instance of ``collections.abc.Sequence``. - Therefore, an important exception is that patterns don't match iterators. - Also, to prevent a common mistake, sequence patterns don't match strings. - -- Sequence patterns support wildcards: ``[x, y, *rest]`` and ``(x, y, - *rest)`` work similar to wildcards in unpacking assignments. The - name after ``*`` may also be ``_``, so ``(x, y, *_)`` matches a sequence - of at least two items without binding the remaining items. - -- Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the - ``"bandwidth"`` and ``"latency"`` values from a dict. Unlike sequence - patterns, extra keys are ignored. A wildcard ``**rest`` is also - supported. (But ``**_`` would be redundant, so it not allowed.) - -- Subpatterns may be captured using the ``as`` keyword:: - - case (Point(x1, y1), Point(x2, y2) as p2): ... - - This binds x1, y1, x2, y2 like you would expect without the ``as`` clause, - and p2 to the entire second item of the subject. - -- Most literals are compared by equality. However, the singletons ``True``, - ``False`` and ``None`` are compared by identity. - -- Named constants may be used in patterns. These named constants must be - dotted names to prevent the constant from being interpreted as a capture - variable:: - - from enum import Enum - class Color(Enum): - RED = 0 - GREEN = 1 - BLUE = 2 - - match color: - case Color.RED: - print("I see red!") - case Color.GREEN: - print("Grass is green") - case Color.BLUE: - print("I'm feeling the blues :(") - -For the full specification see :pep:`634`. Motivation and rationale -are in :pep:`635`, and a longer tutorial is in :pep:`636`. - - -New Features Related to Type Annotations -======================================== - -This section covers major changes affecting :pep:`484` type annotations and -the :mod:`typing` module. - - -PEP 563: Postponed Evaluation of Annotations Becomes Default ------------------------------------------------------------- - -In Python 3.7, postponed evaluation of annotations was added, -to be enabled with a ``from __future__ import annotations`` -directive. In 3.10 this became the default behavior, even -without that future directive. With this being default, all -annotations stored in :attr:`__annotations__` will be strings. -If needed, annotations can be resolved at runtime using -:func:`typing.get_type_hints`. See :pep:`563` for a full -description. Also, the :func:`inspect.signature` will try to -resolve types from now on, and when it fails it will fall back to -showing the string annotations. (Contributed by Batuhan Taskaya -in :issue:`38605`.) - - -PEP 604: New Type Union Operator --------------------------------- - -A new type union operator was introduced which enables the syntax ``X | Y``. -This provides a cleaner way of expressing 'either type X or type Y' instead of -using :data:`typing.Union`, especially in type hints (annotations). - -In previous versions of Python, to apply a type hint for functions accepting -arguments of multiple types, :data:`typing.Union` was used:: - - def square(number: Union[int, float]) -> Union[int, float]: - return number ** 2 - - -Type hints can now be written in a more succinct manner:: - - def square(number: int | float) -> int | float: - return number ** 2 - - -This new syntax is also accepted as the second argument to :func:`isinstance` -and :func:`issubclass`:: - - >>> isinstance(1, int | str) - True - -See :ref:`types-union` and :pep:`604` for more details. - -(Contributed by Maggie Moss and Philippe Prados in :issue:`41428`.) - - -PEP 612: Parameter Specification Variables ------------------------------------------- - -Two new options to improve the information provided to static type checkers for -:pep:`484`\ 's ``Callable`` have been added to the :mod:`typing` module. - -The first is the parameter specification variable. They are used to forward the -parameter types of one callable to another callable -- a pattern commonly -found in higher order functions and decorators. Examples of usage can be found -in :class:`typing.ParamSpec`. Previously, there was no easy way to type annotate -dependency of parameter types in such a precise manner. - -The second option is the new ``Concatenate`` operator. It's used in conjunction -with parameter specification variables to type annotate a higher order callable -which adds or removes parameters of another callable. Examples of usage can -be found in :class:`typing.Concatenate`. - -See :class:`typing.Callable`, :class:`typing.ParamSpec`, -:class:`typing.Concatenate` and :pep:`612` for more details. - -(Contributed by Ken Jin in :issue:`41559`.) - - -PEP 613: TypeAlias Annotation ------------------------------ - -:pep:`484` introduced the concept of type aliases, only requiring them to be -top-level unannotated assignments. This simplicity sometimes made it difficult -for type checkers to distinguish between type aliases and ordinary assignments, -especially when forward references or invalid types were involved. Compare:: - - StrCache = 'Cache[str]' # a type alias - LOG_PREFIX = 'LOG[DEBUG]' # a module constant - -Now the :mod:`typing` module has a special annotation :data:`TypeAlias` to -declare type aliases more explicitly:: - - StrCache: TypeAlias = 'Cache[str]' # a type alias - LOG_PREFIX = 'LOG[DEBUG]' # a module constant - -See :pep:`613` for more details. - -(Contributed by Mikhail Golubev in :issue:`41923`.) - - -Other Language Changes -====================== - -* The :class:`int` type has a new method :meth:`int.bit_count`, returning the - number of ones in the binary expansion of a given integer, also known - as the population count. (Contributed by Niklas Fiekas in :issue:`29882`.) - -* The views returned by :meth:`dict.keys`, :meth:`dict.values` and - :meth:`dict.items` now all have a ``mapping`` attribute that gives a - :class:`types.MappingProxyType` object wrapping the original - dictionary. (Contributed by Dennis Sweeney in :issue:`40890`.) - -* :pep:`618`: The :func:`zip` function now has an optional ``strict`` flag, used - to require that all the iterables have an equal length. - -* Builtin and extension functions that take integer arguments no longer accept - :class:`~decimal.Decimal`\ s, :class:`~fractions.Fraction`\ s and other - objects that can be converted to integers only with a loss (e.g. that have - the :meth:`~object.__int__` method but do not have the - :meth:`~object.__index__` method). - (Contributed by Serhiy Storchaka in :issue:`37999`.) - -* If :func:`object.__ipow__` returns :const:`NotImplemented`, the operator will - correctly fall back to :func:`object.__pow__` and :func:`object.__rpow__` as expected. - (Contributed by Alex Shkop in :issue:`38302`.) - -* Assignment expressions can now be used unparenthesized within set literals - and set comprehensions, as well as in sequence indexes (but not slices). - -* Functions have a new ``__builtins__`` attribute which is used to look for - builtin symbols when a function is executed, instead of looking into - ``__globals__['__builtins__']``. The attribute is initialized from - ``__globals__["__builtins__"]`` if it exists, else from the current builtins. - (Contributed by Mark Shannon in :issue:`42990`.) - - -New Modules -=========== - -* None yet. - - -Improved Modules -================ - -argparse --------- - -Misleading phrase "optional arguments" was replaced with "options" in argparse help. Some tests might require adaptation if they rely on exact output match. -(Contributed by Raymond Hettinger in :issue:`9694`.) - -base64 ------- - -Add :func:`base64.b32hexencode` and :func:`base64.b32hexdecode` to support the -Base32 Encoding with Extended Hex Alphabet. - -codecs ------- - -Add a :func:`codecs.unregister` function to unregister a codec search function. -(Contributed by Hai Shi in :issue:`41842`.) - -collections.abc ---------------- - -The ``__args__`` of the :ref:`parameterized generic ` for -:class:`collections.abc.Callable` are now consistent with :data:`typing.Callable`. -:class:`collections.abc.Callable` generic now flattens type parameters, similar -to what :data:`typing.Callable` currently does. This means that -``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of -``(int, str, str)``; previously this was ``([int, str], str)``. To allow this -change, :class:`types.GenericAlias` can now be subclassed, and a subclass will -be returned when subscripting the :class:`collections.abc.Callable` type. Note -that a :exc:`TypeError` may be raised for invalid forms of parameterizing -:class:`collections.abc.Callable` which may have passed silently in Python 3.9. -(Contributed by Ken Jin in :issue:`42195`.) - -contextlib ----------- - -Add a :func:`contextlib.aclosing` context manager to safely close async generators -and objects representing asynchronously released resources. -(Contributed by Joongi Kim and John Belmonte in :issue:`41229`.) - -Add asynchronous context manager support to :func:`contextlib.nullcontext`. -(Contributed by Tom Gringauz in :issue:`41543`.) - -curses ------- - -The extended color functions added in ncurses 6.1 will be used transparently -by :func:`curses.color_content`, :func:`curses.init_color`, -:func:`curses.init_pair`, and :func:`curses.pair_content`. A new function, -:func:`curses.has_extended_color_support`, indicates whether extended color -support is provided by the underlying ncurses library. -(Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) - -The ``BUTTON5_*`` constants are now exposed in the :mod:`curses` module if -they are provided by the underlying curses library. -(Contributed by Zackery Spytz in :issue:`39273`.) - -.. _distutils-deprecated: - -distutils ---------- - -The entire ``distutils`` package is deprecated, to be removed in Python -3.12. Its functionality for specifying package builds has already been -completely replaced by third-party packages ``setuptools`` and -``packaging``, and most other commonly used APIs are available elsewhere -in the standard library (such as :mod:`platform`, :mod:`shutil`, -:mod:`subprocess` or :mod:`sysconfig`). There are no plans to migrate -any other functionality from ``distutils``, and applications that are -using other functions should plan to make private copies of the code. -Refer to :pep:`632` for discussion. - -The ``bdist_wininst`` command deprecated in Python 3.8 has been removed. -The ``bdist_wheel`` command is now recommended to distribute binary packages -on Windows. -(Contributed by Victor Stinner in :issue:`42802`.) - -doctest -------- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -encodings ---------- - -:func:`encodings.normalize_encoding` now ignores non-ASCII characters. -(Contributed by Hai Shi in :issue:`39337`.) - -gc --- - -Added audit hooks for :func:`gc.get_objects`, :func:`gc.get_referrers` and -:func:`gc.get_referents`. (Contributed by Pablo Galindo in :issue:`43439`.) - -glob ----- - -Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and -:func:`~glob.iglob` which allow to specify the root directory for searching. -(Contributed by Serhiy Storchaka in :issue:`38144`.) - -importlib.metadata ------------------- - -Feature parity with ``importlib_metadata`` 3.7. - -:func:`importlib.metadata.entry_points` now provides a nicer experience -for selecting entry points by group and name through a new -:class:`importlib.metadata.EntryPoints` class. - -Added :func:`importlib.metadata.packages_distributions` for resolving -top-level Python modules and packages to their -:class:`importlib.metadata.Distribution`. - -inspect -------- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -Added *globalns* and *localns* parameters in :func:`~inspect.signature` and -:meth:`inspect.Signature.from_callable` to retrieve the annotations in given -local and global namespaces. -(Contributed by Batuhan Taskaya in :issue:`41960`.) - -linecache ---------- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -os --- - -Added :func:`os.cpu_count()` support for VxWorks RTOS. -(Contributed by Peixing Xin in :issue:`41440`.) - -Added a new function :func:`os.eventfd` and related helpers to wrap the -``eventfd2`` syscall on Linux. -(Contributed by Christian Heimes in :issue:`41001`.) - -Added :func:`os.splice()` that allows to move data between two file -descriptors without copying between kernel address space and user -address space, where one of the file descriptors must refer to a -pipe. (Contributed by Pablo Galindo in :issue:`41625`.) - -Added :data:`~os.O_EVTONLY`, :data:`~os.O_FSYNC`, :data:`~os.O_SYMLINK` -and :data:`~os.O_NOFOLLOW_ANY` for macOS. -(Contributed by Dong-hee Na in :issue:`43106`.) - -pathlib -------- - -Added slice support to :attr:`PurePath.parents `. -(Contributed by Joshua Cannon in :issue:`35498`) - -Added negative indexing support to :attr:`PurePath.parents -`. -(Contributed by Yaroslav Pankovych in :issue:`21041`) - -platform --------- - -Added :func:`platform.freedesktop_os_release()` to retrieve operation system -identification from `freedesktop.org os-release -`_ standard file. -(Contributed by Christian Heimes in :issue:`28468`) - -py_compile ----------- - -Added ``--quiet`` option to command-line interface of :mod:`py_compile`. -(Contributed by Gregory Schevchenko in :issue:`38731`.) - -pyclbr ------- - -Added an ``end_lineno`` attribute to the ``Function`` and ``Class`` -objects in the tree returned by :func:`pyclbr.readline` and -:func:`pyclbr.readline_ex`. It matches the existing (start) ``lineno``. -(Contributed by Aviral Srivastava in :issue:`38307`.) - -shelve ------- - -The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default -instead of :mod:`pickle` protocol ``3`` when creating shelves. -(Contributed by Zackery Spytz in :issue:`34204`.) - -site ----- - -When a module does not define ``__loader__``, fall back to ``__spec__.loader``. -(Contributed by Brett Cannon in :issue:`42133`.) - -socket ------- - -The exception :exc:`socket.timeout` is now an alias of :exc:`TimeoutError`. -(Contributed by Christian Heimes in :issue:`42413`.) - -sys ---- - -Add :data:`sys.orig_argv` attribute: the list of the original command line -arguments passed to the Python executable. -(Contributed by Victor Stinner in :issue:`23427`.) - -Add :data:`sys.stdlib_module_names`, containing the list of the standard library -module names. -(Contributed by Victor Stinner in :issue:`42955`.) - -_thread -------- - -:func:`_thread.interrupt_main` now takes an optional signal number to -simulate (the default is still :data:`signal.SIGINT`). -(Contributed by Antoine Pitrou in :issue:`43356`.) - -threading ---------- - -Added :func:`threading.gettrace` and :func:`threading.getprofile` to -retrieve the functions set by :func:`threading.settrace` and -:func:`threading.setprofile` respectively. -(Contributed by Mario Corchero in :issue:`42251`.) - -Add :data:`threading.__excepthook__` to allow retrieving the original value -of :func:`threading.excepthook` in case it is set to a broken or a different -value. -(Contributed by Mario Corchero in :issue:`42308`.) - -traceback ---------- - -The :func:`~traceback.format_exception`, -:func:`~traceback.format_exception_only`, and -:func:`~traceback.print_exception` functions can now take an exception object -as a positional-only argument. -(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) - -types ------ - -Reintroduced the :data:`types.EllipsisType`, :data:`types.NoneType` -and :data:`types.NotImplementedType` classes, providing a new set -of types readily interpretable by type checkers. -(Contributed by Bas van Beek in :issue:`41810`.) - -typing ------- - -For major changes, see `New Features Related to Type Annotations`_. - -The behavior of :class:`typing.Literal` was changed to conform with :pep:`586` -and to match the behavior of static type checkers specified in the PEP. - -1. ``Literal`` now de-duplicates parameters. -2. Equality comparisons between ``Literal`` objects are now order independent. -3. ``Literal`` comparisons now respects types. For example, - ``Literal[0] == Literal[False]`` previously evaluated to ``True``. It is - now ``False``. To support this change, the internally used type cache now - supports differentiating types. -4. ``Literal`` objects will now raise a :exc:`TypeError` exception during - equality comparisons if one of their parameters are not :term:`immutable`. - Note that declaring ``Literal`` with mutable parameters will not throw - an error:: - - >>> from typing import Literal - >>> Literal[{0}] - >>> Literal[{0}] == Literal[{False}] - Traceback (most recent call last): - File "", line 1, in - TypeError: unhashable type: 'set' - -(Contributed by Yurii Karabas in :issue:`42345`.) - -unittest --------- - -Add new method :meth:`~unittest.TestCase.assertNoLogs` to complement the -existing :meth:`~unittest.TestCase.assertLogs`. (Contributed by Kit Yan Choi -in :issue:`39385`.) - -urllib.parse ------------- - -Python versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as -query parameter separators in :func:`urllib.parse.parse_qs` and -:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with -newer W3C recommendations, this has been changed to allow only a single -separator key, with ``&`` as the default. This change also affects -:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected -functions internally. For more details, please see their respective -documentation. -(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) - -xml ---- - -Add a :class:`~xml.sax.handler.LexicalHandler` class to the -:mod:`xml.sax.handler` module. -(Contributed by Jonathan Gossage and Zackery Spytz in :issue:`35018`.) - -zipimport ---------- -Add methods related to :pep:`451`: :meth:`~zipimport.zipimporter.find_spec`, -:meth:`zipimport.zipimporter.create_module`, and -:meth:`zipimport.zipimporter.exec_module`. -(Contributed by Brett Cannon in :issue:`42131`. - - -Optimizations -============= - -* Constructors :func:`str`, :func:`bytes` and :func:`bytearray` are now faster - (around 30--40% for small objects). - (Contributed by Serhiy Storchaka in :issue:`41334`.) - -* The :mod:`runpy` module now imports fewer modules. - The ``python3 -m module-name`` command startup time is 1.3x faster in - average. - (Contributed by Victor Stinner in :issue:`41006`.) - -* The ``LOAD_ATTR`` instruction now uses new "per opcode cache" mechanism. It - is about 36% faster now for regular attributes and 44% faster for slots. - (Contributed by Pablo Galindo and Yury Selivanov in :issue:`42093` and Guido - van Rossum in :issue:`42927`, based on ideas implemented originally in PyPy - and MicroPython.) - -* When building Python with ``--enable-optimizations`` now - ``-fno-semantic-interposition`` is added to both the compile and link line. - This speeds builds of the Python interpreter created with ``--enable-shared`` - with ``gcc`` by up to 30%. See `this article - `_ - for more details. (Contributed by Victor Stinner and Pablo Galindo in - :issue:`38980`.) - - -* Function parameters and their annotations are no longer computed at runtime, - but rather at compilation time. They are stored as a tuple of strings at the - bytecode level. It is now around 2 times faster to create a function with - parameter annotations. (Contributed by Yurii Karabas and Inada Naoki - in :issue:`42202`) - -* Substring search functions such as ``str1 in str2`` and ``str2.find(str1)`` - now sometimes use Crochemore & Perrin's "Two-Way" string searching - algorithm to avoid quadratic behavior on long strings. (Contributed - by Dennis Sweeney in :issue:`41972`) - -Deprecated -========== - -* Starting in this release, there will be a concerted effort to begin - cleaning up old import semantics that were kept for Python 2.7 - compatibility. Specifically, - :meth:`~importlib.abc.PathEntryFinder.find_loader`/:meth:`~importlib.abc.Finder.find_module` - (superseded by :meth:`~importlib.abc.Finder.find_spec`), - :meth:`~importlib.abc.Loader.load_module` - (superseded by :meth:`~importlib.abc.Loader.exec_module`), - :meth:`~importlib.abc.Loader.module_repr` (which the import system - takes care of for you), the ``__package__`` attribute - (superseded by ``__spec__.parent``), the ``__loader__`` attribute - (superseded by ``__spec__.loader``), and the ``__cached__`` attribute - (superseded by ``__spec__.cached``) will slowly be removed (as well - as other classes and methods in :mod:`importlib`). - :exc:`ImportWarning` and/or :exc:`DeprecationWarning` will be raised - as appropriate to help identify code which needs updating during - this transition. - -* The entire ``distutils`` namespace is deprecated, to be removed in - Python 3.12. Refer to the :ref:`module changes ` - section for more information. - -* Non-integer arguments to :func:`random.randrange` are deprecated. - The :exc:`ValueError` is deprecated in favor of a :exc:`TypeError`. - (Contributed by Serhiy Storchaka and Raymond Hettinger in :issue:`37319`.) - -* The various ``load_module()`` methods of :mod:`importlib` have been - documented as deprecated since Python 3.6, but will now also trigger - a :exc:`DeprecationWarning`. Use - :meth:`~importlib.abc.Loader.exec_module` instead. - (Contributed by Brett Cannon in :issue:`26131`.) - -* :meth:`zimport.zipimporter.load_module` has been deprecated in - preference for :meth:`~zipimport.zipimporter.exec_module`. - (Contributed by Brett Cannon in :issue:`26131`.) - -* The use of :meth:`~importlib.abc.Loader.load_module` by the import - system now triggers an :exc:`ImportWarning` as - :meth:`~importlib.abc.Loader.exec_module` is preferred. - (Contributed by Brett Cannon in :issue:`26131`.) - -* ``sqlite3.OptimizedUnicode`` has been undocumented and obsolete since Python - 3.3, when it was made an alias to :class:`str`. It is now deprecated, - scheduled for removal in Python 3.12. - (Contributed by Erlend E. Aasland in :issue:`42264`.) - -* The undocumented built-in function ``sqlite3.enable_shared_cache`` is now - deprecated, scheduled for removal in Python 3.12. Its use is strongly - discouraged by the SQLite3 documentation. See `the SQLite3 docs - `_ for more details. - If shared cache must be used, open the database in URI mode using the - ``cache=shared`` query parameter. - (Contributed by Erlend E. Aasland in :issue:`24464`.) - - -Removed -======= - -* Removed special methods ``__int__``, ``__float__``, ``__floordiv__``, - ``__mod__``, ``__divmod__``, ``__rfloordiv__``, ``__rmod__`` and - ``__rdivmod__`` of the :class:`complex` class. They always raised - a :exc:`TypeError`. - (Contributed by Serhiy Storchaka in :issue:`41974`.) - -* The ``ParserBase.error()`` method from the private and undocumented ``_markupbase`` - module has been removed. :class:`html.parser.HTMLParser` is the only subclass of - ``ParserBase`` and its ``error()`` implementation has already been removed in - Python 3.5. - (Contributed by Berker Peksag in :issue:`31844`.) - -* Removed the ``unicodedata.ucnhash_CAPI`` attribute which was an internal - PyCapsule object. The related private ``_PyUnicode_Name_CAPI`` structure was - moved to the internal C API. - (Contributed by Victor Stinner in :issue:`42157`.) - -* Removed the ``parser`` module, which was deprecated in 3.9 due to the - switch to the new PEG parser, as well as all the C source and header files - that were only being used by the old parser, including ``node.h``, ``parser.h``, - ``graminit.h`` and ``grammar.h``. - -* Removed the Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, - :c:func:`PyParser_SimpleParseStringFlagsFilename`, - :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` - that were deprecated in 3.9 due to the switch to the new PEG parser. - -* Removed the ``formatter`` module, which was deprecated in Python 3.4. - It is somewhat obsolete, little used, and not tested. It was originally - scheduled to be removed in Python 3.6, but such removals were delayed until - after Python 2.7 EOL. Existing users should copy whatever classes they use - into their code. - (Contributed by Dong-hee Na and Terry J. Reedy in :issue:`42299`.) - -* Removed the :c:func:`PyModule_GetWarningsModule` function that was useless - now due to the _warnings module was converted to a builtin module in 2.6. - (Contributed by Hai Shi in :issue:`42599`.) - -* Remove deprecated aliases to :ref:`collections-abstract-base-classes` from - the :mod:`collections` module. - (Contributed by Victor Stinner in :issue:`37324`.) - -* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's - :doc:`high-level API <../library/asyncio-api-index>` following deprecation - in Python 3.8. The motivation behind this change is multifold: - - 1. This simplifies the high-level API. - 2. The functions in the high-level API have been implicitly getting the - current thread's running event loop since Python 3.7. There isn't a need to - pass the event loop to the API in most normal use cases. - 3. Event loop passing is error-prone especially when dealing with loops - running in different threads. - - Note that the low-level API will still accept ``loop``. - See `Changes in the Python API`_ for examples of how to replace existing code. - - (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley - in :issue:`42392`.) - - -Porting to Python 3.10 -====================== - -This section lists previously described changes and other bugfixes -that may require changes to your code. - - -Changes in the Python API -------------------------- - -* The *etype* parameters of the :func:`~traceback.format_exception`, - :func:`~traceback.format_exception_only`, and - :func:`~traceback.print_exception` functions in the :mod:`traceback` module - have been renamed to *exc*. - (Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) - -* :mod:`atexit`: At Python exit, if a callback registered with - :func:`atexit.register` fails, its exception is now logged. Previously, only - some exceptions were logged, and the last exception was always silently - ignored. - (Contributed by Victor Stinner in :issue:`42639`.) - -* :class:`collections.abc.Callable` generic now flattens type parameters, similar - to what :data:`typing.Callable` currently does. This means that - ``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of - ``(int, str, str)``; previously this was ``([int, str], str)``. Code which - accesses the arguments via :func:`typing.get_args` or ``__args__`` need to account - for this change. Furthermore, :exc:`TypeError` may be raised for invalid forms - of parameterizing :class:`collections.abc.Callable` which may have passed - silently in Python 3.9. - (Contributed by Ken Jin in :issue:`42195`.) - -* :meth:`socket.htons` and :meth:`socket.ntohs` now raise :exc:`OverflowError` - instead of :exc:`DeprecationWarning` if the given parameter will not fit in - a 16-bit unsigned integer. - (Contributed by Erlend E. Aasland in :issue:`42393`.) - -* The ``loop`` parameter has been removed from most of :mod:`asyncio`\ 's - :doc:`high-level API <../library/asyncio-api-index>` following deprecation - in Python 3.8. - - A coroutine that currently look like this:: - - async def foo(loop): - await asyncio.sleep(1, loop=loop) - - Should be replaced with this:: - - async def foo(): - await asyncio.sleep(1) - - If ``foo()`` was specifically designed *not* to run in the current thread's - running event loop (e.g. running in another thread's event loop), consider - using :func:`asyncio.run_coroutine_threadsafe` instead. - - (Contributed by Yurii Karabas, Andrew Svetlov, Yury Selivanov and Kyle Stanley - in :issue:`42392`.) - -* The :data:`types.FunctionType` constructor now inherits the current builtins - if the *globals* dictionary has no ``"__builtins__"`` key, rather than using - ``{"None": None}`` as builtins: same behavior as :func:`eval` and - :func:`exec` functions. Defining a function with ``def function(...): ...`` - in Python is not affected, globals cannot be overriden with this syntax: it - also inherits the current builtins. - (Contributed by Victor Stinner in :issue:`42990`.) - -CPython bytecode changes -======================== - -* The ``MAKE_FUNCTION`` instruction accepts tuple of strings as annotations - instead of dictionary. - (Contributed by Yurii Karabas and Inada Naoki in :issue:`42202`) - -Build Changes -============= - -* The C99 functions :c:func:`snprintf` and :c:func:`vsnprintf` are now required - to build Python. - (Contributed by Victor Stinner in :issue:`36020`.) - -* :mod:`sqlite3` requires SQLite 3.7.15 or higher. (Contributed by Sergey Fedoseev - and Erlend E. Aasland :issue:`40744` and :issue:`40810`.) - -* The :mod:`atexit` module must now always be built as a built-in module. - (Contributed by Victor Stinner in :issue:`42639`.) - -* Added ``--disable-test-modules`` option to the ``configure`` script: - don't build nor install test modules. - (Contributed by Xavier de Gaye, Thomas Petazzoni and Peixing Xin in :issue:`27640`.) - -* Add ``--with-wheel-pkg-dir=PATH`` option to the ``./configure`` script. If - specified, the :mod:`ensurepip` module looks for ``setuptools`` and ``pip`` - wheel packages in this directory: if both are present, these wheel packages - are used instead of ensurepip bundled wheel packages. - - Some Linux distribution packaging policies recommend against bundling - dependencies. For example, Fedora installs wheel packages in the - ``/usr/share/python-wheels/`` directory and don't install the - ``ensurepip._bundled`` package. - - (Contributed by Victor Stinner in :issue:`42856`.) - -* Add a new configure ``--without-static-libpython`` option to not build the - ``libpythonMAJOR.MINOR.a`` static library and not install the ``python.o`` - object file. - - (Contributed by Victor Stinner in :issue:`43103`.) - -* The ``configure`` script now uses the ``pkg-config`` utility, if available, - to detect the location of Tcl/Tk headers and libraries. As before, those - locations can be explicitly specified with the ``--with-tcltk-includes`` - and ``--with-tcltk-libs`` configuration options. - (Contributed by Manolis Stamatogiannakis in :issue:`42603`.) - - -C API Changes -============= - -New Features ------------- - -* The result of :c:func:`PyNumber_Index` now always has exact type :class:`int`. - Previously, the result could have been an instance of a subclass of ``int``. - (Contributed by Serhiy Storchaka in :issue:`40792`.) - -* Add a new :c:member:`~PyConfig.orig_argv` member to the :c:type:`PyConfig` - structure: the list of the original command line arguments passed to the - Python executable. - (Contributed by Victor Stinner in :issue:`23427`.) - -* The :c:func:`PyDateTime_DATE_GET_TZINFO` and - :c:func:`PyDateTime_TIME_GET_TZINFO` macros have been added for accessing - the ``tzinfo`` attributes of :class:`datetime.datetime` and - :class:`datetime.time` objects. - (Contributed by Zackery Spytz in :issue:`30155`.) - -* Add a :c:func:`PyCodec_Unregister` function to unregister a codec - search function. - (Contributed by Hai Shi in :issue:`41842`.) - -* The :c:func:`PyIter_Send` function was added to allow - sending value into iterator without raising ``StopIteration`` exception. - (Contributed by Vladimir Matveev in :issue:`41756`.) - -* Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API. - (Contributed by Alex Gaynor in :issue:`41784`.) - -* Added :c:func:`PyModule_AddObjectRef` function: similar to - :c:func:`PyModule_AddObject` but don't steal a reference to the value on - success. - (Contributed by Victor Stinner in :issue:`1635741`.) - -* Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the - reference count of an object and return the object. - (Contributed by Victor Stinner in :issue:`42262`.) - -* The :c:func:`PyType_FromSpecWithBases` and :c:func:`PyType_FromModuleAndSpec` - functions now accept a single class as the *bases* argument. - (Contributed by Serhiy Storchaka in :issue:`42423`.) - -* The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc`` - slot. - (Contributed by Hai Shi in :issue:`41832`.) - -* The :c:func:`PyType_GetSlot` function can accept static types. - (Contributed by Hai Shi and Petr Viktorin in :issue:`41073`.) - -* Add a new :c:func:`PySet_CheckExact` function to the C-API to check if an - object is an instance of :class:`set` but not an instance of a subtype. - (Contributed by Pablo Galindo in :issue:`43277`.) - -* Added :c:func:`PyErr_SetInterruptEx` which allows passing a signal number - to simulate. - (Contributed by Antoine Pitrou in :issue:`43356`.) - - -Porting to Python 3.10 ----------------------- - -* The ``PY_SSIZE_T_CLEAN`` macro must now be defined to use - :c:func:`PyArg_ParseTuple` and :c:func:`Py_BuildValue` formats which use - ``#``: ``es#``, ``et#``, ``s#``, ``u#``, ``y#``, ``z#``, ``U#`` and ``Z#``. - See :ref:`Parsing arguments and building values - ` and the :pep:`353`. - (Contributed by Victor Stinner in :issue:`40943`.) - -* Since :c:func:`Py_REFCNT()` is changed to the inline static function, - ``Py_REFCNT(obj) = new_refcnt`` must be replaced with ``Py_SET_REFCNT(obj, new_refcnt)``: - see :c:func:`Py_SET_REFCNT()` (available since Python 3.9). For backward - compatibility, this macro can be used:: - - #if PY_VERSION_HEX < 0x030900A4 - # define Py_SET_REFCNT(obj, refcnt) ((Py_REFCNT(obj) = (refcnt)), (void)0) - #endif - - (Contributed by Victor Stinner in :issue:`39573`.) - -* Calling :c:func:`PyDict_GetItem` without :term:`GIL` held had been allowed - for historical reason. It is no longer allowed. - (Contributed by Victor Stinner in :issue:`40839`.) - -* ``PyUnicode_FromUnicode(NULL, size)`` and ``PyUnicode_FromStringAndSize(NULL, size)`` - raise ``DeprecationWarning`` now. Use :c:func:`PyUnicode_New` to allocate - Unicode object without initial data. - (Contributed by Inada Naoki in :issue:`36346`.) - -* The private ``_PyUnicode_Name_CAPI`` structure of the PyCapsule API - ``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. - (Contributed by Victor Stinner in :issue:`42157`.) - -* :c:func:`Py_GetPath`, :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome` and - :c:func:`Py_GetProgramName` functions now return ``NULL`` if called before - :c:func:`Py_Initialize` (before Python is initialized). Use the new - :ref:`Python Initialization Configuration API ` to get the - :ref:`Python Path Configuration. `. - (Contributed by Victor Stinner in :issue:`42260`.) - -* :c:func:`PyList_SET_ITEM`, :c:func:`PyTuple_SET_ITEM` and - :c:func:`PyCell_SET` macros can no longer be used as l-value or r-value. - For example, ``x = PyList_SET_ITEM(a, b, c)`` and - ``PyList_SET_ITEM(a, b, c) = x`` now fail with a compiler error. It prevents - bugs like ``if (PyList_SET_ITEM (a, b, c) < 0) ...`` test. - (Contributed by Zackery Spytz and Victor Stinner in :issue:`30459`.) - -* The non-limited API files ``odictobject.h``, ``parser_interface.h``, - ``picklebufobject.h``, ``pyarena.h``, ``pyctype.h``, ``pydebug.h``, - ``pyfpe.h``, and ``pytime.h`` have been moved to the ``Include/cpython`` - directory. These files must not be included directly, as they are already - included in ``Python.h``: :ref:`Include Files `. If they have - been included directly, consider including ``Python.h`` instead. - (Contributed by Nicholas Sim in :issue:`35134`) - -Deprecated ----------- - -* The ``PyUnicode_InternImmortal()`` function is now deprecated - and will be removed in Python 3.12: use :c:func:`PyUnicode_InternInPlace` - instead. - (Contributed by Victor Stinner in :issue:`41692`.) - -Removed -------- - -* ``PyObject_AsCharBuffer()``, ``PyObject_AsReadBuffer()``, ``PyObject_CheckReadBuffer()``, - and ``PyObject_AsWriteBuffer()`` are removed. Please migrate to new buffer protocol; - :c:func:`PyObject_GetBuffer` and :c:func:`PyBuffer_Release`. - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``Py_UNICODE_str*`` functions manipulating ``Py_UNICODE*`` strings. - (Contributed by Inada Naoki in :issue:`41123`.) - - * ``Py_UNICODE_strlen``: use :c:func:`PyUnicode_GetLength` or - :c:macro:`PyUnicode_GET_LENGTH` - * ``Py_UNICODE_strcat``: use :c:func:`PyUnicode_CopyCharacters` or - :c:func:`PyUnicode_FromFormat` - * ``Py_UNICODE_strcpy``, ``Py_UNICODE_strncpy``: use - :c:func:`PyUnicode_CopyCharacters` or :c:func:`PyUnicode_Substring` - * ``Py_UNICODE_strcmp``: use :c:func:`PyUnicode_Compare` - * ``Py_UNICODE_strncmp``: use :c:func:`PyUnicode_Tailmatch` - * ``Py_UNICODE_strchr``, ``Py_UNICODE_strrchr``: use - :c:func:`PyUnicode_FindChar` - -* Removed ``PyUnicode_GetMax()``. Please migrate to new (:pep:`393`) APIs. - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``PyLong_FromUnicode()``. Please migrate to :c:func:`PyLong_FromUnicodeObject`. - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``PyUnicode_AsUnicodeCopy()``. Please use :c:func:`PyUnicode_AsUCS4Copy` or - :c:func:`PyUnicode_AsWideCharString` - (Contributed by Inada Naoki in :issue:`41103`.) - -* Removed ``_Py_CheckRecursionLimit`` variable: it has been replaced by - ``ceval.recursion_limit`` of the :c:type:`PyInterpreterState` structure. - (Contributed by Victor Stinner in :issue:`41834`.) - -* Removed undocumented macros ``Py_ALLOW_RECURSION`` and - ``Py_END_ALLOW_RECURSION`` and the ``recursion_critical`` field of the - :c:type:`PyInterpreterState` structure. - (Contributed by Serhiy Storchaka in :issue:`41936`.) - -* Removed the undocumented ``PyOS_InitInterrupts()`` function. Initializing - Python already implicitly installs signal handlers: see - :c:member:`PyConfig.install_signal_handlers`. - (Contributed by Victor Stinner in :issue:`41713`.) From 7b65fbce94c3898d17d0f2f9d5f41e8421204b1c Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 31 Mar 2021 15:22:46 +0000 Subject: [PATCH 31/93] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst diff --git a/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst new file mode 100644 index 00000000000000..eb89caa3976db4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst @@ -0,0 +1,2 @@ +Add an Barrier object in synchronization primitives of asyncio Lib +in order to be consistant with Barrier from *threading* and *multiprocessing* libs \ No newline at end of file From 9b60d076853303b5bd54ff88bc26016919a0183a Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 2 Apr 2021 14:57:51 +0200 Subject: [PATCH 32/93] Update 2021-03-31-15-22-45.bpo-43352.nSjMuE.rst --- .../next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst index eb89caa3976db4..e53ba28b64099f 100644 --- a/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst +++ b/Misc/NEWS.d/next/Library/2021-03-31-15-22-45.bpo-43352.nSjMuE.rst @@ -1,2 +1 @@ -Add an Barrier object in synchronization primitives of asyncio Lib -in order to be consistant with Barrier from *threading* and *multiprocessing* libs \ No newline at end of file +Add an Barrier object in synchronization primitives of *asyncio* Lib in order to be consistant with Barrier from *threading* and *multiprocessing* libs* From 576cd53c206f05b586611c6db562bdbd4c71af56 Mon Sep 17 00:00:00 2001 From: Duprat Date: Sun, 11 Apr 2021 15:36:40 +0200 Subject: [PATCH 33/93] Update documentation about add of Barrier object Add Barrier object as a clone of threading.Barrier --- Doc/library/asyncio-api-index.rst | 3 +++ Doc/library/asyncio-sync.rst | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index f558724d4a3ff6..3465a297120920 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -186,6 +186,9 @@ Threading-like synchronization primitives that can be used in Tasks. * - :class:`BoundedSemaphore` - A bounded semaphore. + * - :class:`Barrier` + - A barrier object. + .. rubric:: Examples diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index d12630afc6a326..ac6e84d070850e 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -28,6 +28,7 @@ asyncio has the following basic synchronization primitives: * :class:`Condition` * :class:`Semaphore` * :class:`BoundedSemaphore` +* :class:`Barrier` --------- @@ -325,6 +326,22 @@ BoundedSemaphore a :exc:`ValueError` in :meth:`~Semaphore.release` if it increases the internal counter above the initial *value*. + +Barrier +======= + +.. class:: Barrier(parties, action=None) + + A barrier object. Not thread-safe. + + Create a barrier object for *parties* number of tasks. An *action*, when + provided, is a callable to be called by one of the tasks when they are + released. + + A barrier object: a clone of :class:`threading.Barrier`. + + .. versionadded:: 3.10 + --------- From 6106f21de1685022970a77ab0440e9da4d41fea3 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 30 Apr 2021 14:47:28 +0200 Subject: [PATCH 34/93] Update asyncio-sync.rst Describe when action is called --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index ac6e84d070850e..f7ff876cf61dd0 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -335,7 +335,7 @@ Barrier A barrier object. Not thread-safe. Create a barrier object for *parties* number of tasks. An *action*, when - provided, is a callable to be called by one of the tasks when they are + provided, is a callable to be called by the last waiting task when they are released. A barrier object: a clone of :class:`threading.Barrier`. From 03845614e2bac94aa9ab746751a813aa6cd8f52d Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 13:54:00 +0200 Subject: [PATCH 35/93] Update locks.py first feedbacks from Emmanuel Arias --- Lib/asyncio/locks.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index e7f2561468e8cf..57d998768147ca 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -1,6 +1,7 @@ """Synchronization primitives.""" -__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') +__all__ = ('Lock', 'Event', 'Condition', 'Semaphore', + 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') import collections @@ -431,9 +432,10 @@ def release(self): # since the previous cycle. In addition, a 'resetting' state exists which is # similar to 'draining' except that tasks leave with a BrokenBarrierError, # and a 'broken' state in which all tasks get the exception. - +# +# Asyncio equivalent to threading.Barrier class Barrier(mixins._LoopBoundMixin): - """Asynchronous equivalent to threading.Barrier + """Asyncio equivalent to threading.Barrier Implements a Barrier. Useful for synchronizing a fixed number of tasks at known synchronization @@ -495,10 +497,9 @@ async def _block (self): # It is draining or resetting, wait until done await self._blocking.wait() - #see if the barrier is in a broken state + # see if the barrier is in a broken state if self._state < 0: raise BrokenBarrierError - assert self._state == 0, repr(self) # Optionally run the 'action' and release the tasks waiting # in the barrier. @@ -512,9 +513,7 @@ def _release(self): self._waiting.set() except: #an exception during the _action handler. Break and reraise - self._state = -2 - self._blocking.clear() - self._waiting.set() + self.abort() raise # Wait in the barrier until we are released. Raise an exception @@ -524,7 +523,6 @@ async def _wait(self): # no timeout so if self._state < 0: # resetting or broken raise BrokenBarrierError - assert self._state == 1, repr(self) # If we are the last tasks to exit the barrier, signal any tasks # waiting for the barrier to drain. From 16515ef7caee0934829db8e54cd988929016e172 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 13:56:05 +0200 Subject: [PATCH 36/93] Update test_locks.py Remove "import functools" --- Lib/test/test_asyncio/test_locks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index ec90698b6155aa..85f0a4d4d4428a 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -6,7 +6,6 @@ import asyncio from test.test_asyncio import utils as test_utils -import functools STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' From fa217fdb83bef6c11265ba901bd9ec63b8e5384c Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 16:55:53 +0200 Subject: [PATCH 37/93] Update asyncio-sync.rst Suggestion of Emmanuel Arias: Add sample and description about Barrier Class --- Doc/library/asyncio-sync.rst | 124 +++++++++++++++++++++++++++++++++-- 1 file changed, 118 insertions(+), 6 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index f7ff876cf61dd0..0eab76810948b8 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -330,17 +330,129 @@ BoundedSemaphore Barrier ======= -.. class:: Barrier(parties, action=None) + .. versionadded:: 3.10 A barrier object. Not thread-safe. - Create a barrier object for *parties* number of tasks. An *action*, when - provided, is a callable to be called by the last waiting task when they are - released. + This class, a clone of :class:`threading.Barrier`, provides a simple synchronization + primitive for use by a fixed number of tasks that need to wait for each other. + Each of the task tries to pass the barrier by calling the :meth:`~Barrier.wait` method + and will block until all of the tasks have made their :meth:`~Barrier.wait` calls. + At this point, the tasks are released simultaneously. - A barrier object: a clone of :class:`threading.Barrier`. + The barrier can be reused any number of times for the same number of tasks. - .. versionadded:: 3.10 + .. _asyncio_example_barrier: + + Example:: + + + import asyncio + import os + import random + + DELAYS = (1.0, 4.0, 2.0, 0.5) + + def job_done(): + msg = "ACTION: All files are removed" + print(f'{os.linesep}{msg:-^80s}{os.linesep}') + + async def remove_file(barrier, filename): + print(f'Remove "{filename}" ...') + + # remove filename + await asyncio.sleep(random.choice(DELAYS)) # simulate operation + print(f'\t"{filename}" is removed') + + return await barrier.wait() + + async def remove_folder(barrier, folder): + print(f'Waiting for remove all files from folder "{folder}" ...') + + # wait for all removed files + await barrier.wait() + + # remove folder + await asyncio.sleep(1.0) # simulates operation + print(f'Folder "{folder}" is now removed ...') + + return -1 + + async def main(folder, files, nb_files): + # Create a Barrier object with parties equal to ``nb_files+1`` + # corresponding to the number of all removing items, + # and an action as a print message. + barrier = asyncio.Barrier(nb_files+1, action=job_done) + + # Add remove_file task for each filename. + tasks = [asyncio.create_task(remove_file(barrier, f)) for f in files] + + # Add remove_folder task for the folder. + tasks.append(asyncio.create_task(remove_folder(barrier, folder))) + + # Wait until folder+filenames are removed. + await asyncio.gather(*tasks) + + asyncio.run(main("Datas", [f"file{i+1:02d}.txt" for i in range(10)], 10)) + + +.. class:: Barrier(parties, action=None) + + Create a barrier object for *parties* number of tasks. A callable *action*, when + provided, is called once at the begining of draining. + + .. coroutinemethod:: wait() + + Pass the barrier. When all the tasks party to the barrier have called + this function, they are all released simultaneously. + + The return value is an integer in the range 0 to *parties* -- 1, different + for each task. This can be used to select a task to do some special + housekeeping, e.g.:: + + ... + file_position = await barrier.wait() + if file_position == 0: + # Only one tasks needs to print this + print(f"I have the position #{file_position}") + + If an *action* was provided to the constructor, the last calling task of :meth:`wait` method will + have called it prior to being released. Should this call raise an error, + the barrier is put into the broken state. + + This method may raise a :class:`BrokenBarrierError` exception if the + barrier is broken or reset while a task is waiting. + + .. method:: reset() + + Return the barrier to the default, empty state. Any tasks waiting on it + will receive the :class:`BrokenBarrierError` exception. + + If a barrier is broken it may be better to just leave it and create a new one. + + .. method:: abort() + + Put the barrier into a broken state. This causes any active or future + calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. + Use this for example if one of the taks needs to abort, to avoid infinite waiting tasks. + + .. attribute:: parties + + The number of tasks required to pass the barrier. + + .. attribute:: n_waiting + + The number of tasks currently waiting in the barrier. + + .. attribute:: broken + + A boolean that is ``True`` if the barrier is in the broken state. + + +.. exception:: BrokenBarrierError + + This exception, a subclass of :exc:`RuntimeError`, is raised when the + :class:`Barrier` object is reset or broken. --------- From a97f4a34a71d34c717879a6b0aa477dbb9d70767 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 17 May 2021 21:28:14 +0200 Subject: [PATCH 38/93] Update asyncio-sync.rst bpo-43352 - Add a sample and description for Barrier Object Minor updates as remove crlf --- Doc/library/asyncio-sync.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 0eab76810948b8..263d43d6cd854f 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -346,7 +346,6 @@ Barrier Example:: - import asyncio import os import random @@ -395,7 +394,6 @@ Barrier asyncio.run(main("Datas", [f"file{i+1:02d}.txt" for i in range(10)], 10)) - .. class:: Barrier(parties, action=None) Create a barrier object for *parties* number of tasks. A callable *action*, when From b598470f186e72be2b4772df442e0d879abc247a Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 18 May 2021 08:35:21 +0200 Subject: [PATCH 39/93] Update asyncio-sync.rst bpo-43352 Suppress ` in a comment of the example --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 263d43d6cd854f..2cc437c0abfe7c 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -378,7 +378,7 @@ Barrier return -1 async def main(folder, files, nb_files): - # Create a Barrier object with parties equal to ``nb_files+1`` + # Create a Barrier object with parties equal to nb_files+1 # corresponding to the number of all removing items, # and an action as a print message. barrier = asyncio.Barrier(nb_files+1, action=job_done) From 309810587488696f2d0b1f39e300586699b01a28 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 15:55:25 +0200 Subject: [PATCH 40/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a class BarrierTests to test Barrier object Run i17 tests ncluding one about Barrier does not accept loop parameter --- Lib/test/test_asyncio/test_locks.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 85f0a4d4d4428a..dbe36d6da5362e 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -11,7 +11,7 @@ r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # new line dedicated to repr barrier + r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -958,6 +958,14 @@ def cancel_coros(self): task.cancel() test_utils.run_briefly(self.loop) + def test_barrier_doesnt_accept_loop_parameter(self): + with self.assertRaisesRegex( + TypeError, + rf'As of 3.10, the \*loop\* parameter was removed from ' + rf'Barrier\(\) since it is no longer necessary' + ): + asyncio.Barrier(1, loop=self.loop) + def test_repr(self): async def coro(): try: @@ -967,7 +975,7 @@ async def coro(): else: return await asyncio.sleep(1, True) - self.N = 1011001 + self.N = 999 barrier = asyncio.Barrier(self.N) self.assertTrue("[unset," in repr(barrier)) self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) From 2f55acc14525dcc3fe78ef0f94427cd5e05e98d8 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 16:05:55 +0200 Subject: [PATCH 41/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add the Barrier object in Synchronization section Add a link to a Barrier sample in Examples sub-section Add the BrokenBarrierError Exception in Exception section --- Doc/library/asyncio-api-index.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index 3465a297120920..33c2b119dfb84a 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -194,6 +194,8 @@ Threading-like synchronization primitives that can be used in Tasks. * :ref:`Using asyncio.Event `. +* :ref:`Using asyncio.Barrier `. + * See also the documentation of asyncio :ref:`synchronization primitives `. @@ -214,6 +216,9 @@ Exceptions * - :exc:`asyncio.CancelledError` - Raised when a Task is cancelled. See also :meth:`Task.cancel`. + * - :exc:`asyncio.BrokenBarrierError` + - Raised when a Barrier is broken. See also :meth:`Barrier.wait`. + .. rubric:: Examples From f056865137ef85b399bbd481a37648c082f58902 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 16:09:01 +0200 Subject: [PATCH 42/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a complete documentation inspired from threading.Barrier Add a sample to show how use a Barrier Object --- Doc/library/asyncio-sync.rst | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 2cc437c0abfe7c..63cfcd47f34316 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -334,7 +334,7 @@ Barrier A barrier object. Not thread-safe. - This class, a clone of :class:`threading.Barrier`, provides a simple synchronization + This class, as a clone of :class:`threading.Barrier`, provides a simple synchronization primitive for use by a fixed number of tasks that need to wait for each other. Each of the task tries to pass the barrier by calling the :meth:`~Barrier.wait` method and will block until all of the tasks have made their :meth:`~Barrier.wait` calls. @@ -351,29 +351,31 @@ Barrier import random DELAYS = (1.0, 4.0, 2.0, 0.5) + NB = 10 def job_done(): - msg = "ACTION: All files are removed" + msg = 'ACTION: All files removed' print(f'{os.linesep}{msg:-^80s}{os.linesep}') async def remove_file(barrier, filename): - print(f'Remove "{filename}" ...') + print(f'Remove {filename!r} ...') - # remove filename - await asyncio.sleep(random.choice(DELAYS)) # simulate operation - print(f'\t"{filename}" is removed') + # remove filename (simulation) + await asyncio.sleep(random.choice(DELAYS)) + print(f'\tFile {filename!r} is removed') + # wait for others files return await barrier.wait() async def remove_folder(barrier, folder): - print(f'Waiting for remove all files from folder "{folder}" ...') + print(f'Waiting for remove all files from folder {folder!r} ...') # wait for all removed files await barrier.wait() - # remove folder - await asyncio.sleep(1.0) # simulates operation - print(f'Folder "{folder}" is now removed ...') + # remove folder (simulation) + await asyncio.sleep(1.0) + print(f'Folder {folder!r} is now removed ...') return -1 @@ -392,12 +394,12 @@ Barrier # Wait until folder+filenames are removed. await asyncio.gather(*tasks) - asyncio.run(main("Datas", [f"file{i+1:02d}.txt" for i in range(10)], 10)) + asyncio.run(main("Datas", [f'file{i+1:02d}.txt' for i in range(NB)], NB)) .. class:: Barrier(parties, action=None) Create a barrier object for *parties* number of tasks. A callable *action*, when - provided, is called once at the begining of draining. + provided, is called once just before the release phasis (draining state). .. coroutinemethod:: wait() @@ -414,9 +416,9 @@ Barrier # Only one tasks needs to print this print(f"I have the position #{file_position}") - If an *action* was provided to the constructor, the last calling task of :meth:`wait` method will - have called it prior to being released. Should this call raise an error, - the barrier is put into the broken state. + If an *action* was provided to the constructor, the last calling task of + :meth:`wait` method will have called it prior to being released. + Should this call raise an error, the barrier is put into the broken state. This method may raise a :class:`BrokenBarrierError` exception if the barrier is broken or reset while a task is waiting. @@ -432,7 +434,8 @@ Barrier Put the barrier into a broken state. This causes any active or future calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. - Use this for example if one of the taks needs to abort, to avoid infinite waiting tasks. + Use this for example if one of the taks needs to abort, to avoid infinite + waiting tasks. .. attribute:: parties From d9008f95f9a7198c35ca95112bdf8897b3ff3f04 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 16:14:58 +0200 Subject: [PATCH 43/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a Barrier object as a clone from thraeding Barrier Update of __all__ variable to include this new class and the related BrokenBarrierError exception --- Lib/asyncio/locks.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 57d998768147ca..eec4dafed5a903 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -445,33 +445,37 @@ class Barrier(mixins._LoopBoundMixin): def __init__(self, parties, action=None, *, loop=mixins._marker): """Create a barrier, initialised to 'parties' tasks. - 'action' is a callable which, when supplied, will be called by one of - the tasks after they have all entered the barrier and just prior to - releasing them all. + 'action' is a callable which, when supplied, will be called by + the last task calling the wait() method, + just prior to releasing them all. """ super().__init__(loop=loop) if parties < 1: raise ValueError('parties must be > 0') self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block new tasks while waiting tasks are draining or broken + self._blocking = Event() # used block new tasks while waiting + # tasks are draining or broken self._action = action self._parties = parties - self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken + self._state = 0 # 0 filling, 1, draining, + # -1 resetting, -2 broken self._count = 0 # count waiting tasks def __repr__(self): res = super().__repr__() _wait = 'set' if self._waiting.is_set() else 'unset' _block = 'set' if self._blocking.is_set() else 'unset' - extra = f'{_wait}, count:{self._count}/{self._parties}, {_block}, state:{self._state}' + extra = f'{_wait}, count:{self._count}/{self._parties}' + extra += f', {_block}, state:{self._state}' return f'<{res[1:-1]} [{extra}]>' async def wait(self): """Wait for the barrier. When the specified number of tasks have started waiting, they are all - simultaneously awoken. If an 'action' was provided for the barrier, one - of the tasks will have executed that callback prior to returning. + simultaneously awoken. If an 'action' was provided for the barrier, the + last task calling this method will have executed that callback prior to + returning. Returns an individual index number from 0 to 'parties-1'. """ await self._block() # Block while the barrier drains or resets. @@ -520,6 +524,7 @@ def _release(self): # if the barrier is reset or broken. async def _wait(self): await self._waiting.wait() + # no timeout so if self._state < 0: # resetting or broken raise BrokenBarrierError From 71b923ca099cf7b4635348a284bc1ea355936227 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 19 May 2021 21:42:40 +0200 Subject: [PATCH 44/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a class BarrierTests to test Barrier object Run 17 tests including one about Barrier does not accept loop parameter --- Lib/test/test_asyncio/test_locks.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index dbe36d6da5362e..b0d93ad55b7703 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -69,16 +69,6 @@ def test_lock_doesnt_accept_loop_parameter(self): ): cls(loop=self.loop) - # Barrier object has a positional paramater - # so tests alone - cls = asyncio.Barrier - with self.assertRaisesRegex( - TypeError, - rf'As of 3.10, the \*loop\* parameter was removed from ' - rf'{cls.__name__}\(\) since it is no longer necessary' - ): - cls(1, loop=self.loop) - def test_lock_by_with_statement(self): loop = asyncio.new_event_loop() # don't use TestLoop quirks self.set_event_loop(loop) From 58226e081d6d1475e445a9d8fa227eb1bd0d4cfd Mon Sep 17 00:00:00 2001 From: Duprat Date: Thu, 9 Sep 2021 10:07:23 +0200 Subject: [PATCH 45/93] Update Doc/library/asyncio-sync.rst Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 3fd0d545556c7a..0b806a65d68639 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -368,7 +368,7 @@ Barrier This class, as a clone of :class:`threading.Barrier`, provides a simple synchronization primitive for use by a fixed number of tasks that need to wait for each other. Each of the task tries to pass the barrier by calling the :meth:`~Barrier.wait` method - and will block until all of the tasks have made their :meth:`~Barrier.wait` calls. + and would block until all of the tasks have made their :meth:`~Barrier.wait` calls. At this point, the tasks are released simultaneously. The barrier can be reused any number of times for the same number of tasks. From 0096cdc7fe6cc56166d250d7f9ccea8410506658 Mon Sep 17 00:00:00 2001 From: Duprat Date: Thu, 9 Sep 2021 10:07:43 +0200 Subject: [PATCH 46/93] Update Doc/library/asyncio-sync.rst Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 0b806a65d68639..151e2fee978ea9 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -367,7 +367,7 @@ Barrier This class, as a clone of :class:`threading.Barrier`, provides a simple synchronization primitive for use by a fixed number of tasks that need to wait for each other. - Each of the task tries to pass the barrier by calling the :meth:`~Barrier.wait` method + Each of the tasks tries to pass the barrier by calling the :meth:`~Barrier.wait` method and would block until all of the tasks have made their :meth:`~Barrier.wait` calls. At this point, the tasks are released simultaneously. From f275a8387059098831d56f8cd5847c96b5057a28 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Dec 2021 08:39:11 +0100 Subject: [PATCH 47/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a complete documentation Add a sample to show how use a Barrier Object --- Doc/library/asyncio-sync.rst | 89 ++++++++++++++++++++++++++---------- 1 file changed, 65 insertions(+), 24 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 151e2fee978ea9..1b10d98c8f45b8 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -361,7 +361,7 @@ BoundedSemaphore Barrier ======= - .. versionadded:: 3.10 + .. versionadded:: 3.11 A barrier object. Not thread-safe. @@ -384,14 +384,13 @@ Barrier DELAYS = (1.0, 4.0, 2.0, 0.5) NB = 10 - def job_done(): - msg = 'ACTION: All files removed' - print(f'{os.linesep}{msg:-^80s}{os.linesep}') + async def job_done(): + print(f'{os.linesep}{"All files removed":-^80s}{os.linesep}') async def remove_file(barrier, filename): print(f'Remove {filename!r} ...') - # remove filename (simulation) + # remove filename (simulation with 'random value' secs spleeping) await asyncio.sleep(random.choice(DELAYS)) print(f'\tFile {filename!r} is removed') @@ -402,13 +401,12 @@ Barrier print(f'Waiting for remove all files from folder {folder!r} ...') # wait for all removed files - await barrier.wait() + async with barrier as pos: + # remove folder (simulation with 1 sec sleeping) + await asyncio.sleep(1.0) + print(f'Folder {folder!r} is now removed ...') - # remove folder (simulation) - await asyncio.sleep(1.0) - print(f'Folder {folder!r} is now removed ...') - - return -1 + return pos async def main(folder, files, nb_files): # Create a Barrier object with parties equal to nb_files+1 @@ -425,43 +423,70 @@ Barrier # Wait until folder+filenames are removed. await asyncio.gather(*tasks) - asyncio.run(main("Datas", [f'file{i+1:02d}.txt' for i in range(NB)], NB)) + asyncio.run(main("MyFolder", [f'file{i+1:02d}.txt' for i in range(NB)], NB)) + + As in the example below, the other way to use a Barrier is an :keyword:`async with` + statement:: + + barrier = asyncio.Barrier(2) + + # ... later + try: + async with barrier as i: + # Barrier passed so do the job + ... + except asyncio.BrokenBarrierError: + pass + + which is equivalent to:: + + barrier = asyncio.Barrier(2) + + # ... later + try: + i = await barrier.acquire() + # Barrier passed so do the job + ... + except asyncio.BrokenBarrierError: + pass + + The barrier can be reused any number of times for the same number of tasks. .. class:: Barrier(parties, action=None) - Create a barrier object for *parties* number of tasks. A callable *action*, when - provided, is called once just before the release phasis (draining state). + Create a barrier object for *parties* number of tasks. A task *action*, when + provided, is awaited once just before the release phasis (draining state). .. coroutinemethod:: wait() Pass the barrier. When all the tasks party to the barrier have called this function, they are all released simultaneously. - The return value is an integer in the range 0 to *parties* -- 1, different + The return value is an integer in the range 0 to *parties-1*, different for each task. This can be used to select a task to do some special housekeeping, e.g.:: ... - file_position = await barrier.wait() - if file_position == 0: - # Only one tasks needs to print this - print(f"I have the position #{file_position}") + async with barrier as position: + if position == 0: + # Only one task print this + print('End of *draining phasis*') - If an *action* was provided to the constructor, the last calling task of - :meth:`wait` method will have called it prior to being released. + If an *action* was provided as a task to the constructor, the last calling task of + :meth:`wait` method will have to *await this task* prior to being released. Should this call raise an error, the barrier is put into the broken state. This method may raise a :class:`BrokenBarrierError` exception if the barrier is broken or reset while a task is waiting. - .. method:: reset() + .. coroutinemethod:: reset() Return the barrier to the default, empty state. Any tasks waiting on it will receive the :class:`BrokenBarrierError` exception. If a barrier is broken it may be better to just leave it and create a new one. - .. method:: abort() + .. coroutinemethod:: abort() Put the barrier into a broken state. This causes any active or future calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. @@ -474,12 +499,28 @@ Barrier .. attribute:: n_waiting - The number of tasks currently waiting in the barrier. + The number of tasks currently waiting in the barrier while filling. + + .. attribute:: n_blocked + + The number of tasks currently blocked in the barrier while draining. .. attribute:: broken A boolean that is ``True`` if the barrier is in the broken state. + .. attribute:: draining + + A boolean that is ``True`` if the barrier is in the draining state. + + .. attribute:: filling + + A boolean that is ``True`` if the barrier is in the filling state. + + .. attribute:: resetting + + A boolean that is ``True`` if the barrier is in the resetting state. + .. exception:: BrokenBarrierError From e90d55690eb29e1e6c981e66ed9836e9fcf37a1e Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Dec 2021 08:40:08 +0100 Subject: [PATCH 48/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a Barrier Class --- Lib/asyncio/locks.py | 221 +++++++++++++++++++++++++------------------ 1 file changed, 130 insertions(+), 91 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index eec4dafed5a903..de9bc0fe866970 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -8,7 +8,6 @@ from . import exceptions from . import mixins - class _ContextManagerMixin: async def __aenter__(self): await self.acquire() @@ -421,31 +420,18 @@ def release(self): super().release() -# A barrier class. Inspired in part by the pthread_barrier_* api and -# the CyclicBarrier class from Java. See -# http://sourceware.org/pthreads-win32/manual/pthread_barrier_init.html and -# http://java.sun.com/j2se/1.5.0/docs/api/java/util/concurrent/ -# CyclicBarrier.html -# for information. -# We maintain two main states, 'filling' and 'draining' enabling the barrier -# to be cyclic. Tasks are not allowed into it until it has fully drained -# since the previous cycle. In addition, a 'resetting' state exists which is -# similar to 'draining' except that tasks leave with a BrokenBarrierError, -# and a 'broken' state in which all tasks get the exception. -# -# Asyncio equivalent to threading.Barrier class Barrier(mixins._LoopBoundMixin): """Asyncio equivalent to threading.Barrier Implements a Barrier. Useful for synchronizing a fixed number of tasks at known synchronization points. Tasks block on 'wait()' and are simultaneously awoken once they - have all made that call. + have all made their call. """ def __init__(self, parties, action=None, *, loop=mixins._marker): """Create a barrier, initialised to 'parties' tasks. - 'action' is a callable which, when supplied, will be called by + 'action' is a coroutine which, when supplied, will be called by the last task calling the wait() method, just prior to releasing them all. """ @@ -453,122 +439,154 @@ def __init__(self, parties, action=None, *, loop=mixins._marker): if parties < 1: raise ValueError('parties must be > 0') - self._waiting = Event() # used notify all waiting tasks - self._blocking = Event() # used block new tasks while waiting - # tasks are draining or broken + self._cond = Condition() # notify all tasks when state changes + self._action = action self._parties = parties - self._state = 0 # 0 filling, 1, draining, + self._state = 0 # 0 filling, 1, draining, # -1 resetting, -2 broken - self._count = 0 # count waiting tasks + self._count = 0 # count tasks in Barrier + self._count_wait = 0 # count waiting tasks + self._count_block = 0 # count blocking tasks def __repr__(self): res = super().__repr__() - _wait = 'set' if self._waiting.is_set() else 'unset' - _block = 'set' if self._blocking.is_set() else 'unset' - extra = f'{_wait}, count:{self._count}/{self._parties}' - extra += f', {_block}, state:{self._state}' + extra = 'locked' if self._cond.locked() else 'unlocked' + extra += f', wait:{self._count_wait}/{self._parties}' + extra += f', block:{self._count_block}/{self._parties}' + extra += f', state:{self._state}' return f'<{res[1:-1]} [{extra}]>' + + async def __aenter__(self): + """ wait for the barrier reaches the parties number + when start draining release and return index of waited task + """ + return await self.wait() + + async def __aexit__(self, *args): + pass + async def wait(self): """Wait for the barrier. When the specified number of tasks have started waiting, they are all simultaneously awoken. If an 'action' was provided for the barrier, the last task calling this method will have executed that callback prior to returning. - Returns an individual index number from 0 to 'parties-1'. + Returns an unique and individual index number from 0 to 'parties-1'. + # """ + async with self._cond: + await self._block() # Block while the barrier drains or resets. + try: + index = self._count + self._count += 1 + if index + 1 == self._parties: + # We release the barrier + await self._release() + else: + await self._wait() + return index + finally: + self._count -= 1 + # Wake up any tasks waiting for barrier to drain. + self._exit() + + async def _block(self): + """Block until the barrier is ready for us, + or raise an exception if it is broken. """ - await self._block() # Block while the barrier drains or resets. - index = self._count - self._count += 1 - try: - if index + 1 == self._parties: - # We release the barrier - self._release() - else: - # We wait until someone releases us - await self._wait() - return index - finally: - self._count -= 1 - # Wake up any tasks waiting for barrier to drain. - self._exit() - - # Block until the barrier is ready for us, or raise an exception - # if it is broken. - async def _block (self): - while self._state in (-1, 1): + self._count_block += 1 + while self.draining or self.resetting: # # It is draining or resetting, wait until done - await self._blocking.wait() + await self._cond.wait() + self._count_block -= 1 # see if the barrier is in a broken state - if self._state < 0: + if self.broken: raise BrokenBarrierError # Optionally run the 'action' and release the tasks waiting # in the barrier. - def _release(self): + async def _release(self): + """Optionally run the 'action' and release the tasks waiting + in the barrier. + """ try: if self._action: - self._action() + await self._action() # enter draining state - self._state = 1 - self._blocking.clear() - self._waiting.set() + self._set_draining() + self._cond.notify_all() except: - #an exception during the _action handler. Break and reraise - self.abort() + # an exception occurs during the _action coroutine. + # Break and reraise + self._break() raise - # Wait in the barrier until we are released. Raise an exception - # if the barrier is reset or broken. async def _wait(self): - await self._waiting.wait() + """Wait in the barrier until we are released. Raise an exception + if the barrier is reset or broken. + """ + self._count_wait += 1 + await self._cond.wait_for(lambda: not self.filling) + self._count_wait -= 1 - # no timeout so - if self._state < 0: # resetting or broken + if self.broken or self.resetting: raise BrokenBarrierError - # If we are the last tasks to exit the barrier, signal any tasks - # waiting for the barrier to drain. def _exit(self): + """If we are the last tasks to exit the barrier, signal any tasks + waiting for the barrier to drain. + """ if self._count == 0: - if self._state == 1: # draining - self._state = 0 - elif self._state == -1: # resetting - self._state = 0 - self._waiting.clear() - self._blocking.set() - - # async def reset(self): - def reset(self): + if self.resetting or self.draining: + self._set_filling() + self._cond.notify_all() + + async def reset(self): """Reset the barrier to the initial state. Any tasks currently waiting will get the BrokenBarrier exception raised. """ - if self._count > 0: - if self._state in (0, 1): # filling or draining - # reset the barrier, waking up tasks - self._state = -1 - elif self._state == -2: - # was broken, set it to reset state - # which clears when the last tasks exits - self._state = -1 - self._waiting.set() - self._blocking.clear() - else: - self._state = 0 - + async with self._cond: + if self._count > 0: + if not self.resetting:# self.filling or self.draining + # or self.broken + #reset the barrier, waking up tasks + self._set_resetting() + else: + self._set_filling() + self._cond.notify_all() - # async def abort(self): - def abort(self): + async def abort(self): """Place the barrier into a 'broken' state. Useful in case of error. Any currently waiting tasks and tasks attempting to 'wait()' will have BrokenBarrierError raised. """ + async with self._cond: + self._break() + + def _break(self): + # An internal error was detected. The barrier is set to + # a broken state all parties awakened. + self._set_broken() + self._cond.notify_all() + + def _set_broken(self): + """Set state to broken.""" self._state = -2 - self._waiting.set() - self._blocking.clear() + + def _set_draining(self): + """Set state to draining.""" + self._state = 1 + + def _set_filling(self): + """Set state to filling.""" + self._state = 0 + + def _set_resetting(self): + """Set state to resetting.""" + self._state = -1 @property def parties(self): @@ -578,17 +596,38 @@ def parties(self): @property def n_waiting(self): """Return the number of tasks currently waiting at the barrier.""" - # We don't need synchronization here since this is an ephemeral result - # anyway. It returns the correct value in the steady state. - if self._state == 0: + if self.filling: return self._count return 0 + @property + def n_blocking(self): + """Return the number of tasks currently blocking at the barrier.""" + if self.draining: + return self._count_block + return 0 + @property def broken(self): """Return True if the barrier is in a broken state.""" return self._state == -2 + @property + def draining(self): + """Return True if the barrier is in a broken state.""" + return self._state == 1 + + @property + def filling(self): + """Return True if the barrier is filling.""" + return self._state == 0 + + @property + def resetting(self): + """Return True if the barrier is filling.""" + return self._state == -1 + + # exception raised by the Barrier class class BrokenBarrierError(RuntimeError): pass From 408512db3a215e0674e388c3f99d6d28611ab88c Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 3 Dec 2021 08:40:38 +0100 Subject: [PATCH 49/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a BarrierTests class to run tests --- Lib/test/test_asyncio/test_locks.py | 564 ++++++++++++++++++++-------- 1 file changed, 415 insertions(+), 149 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index b0d93ad55b7703..9b84355908a684 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -11,7 +11,7 @@ r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' - r'(, count:\d+\/\d+)?(, (set|unset))?(, state:(-2|-1|0|1))?' # barrier + r'(, wait:\d+\/\d+)?(, block:\d+\/\d+)?(, state:(-2|-1|0|1))?' # barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -957,82 +957,107 @@ def test_barrier_doesnt_accept_loop_parameter(self): asyncio.Barrier(1, loop=self.loop) def test_repr(self): - async def coro(): - try: - i = await barrier.wait() - except: - ... - else: - return await asyncio.sleep(1, True) - - self.N = 999 barrier = asyncio.Barrier(self.N) - self.assertTrue("[unset," in repr(barrier)) - self.assertTrue(f"count:0/{str(self.N)}" in repr(barrier)) self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(repr(barrier).endswith('unset, state:0]>')) - nb_waits = 3 - tasks = [] - for _ in range(nb_waits): - tasks.append(self.loop.create_task(coro())) - test_utils.run_briefly(self.loop) - self.assertTrue("[unset," in repr(barrier)) + incr = 2 + barrier._count_wait += incr + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(f"wait:{incr}/{self.N}" in repr(barrier)) + self.assertTrue(f"block:0/{self.N}" in repr(barrier)) + barrier._set_filling() self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) - self.assertTrue(repr(barrier).endswith('unset, state:0]>')) + self.assertTrue(repr(barrier).endswith('state:0]>')) - barrier.reset() + barrier._count_block += incr + barrier._count_wait -= incr self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue("[set," in repr(barrier)) - self.assertTrue(f"count:{nb_waits}/{self.N}" in repr(barrier)) - self.assertTrue(repr(barrier).endswith('unset, state:-1]>')) - test_utils.run_briefly(self.loop) + self.assertTrue(f"wait:0/{self.N}" in repr(barrier)) + self.assertTrue(f"block:{incr}/{self.N}" in repr(barrier)) + + barrier._set_draining() self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue("[unset," in repr(barrier)) - self.assertTrue(f"count:0/{self.N}" in repr(barrier)) - self.assertTrue(repr(barrier).endswith('set, state:0]>')) - test_utils.run_briefly(self.loop) + self.assertTrue(repr(barrier).endswith('state:1]>')) - barrier.abort() - test_utils.run_briefly(self.loop) + barrier._set_resetting() self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(f"0/{self.N}" in repr(barrier)) - self.assertTrue(repr(barrier).endswith(':-2]>')) + self.assertTrue(repr(barrier).endswith('state:-1]>')) - self.cancel_coros() + barrier._set_broken() + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue(repr(barrier).endswith('state:-2]>')) + + barrier = asyncio.Barrier(self.N) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue('unlocked' in repr(barrier)) + + self.loop.run_until_complete(barrier._cond.acquire()) + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue('locked' in repr(barrier)) def test_init(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - def test_wait_task_once(self): + def test_filling_one_task(self): + results = [] + async def f(): + i = await barrier.wait() + results.append(i) + self.N = 1 - barrier = asyncio.Barrier(self.N) + barrier = asyncio.Barrier(self.N) + self.loop.run_until_complete(f()) + + self.assertEqual(sum(results), 0) + self.assertEqual(barrier.n_waiting, 0) - r1 = self.loop.run_until_complete(barrier.wait()) + self.assertFalse(barrier.broken) + + def test_filling_one_task_async_with(self): + results = [] + async def f(): + async with barrier as i: + results.append(i) + + self.N = 1 + barrier = asyncio.Barrier(self.N) + self.loop.run_until_complete(f()) + + self.assertEqual(sum(results), 0) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_filling_one_task_twice(self): + self.N = 1 + barrier = asyncio.Barrier(self.N) + r1 = self.loop.create_task(barrier.wait()) + test_utils.run_briefly(self.loop) + + self.assertEqual(barrier.n_waiting, 0) + r2 = self.loop.run_until_complete(barrier.wait()) + + self.assertEqual(r1.result(), r2) + self.assertEqual(barrier.n_waiting, 0) - self.assertEqual(r1, r2) self.assertFalse(barrier.broken) - def test_wait_task_by_task(self): + def test_filling_task_by_task(self): + async def coro(): + await barrier.wait() self.N = 3 barrier = asyncio.Barrier(self.N) - self.assertEqual(barrier.n_waiting, 0) - self.loop.create_task(barrier.wait()) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, 1) - self.loop.create_task(barrier.wait()) - test_utils.run_briefly(self.loop) - self.assertEqual(barrier.n_waiting, 2) - self.loop.create_task(barrier.wait()) - test_utils.run_briefly(self.loop) + for i in range(self.N): + self.assertEqual(barrier.n_waiting, i) + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_wait_tasks_twice(self): + def test_filling_tasks_wait_twice(self): results = [] async def coro(): await barrier.wait() @@ -1042,87 +1067,159 @@ async def coro(): barrier = asyncio.Barrier(self.N) self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertEqual(len(results), self.N*2) self.assertEqual(results.count(True), self.N) self.assertEqual(results.count(False), self.N) + + self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) + + def test_filling_tasks_wait_twice_async_with(self): + results = [] + async def coro(): + async with barrier: + results.append(True) + + async with barrier: + results.append(False) + + barrier = asyncio.Barrier(self.N) + self.loop.run_until_complete(self.run_coros(self.N, coro)) + + self.assertEqual(len(results), self.N*2) + self.assertEqual(results.count(True), self.N) + self.assertEqual(results.count(False), self.N) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) - def test_wait_return_from_wait(self): + def test_filling_tasks_check_return_value(self): results1 = [] results2 = [] async def coro(): await barrier.wait() results1.append(True) + i = await barrier.wait() results2.append(True) return i barrier = asyncio.Barrier(self.N) - res = self.loop.run_until_complete(self.run_coros(self.N, coro)) + res, _ = self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertEqual(len(results1), self.N) self.assertTrue(all(results1)) self.assertEqual(len(results2), self.N) self.assertTrue(all(results2)) - self.assertEqual(sum(res[0]), sum(range(self.N))) + self.assertEqual(sum(res), sum(range(self.N))) + self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) + def test_filling_tasks_check_return_value_async_with(self): + results1 = [] + results2 = [] + async def coro(): + async with barrier: + results1.append(True) - def test_wait_repeat_more(self, more=5): - async def coro(result, value): - ret = await barrier.wait() - result.append(value) + async with barrier as i: + results2.append(True) + return i - results = [[] for _ in range(more)] barrier = asyncio.Barrier(self.N) - for i in range(more): - _ = [self.loop.create_task(coro(results[i], value)) for value in range(self.N)] - test_utils.run_briefly(self.loop) + res, _ = self.loop.run_until_complete(self.run_coros(self.N, coro)) - self.assertEqual(len(results[i]), self.N) - self.assertEqual(sum(results[i]), sum(range(self.N))) - if i > 0: - self.assertEqual(sum(results[i]), sum(results[i-1])) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertEqual(len(results1), self.N) + self.assertTrue(all(results1)) + self.assertEqual(len(results2), self.N) + self.assertTrue(all(results2)) + self.assertEqual(sum(res), sum(range(self.N))) - def test_wait_state_draining(self): - result = [] - state = [] + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_draining_check(self): + results = [] async def coro(): - i = await barrier.wait() - if i == 0: - result.append(True) - # Do we need to add a Barrie.draining() property ? - state.append(repr(barrier).endswith("state:1]>")) + await barrier.wait() + # barrier state change to filling for the last task release + results.append(barrier.draining) - rest = self.N-1 - nb_tasks = self.N + rest barrier = asyncio.Barrier(self.N) - for _ in range(nb_tasks): + for _ in range(self.N): self.loop.create_task(coro()) - for _ in range((nb_tasks//self.N)+1): - test_utils.run_briefly(self.loop) - self.assertEqual(len(result), nb_tasks//self.N) - self.assertTrue(all(result)) - self.assertTrue(all(state)) - self.assertEqual(barrier.n_waiting, rest) + test_utils.run_briefly(self.loop) + + self.assertEqual(len(results), self.N) + self.assertEqual(results[-1], False) + self.assertTrue(all(results[:self.N-1])) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_draining_check_async_with(self): + results = [] + async def coro(): + async with barrier: + # barrier state change to filling for the last task release + results.append(barrier.draining) + + barrier = asyncio.Barrier(self.N) + for _ in range(self.N): + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + + self.assertEqual(len(results), self.N) + self.assertEqual(results[-1], False) + self.assertTrue(all(results[:self.N-1])) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_blocking_tasks_while_draining(self): + rewait = min(2, self.N-1) + results = [] + rewait_n = rewait + async def coro(): + nonlocal rewait_n + + # first time waiting + p = await barrier.wait() + if rewait_n > 0: + rewait_n -= 1 + await barrier.wait() + else: + # test n_blocking + results.append(barrier.n_blocking) + + barrier = asyncio.Barrier(self.N) + _ = [self.loop.create_task(coro()) for _ in range(self.N)] + test_utils.run_briefly(self.loop) + + self.assertEqual(len(results), self.N - rewait) + self.assertEqual(max(results), rewait) + self.assertEqual(results[-1], 0) + test_utils.run_briefly(self.loop) + + self.assertEqual(barrier.n_waiting, rewait) self.assertFalse(barrier.broken) self.cancel_coros() - def test_wait_tasks_cancel_one_task(self): + def test_filling_tasks_cancel_one(self): results = [] async def coro(): await barrier.wait() results.append(True) - self.N=3 + self.N = 3 barrier = asyncio.Barrier(self.N) t1 = self.loop.create_task(coro()) test_utils.run_briefly(self.loop) self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) self.assertEqual(barrier.n_waiting, 2) @@ -1130,60 +1227,43 @@ async def coro(): t1.cancel() test_utils.run_briefly(self.loop) self.assertEqual(barrier.n_waiting, 1) + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) self.assertEqual(barrier.n_waiting, 2) + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) self.assertEqual(len(results), self.N) self.assertTrue(all(results)) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_wait_action_callback(self): - result = [] - async def coro(): - ret = await barrier.wait() - result.append(True) - - result1 = [] - self.N = 2 - barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) - self.assertEqual(len(result1), 1) - self.assertTrue(result1[0]) - self.assertEqual(len(result), 2) - self.assertTrue(all(result)) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_wait_action_callback_more(self): - result = [] + def test_draining_check_action(self): + results = [] + results1 = [] async def coro(): ret = await barrier.wait() - result.append(True) + results.append(True) - result1 = [] + async def action_task(): + results1.append(True) - self.N = 2 - more = 3 - barrier = asyncio.Barrier(self.N, action=lambda: result1.append(True)) - for _ in range(more): - _ = [self.loop.create_task(coro()) for _ in range(self.N)] - test_utils.run_briefly(self.loop) + barrier = asyncio.Barrier(self.N, action=action_task) + for _ in range(self.N): + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + + self.assertEqual(len(results1), 1) + self.assertTrue(results1[0]) + self.assertEqual(len(results), self.N) + self.assertTrue(all(results)) - self.assertEqual(len(result1), more) - self.assertTrue(all(result1)) - self.assertEqual(len(result), self.N*more) - self.assertTrue(all(result)) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_wait_action_callback_error_broken(self): + def test_draining_check_error_on_action(self): ERROR = ZeroDivisionError results1 = [] results2 = [] @@ -1195,24 +1275,63 @@ async def coro(): except: results2.append(True) - def raise_except(): + async def raise_except(): raise ERROR - barrier = asyncio.Barrier(self.N, lambda: raise_except()) - _ = [self.loop.create_task(coro()) for _ in range(self.N)] + barrier = asyncio.Barrier(self.N, action=raise_except) + for _ in range(self.N): + self.loop.create_task(coro()) test_utils.run_briefly(self.loop) + self.assertEqual(len(results1), 1) self.assertFalse(results1[0]) self.assertEqual(len(results2), self.N-1) self.assertTrue(all(results2)) - self.assertEqual(barrier.n_waiting, 0) + self.assertEqual(barrier.n_waiting, 0) self.assertTrue(barrier.broken) - def test_reset(self): + def test_resetting_simple(self): + results = [] + async def coro_reset(): + await barrier.reset() + + barrier = asyncio.Barrier(self.N) + self.loop.create_task(coro_reset()) + test_utils.run_briefly(self.loop) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_resetting_tasks_waiting(self): + results = [] + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + results.append(True) + + async def coro_reset(): + await barrier.reset() + + barrier = asyncio.Barrier(self.N) + for _ in range(self.N-1): + self.loop.create_task(coro()) + test_utils.run_briefly(self.loop) + + self.loop.create_task(coro_reset()) + test_utils.run_briefly(self.loop) + test_utils.run_briefly(self.loop) + + self.assertEqual(len(results), self.N-1) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.resetting) + self.assertFalse(barrier.broken) + + def test_resetting_tasks_and_waiting_again(self): results1 = [] results2 = [] - async def coro(): + async def coro1(): try: await barrier.wait() except asyncio.BrokenBarrierError: @@ -1225,33 +1344,77 @@ async def coro2(): await barrier.wait() results2.append(True) - barrier = asyncio.Barrier(self.N) - _ = [self.loop.create_task(coro()) for _ in range(self.N-1)] + for _ in range(self.N-1): + self.loop.create_task(coro1()) test_utils.run_briefly(self.loop) - barrier.reset() + + self.loop.create_task(barrier.reset()) test_utils.run_briefly(self.loop) + self.loop.create_task(coro2()) test_utils.run_briefly(self.loop) + self.assertFalse(barrier.broken) self.assertEqual(len(results1), self.N-1) self.assertEqual(len(results2), self.N) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_resetting_tasks_and_waiting_again_async_with(self): + results1 = [] + results2 = [] + async def coro1(): + try: + async with barrier: + pass + except asyncio.BrokenBarrierError: + results1.append(True) + finally: + async with barrier: + results2.append(True) + + async def coro2(): + async with barrier: + results2.append(True) + + barrier = asyncio.Barrier(self.N) + for _ in range(self.N-1): + self.loop.create_task(coro1()) + test_utils.run_briefly(self.loop) + + self.loop.create_task(barrier.reset()) + test_utils.run_briefly(self.loop) + + self.loop.create_task(coro2()) + test_utils.run_briefly(self.loop) self.assertFalse(barrier.broken) + self.assertEqual(len(results1), self.N-1) + self.assertEqual(len(results2), self.N) - def test_reset_and_wait(self): + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_resetting_tasks_draining_then_blocking(self): results1 = [] results2 = [] results3 = [] count = 0 + async def coro(): nonlocal count - i = await barrier1.wait() + try: + i = await barrier.wait() + except: + results2.append(True) + count += 1 - if count == self.N: # i == self.N//2: - barrier.reset() + if count == self.N-1: # there is one task still waiting + await barrier.reset() + await barrier.wait() else: try: await barrier.wait() @@ -1260,19 +1423,57 @@ async def coro(): results2.append(True) # Now, pass the barrier again - await barrier1.wait() + await barrier.wait() results3.append(True) + # self.N = 3 barrier = asyncio.Barrier(self.N) - barrier1 = asyncio.Barrier(self.N) self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertFalse(barrier.broken) - self.assertEqual(len(results1), 0) - self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results1), self.N-1) + self.assertEqual(len(results2), 1) self.assertEqual(len(results3), self.N) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + + def test_resetting_last_task_draining(self): + results1 = [] + results2 = [] + results3 = [] + count = 0 + + async def coro(): + nonlocal count + + i = await barrier.wait() + count += 1 + if count == self.N: # last task exited from barrier + await barrier.reset() + await barrier.wait() + else: + try: + await barrier.wait() + results1.append(True) + except Exception as e: + results2.append(True) + + # Now, pass the barrier again + await barrier.wait() + results3.append(True) + + # self.N = 3 + barrier = asyncio.Barrier(self.N) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertFalse(barrier.broken) + self.assertEqual(len(results1), self.N-1) + self.assertEqual(len(results2), 0) + self.assertEqual(len(results3), self.N) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) def test_abort_broken(self): results1 = [] @@ -1287,53 +1488,118 @@ async def coro(): except asyncio.BrokenBarrierError: results2.append(True) except RuntimeError: - barrier.abort() + await barrier.abort() barrier = asyncio.Barrier(self.N) self.loop.run_until_complete(self.run_coros(self.N, coro)) + self.assertTrue(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(barrier.broken) + + def test_abort_broken_async_with(self): + results1 = [] + results2 = [] + async def coro(): + try: + async with barrier as i : + if i == self.N//2: + raise RuntimeError + async with barrier: + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + await barrier.abort() + + barrier = asyncio.Barrier(self.N) + self.loop.run_until_complete(self.run_coros(self.N, coro)) self.assertTrue(barrier.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(barrier.broken) - def test_abort_and_reset(self): + def test_abort_and_resetting(self): results1 = [] results2 = [] results3 = [] async def coro(): try: - i = await barrier.wait() + i = await barrier1.wait() if i == self.N//2: raise RuntimeError - await barrier.wait() + await barrier1.wait() results1.append(True) except asyncio.BrokenBarrierError: results2.append(True) except RuntimeError: - barrier.abort() + await barrier1.abort() # Synchronize and reset the barrier. Must synchronize first so # that everyone has left it when we reset, and after so that no # one enters it before the reset. i = await barrier2.wait() if i == self.N//2: - barrier.reset() + await barrier1.reset() await barrier2.wait() - await barrier.wait() + await barrier1.wait() results3.append(True) - barrier = asyncio.Barrier(self.N) + barrier1 = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) self.loop.run_until_complete(self.run_coros(self.N, coro)) - self.assertFalse(barrier.broken) + + self.assertFalse(barrier1.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertEqual(len(results3), self.N) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertEqual(barrier1.n_waiting, 0) + self.assertFalse(barrier1.broken) + + def test_abort_and_resetting_async_with(self): + results1 = [] + results2 = [] + results3 = [] + async def coro(): + try: + async with barrier1 as i: + if i == self.N//2: + raise RuntimeError + async with barrier1: + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + except RuntimeError: + await barrier1.abort() + + # Synchronize and reset the barrier. Must synchronize first so + # that everyone has left it when we reset, and after so that no + # one enters it before the reset. + async with barrier2 as i: + if i == self.N//2: + await barrier1.reset() + async with barrier2: + async with barrier1: + results3.append(True) + + barrier1 = asyncio.Barrier(self.N) + barrier2 = asyncio.Barrier(self.N) + self.loop.run_until_complete(self.run_coros(self.N, coro)) + + self.assertFalse(barrier1.broken) + self.assertEqual(len(results1), 0) + self.assertEqual(len(results2), self.N-1) + self.assertEqual(len(results3), self.N) + + self.assertEqual(barrier1.n_waiting, 0) + self.assertFalse(barrier1.broken) if __name__ == '__main__': From 49de2239eebb3b0f1c7ab98997b168a2c8dbf520 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 21 Feb 2022 12:03:53 +0100 Subject: [PATCH 50/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (pythonGH-24903) New BarrierTests with 'IsolatedAsyncionTest` --- Lib/test/test_asyncio/test_locks.py | 69 ++++++++++++++--------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index d881693fab9382..e19cc39adbba7b 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -919,7 +919,7 @@ async def test_release_no_waiters(self): self.assertFalse(sem.locked()) -class BarrierTests(test_utils.TestCase): +class BarrierTests(unittest.IsolatedAsyncioTestCase): """ Tests for Barrier objects. """ @@ -933,12 +933,9 @@ async def run_coros(self, n, coro): res = await asyncio.gather(*tasks) return res, tasks - def cancel_coros(self): - for task in asyncio.all_tasks(self.loop): - task.cancel() - test_utils.run_briefly(self.loop) - - def test_barrier_doesnt_accept_loop_parameter(self): + async def test_barrier_doesnt_accept_loop_parameter(self): + """Test here because Barrier has a positional parameter `parties` + """ with self.assertRaisesRegex( TypeError, rf'As of 3.10, the \*loop\* parameter was removed from ' @@ -946,7 +943,7 @@ def test_barrier_doesnt_accept_loop_parameter(self): ): asyncio.Barrier(1, loop=self.loop) - def test_repr(self): + async def test_repr(self): barrier = asyncio.Barrier(self.N) self.assertTrue(RGX_REPR.match(repr(barrier))) @@ -981,68 +978,68 @@ def test_repr(self): self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue('unlocked' in repr(barrier)) - self.loop.run_until_complete(barrier._cond.acquire()) - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue('locked' in repr(barrier)) + async with barrier._cond: + self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertTrue('locked' in repr(barrier)) - def test_init(self): + async def test_barrier_partiest(self): + """check raise ValueError on bad number of parties + """ self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - def test_filling_one_task(self): + async def test_filling_one_task(self): + barrier = asyncio.Barrier(1) results = [] async def f(): i = await barrier.wait() results.append(i) - self.N = 1 - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(f()) - - self.assertEqual(sum(results), 0) + await f() + self.assertEqual(results, [0]) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_filling_one_task_async_with(self): + async def test_filling_one_task_async_with(self): + barrier = asyncio.Barrier(1) results = [] async def f(): async with barrier as i: results.append(i) - self.N = 1 - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(f()) - - self.assertEqual(sum(results), 0) + await f() + self.assertEqual(results, [0]) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_filling_one_task_twice(self): - self.N = 1 - barrier = asyncio.Barrier(self.N) - r1 = self.loop.create_task(barrier.wait()) - test_utils.run_briefly(self.loop) + async def test_filling_one_task_twice(self): + barrier = asyncio.Barrier(1) + r1 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 0) - r2 = self.loop.run_until_complete(barrier.wait()) + r2 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) - self.assertEqual(r1.result(), r2) + self.assertEqual(r1.result(), r2.result()) + self.assertEqual(r1.done(), r2.done()) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_filling_task_by_task(self): - async def coro(): - await barrier.wait() + async def test_filling_task_by_task(self): self.N = 3 barrier = asyncio.Barrier(self.N) + async def coro(): + await barrier.wait() + for i in range(self.N): self.assertEqual(barrier.n_waiting, i) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) + asyncio.create_task(coro()) + asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) From fa4623ccfec451ca542682f2b15b84e18b0d442a Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 22 Feb 2022 10:24:27 +0100 Subject: [PATCH 51/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Comment corrections --- Lib/asyncio/locks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 5c3dd0ead519fb..d4af1a4819b8f0 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -612,7 +612,7 @@ def broken(self): @property def draining(self): - """Return True if the barrier is in a broken state.""" + """Return True if the barrier is draining.""" return self._state == 1 @property @@ -622,7 +622,7 @@ def filling(self): @property def resetting(self): - """Return True if the barrier is filling.""" + """Return True if the barrier is resetting.""" return self._state == -1 From 20fb88e843a4d83d5391e8119f49e24cbcdcf793 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 22 Feb 2022 10:31:28 +0100 Subject: [PATCH 52/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Update Barrier unit tests to use `unittest.IsolatedAsyncioTestCase` --- Lib/test/test_asyncio/test_locks.py | 426 ++++++++++------------------ 1 file changed, 154 insertions(+), 272 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index e19cc39adbba7b..4534827fa23b7f 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -923,28 +923,43 @@ class BarrierTests(unittest.IsolatedAsyncioTestCase): """ Tests for Barrier objects. """ - def setUp(self): - super().setUp() - self.loop = self.new_test_loop() + async def asyncSetUp(self): + await super().asyncSetUp() self.N = 5 - async def run_coros(self, n, coro): - tasks = [self.loop.create_task(coro()) for _ in range(n)] + def make_tasks(self, n, coro): + tasks = [asyncio.create_task(coro()) for _ in range(n)] + return tasks + + async def gather_tasks(self, n, coro): + tasks = self.make_tasks(n, coro) res = await asyncio.gather(*tasks) return res, tasks async def test_barrier_doesnt_accept_loop_parameter(self): - """Test here because Barrier has a positional parameter `parties` - """ + loop = asyncio.get_running_loop() + with self.assertRaisesRegex( TypeError, rf'As of 3.10, the \*loop\* parameter was removed from ' rf'Barrier\(\) since it is no longer necessary' ): - asyncio.Barrier(1, loop=self.loop) + asyncio.Barrier(1, loop=loop) + + async def test_barrier(self): + barrier = asyncio.Barrier(self.N) + self.assertTrue(barrier.filling) + with self.assertRaisesRegex( + TypeError, + "object Barrier can't be used in 'await' expression", + ): + await barrier + + self.assertTrue(barrier.filling) async def test_repr(self): barrier = asyncio.Barrier(self.N) + self.assertTrue(RGX_REPR.match(repr(barrier))) incr = 2 @@ -982,34 +997,36 @@ async def test_repr(self): self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue('locked' in repr(barrier)) - async def test_barrier_partiest(self): - """check raise ValueError on bad number of parties - """ + async def test_barrier_parties(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - async def test_filling_one_task(self): - barrier = asyncio.Barrier(1) + async def test_context_manager(self): + self.N = 2 + barrier = asyncio.Barrier(self.N) results = [] - async def f(): - i = await barrier.wait() - results.append(i) - - await f() - self.assertEqual(results, [0]) + async def coro(): + async with barrier as i: + results.append(i) + + await self.gather_tasks(self.N, coro) + + self.assertListEqual(sorted(results), list(range(self.N))) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - async def test_filling_one_task_async_with(self): + async def test_filling_one_task(self): barrier = asyncio.Barrier(1) results = [] + async def f(): async with barrier as i: results.append(i) await f() + self.assertEqual(len(results), 1) self.assertEqual(results, [0]) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) @@ -1033,37 +1050,19 @@ async def test_filling_one_task_twice(self): async def test_filling_task_by_task(self): self.N = 3 barrier = asyncio.Barrier(self.N) - async def coro(): - await barrier.wait() - - for i in range(self.N): - self.assertEqual(barrier.n_waiting, i) - asyncio.create_task(coro()) - asyncio.sleep(0) - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_filling_tasks_wait_twice(self): - results = [] async def coro(): await barrier.wait() - results.append(True) - await barrier.wait() - results.append(False) - - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) - self.assertEqual(len(results), self.N*2) - self.assertEqual(results.count(True), self.N) - self.assertEqual(results.count(False), self.N) + await self.gather_tasks(self.N, coro) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_filling_tasks_wait_twice_async_with(self): + async def test_filling_tasks_wait_twice(self): + barrier = asyncio.Barrier(self.N) results = [] + async def coro(): async with barrier: results.append(True) @@ -1071,8 +1070,7 @@ async def coro(): async with barrier: results.append(False) - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) + await self.gather_tasks(self.N, coro) self.assertEqual(len(results), self.N*2) self.assertEqual(results.count(True), self.N) @@ -1081,32 +1079,11 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_filling_tasks_check_return_value(self): - results1 = [] - results2 = [] - async def coro(): - await barrier.wait() - results1.append(True) - - i = await barrier.wait() - results2.append(True) - return i - + async def test_filling_tasks_check_return_value(self): barrier = asyncio.Barrier(self.N) - res, _ = self.loop.run_until_complete(self.run_coros(self.N, coro)) - - self.assertEqual(len(results1), self.N) - self.assertTrue(all(results1)) - self.assertEqual(len(results2), self.N) - self.assertTrue(all(results2)) - self.assertEqual(sum(res), sum(range(self.N))) - - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_filling_tasks_check_return_value_async_with(self): results1 = [] results2 = [] + async def coro(): async with barrier: results1.append(True) @@ -1115,48 +1092,27 @@ async def coro(): results2.append(True) return i - barrier = asyncio.Barrier(self.N) - res, _ = self.loop.run_until_complete(self.run_coros(self.N, coro)) + res, _ = await self.gather_tasks(self.N, coro) self.assertEqual(len(results1), self.N) self.assertTrue(all(results1)) self.assertEqual(len(results2), self.N) self.assertTrue(all(results2)) - self.assertEqual(sum(res), sum(range(self.N))) + self.assertListEqual(sorted(res), list(range(self.N))) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_draining_check(self): - results = [] - async def coro(): - await barrier.wait() - # barrier state change to filling for the last task release - results.append(barrier.draining) - + async def test_draining_check(self): barrier = asyncio.Barrier(self.N) - for _ in range(self.N): - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) - - self.assertEqual(len(results), self.N) - self.assertEqual(results[-1], False) - self.assertTrue(all(results[:self.N-1])) - - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_draining_check_async_with(self): results = [] + async def coro(): async with barrier: # barrier state change to filling for the last task release results.append(barrier.draining) - barrier = asyncio.Barrier(self.N) - for _ in range(self.N): - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) + await self.gather_tasks(self.N, coro) self.assertEqual(len(results), self.N) self.assertEqual(results[-1], False) @@ -1165,82 +1121,93 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_blocking_tasks_while_draining(self): - rewait = min(2, self.N-1) + async def test_blocking_tasks_while_draining(self): + rewait = 2 + barrier = asyncio.Barrier(self.N) + barrier_nowaiting = asyncio.Barrier(self.N - rewait) results = [] + states = [] rewait_n = rewait + async def coro(): nonlocal rewait_n # first time waiting p = await barrier.wait() + + states.append(barrier._state) + # after wainting once for all tasks if rewait_n > 0: rewait_n -= 1 + # wait again only for rewait tasks await barrier.wait() else: - # test n_blocking + # task here does not wait so it is blocking (drainig state) results.append(barrier.n_blocking) - barrier = asyncio.Barrier(self.N) - _ = [self.loop.create_task(coro()) for _ in range(self.N)] - test_utils.run_briefly(self.loop) + # wait for end of draining + await barrier_nowaiting.wait() + + # wait for joining other waiting tasks + await barrier.wait() + + await self.gather_tasks(self.N, coro) self.assertEqual(len(results), self.N - rewait) - self.assertEqual(max(results), rewait) + self.assertTrue(all(r == rewait for r in results[:-1])) self.assertEqual(results[-1], 0) - test_utils.run_briefly(self.loop) - - self.assertEqual(barrier.n_waiting, rewait) - self.assertFalse(barrier.broken) - self.cancel_coros() - def test_filling_tasks_cancel_one(self): + async def test_filling_tasks_cancel_one(self): + self.N = 3 + barrier = asyncio.Barrier(self.N) results = [] + async def coro(): await barrier.wait() results.append(True) - self.N = 3 - barrier = asyncio.Barrier(self.N) - t1 = self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) + t1 = asyncio.create_task(coro()) + await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 1) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) + t2 = asyncio.create_task(coro()) + await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 2) t1.cancel() - test_utils.run_briefly(self.loop) + await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 1) + self.assertTrue(t1.cancelled()) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) + t3 = asyncio.create_task(coro()) + await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 2) - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) + t4 = asyncio.create_task(coro()) + await asyncio.gather(t2, t3, t4) + self.assertEqual(len(results), self.N) self.assertTrue(all(results)) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_draining_check_action(self): + async def test_draining_check_action(self): + async def action_task(): + results1.append(True) + barrier = asyncio.Barrier(self.N, action=action_task) results = [] results1 = [] + async def coro(): ret = await barrier.wait() results.append(True) - async def action_task(): - results1.append(True) - - barrier = asyncio.Barrier(self.N, action=action_task) - for _ in range(self.N): - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) + await self.gather_tasks(self.N, coro) + # for _ in range(self.N): + # asyncio.create_task(coro()) + # await asyncio.sleep(0) self.assertEqual(len(results1), 1) self.assertTrue(results1[0]) @@ -1250,25 +1217,28 @@ async def action_task(): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_draining_check_error_on_action(self): + async def test_draining_check_error_on_action(self): ERROR = ZeroDivisionError + async def raise_except(): + await asyncio.sleep(0) + raise ERROR + + barrier = asyncio.Barrier(self.N, action=raise_except) results1 = [] results2 = [] + async def coro(): try: ret = await barrier.wait() except ERROR: results1.append(False) - except: + except asyncio.BrokenBarrierError: results2.append(True) - async def raise_except(): - raise ERROR - - barrier = asyncio.Barrier(self.N, action=raise_except) - for _ in range(self.N): - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) + await self.gather_tasks(self.N, coro) + # tasks = self.make_tasks(self.N, coro) + # await asyncio.sleep(0) + # await asyncio.gather(*tasks) self.assertEqual(len(results1), 1) self.assertFalse(results1[0]) @@ -1278,20 +1248,19 @@ async def raise_except(): self.assertEqual(barrier.n_waiting, 0) self.assertTrue(barrier.broken) - def test_resetting_simple(self): - results = [] - async def coro_reset(): - await barrier.reset() + async def test_reset_barrier(self): + barrier = asyncio.Barrier(1) - barrier = asyncio.Barrier(self.N) - self.loop.create_task(coro_reset()) - test_utils.run_briefly(self.loop) + asyncio.create_task(barrier.reset()) + await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - def test_resetting_tasks_waiting(self): + async def test_reset_barrier_while_waiting(self): + barrier = asyncio.Barrier(self.N) results = [] + async def coro(): try: await barrier.wait() @@ -1301,23 +1270,23 @@ async def coro(): async def coro_reset(): await barrier.reset() - barrier = asyncio.Barrier(self.N) - for _ in range(self.N-1): - self.loop.create_task(coro()) - test_utils.run_briefly(self.loop) + tasks = self.make_tasks(self.N-1, coro) + await asyncio.sleep(0) - self.loop.create_task(coro_reset()) - test_utils.run_briefly(self.loop) - test_utils.run_briefly(self.loop) + asyncio.create_task(coro_reset()) + await asyncio.gather(*tasks) self.assertEqual(len(results), self.N-1) + self.assertTrue(all(results)) self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.resetting) self.assertFalse(barrier.broken) - def test_resetting_tasks_and_waiting_again(self): + async def test_reset_barrier_while_waiting_then_waiting_again(self): + barrier = asyncio.Barrier(self.N) results1 = [] results2 = [] + async def coro1(): try: await barrier.wait() @@ -1328,63 +1297,30 @@ async def coro1(): results2.append(True) async def coro2(): - await barrier.wait() - results2.append(True) + async with barrier: + results2.append(True) - barrier = asyncio.Barrier(self.N) - for _ in range(self.N-1): - self.loop.create_task(coro1()) - test_utils.run_briefly(self.loop) + tasks = self.make_tasks(self.N-1, coro1) - self.loop.create_task(barrier.reset()) - test_utils.run_briefly(self.loop) + asyncio.create_task(barrier.reset()) + await asyncio.sleep(0) - self.loop.create_task(coro2()) - test_utils.run_briefly(self.loop) + asyncio.create_task(coro2()) + await asyncio.sleep(0) + + await asyncio.gather(*tasks) self.assertFalse(barrier.broken) self.assertEqual(len(results1), self.N-1) + self.assertTrue(all(results1)) self.assertEqual(len(results2), self.N) + self.assertTrue(all(results2)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_resetting_tasks_and_waiting_again_async_with(self): - results1 = [] - results2 = [] - async def coro1(): - try: - async with barrier: - pass - except asyncio.BrokenBarrierError: - results1.append(True) - finally: - async with barrier: - results2.append(True) - async def coro2(): - async with barrier: - results2.append(True) + async def test_reset_barrier_while_draining_then_blocking(self): barrier = asyncio.Barrier(self.N) - for _ in range(self.N-1): - self.loop.create_task(coro1()) - test_utils.run_briefly(self.loop) - - self.loop.create_task(barrier.reset()) - test_utils.run_briefly(self.loop) - - self.loop.create_task(coro2()) - test_utils.run_briefly(self.loop) - - self.assertFalse(barrier.broken) - self.assertEqual(len(results1), self.N-1) - self.assertEqual(len(results2), self.N) - - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_resetting_tasks_draining_then_blocking(self): results1 = [] results2 = [] results3 = [] @@ -1413,19 +1349,20 @@ async def coro(): await barrier.wait() results3.append(True) - # self.N = 3 - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) + await self.gather_tasks(self.N, coro) self.assertFalse(barrier.broken) self.assertEqual(len(results1), self.N-1) + self.assertTrue(all(results1)) self.assertEqual(len(results2), 1) + self.assertTrue(results2[0]) self.assertEqual(len(results3), self.N) + self.assertTrue(all(results3)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - def test_resetting_last_task_draining(self): + async def test_reset_barrier_while_draining(self): + barrier = asyncio.Barrier(self.N) results1 = [] results2 = [] results3 = [] @@ -1450,46 +1387,31 @@ async def coro(): await barrier.wait() results3.append(True) - # self.N = 3 - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) + await self.gather_tasks(self.N, coro) self.assertFalse(barrier.broken) + self.assertTrue(all(results1)) self.assertEqual(len(results1), self.N-1) self.assertEqual(len(results2), 0) self.assertEqual(len(results3), self.N) + self.assertTrue(all(results3)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - def test_abort_broken(self): - results1 = [] - results2 = [] - async def coro(): - try: - i = await barrier.wait() - if i == self.N//2: - raise RuntimeError - await barrier.wait() - results1.append(True) - except asyncio.BrokenBarrierError: - results2.append(True) - except RuntimeError: - await barrier.abort() - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) + async def test_abort_barrier(self): + barrier = asyncio.Barrier(1) - self.assertTrue(barrier.broken) - self.assertEqual(len(results1), 0) - self.assertEqual(len(results2), self.N-1) + asyncio.create_task(barrier.abort()) + await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 0) self.assertTrue(barrier.broken) - def test_abort_broken_async_with(self): + async def test_abort_barrier_broken(self): + barrier = asyncio.Barrier(self.N) results1 = [] results2 = [] + async def coro(): try: async with barrier as i : @@ -1502,20 +1424,21 @@ async def coro(): except RuntimeError: await barrier.abort() - barrier = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) + await self.gather_tasks(self.N, coro) self.assertTrue(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(barrier.broken) - def test_abort_and_resetting(self): + async def test_abort_barrier_and_resetting(self): + barrier1 = asyncio.Barrier(self.N) + barrier2 = asyncio.Barrier(self.N) results1 = [] results2 = [] results3 = [] + async def coro(): try: i = await barrier1.wait() @@ -1538,47 +1461,7 @@ async def coro(): await barrier1.wait() results3.append(True) - barrier1 = asyncio.Barrier(self.N) - barrier2 = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) - - self.assertFalse(barrier1.broken) - self.assertEqual(len(results1), 0) - self.assertEqual(len(results2), self.N-1) - self.assertEqual(len(results3), self.N) - - self.assertEqual(barrier1.n_waiting, 0) - self.assertFalse(barrier1.broken) - - def test_abort_and_resetting_async_with(self): - results1 = [] - results2 = [] - results3 = [] - async def coro(): - try: - async with barrier1 as i: - if i == self.N//2: - raise RuntimeError - async with barrier1: - results1.append(True) - except asyncio.BrokenBarrierError: - results2.append(True) - except RuntimeError: - await barrier1.abort() - - # Synchronize and reset the barrier. Must synchronize first so - # that everyone has left it when we reset, and after so that no - # one enters it before the reset. - async with barrier2 as i: - if i == self.N//2: - await barrier1.reset() - async with barrier2: - async with barrier1: - results3.append(True) - - barrier1 = asyncio.Barrier(self.N) - barrier2 = asyncio.Barrier(self.N) - self.loop.run_until_complete(self.run_coros(self.N, coro)) + await self.gather_tasks(self.N, coro) self.assertFalse(barrier1.broken) self.assertEqual(len(results1), 0) @@ -1586,8 +1469,7 @@ async def coro(): self.assertEqual(len(results3), self.N) self.assertEqual(barrier1.n_waiting, 0) - self.assertFalse(barrier1.broken) if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file From 809de71cdd57182ac5f201c05c158b97cfa488ea Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 22 Feb 2022 12:43:22 +0100 Subject: [PATCH 53/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Update Barrier unit tests to use `unittest.IsolatedAsyncioTestCase` --Missing newline at end of file-- --Erase bad comments-- --- Lib/test/test_asyncio/test_locks.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 4534827fa23b7f..604a45f8fbf070 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -920,9 +920,7 @@ async def test_release_no_waiters(self): class BarrierTests(unittest.IsolatedAsyncioTestCase): - """ - Tests for Barrier objects. - """ + async def asyncSetUp(self): await super().asyncSetUp() self.N = 5 @@ -967,6 +965,7 @@ async def test_repr(self): self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue(f"wait:{incr}/{self.N}" in repr(barrier)) self.assertTrue(f"block:0/{self.N}" in repr(barrier)) + barrier._set_filling() self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue(repr(barrier).endswith('state:0]>')) @@ -1318,7 +1317,6 @@ async def coro2(): self.assertEqual(barrier.n_waiting, 0) - async def test_reset_barrier_while_draining_then_blocking(self): barrier = asyncio.Barrier(self.N) results1 = [] @@ -1472,4 +1470,4 @@ async def coro(): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() From f66cd2c845f18b31ed6042e6ca39b501fe93fbe8 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 22 Feb 2022 15:44:19 +0100 Subject: [PATCH 54/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) remove `loop` parameter from `Barrier.__init__` method (ref bpo-46796) --- Lib/asyncio/locks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 79bff134e9f8c9..9dd381df82659a 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -423,13 +423,12 @@ class Barrier(mixins._LoopBoundMixin): have all made their call. """ - def __init__(self, parties, action=None, *, loop=mixins._marker): + def __init__(self, parties, action=None): """Create a barrier, initialised to 'parties' tasks. 'action' is a coroutine which, when supplied, will be called by the last task calling the wait() method, just prior to releasing them all. """ - super().__init__(loop=loop) if parties < 1: raise ValueError('parties must be > 0') From 62272b30af4e053daca9a7c02635bc031a006633 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 22 Feb 2022 16:43:03 +0100 Subject: [PATCH 55/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Remove `test_barrier_doesnt_accept_loop_parameter` - Not necessary here, class Barrier is new Rename 2 tests about 'abort' method --- Lib/test/test_asyncio/test_locks.py | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index e98984b661a6f0..57597a105053e6 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -934,16 +934,6 @@ async def gather_tasks(self, n, coro): res = await asyncio.gather(*tasks) return res, tasks - async def test_barrier_doesnt_accept_loop_parameter(self): - loop = asyncio.get_running_loop() - - with self.assertRaisesRegex( - TypeError, - rf'As of 3.10, the \*loop\* parameter was removed from ' - rf'Barrier\(\) since it is no longer necessary' - ): - asyncio.Barrier(1, loop=loop) - async def test_barrier(self): barrier = asyncio.Barrier(self.N) self.assertTrue(barrier.filling) @@ -1405,7 +1395,7 @@ async def test_abort_barrier(self): self.assertEqual(barrier.n_waiting, 0) self.assertTrue(barrier.broken) - async def test_abort_barrier_broken(self): + async def test_abort_barrier_then_still_broken(self): barrier = asyncio.Barrier(self.N) results1 = [] results2 = [] @@ -1430,7 +1420,7 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) - async def test_abort_barrier_and_resetting(self): + async def test_abort_barrier_then_resetting(self): barrier1 = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) results1 = [] From f7dbf9bbcb66a956b2beb856ec99d50a68c0c779 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 25 Feb 2022 11:14:46 +0100 Subject: [PATCH 56/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) review tests change unmeaning name of tests --- Lib/test/test_asyncio/test_locks.py | 183 ++++++++++++++++++---------- 1 file changed, 121 insertions(+), 62 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 57597a105053e6..bfac7c21dc6a54 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -985,6 +985,7 @@ async def test_repr(self): async with barrier._cond: self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue('locked' in repr(barrier)) + self.assertTrue('unlocked' in repr(barrier)) async def test_barrier_parties(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) @@ -1092,13 +1093,13 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - async def test_draining_check(self): + async def test_draining_state(self): barrier = asyncio.Barrier(self.N) results = [] async def coro(): async with barrier: - # barrier state change to filling for the last task release + # barrier state change to filling for the last task release results.append(barrier.draining) await self.gather_tasks(self.N, coro) @@ -1115,7 +1116,6 @@ async def test_blocking_tasks_while_draining(self): barrier = asyncio.Barrier(self.N) barrier_nowaiting = asyncio.Barrier(self.N - rewait) results = [] - states = [] rewait_n = rewait async def coro(): @@ -1124,7 +1124,6 @@ async def coro(): # first time waiting p = await barrier.wait() - states.append(barrier._state) # after wainting once for all tasks if rewait_n > 0: rewait_n -= 1 @@ -1134,7 +1133,7 @@ async def coro(): # task here does not wait so it is blocking (drainig state) results.append(barrier.n_blocking) - # wait for end of draining + # wait for end of draining `barrier` await barrier_nowaiting.wait() # wait for joining other waiting tasks @@ -1194,9 +1193,6 @@ async def coro(): results.append(True) await self.gather_tasks(self.N, coro) - # for _ in range(self.N): - # asyncio.create_task(coro()) - # await asyncio.sleep(0) self.assertEqual(len(results1), 1) self.assertTrue(results1[0]) @@ -1225,9 +1221,6 @@ async def coro(): results2.append(True) await self.gather_tasks(self.N, coro) - # tasks = self.make_tasks(self.N, coro) - # await asyncio.sleep(0) - # await asyncio.gather(*tasks) self.assertEqual(len(results1), 1) self.assertFalse(results1[0]) @@ -1246,7 +1239,7 @@ async def test_reset_barrier(self): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - async def test_reset_barrier_while_waiting(self): + async def test_reset_barrier_while_tasks_waiting(self): barrier = asyncio.Barrier(self.N) results = [] @@ -1259,9 +1252,11 @@ async def coro(): async def coro_reset(): await barrier.reset() + # N-1 tasks waiting on barrier with N parties tasks = self.make_tasks(self.N-1, coro) await asyncio.sleep(0) + # reset the barrier asyncio.create_task(coro_reset()) await asyncio.gather(*tasks) @@ -1271,7 +1266,68 @@ async def coro_reset(): self.assertFalse(barrier.resetting) self.assertFalse(barrier.broken) - async def test_reset_barrier_while_waiting_then_waiting_again(self): + async def test_reset_barrier_when_tasks_half_draining(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + rest_of_tasks = self.N//2 + + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # catch here waiting tasks + results1.append(True) + else: + # here drained task ouside the barrier + if rest_of_tasks == barrier._count: + # 'd' tasks outside the barrier + await barrier.reset() + + await self.gather_tasks(self.N, coro) + + self.assertEqual(results1, [True]*rest_of_tasks) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.resetting) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_when_tasks_half_draining_half_blocking(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + blocking_tasks = self.N//2 + + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here catch still waiting tasks + results1.append(True) + + # so now waiting again to reach nb_parties + await barrier.wait() + else: + if blocking_tasks == barrier.n_blocking: + # reset now: raise asyncio.BrokenBarrierError for waiting tasks + await barrier.reset() + + # so now waiting again to reach nb_parties + await barrier.wait() + else: + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here no catch - blocked tasks go to wait + results2.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertEqual(results1, [True]*blocking_tasks) + self.assertEqual(results2, []) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.resetting) + self.assertFalse(barrier.broken) + + async def test_reset_barrier_while_tasks_waiting_and_waiting_again(self): barrier = asyncio.Barrier(self.N) results1 = [] results2 = [] @@ -1291,11 +1347,12 @@ async def coro2(): tasks = self.make_tasks(self.N-1, coro1) + # reset barrier, N-1 waiting tasks raise an BrokenBarrierError asyncio.create_task(barrier.reset()) await asyncio.sleep(0) + # complete waiting tasks in the `finally` asyncio.create_task(coro2()) - await asyncio.sleep(0) await asyncio.gather(*tasks) @@ -1307,49 +1364,8 @@ async def coro2(): self.assertEqual(barrier.n_waiting, 0) - async def test_reset_barrier_while_draining_then_blocking(self): - barrier = asyncio.Barrier(self.N) - results1 = [] - results2 = [] - results3 = [] - count = 0 - - async def coro(): - nonlocal count - - try: - i = await barrier.wait() - except: - results2.append(True) - - count += 1 - if count == self.N-1: # there is one task still waiting - await barrier.reset() - await barrier.wait() - else: - try: - await barrier.wait() - results1.append(True) - except Exception as e: - results2.append(True) - - # Now, pass the barrier again - await barrier.wait() - results3.append(True) - - await self.gather_tasks(self.N, coro) - - self.assertFalse(barrier.broken) - self.assertEqual(len(results1), self.N-1) - self.assertTrue(all(results1)) - self.assertEqual(len(results2), 1) - self.assertTrue(results2[0]) - self.assertEqual(len(results3), self.N) - self.assertTrue(all(results3)) - self.assertEqual(barrier.n_waiting, 0) - - async def test_reset_barrier_while_draining(self): + async def test_reset_barrier_while_tasks_draining(self): barrier = asyncio.Barrier(self.N) results1 = [] results2 = [] @@ -1361,18 +1377,26 @@ async def coro(): i = await barrier.wait() count += 1 - if count == self.N: # last task exited from barrier + if count == self.N: + # last task exited from barrier await barrier.reset() + + # wit here to reach the `parties`` await barrier.wait() - else: + else: try: + # second waiting await barrier.wait() + + # N-1 tasks here results1.append(True) except Exception as e: + # never goes here results2.append(True) # Now, pass the barrier again - await barrier.wait() + # last wait, must be completed + k = await barrier.wait() results3.append(True) await self.gather_tasks(self.N, coro) @@ -1395,7 +1419,39 @@ async def test_abort_barrier(self): self.assertEqual(barrier.n_waiting, 0) self.assertTrue(barrier.broken) - async def test_abort_barrier_then_still_broken(self): + async def test_abort_barrier_when_tasks_half_draining_half_blocking(self): + barrier = asyncio.Barrier(self.N) + results1 = [] + results2 = [] + blocking_tasks = self.N//2 + + async def coro(): + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here catch tasks waiting to drain + results1.append(True) + else: + if blocking_tasks == barrier.n_blocking: + # abort now: raise asyncio.BrokenBarrierError for all tasks + await barrier.abort() + else: + try: + await barrier.wait() + except asyncio.BrokenBarrierError: + # here catch blocked tasks (already drained) + results2.append(True) + + await self.gather_tasks(self.N, coro) + + self.assertTrue(barrier.broken) + self.assertEqual(results1, [True]*blocking_tasks) + self.assertEqual(results2, [True]*(self.N-blocking_tasks-1)) + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.resetting) + + async def test_abort_barrier_when_exception(self): + # test from threading.Barrier: see `lock_tests.test_reset` barrier = asyncio.Barrier(self.N) results1 = [] results2 = [] @@ -1417,10 +1473,11 @@ async def coro(): self.assertTrue(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) - + self.assertTrue(all(results2)) self.assertEqual(barrier.n_waiting, 0) - async def test_abort_barrier_then_resetting(self): + async def test_abort_barrier_when_exception_then_resetting(self): + # test from threading.Barrier: see `lock_tests.test_abort_and_reset`` barrier1 = asyncio.Barrier(self.N) barrier2 = asyncio.Barrier(self.N) results1 = [] @@ -1454,7 +1511,9 @@ async def coro(): self.assertFalse(barrier1.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) + self.assertTrue(all(results2)) self.assertEqual(len(results3), self.N) + self.assertTrue(all(results3)) self.assertEqual(barrier1.n_waiting, 0) From 28eba9b5bc9375b5a074fcbf90342610a05c7eb6 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 16 Mar 2022 14:38:16 +0100 Subject: [PATCH 57/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Remove broken, resetting, filling and draining attributes --- Doc/library/asyncio-sync.rst | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 640dd7298346f7..1a2442ee9d2b07 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -489,22 +489,6 @@ Barrier The number of tasks currently blocked in the barrier while draining. - .. attribute:: broken - - A boolean that is ``True`` if the barrier is in the broken state. - - .. attribute:: draining - - A boolean that is ``True`` if the barrier is in the draining state. - - .. attribute:: filling - - A boolean that is ``True`` if the barrier is in the filling state. - - .. attribute:: resetting - - A boolean that is ``True`` if the barrier is in the resetting state. - .. exception:: BrokenBarrierError From cca291861763c6bc7e47da0282eba972d989f2e0 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 16 Mar 2022 14:40:09 +0100 Subject: [PATCH 58/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a new test `test_draining_check_cancelled_action` Update `test_repr` --- Lib/test/test_asyncio/test_locks.py | 133 ++++++++++++++++++---------- 1 file changed, 86 insertions(+), 47 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index bfac7c21dc6a54..962478038d8ef6 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -936,14 +936,14 @@ async def gather_tasks(self, n, coro): async def test_barrier(self): barrier = asyncio.Barrier(self.N) - self.assertTrue(barrier.filling) + self.assertTrue(barrier._is_filling()) with self.assertRaisesRegex( TypeError, "object Barrier can't be used in 'await' expression", ): await barrier - self.assertTrue(barrier.filling) + self.assertTrue(barrier._is_filling()) async def test_repr(self): barrier = asyncio.Barrier(self.N) @@ -951,20 +951,20 @@ async def test_repr(self): self.assertTrue(RGX_REPR.match(repr(barrier))) incr = 2 - barrier._count_wait += incr + barrier._count += incr self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue(f"wait:{incr}/{self.N}" in repr(barrier)) - self.assertTrue(f"block:0/{self.N}" in repr(barrier)) + self.assertTrue(f"block" not in repr(barrier)) barrier._set_filling() self.assertTrue(RGX_REPR.match(repr(barrier))) self.assertTrue(repr(barrier).endswith('state:0]>')) barrier._count_block += incr - barrier._count_wait -= incr + barrier._count -= incr self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(f"wait:0/{self.N}" in repr(barrier)) self.assertTrue(f"block:{incr}/{self.N}" in repr(barrier)) + self.assertTrue(f"wait" not in repr(barrier)) barrier._set_draining() self.assertTrue(RGX_REPR.match(repr(barrier))) @@ -992,7 +992,7 @@ async def test_barrier_parties(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) async def test_context_manager(self): - self.N = 2 + self.N = 3 barrier = asyncio.Barrier(self.N) results = [] @@ -1004,50 +1004,58 @@ async def coro(): self.assertListEqual(sorted(results), list(range(self.N))) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) async def test_filling_one_task(self): barrier = asyncio.Barrier(1) - results = [] async def f(): async with barrier as i: - results.append(i) + return True - await f() + ret = await f() - self.assertEqual(len(results), 1) - self.assertEqual(results, [0]) + self.assertTrue(ret) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) async def test_filling_one_task_twice(self): barrier = asyncio.Barrier(1) - r1 = asyncio.create_task(barrier.wait()) + t1 = asyncio.create_task(barrier.wait()) await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 0) - r2 = asyncio.create_task(barrier.wait()) + t2 = asyncio.create_task(barrier.wait()) await asyncio.sleep(0) - self.assertEqual(r1.result(), r2.result()) - self.assertEqual(r1.done(), r2.done()) + self.assertEqual(t1.result(), t2.result()) + self.assertEqual(t1.done(), t2.done()) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) async def test_filling_task_by_task(self): self.N = 3 barrier = asyncio.Barrier(self.N) - async def coro(): - await barrier.wait() + t1 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 1) + self.assertTrue(barrier._is_filling()) - await self.gather_tasks(self.N, coro) + t2 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + self.assertEqual(barrier.n_waiting, 2) + self.assertTrue(barrier._is_filling()) + + t3 = asyncio.create_task(barrier.wait()) + await asyncio.sleep(0) + + await asyncio.wait([t1, t2, t3]) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) async def test_filling_tasks_wait_twice(self): barrier = asyncio.Barrier(self.N) @@ -1067,7 +1075,7 @@ async def coro(): self.assertEqual(results.count(False), self.N) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) async def test_filling_tasks_check_return_value(self): barrier = asyncio.Barrier(self.N) @@ -1091,7 +1099,7 @@ async def coro(): self.assertListEqual(sorted(res), list(range(self.N))) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) async def test_draining_state(self): barrier = asyncio.Barrier(self.N) @@ -1100,7 +1108,7 @@ async def test_draining_state(self): async def coro(): async with barrier: # barrier state change to filling for the last task release - results.append(barrier.draining) + results.append(barrier._is_draining()) await self.gather_tasks(self.N, coro) @@ -1109,7 +1117,7 @@ async def coro(): self.assertTrue(all(results[:self.N-1])) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) async def test_blocking_tasks_while_draining(self): rewait = 2 @@ -1166,6 +1174,8 @@ async def coro(): t1.cancel() await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 1) + with self.assertRaises(asyncio.CancelledError): + await t1 self.assertTrue(t1.cancelled()) t3 = asyncio.create_task(coro()) @@ -1179,7 +1189,7 @@ async def coro(): self.assertTrue(all(results)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) async def test_draining_check_action(self): async def action_task(): @@ -1200,7 +1210,37 @@ async def coro(): self.assertTrue(all(results)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) + + async def test_draining_check_cancelled_action(self): + async def action_coro(): + asyncio.current_task().cancel() + await asyncio.sleep(0) + + barrier = asyncio.Barrier(self.N, action=action_coro) + results = [] + results1 = [] + results2 = [] + + async def coro(): + try: + await barrier.wait() + results.append(True) + except asyncio.CancelledError: + results1.append(True) + except asyncio.BrokenBarrierError: + results2.append(True) + + #with self.assertRaises(asyncio.CancelledError): + res, t = await self.gather_tasks(self.N, coro) + + self.assertEqual(len(results), 0) + self.assertEqual(results1, [True]) + self.assertEqual(len(results2), self.N-1) + self.assertTrue(all(results2)) + + self.assertEqual(barrier.n_waiting, 0) + self.assertTrue(barrier._is_broken()) async def test_draining_check_error_on_action(self): ERROR = ZeroDivisionError @@ -1214,7 +1254,7 @@ async def raise_except(): async def coro(): try: - ret = await barrier.wait() + await barrier.wait() except ERROR: results1.append(False) except asyncio.BrokenBarrierError: @@ -1222,13 +1262,12 @@ async def coro(): await self.gather_tasks(self.N, coro) - self.assertEqual(len(results1), 1) - self.assertFalse(results1[0]) + self.assertEqual(results1, [False]) self.assertEqual(len(results2), self.N-1) self.assertTrue(all(results2)) self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(barrier.broken) + self.assertTrue(barrier._is_broken()) async def test_reset_barrier(self): barrier = asyncio.Barrier(1) @@ -1237,7 +1276,7 @@ async def test_reset_barrier(self): await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) async def test_reset_barrier_while_tasks_waiting(self): barrier = asyncio.Barrier(self.N) @@ -1263,8 +1302,8 @@ async def coro_reset(): self.assertEqual(len(results), self.N-1) self.assertTrue(all(results)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.resetting) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_resetting()) + self.assertFalse(barrier._is_broken()) async def test_reset_barrier_when_tasks_half_draining(self): barrier = asyncio.Barrier(self.N) @@ -1287,8 +1326,8 @@ async def coro(): self.assertEqual(results1, [True]*rest_of_tasks) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.resetting) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_resetting()) + self.assertFalse(barrier._is_broken()) async def test_reset_barrier_when_tasks_half_draining_half_blocking(self): barrier = asyncio.Barrier(self.N) @@ -1324,8 +1363,8 @@ async def coro(): self.assertEqual(results1, [True]*blocking_tasks) self.assertEqual(results2, []) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.resetting) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_resetting()) + self.assertFalse(barrier._is_broken()) async def test_reset_barrier_while_tasks_waiting_and_waiting_again(self): barrier = asyncio.Barrier(self.N) @@ -1356,7 +1395,7 @@ async def coro2(): await asyncio.gather(*tasks) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) self.assertEqual(len(results1), self.N-1) self.assertTrue(all(results1)) self.assertEqual(len(results2), self.N) @@ -1401,7 +1440,7 @@ async def coro(): await self.gather_tasks(self.N, coro) - self.assertFalse(barrier.broken) + self.assertFalse(barrier._is_broken()) self.assertTrue(all(results1)) self.assertEqual(len(results1), self.N-1) self.assertEqual(len(results2), 0) @@ -1417,7 +1456,7 @@ async def test_abort_barrier(self): await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(barrier.broken) + self.assertTrue(barrier._is_broken()) async def test_abort_barrier_when_tasks_half_draining_half_blocking(self): barrier = asyncio.Barrier(self.N) @@ -1444,11 +1483,11 @@ async def coro(): await self.gather_tasks(self.N, coro) - self.assertTrue(barrier.broken) + self.assertTrue(barrier._is_broken()) self.assertEqual(results1, [True]*blocking_tasks) self.assertEqual(results2, [True]*(self.N-blocking_tasks-1)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.resetting) + self.assertFalse(barrier._is_resetting()) async def test_abort_barrier_when_exception(self): # test from threading.Barrier: see `lock_tests.test_reset` @@ -1470,7 +1509,7 @@ async def coro(): await self.gather_tasks(self.N, coro) - self.assertTrue(barrier.broken) + self.assertTrue(barrier._is_broken()) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertTrue(all(results2)) @@ -1508,7 +1547,7 @@ async def coro(): await self.gather_tasks(self.N, coro) - self.assertFalse(barrier1.broken) + self.assertFalse(barrier1._is_broken()) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertTrue(all(results2)) From 6d3806f92446a20c16d79f175616ae612a447432 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 16 Mar 2022 14:43:59 +0100 Subject: [PATCH 59/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Replace broken, resetting, filling and draining attributes with private methods Update `__repr__` Correction minor bugs on `_count_lock`attribut Remove `_cout_wait` attribut Refactoring code on `_block()`method --- Lib/asyncio/locks.py | 87 +++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 9dd381df82659a..2c6e109d6d5fe8 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -439,17 +439,17 @@ def __init__(self, parties, action=None): self._state = 0 # 0 filling, 1, draining, # -1 resetting, -2 broken self._count = 0 # count tasks in Barrier - self._count_wait = 0 # count waiting tasks self._count_block = 0 # count blocking tasks def __repr__(self): res = super().__repr__() extra = 'locked' if self._cond.locked() else 'unlocked' - extra += f', wait:{self._count_wait}/{self._parties}' - extra += f', block:{self._count_block}/{self._parties}' + if self._count: + extra += f', wait:{self._count}/{self._parties}' + if self._count_block: + extra += f', block:{self._count_block}/{self._parties}' extra += f', state:{self._state}' return f'<{res[1:-1]} [{extra}]>' - async def __aenter__(self): """ wait for the barrier reaches the parties number @@ -489,13 +489,18 @@ async def _block(self): or raise an exception if it is broken. """ self._count_block += 1 - while self.draining or self.resetting: # - # It is draining or resetting, wait until done - await self._cond.wait() + # It is draining or resetting, wait until done + # unless a CancelledError occurs + try: + await self._cond.wait_for(lambda: not (self._is_draining() or\ + self._is_resetting())) + except exceptions.CancelledError: + self._count_block -= 1 + raise self._count_block -= 1 # see if the barrier is in a broken state - if self.broken: + if self._is_broken(): raise BrokenBarrierError # Optionally run the 'action' and release the tasks waiting @@ -511,7 +516,8 @@ async def _release(self): self._set_draining() self._cond.notify_all() except: - # an exception occurs during the _action coroutine. + # an exception occurs during the _action coroutine, + # or the last calling task cancels # Break and reraise self._break() raise @@ -520,11 +526,11 @@ async def _wait(self): """Wait in the barrier until we are released. Raise an exception if the barrier is reset or broken. """ - self._count_wait += 1 - await self._cond.wait_for(lambda: not self.filling) - self._count_wait -= 1 + # wait for end of filling + # unless a CancelledError occurs + await self._cond.wait_for(lambda: not self._is_filling()) - if self.broken or self.resetting: + if self._is_broken() or self._is_resetting(): raise BrokenBarrierError def _exit(self): @@ -532,7 +538,7 @@ def _exit(self): waiting for the barrier to drain. """ if self._count == 0: - if self.resetting or self.draining: + if self._is_resetting() or self._is_draining(): self._set_filling() self._cond.notify_all() @@ -543,8 +549,9 @@ async def reset(self): """ async with self._cond: if self._count > 0: - if not self.resetting:# self.filling or self.draining - # or self.broken + if not self._is_resetting():# self._is_filling() + # or self._is_draining() + # or self.is_broken() #reset the barrier, waking up tasks self._set_resetting() else: @@ -565,22 +572,6 @@ def _break(self): self._set_broken() self._cond.notify_all() - def _set_broken(self): - """Set state to broken.""" - self._state = -2 - - def _set_draining(self): - """Set state to draining.""" - self._state = 1 - - def _set_filling(self): - """Set state to filling.""" - self._state = 0 - - def _set_resetting(self): - """Set state to resetting.""" - self._state = -1 - @property def parties(self): """Return the number of tasks required to trip the barrier.""" @@ -589,34 +580,46 @@ def parties(self): @property def n_waiting(self): """Return the number of tasks currently waiting at the barrier.""" - if self.filling: + if self._is_filling(): return self._count return 0 @property def n_blocking(self): """Return the number of tasks currently blocking at the barrier.""" - if self.draining: + if self._is_draining(): return self._count_block return 0 - @property - def broken(self): + def _set_broken(self): + """Set state to broken.""" + self._state = -2 + + def _set_draining(self): + """Set state to draining.""" + self._state = 1 + + def _set_filling(self): + """Set state to filling.""" + self._state = 0 + + def _set_resetting(self): + """Set state to resetting.""" + self._state = -1 + + def _is_broken(self): """Return True if the barrier is in a broken state.""" return self._state == -2 - @property - def draining(self): + def _is_draining(self): """Return True if the barrier is draining.""" return self._state == 1 - @property - def filling(self): + def _is_filling(self): """Return True if the barrier is filling.""" return self._state == 0 - @property - def resetting(self): + def _is_resetting(self): """Return True if the barrier is resetting.""" return self._state == -1 From a2467f5d2f93ef2fc4560dc0cdd5117d2fe90162 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 16 Mar 2022 16:17:11 +0100 Subject: [PATCH 60/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Suppressed trailing white spaces --- Lib/asyncio/locks.py | 24 ++++++++++---------- Lib/test/test_asyncio/test_locks.py | 35 ++++++++++++++--------------- 2 files changed, 29 insertions(+), 30 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 2c6e109d6d5fe8..6cb308c4834a77 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -436,8 +436,8 @@ def __init__(self, parties, action=None): self._action = action self._parties = parties - self._state = 0 # 0 filling, 1, draining, - # -1 resetting, -2 broken + self._state = 0 # 0 filling, 1, draining, + # -1 resetting, -2 broken self._count = 0 # count tasks in Barrier self._count_block = 0 # count blocking tasks @@ -452,8 +452,8 @@ def __repr__(self): return f'<{res[1:-1]} [{extra}]>' async def __aenter__(self): - """ wait for the barrier reaches the parties number - when start draining release and return index of waited task + """ wait for the barrier reaches the parties number + when start draining release and return index of waited task """ return await self.wait() @@ -469,7 +469,7 @@ async def wait(self): Returns an unique and individual index number from 0 to 'parties-1'. # """ async with self._cond: - await self._block() # Block while the barrier drains or resets. + await self._block() # Block while the barrier drains or resets. try: index = self._count self._count += 1 @@ -492,7 +492,7 @@ async def _block(self): # It is draining or resetting, wait until done # unless a CancelledError occurs try: - await self._cond.wait_for(lambda: not (self._is_draining() or\ + await self._cond.wait_for(lambda: not (self._is_draining() or self._is_resetting())) except exceptions.CancelledError: self._count_block -= 1 @@ -517,7 +517,7 @@ async def _release(self): self._cond.notify_all() except: # an exception occurs during the _action coroutine, - # or the last calling task cancels + # or the last calling task cancels # Break and reraise self._break() raise @@ -526,7 +526,7 @@ async def _wait(self): """Wait in the barrier until we are released. Raise an exception if the barrier is reset or broken. """ - # wait for end of filling + # wait for end of filling # unless a CancelledError occurs await self._cond.wait_for(lambda: not self._is_filling()) @@ -539,7 +539,7 @@ def _exit(self): """ if self._count == 0: if self._is_resetting() or self._is_draining(): - self._set_filling() + self._set_filling() self._cond.notify_all() async def reset(self): @@ -549,11 +549,11 @@ async def reset(self): """ async with self._cond: if self._count > 0: - if not self._is_resetting():# self._is_filling() - # or self._is_draining() + if not self._is_resetting():# self._is_filling() + # or self._is_draining() # or self.is_broken() #reset the barrier, waking up tasks - self._set_resetting() + self._set_resetting() else: self._set_filling() self._cond.notify_all() diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 962478038d8ef6..8ce27d72516a98 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -1007,7 +1007,7 @@ async def coro(): self.assertFalse(barrier._is_broken()) async def test_filling_one_task(self): - barrier = asyncio.Barrier(1) + barrier = asyncio.Barrier(1) async def f(): async with barrier as i: @@ -1122,7 +1122,7 @@ async def coro(): async def test_blocking_tasks_while_draining(self): rewait = 2 barrier = asyncio.Barrier(self.N) - barrier_nowaiting = asyncio.Barrier(self.N - rewait) + barrier_nowaiting = asyncio.Barrier(self.N - rewait) results = [] rewait_n = rewait @@ -1136,7 +1136,7 @@ async def coro(): if rewait_n > 0: rewait_n -= 1 # wait again only for rewait tasks - await barrier.wait() + await barrier.wait() else: # task here does not wait so it is blocking (drainig state) results.append(barrier.n_blocking) @@ -1172,7 +1172,7 @@ async def coro(): self.assertEqual(barrier.n_waiting, 2) t1.cancel() - await asyncio.sleep(0) + await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 1) with self.assertRaises(asyncio.CancelledError): await t1 @@ -1231,8 +1231,7 @@ async def coro(): except asyncio.BrokenBarrierError: results2.append(True) - #with self.assertRaises(asyncio.CancelledError): - res, t = await self.gather_tasks(self.N, coro) + await self.gather_tasks(self.N, coro) self.assertEqual(len(results), 0) self.assertEqual(results1, [True]) @@ -1317,10 +1316,10 @@ async def coro(): # catch here waiting tasks results1.append(True) else: - # here drained task ouside the barrier + # here drained task ouside the barrier if rest_of_tasks == barrier._count: # 'd' tasks outside the barrier - await barrier.reset() + await barrier.reset() await self.gather_tasks(self.N, coro) @@ -1339,7 +1338,7 @@ async def coro(): try: await barrier.wait() except asyncio.BrokenBarrierError: - # here catch still waiting tasks + # here catch still waiting tasks results1.append(True) # so now waiting again to reach nb_parties @@ -1347,10 +1346,10 @@ async def coro(): else: if blocking_tasks == barrier.n_blocking: # reset now: raise asyncio.BrokenBarrierError for waiting tasks - await barrier.reset() + await barrier.reset() # so now waiting again to reach nb_parties - await barrier.wait() + await barrier.wait() else: try: await barrier.wait() @@ -1416,13 +1415,13 @@ async def coro(): i = await barrier.wait() count += 1 - if count == self.N: + if count == self.N: # last task exited from barrier await barrier.reset() - # wit here to reach the `parties`` + # wit here to reach the `parties`` await barrier.wait() - else: + else: try: # second waiting await barrier.wait() @@ -1430,7 +1429,7 @@ async def coro(): # N-1 tasks here results1.append(True) except Exception as e: - # never goes here + # never goes here results2.append(True) # Now, pass the barrier again @@ -1468,12 +1467,12 @@ async def coro(): try: await barrier.wait() except asyncio.BrokenBarrierError: - # here catch tasks waiting to drain + # here catch tasks waiting to drain results1.append(True) else: if blocking_tasks == barrier.n_blocking: # abort now: raise asyncio.BrokenBarrierError for all tasks - await barrier.abort() + await barrier.abort() else: try: await barrier.wait() @@ -1501,7 +1500,7 @@ async def coro(): if i == self.N//2: raise RuntimeError async with barrier: - results1.append(True) + results1.append(True) except asyncio.BrokenBarrierError: results2.append(True) except RuntimeError: From 90e5d04ae55a07d8d500e7bc1e2b3974880d2582 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 16 Mar 2022 17:52:13 +0100 Subject: [PATCH 61/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Remove `n_blocking` attribut Restore `broken` instead of `_is_broken()` Rename all `_is_xxxx' methods to `_xxxx' where `xxxx' in draining, resetting, filling --- Lib/asyncio/locks.py | 54 ++++++++++++++++++++------------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 6cb308c4834a77..d6fb4804f0d5e1 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -492,15 +492,15 @@ async def _block(self): # It is draining or resetting, wait until done # unless a CancelledError occurs try: - await self._cond.wait_for(lambda: not (self._is_draining() or - self._is_resetting())) + await self._cond.wait_for(lambda: not (self._draining() or + self._resetting())) except exceptions.CancelledError: self._count_block -= 1 raise self._count_block -= 1 # see if the barrier is in a broken state - if self._is_broken(): + if self.broken: raise BrokenBarrierError # Optionally run the 'action' and release the tasks waiting @@ -528,9 +528,9 @@ async def _wait(self): """ # wait for end of filling # unless a CancelledError occurs - await self._cond.wait_for(lambda: not self._is_filling()) + await self._cond.wait_for(lambda: not self._filling()) - if self._is_broken() or self._is_resetting(): + if self.broken or self._resetting(): raise BrokenBarrierError def _exit(self): @@ -538,7 +538,7 @@ def _exit(self): waiting for the barrier to drain. """ if self._count == 0: - if self._is_resetting() or self._is_draining(): + if self._resetting() or self._draining(): self._set_filling() self._cond.notify_all() @@ -549,8 +549,8 @@ async def reset(self): """ async with self._cond: if self._count > 0: - if not self._is_resetting():# self._is_filling() - # or self._is_draining() + if not self._resetting():# self._filling() + # or self._draining() # or self.is_broken() #reset the barrier, waking up tasks self._set_resetting() @@ -580,16 +580,26 @@ def parties(self): @property def n_waiting(self): """Return the number of tasks currently waiting at the barrier.""" - if self._is_filling(): + if self._filling(): return self._count return 0 @property - def n_blocking(self): - """Return the number of tasks currently blocking at the barrier.""" - if self._is_draining(): - return self._count_block - return 0 + def broken(self): + """Return True if the barrier is in a broken state.""" + return self._state == -2 + + def _draining(self): + """Return True if the barrier is draining.""" + return self._state == 1 + + def _filling(self): + """Return True if the barrier is filling.""" + return self._state == 0 + + def _resetting(self): + """Return True if the barrier is resetting.""" + return self._state == -1 def _set_broken(self): """Set state to broken.""" @@ -607,22 +617,6 @@ def _set_resetting(self): """Set state to resetting.""" self._state = -1 - def _is_broken(self): - """Return True if the barrier is in a broken state.""" - return self._state == -2 - - def _is_draining(self): - """Return True if the barrier is draining.""" - return self._state == 1 - - def _is_filling(self): - """Return True if the barrier is filling.""" - return self._state == 0 - - def _is_resetting(self): - """Return True if the barrier is resetting.""" - return self._state == -1 - # exception raised by the Barrier class class BrokenBarrierError(RuntimeError): From bea7e7fac9daf8a16f338fe98c77cb81e50c9323 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 16 Mar 2022 18:03:49 +0100 Subject: [PATCH 62/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Use `_count_block` instead of removing `n_blocking` attr Restore `broken` attr and use it instead of `_is_broken()` method Rename all `_is_xxxx' methods to `_xxxx'. --- Lib/test/test_asyncio/test_locks.py | 70 ++++++++++++++--------------- 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 8ce27d72516a98..2cf76ea5b99304 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -936,14 +936,14 @@ async def gather_tasks(self, n, coro): async def test_barrier(self): barrier = asyncio.Barrier(self.N) - self.assertTrue(barrier._is_filling()) + self.assertTrue(barrier._filling()) with self.assertRaisesRegex( TypeError, "object Barrier can't be used in 'await' expression", ): await barrier - self.assertTrue(barrier._is_filling()) + self.assertTrue(barrier._filling()) async def test_repr(self): barrier = asyncio.Barrier(self.N) @@ -1004,7 +1004,7 @@ async def coro(): self.assertListEqual(sorted(results), list(range(self.N))) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_filling_one_task(self): barrier = asyncio.Barrier(1) @@ -1017,7 +1017,7 @@ async def f(): self.assertTrue(ret) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_filling_one_task_twice(self): barrier = asyncio.Barrier(1) @@ -1033,7 +1033,7 @@ async def test_filling_one_task_twice(self): self.assertEqual(t1.done(), t2.done()) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_filling_task_by_task(self): self.N = 3 @@ -1042,12 +1042,12 @@ async def test_filling_task_by_task(self): t1 = asyncio.create_task(barrier.wait()) await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 1) - self.assertTrue(barrier._is_filling()) + self.assertTrue(barrier._filling()) t2 = asyncio.create_task(barrier.wait()) await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 2) - self.assertTrue(barrier._is_filling()) + self.assertTrue(barrier._filling()) t3 = asyncio.create_task(barrier.wait()) await asyncio.sleep(0) @@ -1055,7 +1055,7 @@ async def test_filling_task_by_task(self): await asyncio.wait([t1, t2, t3]) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_filling_tasks_wait_twice(self): barrier = asyncio.Barrier(self.N) @@ -1075,7 +1075,7 @@ async def coro(): self.assertEqual(results.count(False), self.N) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_filling_tasks_check_return_value(self): barrier = asyncio.Barrier(self.N) @@ -1099,7 +1099,7 @@ async def coro(): self.assertListEqual(sorted(res), list(range(self.N))) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_draining_state(self): barrier = asyncio.Barrier(self.N) @@ -1108,7 +1108,7 @@ async def test_draining_state(self): async def coro(): async with barrier: # barrier state change to filling for the last task release - results.append(barrier._is_draining()) + results.append(barrier._draining()) await self.gather_tasks(self.N, coro) @@ -1117,7 +1117,7 @@ async def coro(): self.assertTrue(all(results[:self.N-1])) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_blocking_tasks_while_draining(self): rewait = 2 @@ -1139,7 +1139,7 @@ async def coro(): await barrier.wait() else: # task here does not wait so it is blocking (drainig state) - results.append(barrier.n_blocking) + results.append(barrier._count_block) # wait for end of draining `barrier` await barrier_nowaiting.wait() @@ -1150,9 +1150,7 @@ async def coro(): await self.gather_tasks(self.N, coro) self.assertEqual(len(results), self.N - rewait) - self.assertTrue(all(r == rewait for r in results[:-1])) - self.assertEqual(results[-1], 0) - + self.assertTrue(all(r == rewait for r in results)) async def test_filling_tasks_cancel_one(self): self.N = 3 @@ -1189,7 +1187,7 @@ async def coro(): self.assertTrue(all(results)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_draining_check_action(self): async def action_task(): @@ -1210,7 +1208,7 @@ async def coro(): self.assertTrue(all(results)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_draining_check_cancelled_action(self): async def action_coro(): @@ -1239,7 +1237,7 @@ async def coro(): self.assertTrue(all(results2)) self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(barrier._is_broken()) + self.assertTrue(barrier.broken) async def test_draining_check_error_on_action(self): ERROR = ZeroDivisionError @@ -1266,7 +1264,7 @@ async def coro(): self.assertTrue(all(results2)) self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(barrier._is_broken()) + self.assertTrue(barrier.broken) async def test_reset_barrier(self): barrier = asyncio.Barrier(1) @@ -1275,7 +1273,7 @@ async def test_reset_barrier(self): await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) async def test_reset_barrier_while_tasks_waiting(self): barrier = asyncio.Barrier(self.N) @@ -1301,8 +1299,8 @@ async def coro_reset(): self.assertEqual(len(results), self.N-1) self.assertTrue(all(results)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_resetting()) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier._resetting()) + self.assertFalse(barrier.broken) async def test_reset_barrier_when_tasks_half_draining(self): barrier = asyncio.Barrier(self.N) @@ -1325,8 +1323,8 @@ async def coro(): self.assertEqual(results1, [True]*rest_of_tasks) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_resetting()) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier._resetting()) + self.assertFalse(barrier.broken) async def test_reset_barrier_when_tasks_half_draining_half_blocking(self): barrier = asyncio.Barrier(self.N) @@ -1344,7 +1342,7 @@ async def coro(): # so now waiting again to reach nb_parties await barrier.wait() else: - if blocking_tasks == barrier.n_blocking: + if blocking_tasks == barrier._count_block: # reset now: raise asyncio.BrokenBarrierError for waiting tasks await barrier.reset() @@ -1362,8 +1360,8 @@ async def coro(): self.assertEqual(results1, [True]*blocking_tasks) self.assertEqual(results2, []) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_resetting()) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier._resetting()) + self.assertFalse(barrier.broken) async def test_reset_barrier_while_tasks_waiting_and_waiting_again(self): barrier = asyncio.Barrier(self.N) @@ -1394,7 +1392,7 @@ async def coro2(): await asyncio.gather(*tasks) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) self.assertEqual(len(results1), self.N-1) self.assertTrue(all(results1)) self.assertEqual(len(results2), self.N) @@ -1439,7 +1437,7 @@ async def coro(): await self.gather_tasks(self.N, coro) - self.assertFalse(barrier._is_broken()) + self.assertFalse(barrier.broken) self.assertTrue(all(results1)) self.assertEqual(len(results1), self.N-1) self.assertEqual(len(results2), 0) @@ -1455,7 +1453,7 @@ async def test_abort_barrier(self): await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(barrier._is_broken()) + self.assertTrue(barrier.broken) async def test_abort_barrier_when_tasks_half_draining_half_blocking(self): barrier = asyncio.Barrier(self.N) @@ -1470,7 +1468,7 @@ async def coro(): # here catch tasks waiting to drain results1.append(True) else: - if blocking_tasks == barrier.n_blocking: + if blocking_tasks == barrier._count_block: # abort now: raise asyncio.BrokenBarrierError for all tasks await barrier.abort() else: @@ -1482,11 +1480,11 @@ async def coro(): await self.gather_tasks(self.N, coro) - self.assertTrue(barrier._is_broken()) + self.assertTrue(barrier.broken) self.assertEqual(results1, [True]*blocking_tasks) self.assertEqual(results2, [True]*(self.N-blocking_tasks-1)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._is_resetting()) + self.assertFalse(barrier._resetting()) async def test_abort_barrier_when_exception(self): # test from threading.Barrier: see `lock_tests.test_reset` @@ -1508,7 +1506,7 @@ async def coro(): await self.gather_tasks(self.N, coro) - self.assertTrue(barrier._is_broken()) + self.assertTrue(barrier.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertTrue(all(results2)) @@ -1546,7 +1544,7 @@ async def coro(): await self.gather_tasks(self.N, coro) - self.assertFalse(barrier1._is_broken()) + self.assertFalse(barrier1.broken) self.assertEqual(len(results1), 0) self.assertEqual(len(results2), self.N-1) self.assertTrue(all(results2)) From e97d417043eb7f75d8a295ab16ffdcdf3c143a1b Mon Sep 17 00:00:00 2001 From: Duprat Date: Thu, 17 Mar 2022 10:02:29 +0100 Subject: [PATCH 63/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Description of the cancel behavior of tasks in the Barrier Minor corrections --- Doc/library/asyncio-sync.rst | 63 ++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 35 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 1a2442ee9d2b07..17c0225dc33a57 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -374,12 +374,14 @@ Barrier async def remove_file(barrier, filename): print(f'Remove {filename!r} ...') - # remove filename (simulation with 'random value' secs spleeping) + # remove filename (simulation with 'random value' secs sleeping) await asyncio.sleep(random.choice(DELAYS)) print(f'\tFile {filename!r} is removed') - # wait for others files - return await barrier.wait() + # wait for other files + pos = await barrier.wait() + + return pos async def remove_folder(barrier, folder): print(f'Waiting for remove all files from folder {folder!r} ...') @@ -390,11 +392,11 @@ Barrier await asyncio.sleep(1.0) print(f'Folder {folder!r} is now removed ...') - return pos + return pos async def main(folder, files, nb_files): # Create a Barrier object with parties equal to nb_files+1 - # corresponding to the number of all removing items, + # corresponding to the number of all removing items plus the folder, # and an action as a print message. barrier = asyncio.Barrier(nb_files+1, action=job_done) @@ -409,36 +411,17 @@ Barrier asyncio.run(main("MyFolder", [f'file{i+1:02d}.txt' for i in range(NB)], NB)) - As in the example below, the other way to use a Barrier is an :keyword:`async with` - statement:: - - barrier = asyncio.Barrier(2) + In the previous example, there are the two ways to call the :meth:`~Barrier.wait` method: - # ... later - try: - async with barrier as i: - # Barrier passed so do the job - ... - except asyncio.BrokenBarrierError: - pass + * with a **classic call**: see ``pos = barrier.wait()`` in the 'remove_file' coroutine; - which is equivalent to:: - - barrier = asyncio.Barrier(2) - - # ... later - try: - i = await barrier.acquire() - # Barrier passed so do the job - ... - except asyncio.BrokenBarrierError: - pass + * with a **async context manager call**: see ``async with barrier as pos:`` in the 'remove_folder' coroutine. The barrier can be reused any number of times for the same number of tasks. .. class:: Barrier(parties, action=None) - Create a barrier object for *parties* number of tasks. A task *action*, when + Create a barrier object for *parties* number of tasks. A coroutine *action*, when provided, is awaited once just before the release phasis (draining state). .. coroutinemethod:: wait() @@ -446,8 +429,17 @@ Barrier Pass the barrier. When all the tasks party to the barrier have called this function, they are all released simultaneously. + When a task in the barrier is cancelled, two cases are possible: + + * the task is the last waiting task, an :class:`BrokenBarrierError` + exception occurs, barrier is put into a broken state; + + * the task is waiting (filling state) or blocked (draining state), + this task exits the barrier which stays in the same state. + If the state is filling, count of waiting task decreases by 1. + The return value is an integer in the range 0 to *parties-1*, different - for each task. This can be used to select a task to do some special + for each task. This can be used to select a task to do some special housekeeping, e.g.:: ... @@ -456,12 +448,13 @@ Barrier # Only one task print this print('End of *draining phasis*') - If an *action* was provided as a task to the constructor, the last calling task of - :meth:`wait` method will have to *await this task* prior to being released. - Should this call raise an error, the barrier is put into the broken state. + If an *action* was provided as a coroutine to the constructor, the last calling task of + :meth:`wait` method will have to *await this coroutine* prior to being released. + Should this call raise an error, including cancellation of task, the barrier is put into the broken state. This method may raise a :class:`BrokenBarrierError` exception if the - barrier is broken or reset while a task is waiting. + barrier is broken or reset while a task is waiting. It could raise a :exc:`CancelledError` + if a task is cancelled. .. coroutinemethod:: reset() @@ -485,9 +478,9 @@ Barrier The number of tasks currently waiting in the barrier while filling. - .. attribute:: n_blocked + .. attribute:: broken - The number of tasks currently blocked in the barrier while draining. + A boolean that is ``True`` if the barrier is in the broken state. .. exception:: BrokenBarrierError From b60a9319884de1928ca62cc6cadd70c82513dbdb Mon Sep 17 00:00:00 2001 From: Duprat Date: Thu, 17 Mar 2022 10:16:28 +0100 Subject: [PATCH 64/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Remove trailing spaces --- Lib/asyncio/locks.py | 15 ++++++--------- Lib/test/test_asyncio/test_locks.py | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index d6fb4804f0d5e1..a747546a5a4f55 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -436,10 +436,9 @@ def __init__(self, parties, action=None): self._action = action self._parties = parties - self._state = 0 # 0 filling, 1, draining, - # -1 resetting, -2 broken - self._count = 0 # count tasks in Barrier - self._count_block = 0 # count blocking tasks + self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken + self._count = 0 # count tasks in Barrier + self._count_block = 0 # count blocking tasks def __repr__(self): res = super().__repr__() @@ -450,7 +449,7 @@ def __repr__(self): extra += f', block:{self._count_block}/{self._parties}' extra += f', state:{self._state}' return f'<{res[1:-1]} [{extra}]>' - + async def __aenter__(self): """ wait for the barrier reaches the parties number when start draining release and return index of waited task @@ -485,7 +484,7 @@ async def wait(self): self._exit() async def _block(self): - """Block until the barrier is ready for us, + """Block until the barrier is ready for us, or raise an exception if it is broken. """ self._count_block += 1 @@ -549,9 +548,7 @@ async def reset(self): """ async with self._cond: if self._count > 0: - if not self._resetting():# self._filling() - # or self._draining() - # or self.is_broken() + if not self._resetting(): #reset the barrier, waking up tasks self._set_resetting() else: diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 2cf76ea5b99304..0bcbeb9a838225 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -1012,7 +1012,7 @@ async def test_filling_one_task(self): async def f(): async with barrier as i: return True - + ret = await f() self.assertTrue(ret) @@ -1121,14 +1121,14 @@ async def coro(): async def test_blocking_tasks_while_draining(self): rewait = 2 - barrier = asyncio.Barrier(self.N) + barrier = asyncio.Barrier(self.N) barrier_nowaiting = asyncio.Barrier(self.N - rewait) results = [] rewait_n = rewait - + async def coro(): nonlocal rewait_n - + # first time waiting p = await barrier.wait() @@ -1144,7 +1144,7 @@ async def coro(): # wait for end of draining `barrier` await barrier_nowaiting.wait() - # wait for joining other waiting tasks + # wait for joining other waiting tasks await barrier.wait() await self.gather_tasks(self.N, coro) @@ -1306,7 +1306,7 @@ async def test_reset_barrier_when_tasks_half_draining(self): barrier = asyncio.Barrier(self.N) results1 = [] rest_of_tasks = self.N//2 - + async def coro(): try: await barrier.wait() @@ -1316,7 +1316,7 @@ async def coro(): else: # here drained task ouside the barrier if rest_of_tasks == barrier._count: - # 'd' tasks outside the barrier + # tasks outside the barrier await barrier.reset() await self.gather_tasks(self.N, coro) @@ -1331,7 +1331,7 @@ async def test_reset_barrier_when_tasks_half_draining_half_blocking(self): results1 = [] results2 = [] blocking_tasks = self.N//2 - + async def coro(): try: await barrier.wait() @@ -1417,7 +1417,7 @@ async def coro(): # last task exited from barrier await barrier.reset() - # wit here to reach the `parties`` + # wit here to reach the `parties` await barrier.wait() else: try: @@ -1460,7 +1460,7 @@ async def test_abort_barrier_when_tasks_half_draining_half_blocking(self): results1 = [] results2 = [] blocking_tasks = self.N//2 - + async def coro(): try: await barrier.wait() From 34cfc75f4033e1b2dba8d216f549af203587fa00 Mon Sep 17 00:00:00 2001 From: Duprat Date: Thu, 17 Mar 2022 10:43:56 +0100 Subject: [PATCH 65/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Suppress again trailing whitespaces :o --- Lib/test/test_asyncio/test_locks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 0bcbeb9a838225..f205ac32328db8 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -1305,7 +1305,7 @@ async def coro_reset(): async def test_reset_barrier_when_tasks_half_draining(self): barrier = asyncio.Barrier(self.N) results1 = [] - rest_of_tasks = self.N//2 + rest_of_tasks = self.N//2 async def coro(): try: @@ -1318,7 +1318,7 @@ async def coro(): if rest_of_tasks == barrier._count: # tasks outside the barrier await barrier.reset() - + await self.gather_tasks(self.N, coro) self.assertEqual(results1, [True]*rest_of_tasks) From c7bbebe350fb91a2271a8d70851cdb3fc316a5fd Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 18 Mar 2022 08:17:14 +0100 Subject: [PATCH 66/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 17c0225dc33a57..d26503b9ee4a1c 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -355,7 +355,7 @@ Barrier and would block until all of the tasks have made their :meth:`~Barrier.wait` calls. At this point, the tasks are released simultaneously. - The barrier can be reused any number of times for the same number of tasks. + The barrier can be reused any number of times. .. _asyncio_example_barrier: From fe20bdc8ba64f0705fdb57af7677e4df911b0fca Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 18 Mar 2022 08:18:19 +0100 Subject: [PATCH 67/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index d26503b9ee4a1c..a05f16b550432f 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -349,11 +349,11 @@ Barrier A barrier object. Not thread-safe. - This class, as a clone of :class:`threading.Barrier`, provides a simple synchronization - primitive for use by a fixed number of tasks that need to wait for each other. - Each of the tasks tries to pass the barrier by calling the :meth:`~Barrier.wait` method - and would block until all of the tasks have made their :meth:`~Barrier.wait` calls. - At this point, the tasks are released simultaneously. + A barrier is a simple synchronization primitive that allows to block until a + certain number of tasks are waiting on it. + Tasks can wait on the :meth:`~Barrier.wait` method and would be blocked until + the specified number of tasks end up waiting on :meth:`~Barrier.wait`. + At that point all of the waiting tasks would unblock simultaneously. The barrier can be reused any number of times. From edfd7d4fdaf9c4de21f3ac722966d50a1a000ea5 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 18 Mar 2022 08:18:43 +0100 Subject: [PATCH 68/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index a05f16b550432f..9c65807ce1331b 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -427,7 +427,7 @@ Barrier .. coroutinemethod:: wait() Pass the barrier. When all the tasks party to the barrier have called - this function, they are all released simultaneously. + this function, they are all unblocked simultaneously. When a task in the barrier is cancelled, two cases are possible: From 27e176802c07917d1bd0b7023c162b00ec34584a Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 18 Mar 2022 16:03:17 +0100 Subject: [PATCH 69/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Proposition of a new example Trailing whitespaces Trim to 79 some lines --- Doc/library/asyncio-sync.rst | 91 ++++++++++++++---------------------- 1 file changed, 35 insertions(+), 56 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 9c65807ce1331b..9e99e5f286a307 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -350,8 +350,8 @@ Barrier A barrier object. Not thread-safe. A barrier is a simple synchronization primitive that allows to block until a - certain number of tasks are waiting on it. - Tasks can wait on the :meth:`~Barrier.wait` method and would be blocked until + certain number of tasks are waiting on it. + Tasks can wait on the :meth:`~Barrier.wait` method and would be blocked until the specified number of tasks end up waiting on :meth:`~Barrier.wait`. At that point all of the waiting tasks would unblock simultaneously. @@ -361,68 +361,45 @@ Barrier Example:: - import asyncio - import os - import random + async def example_barrier(): + # action when barrier will unblock + async def done(): + print("calling action 'done'") - DELAYS = (1.0, 4.0, 2.0, 0.5) - NB = 10 + # barrier with 3 parties + b = asyncio.Barrier(3, done) - async def job_done(): - print(f'{os.linesep}{"All files removed":-^80s}{os.linesep}') + # create 2 new waiting tasks + asyncio.create_task(b.wait()) + asyncio.create_task(b.wait()) - async def remove_file(barrier, filename): - print(f'Remove {filename!r} ...') + await asyncio.sleep(0) + print(b) - # remove filename (simulation with 'random value' secs sleeping) - await asyncio.sleep(random.choice(DELAYS)) - print(f'\tFile {filename!r} is removed') + # Now, current task waits + async with b as pos: + print("barrier passed") - # wait for other files - pos = await barrier.wait() + await asyncio.sleep(0) + print(b) - return pos + Result of this example is: - async def remove_folder(barrier, folder): - print(f'Waiting for remove all files from folder {folder!r} ...') + >>> asyncio.run(example_barrier()) + + calling action 'done' + barrier passed + - # wait for all removed files - async with barrier as pos: - # remove folder (simulation with 1 sec sleeping) - await asyncio.sleep(1.0) - print(f'Folder {folder!r} is now removed ...') - - return pos - - async def main(folder, files, nb_files): - # Create a Barrier object with parties equal to nb_files+1 - # corresponding to the number of all removing items plus the folder, - # and an action as a print message. - barrier = asyncio.Barrier(nb_files+1, action=job_done) - - # Add remove_file task for each filename. - tasks = [asyncio.create_task(remove_file(barrier, f)) for f in files] - - # Add remove_folder task for the folder. - tasks.append(asyncio.create_task(remove_folder(barrier, folder))) - - # Wait until folder+filenames are removed. - await asyncio.gather(*tasks) - - asyncio.run(main("MyFolder", [f'file{i+1:02d}.txt' for i in range(NB)], NB)) - - In the previous example, there are the two ways to call the :meth:`~Barrier.wait` method: - - * with a **classic call**: see ``pos = barrier.wait()`` in the 'remove_file' coroutine; - - * with a **async context manager call**: see ``async with barrier as pos:`` in the 'remove_folder' coroutine. + As shown in this example, the :meth:`~Barrier.wait` method: can also be call with + :keyword:`async with` statement. The barrier can be reused any number of times for the same number of tasks. .. class:: Barrier(parties, action=None) - Create a barrier object for *parties* number of tasks. A coroutine *action*, when - provided, is awaited once just before the release phasis (draining state). + Create a barrier object for *parties* number of tasks. A coroutine *action*, + when provided, is awaited once just before the release phasis (draining state). .. coroutinemethod:: wait() @@ -448,13 +425,15 @@ Barrier # Only one task print this print('End of *draining phasis*') - If an *action* was provided as a coroutine to the constructor, the last calling task of - :meth:`wait` method will have to *await this coroutine* prior to being released. - Should this call raise an error, including cancellation of task, the barrier is put into the broken state. + If an *action* was provided as a coroutine to the constructor, the last + calling task of :meth:`wait` method will have to *await this coroutine* + prior to being released. + Should this call raise an error, including cancellation of task, + the barrier is put into the broken state. This method may raise a :class:`BrokenBarrierError` exception if the - barrier is broken or reset while a task is waiting. It could raise a :exc:`CancelledError` - if a task is cancelled. + barrier is broken or reset while a task is waiting. + It could raise a :exc:`CancelledError` if a task is cancelled. .. coroutinemethod:: reset() From 892f389ded87431344b8ea5400bad0f6bffc134b Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 18 Mar 2022 18:00:00 +0100 Subject: [PATCH 70/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add messages in the BrokenBarrierError Refactoring in the empty `except:` block in `of `_release()` (Set list of exceptions, move except block) Rewrite try/except block of `_block()` --- Lib/asyncio/locks.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index a747546a5a4f55..a2653beea2aca5 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -491,16 +491,18 @@ async def _block(self): # It is draining or resetting, wait until done # unless a CancelledError occurs try: - await self._cond.wait_for(lambda: not (self._draining() or - self._resetting())) + try: + await self._cond.wait_for(lambda: not (self._draining() or + self._resetting())) + finally: + # block always will call, even an exception occurs + self._count_block -= 1 except exceptions.CancelledError: - self._count_block -= 1 raise - self._count_block -= 1 # see if the barrier is in a broken state if self.broken: - raise BrokenBarrierError + raise BrokenBarrierError("Barrier aborted") # Optionally run the 'action' and release the tasks waiting # in the barrier. @@ -511,16 +513,17 @@ async def _release(self): try: if self._action: await self._action() - # enter draining state - self._set_draining() - self._cond.notify_all() - except: + except (Exception, exceptions.CancelledError): # an exception occurs during the _action coroutine, # or the last calling task cancels # Break and reraise self._break() raise + # enter draining state + self._set_draining() + self._cond.notify_all() + async def _wait(self): """Wait in the barrier until we are released. Raise an exception if the barrier is reset or broken. @@ -530,7 +533,7 @@ async def _wait(self): await self._cond.wait_for(lambda: not self._filling()) if self.broken or self._resetting(): - raise BrokenBarrierError + raise BrokenBarrierError("Abort or reset of barrier") def _exit(self): """If we are the last tasks to exit the barrier, signal any tasks From 5b8962a3dadd4aed6c043356f8e802fc12dd7281 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 21 Mar 2022 15:41:22 +0100 Subject: [PATCH 71/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903)Update Doc/library/asyncio-sync.rst Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 9e99e5f286a307..8b14c67407cef1 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -411,7 +411,7 @@ Barrier * the task is the last waiting task, an :class:`BrokenBarrierError` exception occurs, barrier is put into a broken state; - * the task is waiting (filling state) or blocked (draining state), + * if the task is waiting (filling state) or blocked (draining state), this task exits the barrier which stays in the same state. If the state is filling, count of waiting task decreases by 1. From 63e0688445066efa98a50ba926ce3a8df3c8ad14 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 21 Mar 2022 15:41:46 +0100 Subject: [PATCH 72/93] Update Doc/library/asybpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903)ncio-sync.rst Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 8b14c67407cef1..692504313a0664 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -413,7 +413,7 @@ Barrier * if the task is waiting (filling state) or blocked (draining state), this task exits the barrier which stays in the same state. - If the state is filling, count of waiting task decreases by 1. + If the state of the barrier is "filling", the number of waiting task decreases by 1. The return value is an integer in the range 0 to *parties-1*, different for each task. This can be used to select a task to do some special From eb422466efa92eb5d1aaf98169519ca46dfdc61e Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 21 Mar 2022 15:42:22 +0100 Subject: [PATCH 73/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903)Update Doc/library/asyncio-sync.rst Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 692504313a0664..ab4784d54eb751 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -415,7 +415,7 @@ Barrier this task exits the barrier which stays in the same state. If the state of the barrier is "filling", the number of waiting task decreases by 1. - The return value is an integer in the range 0 to *parties-1*, different + The return value is an integer in the range of 0 to ``parties-1``, different for each task. This can be used to select a task to do some special housekeeping, e.g.:: From e29d0c0e98f001255b4c242da564d72e2d5d9847 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 21 Mar 2022 15:42:40 +0100 Subject: [PATCH 74/93] Update Doc/library/asyncio-sync.rstbpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index ab4784d54eb751..51c2db4bb3ced8 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -409,7 +409,7 @@ Barrier When a task in the barrier is cancelled, two cases are possible: * the task is the last waiting task, an :class:`BrokenBarrierError` - exception occurs, barrier is put into a broken state; + exception is raised. The barrier is put into a broken state; * if the task is waiting (filling state) or blocked (draining state), this task exits the barrier which stays in the same state. From 6857cf284bb0ff961268422bbc118159ac2a1dae Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 21 Mar 2022 15:43:02 +0100 Subject: [PATCH 75/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 51c2db4bb3ced8..83c41e429e2457 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -399,7 +399,7 @@ Barrier .. class:: Barrier(parties, action=None) Create a barrier object for *parties* number of tasks. A coroutine *action*, - when provided, is awaited once just before the release phasis (draining state). + when provided, is awaited once just before the release occurs (draining state). .. coroutinemethod:: wait() From d4d5b5589f2b71fb2a4791f977ceb71672c47e48 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 21 Mar 2022 15:43:38 +0100 Subject: [PATCH 76/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 83c41e429e2457..af675f085afbab 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -408,7 +408,7 @@ Barrier When a task in the barrier is cancelled, two cases are possible: - * the task is the last waiting task, an :class:`BrokenBarrierError` + * if the task is the last waiting task, a :class:`BrokenBarrierError` exception is raised. The barrier is put into a broken state; * if the task is waiting (filling state) or blocked (draining state), From 5bb6d7e02fadbada82847dabed86bbf90e36dde2 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 21 Mar 2022 15:44:00 +0100 Subject: [PATCH 77/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Co-authored-by: Yury Selivanov --- Doc/library/asyncio-sync.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index af675f085afbab..5b9108c6157e3b 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -391,8 +391,8 @@ Barrier barrier passed - As shown in this example, the :meth:`~Barrier.wait` method: can also be call with - :keyword:`async with` statement. + The example also demonstrates using `async with` as + an alternative to awaiting on ``barrier.wait()``. The barrier can be reused any number of times for the same number of tasks. From 66f023a84af9bd60bb74b9ab37cbc55ae7a769b5 Mon Sep 17 00:00:00 2001 From: Duprat Date: Mon, 21 Mar 2022 17:08:54 +0100 Subject: [PATCH 78/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Suppress duplicated lines Suppress trailing whitespaces --- Doc/library/asyncio-sync.rst | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 5b9108c6157e3b..f95fd219df4940 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -367,7 +367,7 @@ Barrier print("calling action 'done'") # barrier with 3 parties - b = asyncio.Barrier(3, done) + b = asyncio.Barrier(3, done()) # create 2 new waiting tasks asyncio.create_task(b.wait()) @@ -391,11 +391,9 @@ Barrier barrier passed - The example also demonstrates using `async with` as + The example also demonstrates using `async with` as an alternative to awaiting on ``barrier.wait()``. - The barrier can be reused any number of times for the same number of tasks. - .. class:: Barrier(parties, action=None) Create a barrier object for *parties* number of tasks. A coroutine *action*, From e5d229cdc7fb519a74ca1b89843f5367b8ba070e Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 22 Mar 2022 15:35:40 +0100 Subject: [PATCH 79/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Last corrections from @1st1 Modify type of action: coroutine to coroutinefunction --- Doc/library/asyncio-sync.rst | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index f95fd219df4940..1878cb5b93627a 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -367,7 +367,7 @@ Barrier print("calling action 'done'") # barrier with 3 parties - b = asyncio.Barrier(3, done()) + b = asyncio.Barrier(3, done) # create 2 new waiting tasks asyncio.create_task(b.wait()) @@ -391,17 +391,21 @@ Barrier barrier passed - The example also demonstrates using `async with` as - an alternative to awaiting on ``barrier.wait()``. + The example also demonstrates using ``async with`` as an alternative to awaiting + on ``barrier.wait()``. .. class:: Barrier(parties, action=None) - Create a barrier object for *parties* number of tasks. A coroutine *action*, + Create a barrier object for *parties* number of tasks. A coroutinefunction *action*, when provided, is awaited once just before the release occurs (draining state). + Here this coroutinefunction is callable that returns an awaitable. + If the coroutinefunction refers to a coroutine with parameters, you should + use ``functools.partial`` function. + .. coroutinemethod:: wait() - Pass the barrier. When all the tasks party to the barrier have called + Pass the barrier. When all the tasks party to the barrier have called this function, they are all unblocked simultaneously. When a task in the barrier is cancelled, two cases are possible: @@ -411,7 +415,8 @@ Barrier * if the task is waiting (filling state) or blocked (draining state), this task exits the barrier which stays in the same state. - If the state of the barrier is "filling", the number of waiting task decreases by 1. + If the state of the barrier is "filling", the number of waiting task + decreases by 1. The return value is an integer in the range of 0 to ``parties-1``, different for each task. This can be used to select a task to do some special @@ -423,10 +428,9 @@ Barrier # Only one task print this print('End of *draining phasis*') - If an *action* was provided as a coroutine to the constructor, the last + If an *action* was provided as a coroutinefunction to the constructor, the last calling task of :meth:`wait` method will have to *await this coroutine* - prior to being released. - Should this call raise an error, including cancellation of task, + prior to being released. Should this call raise an error, including cancellation of task, the barrier is put into the broken state. This method may raise a :class:`BrokenBarrierError` exception if the From b0d17daac6567b3f480bdf03b1074760dbc94314 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 22 Mar 2022 15:37:29 +0100 Subject: [PATCH 80/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Modifiy action type from coroutine to coroutinefunction Add 2 dedicated tests --- Lib/asyncio/locks.py | 5 ++- Lib/test/test_asyncio/test_locks.py | 58 +++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index a2653beea2aca5..5eb17e2d3bf3ec 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -5,6 +5,7 @@ import collections +from . import coroutines from . import exceptions from . import mixins @@ -417,7 +418,7 @@ def release(self): class Barrier(mixins._LoopBoundMixin): """Asyncio equivalent to threading.Barrier - Implements a Barrier. + Implements a Barrier primitive. Useful for synchronizing a fixed number of tasks at known synchronization points. Tasks block on 'wait()' and are simultaneously awoken once they have all made their call. @@ -431,6 +432,8 @@ def __init__(self, parties, action=None): """ if parties < 1: raise ValueError('parties must be > 0') + if action and not coroutines.iscoroutinefunction(action): + raise TypeError(f"a coroutinefunction was expected for 'action', got {action!r}") self._cond = Condition() # notify all tasks when state changes diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index f205ac32328db8..2cfb03150643b9 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -991,6 +991,42 @@ async def test_barrier_parties(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) + async def test_barrier_action(self): + def bar(): + pass + async def coro_noargs(): + await asyncio.sleep(0) + async def coro(n): + await asyncio.sleep(n) + + with self.assertRaisesRegex( + TypeError, + "a coroutinefunction was expected for 'action', got Date: Tue, 22 Mar 2022 19:30:09 +0100 Subject: [PATCH 81/93] po-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Fix example --- Doc/library/asyncio-sync.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 1878cb5b93627a..35e270234189eb 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -383,9 +383,10 @@ Barrier await asyncio.sleep(0) print(b) - Result of this example is: + asyncio.run(example_barrier()) + + Result of this example is:: - >>> asyncio.run(example_barrier()) calling action 'done' barrier passed From 151ca5d48745b112d28f7ef041b84e77801b5a70 Mon Sep 17 00:00:00 2001 From: Duprat Date: Tue, 22 Mar 2022 19:31:14 +0100 Subject: [PATCH 82/93] po-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Add a test to check multiple reuses of a barrier object. --- Lib/test/test_asyncio/test_locks.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 2cfb03150643b9..b7928424407eaf 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -1246,6 +1246,28 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) + async def test_draining_check_action_more(self): + async def action_task(): + results1.append(True) + barrier = asyncio.Barrier(self.N, action=action_task) + results = [] + results1 = [] + async def coro(): + ret = await barrier.wait() + results.append(True) + + n = 3 + for _ in range(n): + await self.gather_tasks(self.N, coro) + + self.assertEqual(len(results1), n) + self.assertTrue(all(results1)) + self.assertEqual(len(results), self.N*n) + self.assertTrue(all(results)) + + self.assertEqual(barrier.n_waiting, 0) + self.assertFalse(barrier.broken) + async def test_draining_check_action_with_partial(self): import functools async def action_task(n, *args): From 37efb91eb6de4b3d51a59aeb0fd9ce54f1dd556e Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 23 Mar 2022 17:51:26 +0100 Subject: [PATCH 83/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Remove `action` paramater --- Doc/library/asyncio-sync.rst | 30 +---- Lib/asyncio/locks.py | 35 ++---- Lib/test/test_asyncio/test_locks.py | 167 +--------------------------- 3 files changed, 18 insertions(+), 214 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 35e270234189eb..60f6331d13bfd9 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -362,10 +362,6 @@ Barrier Example:: async def example_barrier(): - # action when barrier will unblock - async def done(): - print("calling action 'done'") - # barrier with 3 parties b = asyncio.Barrier(3, done) @@ -388,7 +384,6 @@ Barrier Result of this example is:: - calling action 'done' barrier passed @@ -397,27 +392,17 @@ Barrier .. class:: Barrier(parties, action=None) - Create a barrier object for *parties* number of tasks. A coroutinefunction *action*, - when provided, is awaited once just before the release occurs (draining state). - Here this coroutinefunction is callable that returns an awaitable. - If the coroutinefunction refers to a coroutine with parameters, you should - use ``functools.partial`` function. - + Create a barrier object for *parties* number of tasks. .. coroutinemethod:: wait() Pass the barrier. When all the tasks party to the barrier have called this function, they are all unblocked simultaneously. - When a task in the barrier is cancelled, two cases are possible: - - * if the task is the last waiting task, a :class:`BrokenBarrierError` - exception is raised. The barrier is put into a broken state; - - * if the task is waiting (filling state) or blocked (draining state), - this task exits the barrier which stays in the same state. - If the state of the barrier is "filling", the number of waiting task - decreases by 1. + When a waiting or blocked task in the barrier is cancelled, + this task exits the barrier which stays in the same state. + If the state of the barrier is "filling", the number of waiting task + decreases by 1. The return value is an integer in the range of 0 to ``parties-1``, different for each task. This can be used to select a task to do some special @@ -429,11 +414,6 @@ Barrier # Only one task print this print('End of *draining phasis*') - If an *action* was provided as a coroutinefunction to the constructor, the last - calling task of :meth:`wait` method will have to *await this coroutine* - prior to being released. Should this call raise an error, including cancellation of task, - the barrier is put into the broken state. - This method may raise a :class:`BrokenBarrierError` exception if the barrier is broken or reset while a task is waiting. It could raise a :exc:`CancelledError` if a task is cancelled. diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 5eb17e2d3bf3ec..46f922121ff66c 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -424,24 +424,18 @@ class Barrier(mixins._LoopBoundMixin): have all made their call. """ - def __init__(self, parties, action=None): + def __init__(self, parties): """Create a barrier, initialised to 'parties' tasks. - 'action' is a coroutine which, when supplied, will be called by - the last task calling the wait() method, - just prior to releasing them all. """ if parties < 1: raise ValueError('parties must be > 0') - if action and not coroutines.iscoroutinefunction(action): - raise TypeError(f"a coroutinefunction was expected for 'action', got {action!r}") self._cond = Condition() # notify all tasks when state changes - self._action = action self._parties = parties self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken self._count = 0 # count tasks in Barrier - self._count_block = 0 # count blocking tasks + self._count_block = 0 # count blocked tasks when draining def __repr__(self): res = super().__repr__() @@ -465,9 +459,7 @@ async def __aexit__(self, *args): async def wait(self): """Wait for the barrier. When the specified number of tasks have started waiting, they are all - simultaneously awoken. If an 'action' was provided for the barrier, the - last task calling this method will have executed that callback prior to - returning. + simultaneously awoken. Returns an unique and individual index number from 0 to 'parties-1'. # """ async with self._cond: @@ -507,28 +499,17 @@ async def _block(self): if self.broken: raise BrokenBarrierError("Barrier aborted") - # Optionally run the 'action' and release the tasks waiting - # in the barrier. + # Release the tasks waiting in the barrier. async def _release(self): - """Optionally run the 'action' and release the tasks waiting - in the barrier. + """Enter draining state. Next waiting tasks will be blocked + until the end of draining. """ - try: - if self._action: - await self._action() - except (Exception, exceptions.CancelledError): - # an exception occurs during the _action coroutine, - # or the last calling task cancels - # Break and reraise - self._break() - raise - # enter draining state self._set_draining() self._cond.notify_all() async def _wait(self): - """Wait in the barrier until we are released. Raise an exception + """Wait in the barrier until we are released. Raise an exception if the barrier is reset or broken. """ # wait for end of filling @@ -570,7 +551,7 @@ async def abort(self): self._break() def _break(self): - # An internal error was detected. The barrier is set to + # An internal error was detected. The barrier is set to # a broken state all parties awakened. self._set_broken() self._cond.notify_all() diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index b7928424407eaf..9facb00e4cc683 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -991,41 +991,7 @@ async def test_barrier_parties(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) - async def test_barrier_action(self): - def bar(): - pass - async def coro_noargs(): - await asyncio.sleep(0) - async def coro(n): - await asyncio.sleep(n) - - with self.assertRaisesRegex( - TypeError, - "a coroutinefunction was expected for 'action', got 0: @@ -1174,13 +1140,11 @@ async def coro(): # wait again only for rewait tasks await barrier.wait() else: - # task here does not wait so it is blocking (drainig state) + # task here does not wait so it is blocked (drainig state) results.append(barrier._count_block) - - # wait for end of draining `barrier` + # wait for end of draining state` await barrier_nowaiting.wait() - - # wait for joining other waiting tasks + # wait for other waiting tasks await barrier.wait() await self.gather_tasks(self.N, coro) @@ -1225,127 +1189,6 @@ async def coro(): self.assertEqual(barrier.n_waiting, 0) self.assertFalse(barrier.broken) - async def test_draining_check_action(self): - async def action_task(): - results1.append(True) - barrier = asyncio.Barrier(self.N, action=action_task) - results = [] - results1 = [] - - async def coro(): - ret = await barrier.wait() - results.append(True) - - await self.gather_tasks(self.N, coro) - - self.assertEqual(len(results1), 1) - self.assertTrue(results1[0]) - self.assertEqual(len(results), self.N) - self.assertTrue(all(results)) - - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - async def test_draining_check_action_more(self): - async def action_task(): - results1.append(True) - barrier = asyncio.Barrier(self.N, action=action_task) - results = [] - results1 = [] - async def coro(): - ret = await barrier.wait() - results.append(True) - - n = 3 - for _ in range(n): - await self.gather_tasks(self.N, coro) - - self.assertEqual(len(results1), n) - self.assertTrue(all(results1)) - self.assertEqual(len(results), self.N*n) - self.assertTrue(all(results)) - - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - async def test_draining_check_action_with_partial(self): - import functools - async def action_task(n, *args): - results1.append(True) - barrier = asyncio.Barrier(self.N, action=functools.partial(action_task, 10, 1, 2, 3)) - results = [] - results1 = [] - - async def coro(): - ret = await barrier.wait() - results.append(True) - - await self.gather_tasks(self.N, coro) - - self.assertEqual(len(results1), 1) - self.assertTrue(results1[0]) - self.assertEqual(len(results), self.N) - self.assertTrue(all(results)) - - self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier.broken) - - async def test_draining_check_cancelled_action(self): - async def action_coro(): - asyncio.current_task().cancel() - await asyncio.sleep(0) - - barrier = asyncio.Barrier(self.N, action=action_coro) - results = [] - results1 = [] - results2 = [] - - async def coro(): - try: - await barrier.wait() - results.append(True) - except asyncio.CancelledError: - results1.append(True) - except asyncio.BrokenBarrierError: - results2.append(True) - - await self.gather_tasks(self.N, coro) - - self.assertEqual(len(results), 0) - self.assertEqual(results1, [True]) - self.assertEqual(len(results2), self.N-1) - self.assertTrue(all(results2)) - - self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(barrier.broken) - - async def test_draining_check_error_on_action(self): - ERROR = ZeroDivisionError - async def raise_except(): - await asyncio.sleep(0) - raise ERROR - - barrier = asyncio.Barrier(self.N, action=raise_except) - results1 = [] - results2 = [] - - async def coro(): - try: - await barrier.wait() - except ERROR: - results1.append(False) - except asyncio.BrokenBarrierError: - results2.append(True) - - await self.gather_tasks(self.N, coro) - - self.assertEqual(results1, [False]) - self.assertEqual(len(results2), self.N-1) - self.assertTrue(all(results2)) - - self.assertEqual(barrier.n_waiting, 0) - self.assertTrue(barrier.broken) - async def test_reset_barrier(self): barrier = asyncio.Barrier(1) From 337189e88d6ceb9544b441b5bbe63016501760c6 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 23 Mar 2022 21:53:48 +0100 Subject: [PATCH 84/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Remove `import coroutines` Suppress trailing whitespaces --- Lib/asyncio/locks.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 46f922121ff66c..37a46b1fa4016c 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -5,7 +5,6 @@ import collections -from . import coroutines from . import exceptions from . import mixins @@ -420,7 +419,7 @@ class Barrier(mixins._LoopBoundMixin): Implements a Barrier primitive. Useful for synchronizing a fixed number of tasks at known synchronization - points. Tasks block on 'wait()' and are simultaneously awoken once they + points. Tasks block on 'wait()' and are simultaneously awoken once they have all made their call. """ @@ -461,7 +460,7 @@ async def wait(self): When the specified number of tasks have started waiting, they are all simultaneously awoken. Returns an unique and individual index number from 0 to 'parties-1'. - # """ + """ async with self._cond: await self._block() # Block while the barrier drains or resets. try: @@ -501,7 +500,7 @@ async def _block(self): # Release the tasks waiting in the barrier. async def _release(self): - """Enter draining state. Next waiting tasks will be blocked + """Enter draining state. Next waiting tasks will be blocked until the end of draining. """ # enter draining state From ee59e3ab6889214bcb004c54e16a2462ac3f4215 Mon Sep 17 00:00:00 2001 From: Duprat Date: Wed, 23 Mar 2022 21:54:15 +0100 Subject: [PATCH 85/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) minor correction --- Lib/test/test_asyncio/test_locks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 9facb00e4cc683..a0f82c6431c939 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -1,4 +1,4 @@ -"""Tests for lock.py""" +"""Tests for locks.py""" import unittest from unittest import mock From df5bd42e87e939ab0026cc6ae0e2b099e1a37f05 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 24 Mar 2022 19:53:13 +0200 Subject: [PATCH 86/93] Rewrite repr(), use enum for state, drop _count_blocked used by tests only --- Lib/asyncio/locks.py | 125 +++++++++++----------------- Lib/test/test_asyncio/test_locks.py | 101 ++++++++++++---------- 2 files changed, 106 insertions(+), 120 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 37a46b1fa4016c..23db2968d4279a 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -4,6 +4,7 @@ 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') import collections +import enum from . import exceptions from . import mixins @@ -414,6 +415,14 @@ def release(self): super().release() + +class _BarrierState(enum.Enum): + FILLING = 'filling' + DRAINING = 'draining' + RESETTING = 'resetting' + BROKEN = 'broken' + + class Barrier(mixins._LoopBoundMixin): """Asyncio equivalent to threading.Barrier @@ -424,32 +433,26 @@ class Barrier(mixins._LoopBoundMixin): """ def __init__(self, parties): - """Create a barrier, initialised to 'parties' tasks. - """ + """Create a barrier, initialised to 'parties' tasks.""" if parties < 1: raise ValueError('parties must be > 0') self._cond = Condition() # notify all tasks when state changes self._parties = parties - self._state = 0 # 0 filling, 1, draining, -1 resetting, -2 broken + self._state = _BarrierState.FILLING self._count = 0 # count tasks in Barrier - self._count_block = 0 # count blocked tasks when draining def __repr__(self): res = super().__repr__() - extra = 'locked' if self._cond.locked() else 'unlocked' - if self._count: - extra += f', wait:{self._count}/{self._parties}' - if self._count_block: - extra += f', block:{self._count_block}/{self._parties}' - extra += f', state:{self._state}' + extra = f'{self._state.value}' + if not self.broken: + extra += f', waiters:{self.n_waiting}/{self.parties}' return f'<{res[1:-1]} [{extra}]>' async def __aenter__(self): - """ wait for the barrier reaches the parties number - when start draining release and return index of waited task - """ + # wait for the barrier reaches the parties number + # when start draining release and return index of waited task return await self.wait() async def __aexit__(self, *args): @@ -457,6 +460,7 @@ async def __aexit__(self, *args): async def wait(self): """Wait for the barrier. + When the specified number of tasks have started waiting, they are all simultaneously awoken. Returns an unique and individual index number from 0 to 'parties-1'. @@ -478,71 +482,66 @@ async def wait(self): self._exit() async def _block(self): - """Block until the barrier is ready for us, - or raise an exception if it is broken. - """ - self._count_block += 1 + # Block until the barrier is ready for us, + # or raise an exception if it is broken. + # # It is draining or resetting, wait until done # unless a CancelledError occurs - try: - try: - await self._cond.wait_for(lambda: not (self._draining() or - self._resetting())) - finally: - # block always will call, even an exception occurs - self._count_block -= 1 - except exceptions.CancelledError: - raise + await self._cond.wait_for( + lambda: self._state not in ( + _BarrierState.DRAINING, _BarrierState.RESETTING + ) + ) # see if the barrier is in a broken state - if self.broken: + if self._state is _BarrierState.BROKEN: raise BrokenBarrierError("Barrier aborted") - # Release the tasks waiting in the barrier. async def _release(self): - """Enter draining state. Next waiting tasks will be blocked - until the end of draining. - """ - # enter draining state - self._set_draining() + # Release the tasks waiting in the barrier. + + # Enter draining state. + # Next waiting tasks will be blocked until the end of draining. + self._state = _BarrierState.DRAINING self._cond.notify_all() async def _wait(self): - """Wait in the barrier until we are released. Raise an exception - if the barrier is reset or broken. - """ + # Wait in the barrier until we are released. Raise an exception + # if the barrier is reset or broken. + # wait for end of filling # unless a CancelledError occurs - await self._cond.wait_for(lambda: not self._filling()) + await self._cond.wait_for(lambda: self._state is not _BarrierState.FILLING) - if self.broken or self._resetting(): + if self._state in (_BarrierState.BROKEN, _BarrierState.RESETTING): raise BrokenBarrierError("Abort or reset of barrier") def _exit(self): - """If we are the last tasks to exit the barrier, signal any tasks - waiting for the barrier to drain. - """ + # If we are the last tasks to exit the barrier, signal any tasks + # waiting for the barrier to drain. if self._count == 0: - if self._resetting() or self._draining(): - self._set_filling() + if self._state in (_BarrierState.RESETTING, _BarrierState.DRAINING): + self._state = _BarrierState.FILLING self._cond.notify_all() async def reset(self): """Reset the barrier to the initial state. + Any tasks currently waiting will get the BrokenBarrier exception raised. """ async with self._cond: if self._count > 0: - if not self._resetting(): + if self._state is not _BarrierState.RESETTING: #reset the barrier, waking up tasks - self._set_resetting() + self._state = _BarrierState.RESETTING else: - self._set_filling() + self._state = _BarrierState.FILLING self._cond.notify_all() async def abort(self): """Place the barrier into a 'broken' state. + Useful in case of error. Any currently waiting tasks and tasks attempting to 'wait()' will have BrokenBarrierError raised. """ @@ -552,7 +551,7 @@ async def abort(self): def _break(self): # An internal error was detected. The barrier is set to # a broken state all parties awakened. - self._set_broken() + self._state = _BarrierState.BROKEN self._cond.notify_all() @property @@ -563,42 +562,14 @@ def parties(self): @property def n_waiting(self): """Return the number of tasks currently waiting at the barrier.""" - if self._filling(): + if self._state is _BarrierState.FILLING: return self._count return 0 @property def broken(self): """Return True if the barrier is in a broken state.""" - return self._state == -2 - - def _draining(self): - """Return True if the barrier is draining.""" - return self._state == 1 - - def _filling(self): - """Return True if the barrier is filling.""" - return self._state == 0 - - def _resetting(self): - """Return True if the barrier is resetting.""" - return self._state == -1 - - def _set_broken(self): - """Set state to broken.""" - self._state = -2 - - def _set_draining(self): - """Set state to draining.""" - self._state = 1 - - def _set_filling(self): - """Set state to filling.""" - self._state = 0 - - def _set_resetting(self): - """Set state to resetting.""" - self._state = -1 + return self._state is _BarrierState.BROKEN # exception raised by the Barrier class diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index a0f82c6431c939..7cc2080fbc5eb9 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -9,8 +9,10 @@ STR_RGX_REPR = ( r'^<(?P.*?) object at (?P
.*?)' r'\[(?P' - r'(set|unset|locked|unlocked)(, value:\d)?(, waiters:\d+)?' - r'(, wait:\d+\/\d+)?(, block:\d+\/\d+)?(, state:(-2|-1|0|1))?' # barrier + r'(set|unset|locked|unlocked|filling|draining|resetting|broken)' + r'(, value:\d)?' + r'(, waiters:\d+)?' + r'(, waiters:\d+\/\d+)?' # barrier r')\]>\Z' ) RGX_REPR = re.compile(STR_RGX_REPR) @@ -936,63 +938,72 @@ async def gather_tasks(self, n, coro): async def test_barrier(self): barrier = asyncio.Barrier(self.N) - self.assertTrue(barrier._filling()) + self.assertIn("filling", repr(barrier)) with self.assertRaisesRegex( TypeError, "object Barrier can't be used in 'await' expression", ): await barrier - self.assertTrue(barrier._filling()) + self.assertIn("filling", repr(barrier)) async def test_repr(self): barrier = asyncio.Barrier(self.N) self.assertTrue(RGX_REPR.match(repr(barrier))) + self.assertIn("filling", repr(barrier)) + + waiters = [] + async def wait(barrier): + await barrier.wait() + incr = 2 - barrier._count += incr - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(f"wait:{incr}/{self.N}" in repr(barrier)) - self.assertTrue(f"block" not in repr(barrier)) + for i in range(incr): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) - barrier._set_filling() self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(repr(barrier).endswith('state:0]>')) + self.assertTrue(f"waiters:{incr}/{self.N}" in repr(barrier)) + self.assertIn("filling", repr(barrier)) - barrier._count_block += incr - barrier._count -= incr - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(f"block:{incr}/{self.N}" in repr(barrier)) - self.assertTrue(f"wait" not in repr(barrier)) + # create missing waiters + for i in range(barrier.parties - barrier.n_waiting): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) - barrier._set_draining() self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(repr(barrier).endswith('state:1]>')) + self.assertIn("draining", repr(barrier)) - barrier._set_resetting() - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(repr(barrier).endswith('state:-1]>')) + # add a part of waiters + for i in range(incr): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + # and reset + await barrier.reset() - barrier._set_broken() self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue(repr(barrier).endswith('state:-2]>')) + self.assertIn("resetting", repr(barrier)) + + # add a part of waiters again + for i in range(incr): + waiters.append(asyncio.create_task(wait(barrier))) + await asyncio.sleep(0) + # and abort + await barrier.abort() - barrier = asyncio.Barrier(self.N) self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue('unlocked' in repr(barrier)) + self.assertIn("broken", repr(barrier)) + self.assertTrue(barrier.broken) - async with barrier._cond: - self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertTrue('locked' in repr(barrier)) - self.assertTrue('unlocked' in repr(barrier)) + # suppress unhandled exceptions + await asyncio.gather(*waiters, return_exceptions=True) async def test_barrier_parties(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(0)) self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) self.assertIsInstance(asyncio.Barrier(self.N), asyncio.Barrier) - async def test_context_manager(self): self.N = 3 barrier = asyncio.Barrier(self.N) @@ -1044,12 +1055,12 @@ async def test_filling_task_by_task(self): t1 = asyncio.create_task(barrier.wait()) await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 1) - self.assertTrue(barrier._filling()) + self.assertIn("filling", repr(barrier)) t2 = asyncio.create_task(barrier.wait()) await asyncio.sleep(0) self.assertEqual(barrier.n_waiting, 2) - self.assertTrue(barrier._filling()) + self.assertIn("filling", repr(barrier)) t3 = asyncio.create_task(barrier.wait()) await asyncio.sleep(0) @@ -1110,7 +1121,7 @@ async def test_draining_state(self): async def coro(): async with barrier: # barrier state change to filling for the last task release - results.append(barrier._draining()) + results.append("draining" in repr(barrier)) await self.gather_tasks(self.N, coro) @@ -1127,6 +1138,7 @@ async def test_blocking_tasks_while_draining(self): barrier_nowaiting = asyncio.Barrier(self.N - rewait) results = [] rewait_n = rewait + counter = 0 async def coro(): nonlocal rewait_n @@ -1140,18 +1152,15 @@ async def coro(): # wait again only for rewait tasks await barrier.wait() else: - # task here does not wait so it is blocked (drainig state) - results.append(barrier._count_block) # wait for end of draining state` await barrier_nowaiting.wait() # wait for other waiting tasks await barrier.wait() + # a success means that barrier_nowaiting + # was waited for exactly N-rewait=3 times await self.gather_tasks(self.N, coro) - self.assertEqual(len(results), self.N - rewait) - self.assertTrue(all(r == rewait for r in results)) - async def test_filling_tasks_cancel_one(self): self.N = 3 barrier = asyncio.Barrier(self.N) @@ -1222,7 +1231,7 @@ async def coro_reset(): self.assertEqual(len(results), self.N-1) self.assertTrue(all(results)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._resetting()) + self.assertNotIn("resetting", repr(barrier)) self.assertFalse(barrier.broken) async def test_reset_barrier_when_tasks_half_draining(self): @@ -1246,7 +1255,7 @@ async def coro(): self.assertEqual(results1, [True]*rest_of_tasks) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._resetting()) + self.assertNotIn("resetting", repr(barrier)) self.assertFalse(barrier.broken) async def test_reset_barrier_when_tasks_half_draining_half_blocking(self): @@ -1254,8 +1263,10 @@ async def test_reset_barrier_when_tasks_half_draining_half_blocking(self): results1 = [] results2 = [] blocking_tasks = self.N//2 + count = 0 async def coro(): + nonlocal count try: await barrier.wait() except asyncio.BrokenBarrierError: @@ -1265,7 +1276,8 @@ async def coro(): # so now waiting again to reach nb_parties await barrier.wait() else: - if blocking_tasks == barrier._count_block: + count += 1 + if count > blocking_tasks: # reset now: raise asyncio.BrokenBarrierError for waiting tasks await barrier.reset() @@ -1283,7 +1295,7 @@ async def coro(): self.assertEqual(results1, [True]*blocking_tasks) self.assertEqual(results2, []) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._resetting()) + self.assertNotIn("resetting", repr(barrier)) self.assertFalse(barrier.broken) async def test_reset_barrier_while_tasks_waiting_and_waiting_again(self): @@ -1383,15 +1395,18 @@ async def test_abort_barrier_when_tasks_half_draining_half_blocking(self): results1 = [] results2 = [] blocking_tasks = self.N//2 + count = 0 async def coro(): + nonlocal count try: await barrier.wait() except asyncio.BrokenBarrierError: # here catch tasks waiting to drain results1.append(True) else: - if blocking_tasks == barrier._count_block: + count += 1 + if count > blocking_tasks: # abort now: raise asyncio.BrokenBarrierError for all tasks await barrier.abort() else: @@ -1407,7 +1422,7 @@ async def coro(): self.assertEqual(results1, [True]*blocking_tasks) self.assertEqual(results2, [True]*(self.N-blocking_tasks-1)) self.assertEqual(barrier.n_waiting, 0) - self.assertFalse(barrier._resetting()) + self.assertNotIn("resetting", repr(barrier)) async def test_abort_barrier_when_exception(self): # test from threading.Barrier: see `lock_tests.test_reset` From f1acd93a63969e227795edc402a8c42441ecf61e Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 24 Mar 2022 20:08:04 +0200 Subject: [PATCH 87/93] Inline method --- Lib/asyncio/locks.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index 23db2968d4279a..c4ba275a3fb9c1 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -546,13 +546,8 @@ async def abort(self): attempting to 'wait()' will have BrokenBarrierError raised. """ async with self._cond: - self._break() - - def _break(self): - # An internal error was detected. The barrier is set to - # a broken state all parties awakened. - self._state = _BarrierState.BROKEN - self._cond.notify_all() + self._state = _BarrierState.BROKEN + self._cond.notify_all() @property def parties(self): From 1bb32971074705d2e529155096c230b198b9c52c Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Thu, 24 Mar 2022 20:18:36 +0200 Subject: [PATCH 88/93] Move BrokenBarrierError to exceptions.py --- Lib/asyncio/exceptions.py | 7 ++++++- Lib/asyncio/locks.py | 11 +++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Lib/asyncio/exceptions.py b/Lib/asyncio/exceptions.py index c764c9ffcfc199..5ece595aad6475 100644 --- a/Lib/asyncio/exceptions.py +++ b/Lib/asyncio/exceptions.py @@ -1,7 +1,8 @@ """asyncio exceptions.""" -__all__ = ('CancelledError', 'InvalidStateError', 'TimeoutError', +__all__ = ('BrokenBarrierError', + 'CancelledError', 'InvalidStateError', 'TimeoutError', 'IncompleteReadError', 'LimitOverrunError', 'SendfileNotAvailableError') @@ -55,3 +56,7 @@ def __init__(self, message, consumed): def __reduce__(self): return type(self), (self.args[0], self.consumed) + + +class BrokenBarrierError(RuntimeError): + """Barrier is broken by barrier.abort() call.""" diff --git a/Lib/asyncio/locks.py b/Lib/asyncio/locks.py index c4ba275a3fb9c1..a18b109c726a88 100644 --- a/Lib/asyncio/locks.py +++ b/Lib/asyncio/locks.py @@ -1,7 +1,7 @@ """Synchronization primitives.""" __all__ = ('Lock', 'Event', 'Condition', 'Semaphore', - 'BoundedSemaphore', 'Barrier', 'BrokenBarrierError') + 'BoundedSemaphore', 'Barrier') import collections import enum @@ -495,7 +495,7 @@ async def _block(self): # see if the barrier is in a broken state if self._state is _BarrierState.BROKEN: - raise BrokenBarrierError("Barrier aborted") + raise exceptions.BrokenBarrierError("Barrier aborted") async def _release(self): # Release the tasks waiting in the barrier. @@ -514,7 +514,7 @@ async def _wait(self): await self._cond.wait_for(lambda: self._state is not _BarrierState.FILLING) if self._state in (_BarrierState.BROKEN, _BarrierState.RESETTING): - raise BrokenBarrierError("Abort or reset of barrier") + raise exceptions.BrokenBarrierError("Abort or reset of barrier") def _exit(self): # If we are the last tasks to exit the barrier, signal any tasks @@ -565,8 +565,3 @@ def n_waiting(self): def broken(self): """Return True if the barrier is in a broken state.""" return self._state is _BarrierState.BROKEN - - -# exception raised by the Barrier class -class BrokenBarrierError(RuntimeError): - pass From d82f1a52e685d3f203692b60b1bcc976fa40c6a4 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 25 Mar 2022 13:03:33 +0100 Subject: [PATCH 89/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Update sample with new `__repr__` --- Doc/library/asyncio-sync.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 60f6331d13bfd9..6e15a122dd551e 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -374,6 +374,7 @@ Barrier # Now, current task waits async with b as pos: + print(b) print("barrier passed") await asyncio.sleep(0) @@ -383,9 +384,10 @@ Barrier Result of this example is:: - + + barrier passed - + The example also demonstrates using ``async with`` as an alternative to awaiting on ``barrier.wait()``. From f89dc5f431f3ffb20f01c0fc7be9a1d534fb2226 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 25 Mar 2022 13:03:51 +0100 Subject: [PATCH 90/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Missing a newline --- Lib/test/test_asyncio/test_locks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 7cc2080fbc5eb9..4f47b30ef01341 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -1004,6 +1004,7 @@ async def test_barrier_parties(self): self.assertRaises(ValueError, lambda: asyncio.Barrier(-4)) self.assertIsInstance(asyncio.Barrier(self.N), asyncio.Barrier) + async def test_context_manager(self): self.N = 3 barrier = asyncio.Barrier(self.N) From 896e27412fed83428d6ddbb5d43b67ba18d55f3f Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 25 Mar 2022 16:13:37 +0200 Subject: [PATCH 91/93] Tune docs --- Doc/library/asyncio-sync.rst | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 6e15a122dd551e..b7f472fd40a732 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -345,16 +345,19 @@ BoundedSemaphore Barrier ======= - .. versionadded:: 3.11 +.. class:: Barrier(parties, action=None) A barrier object. Not thread-safe. - A barrier is a simple synchronization primitive that allows to block until a - certain number of tasks are waiting on it. + A barrier is a simple synchronization primitive that allows to block until + *parties* number of tasks are waiting on it. Tasks can wait on the :meth:`~Barrier.wait` method and would be blocked until the specified number of tasks end up waiting on :meth:`~Barrier.wait`. At that point all of the waiting tasks would unblock simultaneously. + :keyword:`async with` can be used as an alternative to awaiting on + :meth:`~Barrier.wait`. + The barrier can be reused any number of times. .. _asyncio_example_barrier: @@ -363,7 +366,7 @@ Barrier async def example_barrier(): # barrier with 3 parties - b = asyncio.Barrier(3, done) + b = asyncio.Barrier(3) # create 2 new waiting tasks asyncio.create_task(b.wait()) @@ -372,10 +375,10 @@ Barrier await asyncio.sleep(0) print(b) - # Now, current task waits - async with b as pos: - print(b) - print("barrier passed") + # The third .wait() call passes the barrier + awaut b.wait() + print(b) + print("barrier passed") await asyncio.sleep(0) print(b) @@ -384,17 +387,12 @@ Barrier Result of this example is:: - - + + barrier passed - + - The example also demonstrates using ``async with`` as an alternative to awaiting - on ``barrier.wait()``. - -.. class:: Barrier(parties, action=None) - - Create a barrier object for *parties* number of tasks. + .. versionadded:: 3.11 .. coroutinemethod:: wait() From b7bdf402528ee0410d7bb647531be49b2c63a812 Mon Sep 17 00:00:00 2001 From: Duprat Date: Fri, 25 Mar 2022 17:01:10 +0100 Subject: [PATCH 92/93] bpo-43352: Add a Barrier object to asyncio synchronized primitives (GH-24903) Minor correction in example --- Doc/library/asyncio-sync.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index b7f472fd40a732..141733ee2c8001 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -376,7 +376,7 @@ Barrier print(b) # The third .wait() call passes the barrier - awaut b.wait() + await b.wait() print(b) print("barrier passed") From 3c3029003f04fa292a238818ae0ddf3f057054b9 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 25 Mar 2022 19:16:11 +0200 Subject: [PATCH 93/93] Bump --- Lib/test/test_asyncio/test_locks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_asyncio/test_locks.py b/Lib/test/test_asyncio/test_locks.py index 4f47b30ef01341..6204a0fc4aea25 100644 --- a/Lib/test/test_asyncio/test_locks.py +++ b/Lib/test/test_asyncio/test_locks.py @@ -951,7 +951,6 @@ async def test_repr(self): barrier = asyncio.Barrier(self.N) self.assertTrue(RGX_REPR.match(repr(barrier))) - self.assertIn("filling", repr(barrier)) waiters = []